You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2012/01/31 20:21:03 UTC
svn commit: r1238754 - in /incubator/accumulo/trunk: ./ src/core/
src/core/src/main/java/org/apache/accumulo/core/iterators/
src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/
src/examples/wikisearch/ingest/src...
Author: ecn
Date: Tue Jan 31 19:21:02 2012
New Revision: 1238754
URL: http://svn.apache.org/viewvc?rev=1238754&view=rev
Log:
ACCUMULO-357, ACCUMULO-354, ACCUMULO-356, merge to trunk
Modified:
incubator/accumulo/trunk/ (props changed)
incubator/accumulo/trunk/src/core/ (props changed)
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java
incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/TextIndexTest.java
incubator/accumulo/trunk/src/server/ (props changed)
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/security/ZKAuthenticator.java
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/functional/BulkSplitOptimizationTest.java
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/security/TableOp.java
incubator/accumulo/trunk/test/system/auto/TestUtils.py
Propchange: incubator/accumulo/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 31 19:21:02 2012
@@ -1,3 +1,3 @@
/incubator/accumulo/branches/1.3:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611,1228195,1230180,1230736,1231043,1236873
/incubator/accumulo/branches/1.3.5rc:1209938
-/incubator/accumulo/branches/1.4:1201902-1237930
+/incubator/accumulo/branches/1.4:1201902-1238753
Propchange: incubator/accumulo/trunk/src/core/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 31 19:21:02 2012
@@ -1,3 +1,3 @@
-/incubator/accumulo/branches/1.3.5rc/src/core:1209938
/incubator/accumulo/branches/1.3/src/core:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215
-/incubator/accumulo/branches/1.4/src/core:1201902-1237930
+/incubator/accumulo/branches/1.3.5rc/src/core:1209938
+/incubator/accumulo/branches/1.4/src/core:1201902-1238753
Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java?rev=1238754&r1=1238753&r2=1238754&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java (original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java Tue Jan 31 19:21:02 2012
@@ -40,7 +40,7 @@ public abstract class TypedValueCombiner
private boolean lossy = false;
protected static final String LOSSY = "lossy";
-
+
/**
* A Java Iterator that translates an Iterator<Value> to an Iterator<V> using the decode method of an Encoder.
*/
@@ -68,19 +68,19 @@ public abstract class TypedValueCombiner
}
V next = null;
+ boolean hasNext = false;
+
@Override
public boolean hasNext() {
- if (next != null)
+ if (hasNext)
return true;
-
- while (true)
- {
+
+ while (true) {
if (!source.hasNext())
return false;
- try
- {
+ try {
next = encoder.decode(source.next().get());
- return true;
+ return hasNext = true;
} catch (ValueFormatException vfe) {
if (!lossy)
throw vfe;
@@ -90,10 +90,11 @@ public abstract class TypedValueCombiner
@Override
public V next() {
- if (!hasNext())
+ if (!hasNext && !hasNext())
throw new NoSuchElementException();
V toRet = next;
next = null;
+ hasNext = false;
return toRet;
}
@@ -192,7 +193,7 @@ public abstract class TypedValueCombiner
super.init(source, options, env);
setLossyness(options);
}
-
+
private void setLossyness(Map<String,String> options) {
String loss = options.get(LOSSY);
if (loss == null)
@@ -214,7 +215,7 @@ public abstract class TypedValueCombiner
setLossyness(options);
return true;
}
-
+
/**
* A convenience method to set the "lossy" option on a TypedValueCombiner. If true, the combiner will ignore any values which fail to decode. Otherwise, the
* combiner will throw an error which will interrupt the action (and prevent potential data loss). False is the default behavior.
Modified: incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java?rev=1238754&r1=1238753&r2=1238754&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java (original)
+++ incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java Tue Jan 31 19:21:02 2012
@@ -88,6 +88,7 @@ public class WikipediaIngester extends C
columns.add(new Column("fi\0" + family));
}
TextIndexCombiner.setColumns(setting, columns);
+ TextIndexCombiner.setLossyness(setting, true);
tops.attachIterator(tableName, setting, EnumSet.allOf(IteratorScope.class));
}
@@ -102,6 +103,7 @@ public class WikipediaIngester extends C
// Add the UID combiner
IteratorSetting setting = new IteratorSetting(19, "UIDAggregator", GlobalIndexUidCombiner.class);
GlobalIndexUidCombiner.setCombineAllColumns(setting, true);
+ GlobalIndexUidCombiner.setLossyness(setting, true);
tops.attachIterator(indexTableName, setting, EnumSet.allOf(IteratorScope.class));
}
@@ -110,6 +112,7 @@ public class WikipediaIngester extends C
// Add the UID combiner
IteratorSetting setting = new IteratorSetting(19, "UIDAggregator", GlobalIndexUidCombiner.class);
GlobalIndexUidCombiner.setCombineAllColumns(setting, true);
+ GlobalIndexUidCombiner.setLossyness(setting, true);
tops.attachIterator(reverseIndexTableName, setting, EnumSet.allOf(IteratorScope.class));
}
Modified: incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/TextIndexTest.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/TextIndexTest.java?rev=1238754&r1=1238753&r2=1238754&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/TextIndexTest.java (original)
+++ incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/TextIndexTest.java Tue Jan 31 19:21:02 2012
@@ -139,4 +139,47 @@ public class TextIndexTest {
Assert.assertTrue(offsets.get(4) == 15);
Assert.assertTrue(offsets.get(5) == 19);
}
+
+ @Test
+ public void testEmptyValue() throws InvalidProtocolBufferException {
+ Builder builder = createBuilder();
+ builder.addWordOffset(13);
+ builder.addWordOffset(15);
+ builder.addWordOffset(19);
+ builder.setNormalizedTermFrequency(0.12f);
+
+ values.add(new Value("".getBytes()));
+ values.add(new Value(builder.build().toByteArray()));
+ values.add(new Value("".getBytes()));
+
+ builder = createBuilder();
+ builder.addWordOffset(1);
+ builder.addWordOffset(5);
+ builder.setNormalizedTermFrequency(0.1f);
+
+ values.add(new Value(builder.build().toByteArray()));
+ values.add(new Value("".getBytes()));
+
+ builder = createBuilder();
+ builder.addWordOffset(3);
+ builder.setNormalizedTermFrequency(0.05f);
+
+ values.add(new Value(builder.build().toByteArray()));
+ values.add(new Value("".getBytes()));
+
+ Value result = combiner.reduce(new Key(), values.iterator());
+
+ TermWeight.Info info = TermWeight.Info.parseFrom(result.get());
+
+ Assert.assertTrue(info.getNormalizedTermFrequency() == 0.27f);
+
+ List<Integer> offsets = info.getWordOffsetList();
+ Assert.assertTrue(offsets.size() == 6);
+ Assert.assertTrue(offsets.get(0) == 1);
+ Assert.assertTrue(offsets.get(1) == 3);
+ Assert.assertTrue(offsets.get(2) == 5);
+ Assert.assertTrue(offsets.get(3) == 13);
+ Assert.assertTrue(offsets.get(4) == 15);
+ Assert.assertTrue(offsets.get(5) == 19);
+ }
}
Propchange: incubator/accumulo/trunk/src/server/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 31 19:21:02 2012
@@ -1,3 +1,3 @@
-/incubator/accumulo/branches/1.3.5rc/src/server:1209938
/incubator/accumulo/branches/1.3/src/server:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611
-/incubator/accumulo/branches/1.4/src/server:1201902-1237930
+/incubator/accumulo/branches/1.3.5rc/src/server:1209938
+/incubator/accumulo/branches/1.4/src/server:1201902-1238753
Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1238754&r1=1238753&r2=1238754&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java (original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java Tue Jan 31 19:21:02 2012
@@ -1490,22 +1490,23 @@ public class Master implements LiveTServ
for (MergeStats stats : mergeStatsCache.values()) {
try {
MergeState update = stats.nextMergeState();
- if (update != stats.getMergeInfo().getState()) {
- if (update == MergeState.MERGING) {
- if (stats.verifyMergeConsistency(getConnector(), Master.this)) {
- try {
- if (stats.getMergeInfo().isDelete())
- deleteTablets(stats.getMergeInfo());
- else
- mergeMetadataRecords(stats.getMergeInfo());
- setMergeState(stats.getMergeInfo(), update = MergeState.COMPLETE);
- } catch (Exception ex) {
- log.error("Unable merge metadata table records", ex);
+ if (update == MergeState.MERGING) {
+ if (stats.verifyMergeConsistency(getConnector(), Master.this)) {
+ try {
+ if (stats.getMergeInfo().isDelete()) {
+ deleteTablets(stats.getMergeInfo());
+ } else {
+ mergeMetadataRecords(stats.getMergeInfo());
}
+ setMergeState(stats.getMergeInfo(), update = MergeState.COMPLETE);
+ } catch (Exception ex) {
+ log.error("Unable merge metadata table records", ex);
}
}
- if (update == MergeState.COMPLETE)
- update = MergeState.NONE;
+ }
+ if (update == MergeState.COMPLETE)
+ update = MergeState.NONE;
+ if (update != stats.getMergeInfo().getState()) {
setMergeState(stats.getMergeInfo(), update);
}
} catch (Exception ex) {
Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java?rev=1238754&r1=1238753&r2=1238754&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java (original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java Tue Jan 31 19:21:02 2012
@@ -90,9 +90,14 @@ public class MergeStats {
public MergeState nextMergeState() throws Exception {
MergeState state = info.getState();
+ log.info("Computing next merge state for " + this.info.getRange() + " which is presently " + state);
if (state == MergeState.STARTED) {
state = MergeState.SPLITTING;
}
+ if (total == 0) {
+ log.info("failed to see any tablets for this range, ignoring");
+ return state;
+ }
if (state == MergeState.SPLITTING) {
log.info(hosted + " are hosted, total " + total);
if (!info.isDelete() && total == 1) {
@@ -171,9 +176,11 @@ public class MergeStats {
break;
}
verify.update(tls.extent, tls.getState(master.onlineTabletServers()), tls.chopped);
+ // check that the prevRow matches the previous row
if (pr != null && (tls.extent.getPrevEndRow() == null || !tls.extent.getPrevEndRow().equals(pr)))
return false;
pr = tls.extent.getEndRow();
+ // stop when we've seen the tablet just beyond our range
if (tls.extent.getPrevEndRow() != null && extent.getEndRow() != null && tls.extent.getPrevEndRow().compareTo(extent.getEndRow()) > 0) {
break;
}
@@ -184,7 +191,8 @@ public class MergeStats {
public static void main(String[] args) throws Exception {
Instance instance = HdfsZooInstance.getInstance();
Map<String,String> tableIdMap = instance.getConnector(SecurityConstants.getSystemCredentials()).tableOperations().tableIdMap();
- for (String tableId : tableIdMap.keySet()) {
+ for (String table : tableIdMap.keySet()) {
+ String tableId = tableIdMap.get(table);
String path = ZooUtil.getRoot(instance.getInstanceID()) + Constants.ZTABLES + "/" + tableId.toString() + "/merge";
MergeInfo info = new MergeInfo();
if (ZooReaderWriter.getInstance().exists(path)) {
@@ -193,7 +201,7 @@ public class MergeStats {
in.reset(data, data.length);
info.readFields(in);
}
- System.out.println(String.format("%25s %10s %10s %s", tableIdMap.get(tableId), info.state, info.operation, info.range));
+ System.out.println(String.format("%25s %10s %10s %s", table, info.state, info.operation, info.range));
}
}
}
\ No newline at end of file
Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/security/ZKAuthenticator.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/security/ZKAuthenticator.java?rev=1238754&r1=1238753&r2=1238754&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/security/ZKAuthenticator.java (original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/security/ZKAuthenticator.java Tue Jan 31 19:21:02 2012
@@ -370,7 +370,10 @@ public final class ZKAuthenticator imple
if (Tool.convertSystemPermissions(perms).contains(permission))
return true;
zooCache.clear(ZKUserPath + "/" + user + ZKUserSysPerms);
- return Tool.convertSystemPermissions(zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms)).contains(permission);
+ perms = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
+ if (perms == null)
+ return false;
+ return Tool.convertSystemPermissions(perms).contains(permission);
}
throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST); // user doesn't exist
}
Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/functional/BulkSplitOptimizationTest.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/functional/BulkSplitOptimizationTest.java?rev=1238754&r1=1238753&r2=1238754&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/functional/BulkSplitOptimizationTest.java (original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/functional/BulkSplitOptimizationTest.java Tue Jan 31 19:21:02 2012
@@ -23,10 +23,11 @@ import java.util.Map;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.UtilWaitThread;
-import org.apache.accumulo.server.test.CreateRFiles;
+import org.apache.accumulo.server.test.CreateMapFiles;
import org.apache.accumulo.server.test.VerifyIngest;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
/**
* This test verifies that when a lot of files are bulk imported into a table with one tablet and then splits that not all map files go to the children tablets.
@@ -42,8 +43,8 @@ public class BulkSplitOptimizationTest e
@Override
public void cleanup() throws Exception {
FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
- fs.delete(new Path("/testrf"), true);
- fs.delete(new Path("/testrf_failures"), true);
+ fs.delete(new Path("/tmp/testmf"), true);
+ fs.delete(new Path("/tmp/testmf_failures"), true);
}
@Override
@@ -61,14 +62,14 @@ public class BulkSplitOptimizationTest e
public void run() throws Exception {
FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
- fs.delete(new Path("/testrf"), true);
+ fs.delete(new Path("/tmp/testmf"), true);
- CreateRFiles.main(new String[] {"testrf", "8", "0", "100000", "99"});
+ CreateMapFiles.main(new String[] {"tmp/testmf", "8", "0", "100000", "99"});
- bulkImport(fs, TABLE_NAME, "/testrf");
+ bulkImport(fs, TABLE_NAME, "/tmp/testmf");
checkSplits(TABLE_NAME, 0, 0);
- checkRFiles(TABLE_NAME, 1, 1, 100, 100);
+ checkMapFiles(TABLE_NAME, 1, 1, 100, 100);
// initiate splits
getConnector().tableOperations().setProperty(TABLE_NAME, Property.TABLE_SPLIT_THRESHOLD.getKey(), "100K");
@@ -85,6 +86,6 @@ public class BulkSplitOptimizationTest e
VerifyIngest.main(new String[] {"-timestamp", "1", "-size", "50", "-random", "56", "100000", "0", "1"});
// ensure each tablet does not have all map files
- checkRFiles(TABLE_NAME, 50, 100, 1, 4);
+ checkMapFiles(TABLE_NAME, 50, 100, 1, 4);
}
}
Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/security/TableOp.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/security/TableOp.java?rev=1238754&r1=1238753&r2=1238754&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/security/TableOp.java (original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/security/TableOp.java Tue Jan 31 19:21:02 2012
@@ -183,7 +183,9 @@ public class TableOp extends Test {
}
for (String s : SecurityHelper.getAuthsArray())
SecurityHelper.increaseAuthMap(state, s, 1);
-
+ fs.delete(dir, true);
+ fs.delete(fail, true);
+
if (!hasPerm)
throw new AccumuloException("Bulk Import succeeded when it should have failed: " + dir + " table " + tableName);
break;
Modified: incubator/accumulo/trunk/test/system/auto/TestUtils.py
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/test/system/auto/TestUtils.py?rev=1238754&r1=1238753&r2=1238754&view=diff
==============================================================================
--- incubator/accumulo/trunk/test/system/auto/TestUtils.py (original)
+++ incubator/accumulo/trunk/test/system/auto/TestUtils.py Tue Jan 31 19:21:02 2012
@@ -41,7 +41,7 @@ FUZZ=os.getpid() % 997
ACCUMULO_HOME = os.path.dirname(__file__)
ACCUMULO_HOME = os.path.join(ACCUMULO_HOME, *(os.path.pardir,)*3)
ACCUMULO_HOME = os.path.realpath(ACCUMULO_HOME)
-ACCUMULO_DIR = "/accumulo-" + ID
+ACCUMULO_DIR = "/user/" + os.getlogin() + "/accumulo-" + ID
SITE = "test-" + ID
WALOG = os.path.join(ACCUMULO_HOME, 'walogs', ID)