You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2012/01/25 21:33:25 UTC
svn commit: r1235920 - in /incubator/accumulo/trunk: ./ src/core/
src/core/src/main/java/org/apache/accumulo/core/client/ src/server/
src/server/src/main/java/org/apache/accumulo/server/logger/
src/server/src/main/java/org/apache/accumulo/server/test/r...
Author: kturner
Date: Wed Jan 25 20:33:25 2012
New Revision: 1235920
URL: http://svn.apache.org/viewvc?rev=1235920&view=rev
Log:
ACCUMULO-334 merged from 1.4
Modified:
incubator/accumulo/trunk/ (props changed)
incubator/accumulo/trunk/src/core/ (props changed)
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
incubator/accumulo/trunk/src/server/ (props changed)
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/BulkPlusOne.java
Propchange: incubator/accumulo/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jan 25 20:33:25 2012
@@ -1,3 +1,3 @@
/incubator/accumulo/branches/1.3:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611,1228195,1230180,1230736,1231043
/incubator/accumulo/branches/1.3.5rc:1209938
-/incubator/accumulo/branches/1.4:1201902-1235526
+/incubator/accumulo/branches/1.4:1201902-1235917
Propchange: incubator/accumulo/trunk/src/core/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jan 25 20:33:25 2012
@@ -1,3 +1,3 @@
-/incubator/accumulo/branches/1.3/src/core:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215
/incubator/accumulo/branches/1.3.5rc/src/core:1209938
-/incubator/accumulo/branches/1.4/src/core:1201902-1235526
+/incubator/accumulo/branches/1.3/src/core:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215
+/incubator/accumulo/branches/1.4/src/core:1201902-1235917
Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java?rev=1235920&r1=1235919&r2=1235920&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java (original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java Wed Jan 25 20:33:25 2012
@@ -16,7 +16,6 @@
*/
package org.apache.accumulo.core.client;
-import java.util.EnumSet;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
@@ -24,7 +23,6 @@ import java.util.Set;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.util.ArgumentChecker;
import org.apache.accumulo.core.util.Pair;
@@ -128,7 +126,7 @@ public class IteratorSetting {
this.properties.clear();
addOptions(properties);
}
-
+
/**
* @return <tt>true</tt> if this iterator has configuration parameters.
*/
@@ -159,8 +157,6 @@ public class IteratorSetting {
* the distinguishing name for the iterator
* @param iteratorClass
* the fully qualified class name for the iterator
- * @param scopes
- * the scopes of the iterator
* @param properties
* any properties for the iterator
*/
@@ -186,6 +182,7 @@ public class IteratorSetting {
}
/**
+
* Constructs an iterator setting using the given class's SimpleName for the iterator name and configured for the specified scopes with the specified
* parameters.
*
@@ -193,8 +190,6 @@ public class IteratorSetting {
* the priority for the iterator @see {@link #setPriority(int)}
* @param iteratorClass
* the class for the iterator
- * @param scopes
- * the scopes of the iterator
* @param properties
* any properties for the iterator
*/
@@ -218,25 +213,6 @@ public class IteratorSetting {
}
/**
- * Constructs an iterator setting configured for the specified scopes with the specified parameters.
- *
- * @param priority
- * the priority for the iterator @see {@link #setPriority(int)}
- * @param name
- * the distinguishing name for the iterator
- * @param iteratorClass
- * the class for the iterator
- * @param scopes
- * the scopes of the iterator
- * @param properties
- * any properties for the iterator
- */
- public IteratorSetting(int priority, String name, Class<? extends SortedKeyValueIterator<Key,Value>> iteratorClass, EnumSet<IteratorScope> scopes,
- Map<String,String> properties) {
- this(priority, name, iteratorClass.getName(), properties);
- }
-
- /**
* Add another option to the iterator.
*
* @param option
@@ -308,7 +284,7 @@ public class IteratorSetting {
sb.append(properties);
return sb.toString();
}
-
+
/**
* A convenience class for passing column family and column qualifiers to iterator configuration methods.
*/
@@ -334,7 +310,7 @@ public class IteratorSetting {
return getFirst();
}
- public Text getColumnQualifierf() {
+ public Text getColumnQualifier() {
return getSecond();
}
Propchange: incubator/accumulo/trunk/src/server/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jan 25 20:33:25 2012
@@ -1,3 +1,3 @@
-/incubator/accumulo/branches/1.3/src/server:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611
/incubator/accumulo/branches/1.3.5rc/src/server:1209938
-/incubator/accumulo/branches/1.4/src/server:1201902-1235526
+/incubator/accumulo/branches/1.3/src/server:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611
+/incubator/accumulo/branches/1.4/src/server:1201902-1235917
Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java?rev=1235920&r1=1235919&r2=1235920&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java (original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java Wed Jan 25 20:33:25 2012
@@ -18,7 +18,12 @@ package org.apache.accumulo.server.logge
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.file.FileUtil;
import org.apache.accumulo.core.util.CachedConfiguration;
@@ -38,7 +43,7 @@ import org.apache.hadoop.io.Text;
public class LogReader {
public static void usage() {
- System.err.println("Usage : " + LogReader.class.getName() + " [-r <row>] [-m <maxColumns] <log file>");
+ System.err.println("Usage : " + LogReader.class.getName() + " [-r <row>] [-m <maxColumns] [-t <key extent>] [-p <row regex>] <log file>");
}
/**
@@ -55,9 +60,13 @@ public class LogReader {
FileSystem local = TraceFileSystem.wrap(FileSystem.getLocal(conf));
Option rowOpt = new Option("r", "--row", true, "search for a specific row");
Option maxOpt = new Option("m", "--max-mutations", true, "the maximum number of mutations to print per log entry");
+ Option tabletOpt = new Option("t", "--tablet", true, "key extent");
+ Option rowPatternOpt = new Option("p", "--row-pattern", true, "search for a row that matches the given regex");
Options options = new Options();
options.addOption(rowOpt);
options.addOption(maxOpt);
+ options.addOption(tabletOpt);
+ options.addOption(rowPatternOpt);
CommandLine cl;
try {
cl = new BasicParser().parse(options, args);
@@ -66,6 +75,8 @@ public class LogReader {
return;
}
+ Matcher rowMatcher = null;
+ KeyExtent ke = null;
Text row = null;
int max = 5;
String[] files = cl.getArgs();
@@ -77,7 +88,18 @@ public class LogReader {
row = new Text(cl.getOptionValue(rowOpt.getOpt()));
if (cl.hasOption(maxOpt.getOpt()))
max = Integer.parseInt(cl.getOptionValue(maxOpt.getOpt()));
+ if (cl.hasOption(tabletOpt.getOpt())) {
+ String extent = cl.getOptionValue(tabletOpt.getOpt());
+ String sa[] = extent.split(";");
+ ke = new KeyExtent(new Text(sa[0]), new Text(sa[1]), new Text(sa[2]));
+ }
+ if (cl.hasOption(rowPatternOpt.getOpt())) {
+ Pattern pattern = Pattern.compile(cl.getOptionValue(rowPatternOpt.getOpt()));
+ rowMatcher = pattern.matcher("");
+ }
+ Set<Integer> tabletIds = new HashSet<Integer>();
+
for (String file : files) {
Path path = new Path(file);
@@ -88,40 +110,61 @@ public class LogReader {
// read log entries from a simple hdfs file
org.apache.hadoop.io.SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(file), conf);
while (reader.next(key, value)) {
- printLogEvent(key, value, row, max);
+ printLogEvent(key, value, row, rowMatcher, ke, tabletIds, max);
}
} else if (local.isFile(path)) {
// read log entries from a simple file
org.apache.hadoop.io.SequenceFile.Reader reader = new SequenceFile.Reader(local, new Path(file), conf);
while (reader.next(key, value)) {
- printLogEvent(key, value, row, max);
+ printLogEvent(key, value, row, rowMatcher, ke, tabletIds, max);
}
} else {
try {
// read the log entries sorted in a map file
MultiReader input = new MultiReader(fs, conf, file);
while (input.next(key, value)) {
- printLogEvent(key, value, row, max);
+ printLogEvent(key, value, row, rowMatcher, ke, tabletIds, max);
}
} catch (FileNotFoundException ex) {
SequenceFile.Reader input = new SequenceFile.Reader(local, new Path(file), conf);
while (input.next(key, value)) {
- printLogEvent(key, value, row, max);
+ printLogEvent(key, value, row, rowMatcher, ke, tabletIds, max);
}
}
}
}
}
- public static void printLogEvent(LogFileKey key, LogFileValue value, Text row, int maxMutations) {
- if (row != null) {
+ public static void printLogEvent(LogFileKey key, LogFileValue value, Text row, Matcher rowMatcher, KeyExtent ke, Set<Integer> tabletIds, int maxMutations) {
+
+ if (ke != null) {
+ if (key.event == LogEvents.DEFINE_TABLET) {
+ if (key.tablet.equals(ke)) {
+ tabletIds.add(key.tid);
+ } else {
+ return;
+ }
+ } else if (!tabletIds.contains(key.tid)) {
+ return;
+ }
+ }
+
+ if (row != null || rowMatcher != null) {
if (key.event == LogEvents.MUTATION || key.event == LogEvents.MANY_MUTATIONS) {
boolean found = false;
for (Mutation m : value.mutations) {
- if (new Text(m.getRow()).equals(row)) {
+ if (row != null && new Text(m.getRow()).equals(row)) {
found = true;
break;
}
+
+ if (rowMatcher != null) {
+ rowMatcher.reset(new String(m.getRow()));
+ if (rowMatcher.matches()) {
+ found = true;
+ break;
+ }
+ }
}
if (!found)
Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/BulkPlusOne.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/BulkPlusOne.java?rev=1235920&r1=1235919&r2=1235920&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/BulkPlusOne.java (original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/BulkPlusOne.java Wed Jan 25 20:33:25 2012
@@ -79,7 +79,7 @@ public class BulkPlusOne extends BulkTes
List<Integer> rows = new ArrayList<Integer>(startRows);
rows.add(LOTS);
-
+
for (int i = 0; i < parts; i++) {
String fileName = dir + "/" + String.format("part_%d.", i) + RFile.EXTENSION;
FileSKVWriter f = FileOperations.getInstance().openWriter(fileName, fs, fs.getConf(), defaultConfiguration);
@@ -89,7 +89,7 @@ public class BulkPlusOne extends BulkTes
for (int j = start; j < end; j++) {
Text row = new Text(String.format(FMT, j));
for (Column col : COLNAMES) {
- f.append(new Key(row, col.getColumnFamily(), col.getColumnQualifierf()), value);
+ f.append(new Key(row, col.getColumnFamily(), col.getColumnQualifier()), value);
}
f.append(new Key(row, MARKER_CF, new Text(markerColumnFamily)), value);
}
@@ -101,6 +101,7 @@ public class BulkPlusOne extends BulkTes
FileStatus[] failures = fs.listStatus(fail);
if (failures != null && failures.length > 0)
throw new Exception("Failures " + Arrays.asList(failures) + " found importing files from " + dir);
+ log.debug("Finished bulk import, start rows " + printRows + " last row " + String.format(FMT, LOTS - 1) + " marker " + markerColumnFamily);
}
@Override