You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by bi...@apache.org on 2012/01/30 19:36:35 UTC
svn commit: r1237918 - in /incubator/accumulo/trunk: ./ lib/ext/ src/core/
src/core/src/main/java/org/apache/accumulo/core/util/shell/
src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/aggregator/
src/examples/wikisea...
Author: billie
Date: Mon Jan 30 18:36:34 2012
New Revision: 1237918
URL: http://svn.apache.org/viewvc?rev=1237918&view=rev
Log:
ACCUMULO-354 replaced example aggregators with combiners - merged to trunk
Added:
incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/GlobalIndexUidCombiner.java
- copied unchanged from r1237899, incubator/accumulo/branches/1.4/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/GlobalIndexUidCombiner.java
incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/TextIndexCombiner.java
- copied unchanged from r1237899, incubator/accumulo/branches/1.4/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/TextIndexCombiner.java
incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/
- copied from r1237899, incubator/accumulo/branches/1.4/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/
incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/GlobalIndexUidTest.java
- copied unchanged from r1237899, incubator/accumulo/branches/1.4/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/GlobalIndexUidTest.java
incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/TextIndexTest.java
- copied unchanged from r1237899, incubator/accumulo/branches/1.4/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/TextIndexTest.java
Removed:
incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/aggregator/
incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/TotalAggregatingIterator.java
incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/aggregator/
Modified:
incubator/accumulo/trunk/ (props changed)
incubator/accumulo/trunk/lib/ext/ (props changed)
incubator/accumulo/trunk/pom.xml
incubator/accumulo/trunk/src/core/ (props changed)
incubator/accumulo/trunk/src/core/pom.xml
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java
incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/util/TextUtil.java
incubator/accumulo/trunk/src/examples/wikisearch/query-war/src/main/webapp/ui.jsp
incubator/accumulo/trunk/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/query/Query.java
incubator/accumulo/trunk/src/examples/wikisearch/query/src/main/resources/META-INF/ (props changed)
incubator/accumulo/trunk/src/server/ (props changed)
Propchange: incubator/accumulo/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Jan 30 18:36:34 2012
@@ -1,3 +1,3 @@
/incubator/accumulo/branches/1.3:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611,1228195,1230180,1230736,1231043,1236873
/incubator/accumulo/branches/1.3.5rc:1209938
-/incubator/accumulo/branches/1.4:1201902-1237856
+/incubator/accumulo/branches/1.4:1201902-1237899
Propchange: incubator/accumulo/trunk/lib/ext/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Mon Jan 30 18:36:34 2012
@@ -0,0 +1 @@
+*.jar
Modified: incubator/accumulo/trunk/pom.xml
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/pom.xml?rev=1237918&r1=1237917&r2=1237918&view=diff
==============================================================================
--- incubator/accumulo/trunk/pom.xml (original)
+++ incubator/accumulo/trunk/pom.xml Mon Jan 30 18:36:34 2012
@@ -522,6 +522,11 @@
<artifactId>commons-jci-fam</artifactId>
<version>1.0</version>
</dependency>
+ <dependency>
+ <groupId>org.python</groupId>
+ <artifactId>jython</artifactId>
+ <version>2.5.0</version>
+ </dependency>
</dependencies>
</dependencyManagement>
Propchange: incubator/accumulo/trunk/src/core/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Jan 30 18:36:34 2012
@@ -1,3 +1,3 @@
-/incubator/accumulo/branches/1.3.5rc/src/core:1209938
/incubator/accumulo/branches/1.3/src/core:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215
-/incubator/accumulo/branches/1.4/src/core:1201902-1237856
+/incubator/accumulo/branches/1.3.5rc/src/core:1209938
+/incubator/accumulo/branches/1.4/src/core:1201902-1237899
Modified: incubator/accumulo/trunk/src/core/pom.xml
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/pom.xml?rev=1237918&r1=1237917&r2=1237918&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/pom.xml (original)
+++ incubator/accumulo/trunk/src/core/pom.xml Mon Jan 30 18:36:34 2012
@@ -111,6 +111,10 @@
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.python</groupId>
+ <artifactId>jython</artifactId>
+ </dependency>
</dependencies>
</project>
Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java?rev=1237918&r1=1237917&r2=1237918&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java (original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java Mon Jan 30 18:36:34 2012
@@ -171,7 +171,7 @@ public class Shell {
private Token rootToken;
public final Map<String,Command> commandFactory = new TreeMap<String,Command>();
- private boolean configError = false;
+ boolean configError = false;
// Global options flags
public static final String userOption = "u";
@@ -193,6 +193,14 @@ public class Shell {
private long authTimeout;
private long lastUserActivity = System.currentTimeMillis();
+ public Shell() throws IOException {
+ this.reader = new ConsoleReader();
+ }
+
+ public Shell(ConsoleReader reader) {
+ this.reader = reader;
+ }
+
@SuppressWarnings("deprecation")
// Not for client use
public void config(String... args) {
@@ -312,7 +320,6 @@ public class Shell {
DistributedTrace.enable(instance, new ZooReader(instance), "shell", InetAddress.getLocalHost().getHostName());
}
- this.reader = new ConsoleReader();
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void start() {
@@ -409,8 +416,6 @@ public class Shell {
log.warn("Unable to load history file at " + historyPath);
}
- ShellCompletor userCompletor = null;
-
if (execFile != null) {
java.util.Scanner scanner = new java.util.Scanner(new File(execFile));
while (scanner.hasNextLine())
@@ -446,6 +451,23 @@ public class Shell {
}
}
+ ShellCompletor userCompletor = null;
+
+ public boolean execCommand(String line) {
+ try {
+ System.out.println("trying to execute line in shell: " + line);
+ execCommand(line, disableAuthTimeout, false);
+ } catch (IOException e) {
+ System.out.println("got IOException");
+ return false;
+ }
+ if (exitCode > 0) {
+ System.out.println("got exit code " + exitCode);
+ return false;
+ }
+ return true;
+ }
+
public void printInfo() throws IOException {
reader.printString("\n" + SHELL_DESCRIPTION + "\n" + "- \n" + "- version: " + Constants.VERSION + "\n" + "- instance name: "
+ connector.getInstance().getInstanceName() + "\n" + "- instance id: " + connector.getInstance().getInstanceID() + "\n" + "- \n"
@@ -487,7 +509,7 @@ public class Shell {
reader.printString(sb.toString());
}
- private String getDefaultPrompt() {
+ String getDefaultPrompt() {
return connector.whoami() + "@" + connector.getInstance().getInstanceName() + (getTableName().isEmpty() ? "" : " ") + getTableName() + "> ";
}
Modified: incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java?rev=1237918&r1=1237917&r2=1237918&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java (original)
+++ incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java Mon Jan 30 18:36:34 2012
@@ -19,6 +19,7 @@ package org.apache.accumulo.examples.wik
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.EnumSet;
import java.util.List;
import java.util.Set;
import java.util.SortedSet;
@@ -29,15 +30,17 @@ import java.util.regex.Pattern;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.IteratorSetting.Column;
import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.admin.TableOperations;
import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat;
-import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
-import org.apache.accumulo.core.iterators.aggregation.NumSummation;
-import org.apache.accumulo.core.iterators.aggregation.conf.AggregatorConfiguration;
+import org.apache.accumulo.core.iterators.user.SummingCombiner;
+import org.apache.accumulo.examples.wikisearch.iterator.GlobalIndexUidCombiner;
+import org.apache.accumulo.examples.wikisearch.iterator.TextIndexCombiner;
import org.apache.accumulo.examples.wikisearch.reader.AggregatingRecordReader;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
@@ -52,8 +55,6 @@ import org.apache.hadoop.mapreduce.lib.i
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
-
-@SuppressWarnings("deprecation")
public class WikipediaIngester extends Configured implements Tool {
public final static String INGEST_LANGUAGE = "wikipedia.ingest_language";
@@ -74,23 +75,21 @@ public class WikipediaIngester extends C
// create the shard table
if (!tops.exists(tableName)) {
- // Set a text index aggregator on the given field names. No aggregator is set if the option is not supplied
+ // Set a text index combiner on the given field names. No combiner is set if the option is not supplied
String textIndexFamilies = WikipediaMapper.TOKENS_FIELD_NAME;
+ tops.create(tableName);
if (textIndexFamilies.length() > 0) {
- System.out.println("Adding content aggregator on the fields: " + textIndexFamilies);
-
- // Create and set the aggregators in one shot
- List<AggregatorConfiguration> aggregators = new ArrayList<AggregatorConfiguration>();
+ System.out.println("Adding content combiner on the fields: " + textIndexFamilies);
+ IteratorSetting setting = new IteratorSetting(10, TextIndexCombiner.class);
+ List<Column> columns = new ArrayList<Column>();
for (String family : StringUtils.split(textIndexFamilies, ',')) {
- aggregators.add(new AggregatorConfiguration(new Text("fi\0" + family), org.apache.accumulo.examples.wikisearch.aggregator.TextIndexAggregator.class.getName()));
+ columns.add(new Column("fi\0" + family));
}
+ TextIndexCombiner.setColumns(setting, columns);
- tops.create(tableName);
- tops.addAggregators(tableName, aggregators);
- } else {
- tops.create(tableName);
+ tops.attachIterator(tableName, setting, EnumSet.allOf(IteratorScope.class));
}
// Set the locality group for the full content column family
@@ -100,34 +99,27 @@ public class WikipediaIngester extends C
if (!tops.exists(indexTableName)) {
tops.create(indexTableName);
- // Add the UID aggregator
- for (IteratorScope scope : IteratorScope.values()) {
- String stem = String.format("%s%s.%s", Property.TABLE_ITERATOR_PREFIX, scope.name(), "UIDAggregator");
- tops.setProperty(indexTableName, stem, "19,org.apache.accumulo.examples.wikisearch.iterator.TotalAggregatingIterator");
- stem += ".opt.";
- tops.setProperty(indexTableName, stem + "*", "org.apache.accumulo.examples.wikisearch.aggregator.GlobalIndexUidAggregator");
-
- }
+ // Add the UID combiner
+ IteratorSetting setting = new IteratorSetting(19, "UIDAggregator", GlobalIndexUidCombiner.class);
+ GlobalIndexUidCombiner.setCombineAllColumns(setting, true);
+ tops.attachIterator(indexTableName, setting, EnumSet.allOf(IteratorScope.class));
}
if (!tops.exists(reverseIndexTableName)) {
tops.create(reverseIndexTableName);
- // Add the UID aggregator
- for (IteratorScope scope : IteratorScope.values()) {
- String stem = String.format("%s%s.%s", Property.TABLE_ITERATOR_PREFIX, scope.name(), "UIDAggregator");
- tops.setProperty(reverseIndexTableName, stem, "19,org.apache.accumulo.examples.wikisearch.iterator.TotalAggregatingIterator");
- stem += ".opt.";
- tops.setProperty(reverseIndexTableName, stem + "*", "org.apache.accumulo.examples.wikisearch.aggregator.GlobalIndexUidAggregator");
-
- }
+ // Add the UID combiner
+ IteratorSetting setting = new IteratorSetting(19, "UIDAggregator", GlobalIndexUidCombiner.class);
+ GlobalIndexUidCombiner.setCombineAllColumns(setting, true);
+ tops.attachIterator(reverseIndexTableName, setting, EnumSet.allOf(IteratorScope.class));
}
if (!tops.exists(metadataTableName)) {
- // Add the NumSummation aggregator for the frequency column
- List<AggregatorConfiguration> aggregators = new ArrayList<AggregatorConfiguration>();
- aggregators.add(new AggregatorConfiguration(new Text("f"), NumSummation.class.getName()));
+ // Add the SummingCombiner with VARLEN encoding for the frequency column
tops.create(metadataTableName);
- tops.addAggregators(metadataTableName, aggregators);
+ IteratorSetting setting = new IteratorSetting(10, SummingCombiner.class);
+ SummingCombiner.setColumns(setting, Collections.singletonList(new Column("f")));
+ SummingCombiner.setEncodingType(setting, SummingCombiner.Type.VARLEN);
+ tops.attachIterator(metadataTableName, setting, EnumSet.allOf(IteratorScope.class));
}
}
@@ -136,7 +128,7 @@ public class WikipediaIngester extends C
Job job = new Job(getConf(), "Ingest Wikipedia");
Configuration conf = job.getConfiguration();
conf.set("mapred.map.tasks.speculative.execution", "false");
-
+
String tablename = WikipediaConfiguration.getTableName(conf);
String zookeepers = WikipediaConfiguration.getZookeepers(conf);
Modified: incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/util/TextUtil.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/util/TextUtil.java?rev=1237918&r1=1237917&r2=1237918&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/util/TextUtil.java (original)
+++ incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/util/TextUtil.java Mon Jan 30 18:36:34 2012
@@ -18,8 +18,9 @@ package org.apache.accumulo.examples.wik
import java.nio.ByteBuffer;
import java.nio.charset.CharacterCodingException;
+
+import org.apache.accumulo.core.iterators.user.SummingCombiner;
import org.apache.hadoop.io.Text;
-import org.apache.accumulo.core.iterators.aggregation.LongSummation;
public class TextUtil {
@@ -43,7 +44,7 @@ public class TextUtil {
public static void textAppend(Text t, long s) {
t.append(nullByte, 0, 1);
- t.append(LongSummation.longToBytes(s), 0, 8);
+ t.append(SummingCombiner.FIXED_LEN_ENCODER.encode(s), 0, 8);
}
private static final byte[] nullByte = {0};
Modified: incubator/accumulo/trunk/src/examples/wikisearch/query-war/src/main/webapp/ui.jsp
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/examples/wikisearch/query-war/src/main/webapp/ui.jsp?rev=1237918&r1=1237917&r2=1237918&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/examples/wikisearch/query-war/src/main/webapp/ui.jsp (original)
+++ incubator/accumulo/trunk/src/examples/wikisearch/query-war/src/main/webapp/ui.jsp Mon Jan 30 18:36:34 2012
@@ -62,6 +62,8 @@
<p>The search syntax is boolean logic, for example: TEXT == 'boy' and TITLE =~ 'Autism'. The supported operators are:
==, !=, <, >, ≤, ≥, =~, and !~. Likewise grouping can be performed using parentheses and predicates can be
joined using and, or, and not.
+ <p>To highlight the cell-level access control of Apache Accumulo, the "authorization" required for a particular cell is the language
+ of the associated wikipedia article.
</div>
<div id="d">
<form id="FORM" name="queryForm" method="get" target="results" onsubmit="return setAction()">
@@ -73,6 +75,8 @@
</div>
<br />
<div class="center_input">
+ <label>Authorizations: </label>
+ <br />
<label>All</label><input type="checkbox" name="auths" value="all" />
</div>
<div class="center_input">
Modified: incubator/accumulo/trunk/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/query/Query.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/query/Query.java?rev=1237918&r1=1237917&r2=1237918&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/query/Query.java (original)
+++ incubator/accumulo/trunk/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/query/Query.java Mon Jan 30 18:36:34 2012
@@ -40,7 +40,6 @@ import javax.xml.transform.TransformerFa
import javax.xml.transform.stream.StreamResult;
import javax.xml.transform.stream.StreamSource;
-
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.ZooKeeperInstance;
@@ -49,7 +48,6 @@ import org.apache.accumulo.examples.wiki
import org.apache.accumulo.examples.wikisearch.sample.Results;
import org.apache.log4j.Logger;
-
@Stateless
@Local(IQuery.class)
public class Query implements IQuery {
@@ -192,12 +190,9 @@ public class Query implements IQuery {
// Create list of auths
List<String> authorizations = new ArrayList<String>();
- if (auths == null || "".equals(auths)) {
- authorizations.add("all");
- } else {
+ if (auths != null && auths.length() > 0)
for (String a : auths.split(","))
authorizations.add(a);
- }
ContentLogic table = new ContentLogic();
table.setTableName(tableName);
return table.runQuery(connector, query, authorizations);
@@ -229,8 +224,9 @@ public class Query implements IQuery {
// Create list of auths
List<String> authorizations = new ArrayList<String>();
- for (String a : auths.split(","))
- authorizations.add(a);
+ if (auths != null && auths.length() > 0)
+ for (String a : auths.split(","))
+ authorizations.add(a);
QueryLogic table = new QueryLogic();
table.setTableName(tableName);
Propchange: incubator/accumulo/trunk/src/examples/wikisearch/query/src/main/resources/META-INF/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Mon Jan 30 18:36:34 2012
@@ -0,0 +1 @@
+ejb-jar.xml
Propchange: incubator/accumulo/trunk/src/server/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Jan 30 18:36:34 2012
@@ -1,3 +1,3 @@
-/incubator/accumulo/branches/1.3.5rc/src/server:1209938
/incubator/accumulo/branches/1.3/src/server:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611
-/incubator/accumulo/branches/1.4/src/server:1201902-1237856
+/incubator/accumulo/branches/1.3.5rc/src/server:1209938
+/incubator/accumulo/branches/1.4/src/server:1201902-1237899