You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2012/01/10 18:37:48 UTC
svn commit: r1229653 - in /incubator/accumulo/branches/1.4: ./ conf/
src/assemble/ src/wikisearch/ingest/bin/
src/wikisearch/ingest/src/main/java/org/apache/accumulo/wikisearch/ingest/
src/wikisearch/query-war/src/main/webapp/ src/wikisearch/query/src/...
Author: ecn
Date: Tue Jan 10 17:37:48 2012
New Revision: 1229653
URL: http://svn.apache.org/viewvc?rev=1229653&view=rev
Log:
ACCUMULO-285 found lots of issues during 1.4 testing of wikisearch
Modified:
incubator/accumulo/branches/1.4/README
incubator/accumulo/branches/1.4/conf/accumulo-site.xml.example
incubator/accumulo/branches/1.4/src/assemble/dist.xml
incubator/accumulo/branches/1.4/src/wikisearch/ingest/bin/ingest.sh
incubator/accumulo/branches/1.4/src/wikisearch/ingest/src/main/java/org/apache/accumulo/wikisearch/ingest/WikipediaIngester.java
incubator/accumulo/branches/1.4/src/wikisearch/query-war/src/main/webapp/style.xsl
incubator/accumulo/branches/1.4/src/wikisearch/query-war/src/main/webapp/ui.jsp
incubator/accumulo/branches/1.4/src/wikisearch/query/src/assembly/dist.xml
incubator/accumulo/branches/1.4/src/wikisearch/query/src/test/java/org/apache/accumulo/wikisearch/logic/TestQueryLogic.java
Modified: incubator/accumulo/branches/1.4/README
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/README?rev=1229653&r1=1229652&r2=1229653&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/README (original)
+++ incubator/accumulo/branches/1.4/README Tue Jan 10 17:37:48 2012
@@ -8,7 +8,7 @@ However, if you only have source code, o
have maven configured to get Accumulo prerequisites from repositories. See
the pom.xml file for the necessary components.
-Run "mvn package && mvn assembly:single"
+Run "mvn package && mvn assembly:single -N"
If you are running on another Unix-like operating system (OSX, etc) then
you may wish to build the native libraries. They are not strictly necessary
@@ -19,7 +19,7 @@ but having them available suppresses a r
******************************************************************************
2. Deployment
-Copy the accumulo tar file produced by "mvn package && mvn assembly:single"
+Copy the accumulo tar file produced by "mvn package && mvn assembly:single -N"
from the target/ directory to the desired destination, then untar it (e.g.
tar xvzf accumulo-1.4.0-incubating-SNAPSHOT-dist.tar.gz).
@@ -61,6 +61,8 @@ These directories must be at the same lo
Note that zookeeper must be installed on every machine, but it should not be
run on every machine.
+Create the $ACCUMULO_LOG_DIR on every machine in the slaves file.
+
* Note that you will be specifying the Java heap space in accumulo-env.sh.
You should make sure that the total heap space used for the accumulo tserver,
logger and the hadoop datanode and tasktracker is less than the available
Modified: incubator/accumulo/branches/1.4/conf/accumulo-site.xml.example
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/conf/accumulo-site.xml.example?rev=1229653&r1=1229652&r2=1229653&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/conf/accumulo-site.xml.example (original)
+++ incubator/accumulo/branches/1.4/conf/accumulo-site.xml.example Tue Jan 10 17:37:48 2012
@@ -65,6 +65,19 @@
</property>
<property>
+ <name>trace.password</name>
+ <!--
+ change this to the root user's password, and/or change the user below
+ -->
+ <value>secret</value>
+ </property>
+
+ <property>
+ <name>trace.user</name>
+ <value>root</value>
+ </property>
+
+ <property>
<name>general.classpaths</name>
<value>
$ACCUMULO_HOME/src/server/target/classes/,
Modified: incubator/accumulo/branches/1.4/src/assemble/dist.xml
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/assemble/dist.xml?rev=1229653&r1=1229652&r2=1229653&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/assemble/dist.xml (original)
+++ incubator/accumulo/branches/1.4/src/assemble/dist.xml Tue Jan 10 17:37:48 2012
@@ -80,6 +80,7 @@
<fileSet>
<includes>
<include>[A-Z]*</include>
+ <include>pom.xml</include>
<include>contrib/**</include>
<include>docs/**</include>
<include>lib/**</include>
@@ -94,6 +95,7 @@
<!-- the following files ignored because they should not be in the tarball -->
<exclude>**/.*</exclude>
<exclude>**/test-*</exclude>
+ <exclude>**/wikipedia.xml</exclude>
<exclude>**/walkers.txt</exclude>
<exclude>**/ingesters.txt</exclude>
<exclude>**/continuous-env.sh</exclude>
Modified: incubator/accumulo/branches/1.4/src/wikisearch/ingest/bin/ingest.sh
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/wikisearch/ingest/bin/ingest.sh?rev=1229653&r1=1229652&r2=1229653&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/wikisearch/ingest/bin/ingest.sh (original)
+++ incubator/accumulo/branches/1.4/src/wikisearch/ingest/bin/ingest.sh Tue Jan 10 17:37:48 2012
@@ -66,9 +66,9 @@ LIBJARS=`echo $CLASSPATH | sed 's/:/,/g'
#
# Map/Reduce job
#
-JAR=$SCRIPT_DIR/../lib/accumulo-sample-ingest-1.5.0-incubating-SNAPSHOT.jar
+JAR=$SCRIPT_DIR/../lib/wikisearch-ingest-1.4.0-incubating-SNAPSHOT.jar
CONF=$SCRIPT_DIR/../conf/wikipedia.xml
HDFS_DATA_DIR=$1
export HADOOP_CLASSPATH=$CLASSPATH
-echo "hadoop jar $JAR ingest.WikipediaIngester -libjars $LIBJARS -conf $CONF -Dwikipedia.input=${HDFS_DATA_DIR}"
-hadoop jar $JAR ingest.WikipediaIngester -libjars $LIBJARS -conf $CONF -Dwikipedia.input=${HDFS_DATA_DIR}
+echo "hadoop jar $JAR org.apache.accumulo.wikisearch.ingest.WikipediaIngester -libjars $LIBJARS -conf $CONF -Dwikipedia.input=${HDFS_DATA_DIR}"
+hadoop jar $JAR org.apache.accumulo.wikisearch.ingest.WikipediaIngester -libjars $LIBJARS -conf $CONF -Dwikipedia.input=${HDFS_DATA_DIR}
Modified: incubator/accumulo/branches/1.4/src/wikisearch/ingest/src/main/java/org/apache/accumulo/wikisearch/ingest/WikipediaIngester.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/wikisearch/ingest/src/main/java/org/apache/accumulo/wikisearch/ingest/WikipediaIngester.java?rev=1229653&r1=1229652&r2=1229653&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/wikisearch/ingest/src/main/java/org/apache/accumulo/wikisearch/ingest/WikipediaIngester.java (original)
+++ incubator/accumulo/branches/1.4/src/wikisearch/ingest/src/main/java/org/apache/accumulo/wikisearch/ingest/WikipediaIngester.java Tue Jan 10 17:37:48 2012
@@ -103,9 +103,9 @@ public class WikipediaIngester extends C
// Add the UID aggregator
for (IteratorScope scope : IteratorScope.values()) {
String stem = String.format("%s%s.%s", Property.TABLE_ITERATOR_PREFIX, scope.name(), "UIDAggregator");
- tops.setProperty(indexTableName, stem, "19,iterator.TotalAggregatingIterator");
+ tops.setProperty(indexTableName, stem, "19,org.apache.accumulo.wikisearch.iterator.TotalAggregatingIterator");
stem += ".opt.";
- tops.setProperty(indexTableName, stem + "*", "aggregator.GlobalIndexUidAggregator");
+ tops.setProperty(indexTableName, stem + "*", "org.apache.accumulo.wikisearch.aggregator.GlobalIndexUidAggregator");
}
}
@@ -115,9 +115,9 @@ public class WikipediaIngester extends C
// Add the UID aggregator
for (IteratorScope scope : IteratorScope.values()) {
String stem = String.format("%s%s.%s", Property.TABLE_ITERATOR_PREFIX, scope.name(), "UIDAggregator");
- tops.setProperty(reverseIndexTableName, stem, "19,iterator.TotalAggregatingIterator");
+ tops.setProperty(reverseIndexTableName, stem, "19,org.apache.accumulo.wikisearch.iterator.TotalAggregatingIterator");
stem += ".opt.";
- tops.setProperty(reverseIndexTableName, stem + "*", "aggregator.GlobalIndexUidAggregator");
+ tops.setProperty(reverseIndexTableName, stem + "*", "org.apache.accumulo.wikisearch.aggregator.GlobalIndexUidAggregator");
}
}
Modified: incubator/accumulo/branches/1.4/src/wikisearch/query-war/src/main/webapp/style.xsl
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/wikisearch/query-war/src/main/webapp/style.xsl?rev=1229653&r1=1229652&r2=1229653&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/wikisearch/query-war/src/main/webapp/style.xsl (original)
+++ incubator/accumulo/branches/1.4/src/wikisearch/query-war/src/main/webapp/style.xsl Tue Jan 10 17:37:48 2012
@@ -29,7 +29,7 @@
<td><xsl:value-of select="field[@name = 'COMMENTS']/text()" /></td>
<xsl:variable name="pointer" select="field[@name ='DOCUMENT']/text()" />
<xsl:variable name="href">
- <xsl:text>/accumulo-sample/rest/Query/content?query=</xsl:text><xsl:copy-of select="$pointer"/><xsl:text>&auths=all</xsl:text>
+ <xsl:text>/accumulo-wikisearch/rest/Query/content?query=</xsl:text><xsl:copy-of select="$pointer"/><xsl:text>&auths=all</xsl:text>
</xsl:variable>
<xsl:variable name="link">
<xsl:element name="a">
Modified: incubator/accumulo/branches/1.4/src/wikisearch/query-war/src/main/webapp/ui.jsp
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/wikisearch/query-war/src/main/webapp/ui.jsp?rev=1229653&r1=1229652&r2=1229653&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/wikisearch/query-war/src/main/webapp/ui.jsp (original)
+++ incubator/accumulo/branches/1.4/src/wikisearch/query-war/src/main/webapp/ui.jsp Tue Jan 10 17:37:48 2012
@@ -116,7 +116,7 @@
}
}
//Build the new query
- var existingAction = "/accumulo-sample/rest/Query/html";
+ var existingAction = "/accumulo-wikisearch/rest/Query/html";
var query = f.query.value;
var newAction = existingAction + "?query=" + query + "&auths=" + authString;
Modified: incubator/accumulo/branches/1.4/src/wikisearch/query/src/assembly/dist.xml
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/wikisearch/query/src/assembly/dist.xml?rev=1229653&r1=1229652&r2=1229653&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/wikisearch/query/src/assembly/dist.xml (original)
+++ incubator/accumulo/branches/1.4/src/wikisearch/query/src/assembly/dist.xml Tue Jan 10 17:37:48 2012
@@ -30,12 +30,5 @@
</excludes>
<fileMode>0644</fileMode>
</fileSet>
- <fileSet>
- <directory>target</directory>
- <outputDirectory>deploy</outputDirectory>
- <includes>
- <include>${project.name}-${project.version}.jar</include>
- </includes>
- </fileSet>
</fileSets>
</assembly>
Modified: incubator/accumulo/branches/1.4/src/wikisearch/query/src/test/java/org/apache/accumulo/wikisearch/logic/TestQueryLogic.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/wikisearch/query/src/test/java/org/apache/accumulo/wikisearch/logic/TestQueryLogic.java?rev=1229653&r1=1229652&r2=1229653&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/wikisearch/query/src/test/java/org/apache/accumulo/wikisearch/logic/TestQueryLogic.java (original)
+++ incubator/accumulo/branches/1.4/src/wikisearch/query/src/test/java/org/apache/accumulo/wikisearch/logic/TestQueryLogic.java Tue Jan 10 17:37:48 2012
@@ -30,7 +30,6 @@ import org.apache.accumulo.core.client.B
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.mock.MockInstance;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
@@ -72,6 +71,8 @@ public class TestQueryLogic {
private static final String RINDEX_TABLE_NAME = "wikiReverseIndex";
+ private static final String TABLE_NAMES[] = {METADATA_TABLE_NAME, TABLE_NAME, RINDEX_TABLE_NAME, INDEX_TABLE_NAME};
+
private class MockAccumuloRecordWriter extends RecordWriter<Text,Mutation> {
@Override
public void write(Text key, Mutation value) throws IOException, InterruptedException {
@@ -115,31 +116,13 @@ public class TestQueryLogic {
MockInstance i = new MockInstance();
c = i.getConnector("root", "pass");
- try{
- c.tableOperations().delete(METADATA_TABLE_NAME);
- } catch (TableNotFoundException tnfe) {}
- ;
- try{
- c.tableOperations().delete(TABLE_NAME);
- } catch (TableNotFoundException tnfe) {}
- ;
- try{
- c.tableOperations().delete(INDEX_TABLE_NAME);
- } catch (TableNotFoundException tnfe) {}
- ;
- try{
- c.tableOperations().delete(RINDEX_TABLE_NAME);
- } catch (TableNotFoundException tnfe) {}
- ;
- c.tableOperations().create(METADATA_TABLE_NAME);
- c.tableOperations().create(TABLE_NAME);
- c.tableOperations().create(INDEX_TABLE_NAME);
- c.tableOperations().create(RINDEX_TABLE_NAME);
-
- writerMap.put(new Text(METADATA_TABLE_NAME), c.createBatchWriter(METADATA_TABLE_NAME, 1000L, 1000L, 1));
- writerMap.put(new Text(TABLE_NAME), c.createBatchWriter(TABLE_NAME, 1000L, 1000L, 1));
- writerMap.put(new Text(INDEX_TABLE_NAME), c.createBatchWriter(INDEX_TABLE_NAME, 1000L, 1000L, 1));
- writerMap.put(new Text(RINDEX_TABLE_NAME), c.createBatchWriter(RINDEX_TABLE_NAME, 1000L, 1000L, 1));
+ for (String table : TABLE_NAMES) {
+ try {
+ c.tableOperations().delete(table);
+ } catch (Exception ex) {}
+ c.tableOperations().create(table);
+ writerMap.put(new Text(table), c.createBatchWriter(table, 1000L, 1000L, 1));
+ }
TaskAttemptID id = new TaskAttemptID();
TaskAttemptContext context = new TaskAttemptContext(conf, id);