You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2013/11/26 16:47:50 UTC
[01/39] git commit: ACCUMULO-804 changed use of SocketInputStream for
hadoop 2 compatibility
Updated Branches:
refs/heads/1.6.0-SNAPSHOT f478737a1 -> 6c80fa503
ACCUMULO-804 changed use of SocketInputStream for hadoop 2 compatibility
git-svn-id: https://svn.apache.org/repos/asf/accumulo/trunk@1398373 13f79535-47bb-0310-9956-ffa450edef68
(cherry picked from commit ca1494d1508a3b1404ae9989d4211b00aa3ce307)
Reason: Hadoop2 compat
Author: Billie Rinaldi <bi...@apache.org>
Ref: ACCUMULO-1792
Signed-off-by: Eric Newton <er...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/fc3fdf33
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/fc3fdf33
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/fc3fdf33
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: fc3fdf33716aa7105e1792c81dfb452fbf8c7944
Parents: aff0d64
Author: Billie Rinaldi <bi...@apache.org>
Authored: Mon Oct 15 16:32:24 2012 +0000
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Nov 25 16:06:41 2013 -0500
----------------------------------------------------------------------
.../java/org/apache/accumulo/core/util/TTimeoutTransport.java | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fc3fdf33/src/core/src/main/java/org/apache/accumulo/core/util/TTimeoutTransport.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/util/TTimeoutTransport.java b/src/core/src/main/java/org/apache/accumulo/core/util/TTimeoutTransport.java
index 6f74451..3c1fa6a 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/util/TTimeoutTransport.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/util/TTimeoutTransport.java
@@ -25,8 +25,7 @@ import java.net.Socket;
import java.net.SocketAddress;
import java.nio.channels.spi.SelectorProvider;
-import org.apache.hadoop.net.SocketInputStream;
-import org.apache.hadoop.net.SocketOutputStream;
+import org.apache.hadoop.net.NetUtils;
import org.apache.thrift.transport.TIOStreamTransport;
import org.apache.thrift.transport.TTransport;
@@ -37,8 +36,8 @@ public class TTimeoutTransport {
socket.setSoLinger(false, 0);
socket.setTcpNoDelay(true);
socket.connect(addr);
- InputStream input = new BufferedInputStream(new SocketInputStream(socket, timeoutMillis), 1024 * 10);
- OutputStream output = new BufferedOutputStream(new SocketOutputStream(socket, timeoutMillis), 1024 * 10);
+ InputStream input = new BufferedInputStream(NetUtils.getInputStream(socket, timeoutMillis), 1024 * 10);
+ OutputStream output = new BufferedOutputStream(NetUtils.getOutputStream(socket, timeoutMillis), 1024 * 10);
return new TIOStreamTransport(input, output);
}
}
[03/39] ACCUMULO-600 removed wikisearch from trunk
Posted by ec...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/test/resources/enwiki-20110901-001.xml
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/test/resources/enwiki-20110901-001.xml b/src/examples/wikisearch/query/src/test/resources/enwiki-20110901-001.xml
deleted file mode 100644
index 41d146a..0000000
--- a/src/examples/wikisearch/query/src/test/resources/enwiki-20110901-001.xml
+++ /dev/null
@@ -1,153 +0,0 @@
-<mediawiki xmlns="http://www.mediawiki.org/xml/export-0.5/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.mediawiki.org/xml/export-0.5/ http://www.mediawiki.org/xml/export-0.5.xsd" version="0.5" xml:lang="en">
- <siteinfo>
- <sitename>Wikipedia</sitename>
- <base>http://en.wikipedia.org/wiki/Main_Page</base>
- <generator>MediaWiki 1.17wmf1</generator>
- <case>first-letter</case>
- <namespaces>
- <namespace key="-2" case="first-letter">Media</namespace>
- <namespace key="-1" case="first-letter">Special</namespace>
- <namespace key="0" case="first-letter" />
- <namespace key="1" case="first-letter">Talk</namespace>
- <namespace key="2" case="first-letter">User</namespace>
- <namespace key="3" case="first-letter">User talk</namespace>
- <namespace key="4" case="first-letter">Wikipedia</namespace>
- <namespace key="5" case="first-letter">Wikipedia talk</namespace>
- <namespace key="6" case="first-letter">File</namespace>
- <namespace key="7" case="first-letter">File talk</namespace>
- <namespace key="8" case="first-letter">MediaWiki</namespace>
- <namespace key="9" case="first-letter">MediaWiki talk</namespace>
- <namespace key="10" case="first-letter">Template</namespace>
- <namespace key="11" case="first-letter">Template talk</namespace>
- <namespace key="12" case="first-letter">Help</namespace>
- <namespace key="13" case="first-letter">Help talk</namespace>
- <namespace key="14" case="first-letter">Category</namespace>
- <namespace key="15" case="first-letter">Category talk</namespace>
- <namespace key="100" case="first-letter">Portal</namespace>
- <namespace key="101" case="first-letter">Portal talk</namespace>
- <namespace key="108" case="first-letter">Book</namespace>
- <namespace key="109" case="first-letter">Book talk</namespace>
- </namespaces>
- </siteinfo>
- <page>
- <title>Abacus</title>
- <id>655</id>
- <revision>
- <id>34350</id>
- <timestamp>2002-02-25T15:43:11Z</timestamp>
- <contributor>
- <ip>Conversion script</ip>
- </contributor>
- <minor />
- <comment>Automated conversion</comment>
- <text xml:space="preserve">1. An '''abacus''' is a counting frame, typically wooden with balls sliding on wires. It was first used before the adoption of the ten-digit [[Arabic numerals | Arabic numeral]] system and is still widely used by small merchants in [[China]]. The Roman abacus contains seven long and seven shorter rods or bars, the former having four perforated beads running on them and the latter one. The bar marked 1 indicates units, X tens, and so on up to millions. The beads on the shorter bars denote fives,--five units, five tens, etc. The rod O and corresponding short rod are for marking ounces; and the short quarter rods for fractions of an ounce. Computations are made with it by means of balls of bone or ivory running on slender bamboo rods, similar to the simpler board, fitted up with beads strung on wires, which has been employed in teaching the rudiments of arithmetic in English schools.
-
-The '''Suan'''4-'''Pan'''2 (&#31639;&#30436;) of the Chinese closely resembles the Roman abacus in its construction and use. The Chinese abacus is usally around eight inches tall and it comes in various width depending on application, it usually has more than seven rods. There are two beads on each rod in the upper deck and five beads each in the bottom. The beads are usually round and made of hard wood. The abacus can be reset to the starting position instantly by a quick jerk along the horizontal axis to spin all the beads away from the horizontal beam at the center. The beads are counted by moving them up or down towards the beam. Chinese abacus does more than just counting. Unlike the simple counting board used in elimentary schools, very efficient Suan4-Pan2 techniques were developed to do multiplication, division, addition, substraction, square root and cubic root at high speed. The beads and rods were often lubricated to ensure speed. When all five beads in the
lower deck are moved up, they are reset to the original position, and one bead in the top deck is moved down as a carry. When both beads in the upper deck are moved down, they are reset and a bead on the adjacent rod on the left is moved up as a carry. The result of the computation is read off from the beads clustered near the separator beam between the upper and lower deck. In a sense, the abacus works as a 5-2-5-2-5-2... based number system in which carries and shiftings are similiar to the decimal number system. Since each rod represents a digit in a decimal number, the computation capacity of the abacus is only limited by the number of rods on the abacus. When a mathematician runs out of rods, he simply adds another abacus to the left of the row. In theory, the abacus can be expanded infinitely.
-
-As recently as the late 1960s, abacus arithmetics were still being taught in school (e.g. in Hong Kong). When hand held calculators became popular, nobody wanted to learn how to operate an abacus any more. In the early days of handheld calculators, news about abacus operators beating electronic calculator in arithmetics competitions in both speed and accuracy often appeared in the media. The main reason being that early calculators were often plagued by rounding and overflow errors. (Most handheld calculators can only handle 8 to 10 significant digits, the abacus is virtually limitless in precision.) Inexperienced operators might contribute to the loss too. But when calculators' functionality improved, everyone knew that the abacus could never compute complex functions (e.g. trignometry) faster than a calculator. The older generation (those who were born before the early 1950s) still used it for a while, but electronic calculators gradually displaced abacus in Hong Kong over th
e past four decades. Abacus is hardly seen in Hong Kong nowadays. However, abacuses are still being used in China and Japan. The [[slide rule]]s also suffered a similar demise.
-
-The Suan4-Pan2 is closely tied to the [[[Chinese numerals|Chinese "Hua1 Ma3" numbering system]]].
-
-The Japanese eliminated one bead each from the upper and lower deck in each column of the Chinese abacus, because these beads are redundent. That makes the Japanese '''soroban''' (&#21313;&#38706;&#30436;) more like the Roman abacus. The soroban is about 3 inches tall. The beans on a soroban are usually double cone shape.
-
-Many sources also mentioned use of abacus in ancient Mayan culture.
-The Mesoamerican abacus is closely tied to the base-20 [[Mayan numerals]] system.
-
-External Ref:
-[[http://www.ee.ryerson.ca/~elf/abacus/ Abacus]],
-[[http://www.soroban.com/ Soroban]],
-[[http://www.sungwh.freeserve.co.uk/sapienti/abacus01.htm Suan Pan]],
-[[http://hawk.hama-med.ac.jp/dbk/abacus.html Mesoamerican abacus]],
-[[http://www.dotpoint.com/xnumber/pic_roman_abacus.htm Roman abacus]]
-
-----
-
-2. (From the Greek ''abax'', a slab; or French ''abaque'', tailloir), in architecture, the upper member of the capital of a column. Its chief function is to provide a larger supporting surface for the architrave or arch it has to carry. In the Greek [[Doric]] order the abacus is a plain square slab. In the Roman and Renaissance Doric orders it is crowned by a moulding. In the Archaic-Greek [[Ionic]] order, owing to the greater width of the capital, the abacus is rectangular in plan, and consists of a carved [[ovolo]] moulding. In later examples the abacus is square, except where there are angle [[volute]]s, when it is slightly curved over the same. In the Roman and Renaissance Ionic capital, the abacus is square with a fillet On the top of an ogee moulding, but curved over angle volutes. In the Greek [[Corinthian]] order the abacus is moulded, its sides are concave and its angles canted (except in one or two exceptional Greek capitals, where it is brought to a sharp angle); a
nd the same shape is adopted in the Roman and Renaissance Corinthian and Composite capitals, in some cases with the ovolo moulding carved. In Romanesque architecture the abacus is square with the lower edge splayed off and moulded or carved, and the same was retained in France during the medieval period; but in England, in Early English work, a circular deeply moulded abacus was introduced, which in the 14th and 15th centuries was transformed into an octagonal one. The diminutive of abacus, [[abaciscus]], is applied in architecture to the chequers or squares of a tessellated pavement.
-
-----
-
-3. (possibly defunct) The name of abacus is also given, in [[logic]], to an instrument, often called the "logical machine", analogous to the mathematical abacus. It is constructed to show all the possible combinations of a set of logical terms with their negatives, and, further, the way in which these combinations are affected by the addition of attributes or other limiting words, i.e., to simplify mechanically the solution of logical problems. These instruments are all more or less elaborate developments of the "logical slate", on which were written in vertical columns all the combinations of symbols or letters which could be made logically out of a definite number of terms. These were compared with any given premises, and those which were incompatible were crossed off. In the abacus the combinations are inscribed each on a single slip of wood or similar substance, which is moved by a key; incompatible combinations can thus be mechanically removed at will, i
n accordance with any given series of premises.
-
-----
-
-see also:
-* [[slide rule]]
-
-[[talk:Abacus|Talk]]
-</text>
- </revision>
- </page>
- <page>
- <title>Acid</title>
- <id>656</id>
- <revision>
- <id>46344</id>
- <timestamp>2002-02-25T15:43:11Z</timestamp>
- <contributor>
- <ip>Conversion script</ip>
- </contributor>
- <minor />
- <comment>Automated conversion</comment>
- <text xml:space="preserve">An '''acid''' is a chemical generally defined by its reactions with complementary chemicals, designated [[base]]s. See [[Acid-base reaction theories]].
-
-Some of the stronger acids include the hydrohalic acids - HCl, HBr, and HI - and the oxyacids, which tend to contain central atoms in high oxidation states surrounded by oxygen - including HNO<sub>3</sub> and H<sub>2</sub>SO<sub>4</sub>.
-
-
-Acidity is typically measured using the [[pH]] scale.
-
-----
-See also:
-
-"Acid" is also a slang word referring to [[LSD]].
-
-'''ACID''' is an acronym that expands to four essential properties of a [[database management system]].
-See [[ACID properties]].
-</text>
- </revision>
- </page>
- <page>
- <title>Asphalt</title>
- <id>657</id>
- <revision>
- <id>29335</id>
- <timestamp>2002-02-25T15:43:11Z</timestamp>
- <contributor>
- <ip>Conversion script</ip>
- </contributor>
- <minor />
- <comment>Automated conversion</comment>
- <text xml:space="preserve">'''Asphalt''' (also called [[bitumen]]) is a material that occurs naturally in most crude [[petroleum]]s. It is commonly used to build the surface of roads.
-</text>
- </revision>
- </page>
- <page>
- <title>Acronym</title>
- <id>658</id>
- <redirect />
- <revision>
- <id>60824</id>
- <timestamp>2002-02-25T15:43:11Z</timestamp>
- <contributor>
- <ip>Conversion script</ip>
- </contributor>
- <minor />
- <comment>Automated conversion</comment>
- <text xml:space="preserve">An '''acronym''' is an [[abbreviation]], often composed of the initial letters of the words in a short phrase, that is treated as word (often, a piece of jargon or the proper name of an organization). For example, SAM for [[''s''urface-to-''a''ir ''m''issile]] and [[NATO]] for the [[North Atlantic Treaty Organization]]. In its original meaning, acronyms were restricted to ''pronouncible'' abbreviations (what might be called ''true'' acronyms), though common usage permits calling unpronouncable abbreviations acronyms as well. Sometimes conjuntions and prepositions (such as and or to) contribute letters to make the acronym pronouncible, in contradiction to the normal [[English language|English]] rule for abbreviations.
-
-Often, an acronym will come into such wide use that people think of it as a word in itself, forget that it started out as an acronym, and write in in small letters. Examples include [[quasar]] (''q''uasi-''s''tellar ''r''adio ''s''ource), [[laser]] (''l''ight ''a''mplification by ''s''timulated ''e''mission of ''r''adiation) and radar (''r''adio ''d''etection ''a''nd ''r''anging).
-
-Non-pronouncible abbreviations formed from initials (such as IBM for International Business Machines) are sometimes called '''[[initialism]]s'''.
-
-Some lists of acronyms in use:
-
-*[[Internet slang|acronyms used on the Internet]]
-*[[Acronym/List|list of acronyms]]
-*[[Acronym/Medical List|list of medical acronyms]]
-
-A large list of acronyms may be found at http://www.acronymfinder.com/
-
-[[talk:Acronym|/Talk]]
-</text>
- </revision>
- </page>
-</mediawiki>
[35/39] git commit: ACCUMULO-1792 Update commons-io dependency for
Hadoop2.
Posted by ec...@apache.org.
ACCUMULO-1792 Update commons-io dependency for Hadoop2.
Based on the discussion around ACCUMULO-1244, we can update to 2.1 while not marking commons-io as provided to eliminated classpath issues on hadoop 2 and bring a copy in lib/ for hadoop 0.20.
Signed-off-by: Eric Newton <er...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5e3967fa
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5e3967fa
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5e3967fa
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 5e3967fa08a36b386cdd0b40a04b5cd7c1e331de
Parents: d5f1657
Author: Sean Busbey <bu...@cloudera.com>
Authored: Mon Oct 28 00:44:46 2013 -0400
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Nov 25 16:06:44 2013 -0500
----------------------------------------------------------------------
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5e3967fa/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e4c06fa..ca550d5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -786,7 +786,7 @@
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
- <version>1.4</version>
+ <version>2.1</version>
</dependency>
<dependency>
<groupId>commons-collections</groupId>
[20/39] git commit: ACCUMULO-286 fixed mysteriously broken
parameterization in ContextFactory
Posted by ec...@apache.org.
ACCUMULO-286 fixed mysteriously broken parameterization in ContextFactory
git-svn-id: https://svn.apache.org/repos/asf/incubator/accumulo/trunk@1229638 13f79535-47bb-0310-9956-ffa450edef68
(cherry picked from commit dd31ed78ac3c7d8965c050b668166d1faa510aa6)
Reason: Testing
Author: Billie Rinaldi <bi...@apache.org>
Ref: ACCUMULO-1792
Signed-off-by: Eric Newton <er...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6bba9406
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6bba9406
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6bba9406
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 6bba9406c6dcc08c64149bb6cb801086c15558ee
Parents: 792af9a
Author: Billie Rinaldi <bi...@apache.org>
Authored: Tue Jan 10 17:01:38 2012 +0000
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Nov 25 16:06:42 2013 -0500
----------------------------------------------------------------------
.../java/org/apache/accumulo/core/util/ContextFactory.java | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6bba9406/src/core/src/main/java/org/apache/accumulo/core/util/ContextFactory.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/util/ContextFactory.java b/src/core/src/main/java/org/apache/accumulo/core/util/ContextFactory.java
index 67819da..61e853d 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/util/ContextFactory.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/util/ContextFactory.java
@@ -147,16 +147,15 @@ public class ContextFactory {
}
}
- @SuppressWarnings("unchecked")
+ @SuppressWarnings({"unchecked", "rawtypes"})
public static <K1,V1,K2,V2> Mapper<K1,V1,K2,V2>.Context createMapContext(Mapper<K1,V1,K2,V2> m, TaskAttemptContext tac, RecordReader<K1,V1> reader,
RecordWriter<K2,V2> writer, InputSplit split) {
try {
if (useV21) {
Object basis = MAP_CONTEXT_IMPL_CONSTRUCTOR.newInstance(tac.getConfiguration(), tac.getTaskAttemptID(), reader, writer, null, null, split);
- return (Mapper<K1,V1,K2,V2>.Context) MAP_CONTEXT_CONSTRUCTOR.newInstance((Mapper<K1,V1,K2,V2>) MAP_CONSTRUCTOR.newInstance(), basis);
+ return (Mapper.Context) MAP_CONTEXT_CONSTRUCTOR.newInstance((Mapper<K1,V1,K2,V2>) MAP_CONSTRUCTOR.newInstance(), basis);
} else {
- return (Mapper<K1,V1,K2,V2>.Context) MAP_CONTEXT_CONSTRUCTOR.newInstance(m, tac.getConfiguration(), tac.getTaskAttemptID(), reader, writer, null, null,
- split);
+ return (Mapper.Context) MAP_CONTEXT_CONSTRUCTOR.newInstance(m, tac.getConfiguration(), tac.getTaskAttemptID(), reader, writer, null, null, split);
}
} catch (InstantiationException e) {
throw new IllegalArgumentException("Can't create object", e);
[12/39] ACCUMULO-600 removed wikisearch from trunk
Posted by ec...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/reader/AggregatingRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/reader/AggregatingRecordReaderTest.java b/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/reader/AggregatingRecordReaderTest.java
deleted file mode 100644
index c842da7..0000000
--- a/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/reader/AggregatingRecordReaderTest.java
+++ /dev/null
@@ -1,287 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.reader;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.File;
-import java.io.FileWriter;
-import java.io.StringReader;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.xpath.XPath;
-import javax.xml.xpath.XPathExpression;
-import javax.xml.xpath.XPathFactory;
-
-import org.apache.accumulo.core.util.ContextFactory;
-import org.apache.accumulo.examples.wikisearch.ingest.WikipediaInputFormat.WikipediaInputSplit;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.lib.input.FileSplit;
-import org.junit.Before;
-import org.junit.Test;
-import org.w3c.dom.Document;
-import org.xml.sax.ErrorHandler;
-import org.xml.sax.InputSource;
-import org.xml.sax.SAXException;
-import org.xml.sax.SAXParseException;
-
-public class AggregatingRecordReaderTest {
-
- public static class MyErrorHandler implements ErrorHandler {
-
- @Override
- public void error(SAXParseException exception) throws SAXException {
- // System.out.println(exception.getMessage());
- }
-
- @Override
- public void fatalError(SAXParseException exception) throws SAXException {
- // System.out.println(exception.getMessage());
- }
-
- @Override
- public void warning(SAXParseException exception) throws SAXException {
- // System.out.println(exception.getMessage());
- }
-
- }
-
- private static final String xml1 = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" + "<doc>\n" + " <a>A</a>\n" + " <b>B</b>\n" + "</doc>\n"
- + "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" + "<doc>\n" + " <a>C</a>\n" + " <b>D</b>\n" + "</doc>\n" + "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
- + "<doc>\n" + " <a>E</a>\n" + " <b>F</b>\n" + "</doc>\n";
-
- private static final String xml2 = " <b>B</b>\n" + "</doc>\n" + "<doc>\n" + " <a>C</a>\n" + " <b>D</b>\n" + "</doc>\n" + "<doc>\n" + " <a>E</a>\n"
- + " <b>F</b>\n" + "</doc>\n";
-
- private static final String xml3 = "<doc>\n" + " <a>A</a>\n" + " <b>B</b>\n" + "</doc>\n" + "<doc>\n" + " <a>C</a>\n" + " <b>D</b>\n" + "</doc>\n"
- + "<doc>\n" + " <a>E</a>\n";
-
- private static final String xml4 = "<doc>" + " <a>A</a>" + " <b>B</b>" + "</doc>" + "<doc>" + " <a>C</a>" + " <b>D</b>" + "</doc>" + "<doc>"
- + " <a>E</a>" + " <b>F</b>" + "</doc>";
-
- private static final String xml5 = "<doc attr=\"G\">" + " <a>A</a>" + " <b>B</b>" + "</doc>" + "<doc>" + " <a>C</a>" + " <b>D</b>" + "</doc>"
- + "<doc attr=\"H\"/>" + "<doc>" + " <a>E</a>" + " <b>F</b>" + "</doc>" + "<doc attr=\"I\"/>";
-
- private Configuration conf = null;
- private TaskAttemptContext ctx = null;
- private static DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
- private XPathFactory xpFactory = XPathFactory.newInstance();
- private XPathExpression EXPR_A = null;
- private XPathExpression EXPR_B = null;
- private XPathExpression EXPR_ATTR = null;
-
- @Before
- public void setUp() throws Exception {
- conf = new Configuration();
- conf.set(AggregatingRecordReader.START_TOKEN, "<doc");
- conf.set(AggregatingRecordReader.END_TOKEN, "</doc>");
- conf.set(AggregatingRecordReader.RETURN_PARTIAL_MATCHES, Boolean.toString(true));
- ctx = ContextFactory.createTaskAttemptContext(conf);
- XPath xp = xpFactory.newXPath();
- EXPR_A = xp.compile("/doc/a");
- EXPR_B = xp.compile("/doc/b");
- EXPR_ATTR = xp.compile("/doc/@attr");
- }
-
- public File createFile(String data) throws Exception {
- // Write out test file
- File f = File.createTempFile("aggReaderTest", ".xml");
- f.deleteOnExit();
- FileWriter writer = new FileWriter(f);
- writer.write(data);
- writer.flush();
- writer.close();
- return f;
- }
-
- private void testXML(Text xml, String aValue, String bValue, String attrValue) throws Exception {
- StringReader reader = new StringReader(xml.toString());
- InputSource source = new InputSource(reader);
-
- DocumentBuilder parser = factory.newDocumentBuilder();
- parser.setErrorHandler(new MyErrorHandler());
- Document root = parser.parse(source);
- assertNotNull(root);
-
- reader = new StringReader(xml.toString());
- source = new InputSource(reader);
- assertEquals(EXPR_A.evaluate(source), aValue);
-
- reader = new StringReader(xml.toString());
- source = new InputSource(reader);
- assertEquals(EXPR_B.evaluate(source), bValue);
-
- reader = new StringReader(xml.toString());
- source = new InputSource(reader);
- assertEquals(EXPR_ATTR.evaluate(source), attrValue);
- }
-
- @Test
- public void testIncorrectArgs() throws Exception {
- File f = createFile(xml1);
-
- // Create FileSplit
- Path p = new Path(f.toURI().toString());
- WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null), 0);
- AggregatingRecordReader reader = new AggregatingRecordReader();
- try {
- // Clear the values for BEGIN and STOP TOKEN
- conf.set(AggregatingRecordReader.START_TOKEN, null);
- conf.set(AggregatingRecordReader.END_TOKEN, null);
- reader.initialize(split, ctx);
- // If we got here, then the code didnt throw an exception
- fail();
- } catch (Exception e) {
- // Do nothing, we succeeded
- f = null;
- }
- reader.close();
- }
-
- @Test
- public void testCorrectXML() throws Exception {
- File f = createFile(xml1);
-
- // Create FileSplit
- Path p = new Path(f.toURI().toString());
- WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null), 0);
-
- // Initialize the RecordReader
- AggregatingRecordReader reader = new AggregatingRecordReader();
- reader.initialize(split, ctx);
- assertTrue(reader.nextKeyValue());
- testXML(reader.getCurrentValue(), "A", "B", "");
- assertTrue(reader.nextKeyValue());
- testXML(reader.getCurrentValue(), "C", "D", "");
- assertTrue(reader.nextKeyValue());
- testXML(reader.getCurrentValue(), "E", "F", "");
- assertTrue(!reader.nextKeyValue());
-
- }
-
- @Test
- public void testPartialXML() throws Exception {
- File f = createFile(xml2);
-
- // Create FileSplit
- Path p = new Path(f.toURI().toString());
- WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null), 0);
-
- // Initialize the RecordReader
- AggregatingRecordReader reader = new AggregatingRecordReader();
- reader.initialize(split, ctx);
- assertTrue(reader.nextKeyValue());
- testXML(reader.getCurrentValue(), "C", "D", "");
- assertTrue(reader.nextKeyValue());
- testXML(reader.getCurrentValue(), "E", "F", "");
- assertTrue(!reader.nextKeyValue());
- }
-
- public void testPartialXML2WithNoPartialRecordsReturned() throws Exception {
- conf.set(AggregatingRecordReader.RETURN_PARTIAL_MATCHES, Boolean.toString(false));
- File f = createFile(xml3);
-
- // Create FileSplit
- Path p = new Path(f.toURI().toString());
- WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null), 0);
-
- // Initialize the RecordReader
- AggregatingRecordReader reader = new AggregatingRecordReader();
- reader.initialize(split, ctx);
- assertTrue(reader.nextKeyValue());
- testXML(reader.getCurrentValue(), "A", "B", "");
- assertTrue(reader.nextKeyValue());
- testXML(reader.getCurrentValue(), "C", "D", "");
- assertTrue(!reader.nextKeyValue());
- }
-
- @Test
- public void testPartialXML2() throws Exception {
- File f = createFile(xml3);
-
- // Create FileSplit
- Path p = new Path(f.toURI().toString());
- WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null), 0);
-
- // Initialize the RecordReader
- AggregatingRecordReader reader = new AggregatingRecordReader();
- reader.initialize(split, ctx);
- assertTrue(reader.nextKeyValue());
- testXML(reader.getCurrentValue(), "A", "B", "");
- assertTrue(reader.nextKeyValue());
- testXML(reader.getCurrentValue(), "C", "D", "");
- assertTrue(reader.nextKeyValue());
- try {
- testXML(reader.getCurrentValue(), "E", "", "");
- fail("Fragment returned, and it somehow passed XML parsing.");
- } catch (SAXParseException e) {
- // ignore
- }
- assertTrue(!reader.nextKeyValue());
- }
-
- @Test
- public void testLineSplitting() throws Exception {
- File f = createFile(xml4);
-
- // Create FileSplit
- Path p = new Path(f.toURI().toString());
- WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null), 0);
-
- // Initialize the RecordReader
- AggregatingRecordReader reader = new AggregatingRecordReader();
- reader.initialize(split, ctx);
- assertTrue(reader.nextKeyValue());
- testXML(reader.getCurrentValue(), "A", "B", "");
- assertTrue(reader.nextKeyValue());
- testXML(reader.getCurrentValue(), "C", "D", "");
- assertTrue(reader.nextKeyValue());
- testXML(reader.getCurrentValue(), "E", "F", "");
- assertTrue(!reader.nextKeyValue());
- }
-
- @Test
- public void testNoEndTokenHandling() throws Exception {
- File f = createFile(xml5);
- // Create FileSplit
- Path p = new Path(f.toURI().toString());
- WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null), 0);
-
- // Initialize the RecordReader
- AggregatingRecordReader reader = new AggregatingRecordReader();
- reader.initialize(split, ctx);
- assertTrue("Not enough records returned.", reader.nextKeyValue());
- testXML(reader.getCurrentValue(), "A", "B", "G");
- assertTrue("Not enough records returned.", reader.nextKeyValue());
- testXML(reader.getCurrentValue(), "C", "D", "");
- assertTrue("Not enough records returned.", reader.nextKeyValue());
- testXML(reader.getCurrentValue(), "", "", "H");
- assertTrue("Not enough records returned.", reader.nextKeyValue());
- testXML(reader.getCurrentValue(), "E", "F", "");
- assertTrue("Not enough records returned.", reader.nextKeyValue());
- testXML(reader.getCurrentValue(), "", "", "I");
- assertTrue("Too many records returned.", !reader.nextKeyValue());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/test/resources/enwiki-20110901-001.xml
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/test/resources/enwiki-20110901-001.xml b/src/examples/wikisearch/ingest/src/test/resources/enwiki-20110901-001.xml
deleted file mode 100644
index 41d146a..0000000
--- a/src/examples/wikisearch/ingest/src/test/resources/enwiki-20110901-001.xml
+++ /dev/null
@@ -1,153 +0,0 @@
-<mediawiki xmlns="http://www.mediawiki.org/xml/export-0.5/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.mediawiki.org/xml/export-0.5/ http://www.mediawiki.org/xml/export-0.5.xsd" version="0.5" xml:lang="en">
- <siteinfo>
- <sitename>Wikipedia</sitename>
- <base>http://en.wikipedia.org/wiki/Main_Page</base>
- <generator>MediaWiki 1.17wmf1</generator>
- <case>first-letter</case>
- <namespaces>
- <namespace key="-2" case="first-letter">Media</namespace>
- <namespace key="-1" case="first-letter">Special</namespace>
- <namespace key="0" case="first-letter" />
- <namespace key="1" case="first-letter">Talk</namespace>
- <namespace key="2" case="first-letter">User</namespace>
- <namespace key="3" case="first-letter">User talk</namespace>
- <namespace key="4" case="first-letter">Wikipedia</namespace>
- <namespace key="5" case="first-letter">Wikipedia talk</namespace>
- <namespace key="6" case="first-letter">File</namespace>
- <namespace key="7" case="first-letter">File talk</namespace>
- <namespace key="8" case="first-letter">MediaWiki</namespace>
- <namespace key="9" case="first-letter">MediaWiki talk</namespace>
- <namespace key="10" case="first-letter">Template</namespace>
- <namespace key="11" case="first-letter">Template talk</namespace>
- <namespace key="12" case="first-letter">Help</namespace>
- <namespace key="13" case="first-letter">Help talk</namespace>
- <namespace key="14" case="first-letter">Category</namespace>
- <namespace key="15" case="first-letter">Category talk</namespace>
- <namespace key="100" case="first-letter">Portal</namespace>
- <namespace key="101" case="first-letter">Portal talk</namespace>
- <namespace key="108" case="first-letter">Book</namespace>
- <namespace key="109" case="first-letter">Book talk</namespace>
- </namespaces>
- </siteinfo>
- <page>
- <title>Abacus</title>
- <id>655</id>
- <revision>
- <id>34350</id>
- <timestamp>2002-02-25T15:43:11Z</timestamp>
- <contributor>
- <ip>Conversion script</ip>
- </contributor>
- <minor />
- <comment>Automated conversion</comment>
- <text xml:space="preserve">1. An '''abacus''' is a counting frame, typically wooden with balls sliding on wires. It was first used before the adoption of the ten-digit [[Arabic numerals | Arabic numeral]] system and is still widely used by small merchants in [[China]]. The Roman abacus contains seven long and seven shorter rods or bars, the former having four perforated beads running on them and the latter one. The bar marked 1 indicates units, X tens, and so on up to millions. The beads on the shorter bars denote fives,--five units, five tens, etc. The rod O and corresponding short rod are for marking ounces; and the short quarter rods for fractions of an ounce. Computations are made with it by means of balls of bone or ivory running on slender bamboo rods, similar to the simpler board, fitted up with beads strung on wires, which has been employed in teaching the rudiments of arithmetic in English schools.
-
-The '''Suan'''4-'''Pan'''2 (&#31639;&#30436;) of the Chinese closely resembles the Roman abacus in its construction and use. The Chinese abacus is usally around eight inches tall and it comes in various width depending on application, it usually has more than seven rods. There are two beads on each rod in the upper deck and five beads each in the bottom. The beads are usually round and made of hard wood. The abacus can be reset to the starting position instantly by a quick jerk along the horizontal axis to spin all the beads away from the horizontal beam at the center. The beads are counted by moving them up or down towards the beam. Chinese abacus does more than just counting. Unlike the simple counting board used in elimentary schools, very efficient Suan4-Pan2 techniques were developed to do multiplication, division, addition, substraction, square root and cubic root at high speed. The beads and rods were often lubricated to ensure speed. When all five beads in the
lower deck are moved up, they are reset to the original position, and one bead in the top deck is moved down as a carry. When both beads in the upper deck are moved down, they are reset and a bead on the adjacent rod on the left is moved up as a carry. The result of the computation is read off from the beads clustered near the separator beam between the upper and lower deck. In a sense, the abacus works as a 5-2-5-2-5-2... based number system in which carries and shiftings are similiar to the decimal number system. Since each rod represents a digit in a decimal number, the computation capacity of the abacus is only limited by the number of rods on the abacus. When a mathematician runs out of rods, he simply adds another abacus to the left of the row. In theory, the abacus can be expanded infinitely.
-
-As recently as the late 1960s, abacus arithmetics were still being taught in school (e.g. in Hong Kong). When hand held calculators became popular, nobody wanted to learn how to operate an abacus any more. In the early days of handheld calculators, news about abacus operators beating electronic calculator in arithmetics competitions in both speed and accuracy often appeared in the media. The main reason being that early calculators were often plagued by rounding and overflow errors. (Most handheld calculators can only handle 8 to 10 significant digits, the abacus is virtually limitless in precision.) Inexperienced operators might contribute to the loss too. But when calculators' functionality improved, everyone knew that the abacus could never compute complex functions (e.g. trignometry) faster than a calculator. The older generation (those who were born before the early 1950s) still used it for a while, but electronic calculators gradually displaced abacus in Hong Kong over th
e past four decades. Abacus is hardly seen in Hong Kong nowadays. However, abacuses are still being used in China and Japan. The [[slide rule]]s also suffered a similar demise.
-
-The Suan4-Pan2 is closely tied to the [[[Chinese numerals|Chinese "Hua1 Ma3" numbering system]]].
-
-The Japanese eliminated one bead each from the upper and lower deck in each column of the Chinese abacus, because these beads are redundent. That makes the Japanese '''soroban''' (&#21313;&#38706;&#30436;) more like the Roman abacus. The soroban is about 3 inches tall. The beans on a soroban are usually double cone shape.
-
-Many sources also mentioned use of abacus in ancient Mayan culture.
-The Mesoamerican abacus is closely tied to the base-20 [[Mayan numerals]] system.
-
-External Ref:
-[[http://www.ee.ryerson.ca/~elf/abacus/ Abacus]],
-[[http://www.soroban.com/ Soroban]],
-[[http://www.sungwh.freeserve.co.uk/sapienti/abacus01.htm Suan Pan]],
-[[http://hawk.hama-med.ac.jp/dbk/abacus.html Mesoamerican abacus]],
-[[http://www.dotpoint.com/xnumber/pic_roman_abacus.htm Roman abacus]]
-
-----
-
-2. (From the Greek ''abax'', a slab; or French ''abaque'', tailloir), in architecture, the upper member of the capital of a column. Its chief function is to provide a larger supporting surface for the architrave or arch it has to carry. In the Greek [[Doric]] order the abacus is a plain square slab. In the Roman and Renaissance Doric orders it is crowned by a moulding. In the Archaic-Greek [[Ionic]] order, owing to the greater width of the capital, the abacus is rectangular in plan, and consists of a carved [[ovolo]] moulding. In later examples the abacus is square, except where there are angle [[volute]]s, when it is slightly curved over the same. In the Roman and Renaissance Ionic capital, the abacus is square with a fillet On the top of an ogee moulding, but curved over angle volutes. In the Greek [[Corinthian]] order the abacus is moulded, its sides are concave and its angles canted (except in one or two exceptional Greek capitals, where it is brought to a sharp angle); a
nd the same shape is adopted in the Roman and Renaissance Corinthian and Composite capitals, in some cases with the ovolo moulding carved. In Romanesque architecture the abacus is square with the lower edge splayed off and moulded or carved, and the same was retained in France during the medieval period; but in England, in Early English work, a circular deeply moulded abacus was introduced, which in the 14th and 15th centuries was transformed into an octagonal one. The diminutive of abacus, [[abaciscus]], is applied in architecture to the chequers or squares of a tessellated pavement.
-
-----
-
-3. (possibly defunct) The name of abacus is also given, in [[logic]], to an instrument, often called the "logical machine", analogous to the mathematical abacus. It is constructed to show all the possible combinations of a set of logical terms with their negatives, and, further, the way in which these combinations are affected by the addition of attributes or other limiting words, i.e., to simplify mechanically the solution of logical problems. These instruments are all more or less elaborate developments of the "logical slate", on which were written in vertical columns all the combinations of symbols or letters which could be made logically out of a definite number of terms. These were compared with any given premises, and those which were incompatible were crossed off. In the abacus the combinations are inscribed each on a single slip of wood or similar substance, which is moved by a key; incompatible combinations can thus be mechanically removed at will, i
n accordance with any given series of premises.
-
-----
-
-see also:
-* [[slide rule]]
-
-[[talk:Abacus|Talk]]
-</text>
- </revision>
- </page>
- <page>
- <title>Acid</title>
- <id>656</id>
- <revision>
- <id>46344</id>
- <timestamp>2002-02-25T15:43:11Z</timestamp>
- <contributor>
- <ip>Conversion script</ip>
- </contributor>
- <minor />
- <comment>Automated conversion</comment>
- <text xml:space="preserve">An '''acid''' is a chemical generally defined by its reactions with complementary chemicals, designated [[base]]s. See [[Acid-base reaction theories]].
-
-Some of the stronger acids include the hydrohalic acids - HCl, HBr, and HI - and the oxyacids, which tend to contain central atoms in high oxidation states surrounded by oxygen - including HNO<sub>3</sub> and H<sub>2</sub>SO<sub>4</sub>.
-
-
-Acidity is typically measured using the [[pH]] scale.
-
-----
-See also:
-
-"Acid" is also a slang word referring to [[LSD]].
-
-'''ACID''' is an acronym that expands to four essential properties of a [[database management system]].
-See [[ACID properties]].
-</text>
- </revision>
- </page>
- <page>
- <title>Asphalt</title>
- <id>657</id>
- <revision>
- <id>29335</id>
- <timestamp>2002-02-25T15:43:11Z</timestamp>
- <contributor>
- <ip>Conversion script</ip>
- </contributor>
- <minor />
- <comment>Automated conversion</comment>
- <text xml:space="preserve">'''Asphalt''' (also called [[bitumen]]) is a material that occurs naturally in most crude [[petroleum]]s. It is commonly used to build the surface of roads.
-</text>
- </revision>
- </page>
- <page>
- <title>Acronym</title>
- <id>658</id>
- <redirect />
- <revision>
- <id>60824</id>
- <timestamp>2002-02-25T15:43:11Z</timestamp>
- <contributor>
- <ip>Conversion script</ip>
- </contributor>
- <minor />
- <comment>Automated conversion</comment>
- <text xml:space="preserve">An '''acronym''' is an [[abbreviation]], often composed of the initial letters of the words in a short phrase, that is treated as word (often, a piece of jargon or the proper name of an organization). For example, SAM for [[''s''urface-to-''a''ir ''m''issile]] and [[NATO]] for the [[North Atlantic Treaty Organization]]. In its original meaning, acronyms were restricted to ''pronouncible'' abbreviations (what might be called ''true'' acronyms), though common usage permits calling unpronouncable abbreviations acronyms as well. Sometimes conjuntions and prepositions (such as and or to) contribute letters to make the acronym pronouncible, in contradiction to the normal [[English language|English]] rule for abbreviations.
-
-Often, an acronym will come into such wide use that people think of it as a word in itself, forget that it started out as an acronym, and write in in small letters. Examples include [[quasar]] (''q''uasi-''s''tellar ''r''adio ''s''ource), [[laser]] (''l''ight ''a''mplification by ''s''timulated ''e''mission of ''r''adiation) and radar (''r''adio ''d''etection ''a''nd ''r''anging).
-
-Non-pronouncible abbreviations formed from initials (such as IBM for International Business Machines) are sometimes called '''[[initialism]]s'''.
-
-Some lists of acronyms in use:
-
-*[[Internet slang|acronyms used on the Internet]]
-*[[Acronym/List|list of acronyms]]
-*[[Acronym/Medical List|list of medical acronyms]]
-
-A large list of acronyms may be found at http://www.acronymfinder.com/
-
-[[talk:Acronym|/Talk]]
-</text>
- </revision>
- </page>
-</mediawiki>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/pom.xml
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/pom.xml b/src/examples/wikisearch/pom.xml
deleted file mode 100644
index 1ea9a2a..0000000
--- a/src/examples/wikisearch/pom.xml
+++ /dev/null
@@ -1,253 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <artifactId>accumulo-examples</artifactId>
- <groupId>org.apache.accumulo</groupId>
- <version>1.4.5-SNAPSHOT</version>
- <relativePath>../</relativePath>
- </parent>
- <artifactId>accumulo-wikisearch</artifactId>
- <packaging>pom</packaging>
- <name>accumulo-wikisearch</name>
-
- <modules>
- <module>ingest</module>
- <module>query</module>
- <module>query-war</module>
- </modules>
-
- <repositories>
- <repository>
- <id>central</id>
- <name>Maven Repository Switchboard</name>
- <layout>default</layout>
- <url>http://repo1.maven.org/maven2</url>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- </repository>
- <repository>
- <id>java.net</id>
- <name>java.net</name>
- <layout>default</layout>
- <url>https://maven.java.net/content/groups/public</url>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- </repository>
- </repositories>
-
- <build>
- <defaultGoal>package</defaultGoal>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-enforcer-plugin</artifactId>
- <executions>
- <execution>
- <id>enforce-mvn</id>
- <goals>
- <goal>enforce</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <artifactId>maven-clean-plugin</artifactId>
- <configuration>
- <filesets>
- <fileset>
- <directory>lib</directory>
- </fileset>
- <fileset>
- <directory>target</directory>
- </fileset>
- </filesets>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <outputDirectory>lib</outputDirectory>
- <archive>
- <manifest>
- <addDefaultImplementationEntries>true</addDefaultImplementationEntries>
- </manifest>
- <manifestEntries>
- <Implementation-Build>${buildNumber}</Implementation-Build>
- <Implementation-Timestamp>${timestamp}</Implementation-Timestamp>
- </manifestEntries>
- </archive>
- <includes>
- <include>**/**</include>
- </includes>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-resources-plugin</artifactId>
- <configuration>
- <encoding>UTF-8</encoding>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-javadoc-plugin</artifactId>
- <configuration>
- <encoding>UTF-8</encoding>
- <quiet>true</quiet>
- <jarOutputDirectory>lib</jarOutputDirectory>
- <reportOutputDirectory>docs</reportOutputDirectory>
- <javadocVersion>1.6</javadocVersion>
- <additionalJOption>-J-Xmx512m</additionalJOption>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-source-plugin</artifactId>
- <configuration>
- <outputDirectory>lib</outputDirectory>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-surefire-plugin</artifactId>
- </plugin>
- <plugin>
- <inherited>false</inherited>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>copy-dependencies</id>
- <phase>process-resources</phase>
- <goals>
- <goal>copy-dependencies</goal>
- </goals>
- <configuration>
- <outputDirectory>../../lib</outputDirectory>
- <!-- just grab the non-provided runtime dependencies -->
- <includeArtifactIds>commons-collections,commons-configuration,commons-io,commons-lang,jline,log4j,libthrift,commons-jci-core,commons-jci-fam,commons-logging,commons-logging-api,cloudtrace</includeArtifactIds>
- <excludeGroupIds>accumulo</excludeGroupIds>
- <excludeTransitive>true</excludeTransitive>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <version.commons-lang>2.4</version.commons-lang>
- <version.accumulo>${project.version}</version.accumulo>
- <version.commons-jexl>2.0.1</version.commons-jexl>
- <version.commons-codec>1.5</version.commons-codec>
- <version.ejb-spec-api>1.0.1.Final</version.ejb-spec-api>
- <version.jaxrs>2.1.0.GA</version.jaxrs>
- <version.kryo>1.04</version.kryo>
- <version.log4j>1.2.16</version.log4j>
- <version.log4j-extras>1.0</version.log4j-extras>
- <version.lucene>3.0.2</version.lucene>
- <version.lucene-analyzers>3.0.2</version.lucene-analyzers>
- <version.lucene-wikipedia>3.0.2</version.lucene-wikipedia>
- <version.protobuf>2.3.0</version.protobuf>
- <version.googlecollections>1.0</version.googlecollections>
- <version.libthrift>0.6.1</version.libthrift>
- <version.zookeeper>3.3.1</version.zookeeper>
- <version.minlog>1.2</version.minlog>
- </properties>
-
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- <version>${version.commons-codec}</version>
- </dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- <version>${version.commons-lang}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-core</artifactId>
- <version>${version.accumulo}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>zookeeper</artifactId>
- <version>${version.zookeeper}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libthrift</artifactId>
- <version>${version.libthrift}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>cloudtrace</artifactId>
- <version>${version.accumulo}</version>
- </dependency>
- <dependency>
- <groupId>com.google.collections</groupId>
- <artifactId>google-collections</artifactId>
- <version>${version.googlecollections}</version>
- </dependency>
- <dependency>
- <groupId>com.googlecode</groupId>
- <artifactId>kryo</artifactId>
- <version>${version.kryo}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-core</artifactId>
- <version>${version.lucene}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-analyzers</artifactId>
- <version>${version.lucene-analyzers}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-wikipedia</artifactId>
- <version>${version.lucene-wikipedia}</version>
- </dependency>
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- <version>${version.protobuf}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-jexl</artifactId>
- <version>${version.commons-jexl}</version>
- </dependency>
- <dependency>
- <groupId>com.googlecode</groupId>
- <artifactId>minlog</artifactId>
- <version>${version.minlog}</version>
- </dependency>
- <dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-server</artifactId>
- <version>1.11</version>
- </dependency>
- </dependencies>
- </dependencyManagement>
-
-</project>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query-war/pom.xml
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query-war/pom.xml b/src/examples/wikisearch/query-war/pom.xml
deleted file mode 100644
index 485d584..0000000
--- a/src/examples/wikisearch/query-war/pom.xml
+++ /dev/null
@@ -1,66 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <artifactId>accumulo-wikisearch</artifactId>
- <groupId>org.apache.accumulo</groupId>
- <version>1.4.5-SNAPSHOT</version>
- </parent>
-
- <artifactId>wikisearch-query-war</artifactId>
- <packaging>war</packaging>
- <name>wikisearch-query-war</name>
-
- <dependencies>
- <dependency>
- <groupId>javax</groupId>
- <artifactId>javaee-web-api</artifactId>
- <version>6.0</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
-
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <source>1.6</source>
- <target>1.6</target>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-war-plugin</artifactId>
- <version>2.1.1</version>
- <configuration>
- <failOnMissingWebXml>true</failOnMissingWebXml>
- </configuration>
- </plugin>
- </plugins>
- </build>
-
-</project>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query-war/src/main/webapp/WEB-INF/jboss-web.xml
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query-war/src/main/webapp/WEB-INF/jboss-web.xml b/src/examples/wikisearch/query-war/src/main/webapp/WEB-INF/jboss-web.xml
deleted file mode 100644
index 6765033..0000000
--- a/src/examples/wikisearch/query-war/src/main/webapp/WEB-INF/jboss-web.xml
+++ /dev/null
@@ -1,20 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<jboss-web>
- <context-root>/accumulo-wikisearch</context-root>
-</jboss-web>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query-war/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query-war/src/main/webapp/WEB-INF/web.xml b/src/examples/wikisearch/query-war/src/main/webapp/WEB-INF/web.xml
deleted file mode 100644
index a4ff03a..0000000
--- a/src/examples/wikisearch/query-war/src/main/webapp/WEB-INF/web.xml
+++ /dev/null
@@ -1,57 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<web-app xmlns="http://java.sun.com/xml/ns/javaee"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://java.sun.com/xml/ns/javaee
- http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd" version="3.0">
-
- <context-param>
- <param-name>resteasy.jndi.resources</param-name>
- <param-value>Query/local</param-value>
- </context-param>
-
- <listener>
- <listener-class>org.jboss.resteasy.plugins.server.servlet.ResteasyBootstrap</listener-class>
- </listener>
-
- <servlet>
- <servlet-name>Resteasy</servlet-name>
- <servlet-class>org.jboss.resteasy.plugins.server.servlet.HttpServletDispatcher</servlet-class>
- </servlet>
-
- <servlet-mapping>
- <servlet-name>Resteasy</servlet-name>
- <url-pattern>/rest/*</url-pattern>
- </servlet-mapping>
-
- <context-param>
- <param-name>resteasy.servlet.mapping.prefix</param-name>
- <param-value>/rest</param-value>
- </context-param>
-
- <servlet>
- <servlet-name>UserInterface</servlet-name>
- <jsp-file>/ui.jsp</jsp-file>
- </servlet>
-
- <servlet-mapping>
- <servlet-name>UserInterface</servlet-name>
- <url-pattern>/ui/*</url-pattern>
- </servlet-mapping>
-
-</web-app>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query-war/src/main/webapp/style.xsl
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query-war/src/main/webapp/style.xsl b/src/examples/wikisearch/query-war/src/main/webapp/style.xsl
deleted file mode 100644
index 8b19cde..0000000
--- a/src/examples/wikisearch/query-war/src/main/webapp/style.xsl
+++ /dev/null
@@ -1,47 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
-<xsl:output method="html" indent="yes" encoding="UTF-8" omit-xml-declaration="yes" />
- <xsl:template match="/results">
- <html>
- <table border="1">
- <tr><th>Id</th><th>Title</th><th>Timestamp</th><th>Comments</th><th>Document Link</th></tr>
- <xsl:for-each select="document">
- <tr>
- <td><xsl:value-of select="field[@name = 'ID']/text()" /></td>
- <td><xsl:value-of select="field[@name = 'TITLE']/text()" /></td>
- <td><xsl:value-of select="field[@name = 'TIMESTAMP']/text()" /></td>
- <td><xsl:value-of select="field[@name = 'COMMENTS']/text()" /></td>
- <xsl:variable name="pointer" select="field[@name ='DOCUMENT']/text()" />
- <xsl:variable name="href">
- <xsl:text>/accumulo-wikisearch/rest/Query/content?query=</xsl:text><xsl:copy-of select="$pointer"/><xsl:text>&auths=all</xsl:text>
- </xsl:variable>
- <xsl:variable name="link">
- <xsl:element name="a">
- <xsl:attribute name="href"><xsl:copy-of select="$href" /></xsl:attribute>
- <xsl:attribute name="target">_blank</xsl:attribute>
- <xsl:text>View Document</xsl:text>
- </xsl:element>
- </xsl:variable>
- <td><xsl:copy-of select="$link"/></td>
- </tr>
- </xsl:for-each>
- </table>
- </html>
- </xsl:template>
-</xsl:stylesheet>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query-war/src/main/webapp/ui.jsp
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query-war/src/main/webapp/ui.jsp b/src/examples/wikisearch/query-war/src/main/webapp/ui.jsp
deleted file mode 100644
index 6719e10..0000000
--- a/src/examples/wikisearch/query-war/src/main/webapp/ui.jsp
+++ /dev/null
@@ -1,131 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<%@page contentType="text/html" pageEncoding="UTF-8"%>
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
- "http://www.w3.org/TR/html4/loose.dtd">
-
-<html>
- <head>
- <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
- <title>Wiki Search Page</title>
- <style type="text/css">
- #comments {
- width: 90%;
- margin: auto;
- }
- h1 {
- text-align: center;
- }
- #d {
- width: 80%;
- margin: auto;
- }
- .center_box {
- width: 70%;
- margin: auto;
- }
- .center_input {
- width: 30%;
- margin: auto;
- }
- </style>
- </head>
- <body>
- <div id="comments">
- <h1>Wiki Search using Apache Accumulo</h1>
- <p>This sample application demonstrates the ability to use search documents using Apache Accumulo. The associated ingest software
- extracts the id, title, timestamp, and comments from each wikipedia article. In addition, the wikipedia text has been tokenized
- and is available for searching. You can enter a boolean expression into the search box below and select the particular set of
- wikipedia languages you want to search.</p>
- <p> Fields available for searching:
- <ol>
- <li>TEXT</li>
- <li>ID</li>
- <li>TITLE</li>
- <li>TIMESTAMP</li>
- <li>COMMENTS</li>
- </ol>
- <p>The search syntax is boolean logic, for example: TEXT == 'boy' and TITLE =~ 'Autism'. The supported operators are:
- ==, !=, <, >, ≤, ≥, =~, and !~. Likewise grouping can be performed using parentheses and predicates can be
- joined using and, or, and not.
- <p>To highlight the cell-level access control of Apache Accumulo, the "authorization" required for a particular cell is the language
- of the associated wikipedia article.
- </div>
- <div id="d">
- <form id="FORM" name="queryForm" method="get" target="results" onsubmit="return setAction()">
- <br />
- <br />
- <div class="center_box">
- <label>Query: </label>
- <input id="QUERY" type="text" name="query" size="100" maxlength="300"/>
- </div>
- <br />
- <div class="center_input">
- <label>Authorizations: </label>
- <br />
- <label>All</label><input type="checkbox" name="auths" value="all" />
- </div>
- <div class="center_input">
- <label>Arabic</label> <input type="checkbox" name="auths" value="arwiki" />
- <label>Brazilian</label> <input type="checkbox" name="auths" value="brwiki" />
- <label>Chinese</label> <input type="checkbox" name="auths" value="zhwiki" />
- </div>
- <div class="center_input">
- <label>Dutch</label> <input type="checkbox" name="auths" value="nlwiki" />
- <label>English</label> <input type="checkbox" name="auths" value="enwiki" />
- <label>Farsi</label> <input type="checkbox" name="auths" value="fawiki" />
- </div>
- <div class="center_input">
- <label>French</label> <input type="checkbox" name="auths" value="frwiki" />
- <label>German</label> <input type="checkbox" name="auths" value="dewiki" />
- <label>Greek</label> <input type="checkbox" name="auths" value="elwiki" />
- </div>
- <div class="center_input">
- <label>Italian</label> <input type="checkbox" name="auths" value="itwiki" />
- <label>Spanish</label> <input type="checkbox" name="auths" value="eswiki" />
- <label>Russia</label>n <input type="checkbox" name="auths" value="ruwiki" /><br />
- </div>
- <div class="center_input">
- <input type="submit" name="Submit Query" />
- </div>
- </form>
- <br />
- <br />
- <iframe name="results" width="90%" height="400" scrolling="yes" >
- </iframe>
- </div>
- <script type="text/javascript">
- function setAction() {
- var f = document.forms[0];
- var authString = "";
- var sep = "";
- for (var i=0; i<f.auths.length; i++) {
- if (f.auths[i].checked) {
- authString = authString + sep + f.auths[i].value;
- sep = ",";
- }
- }
- //Build the new query
- var existingAction = "/accumulo-wikisearch/rest/Query/html";
- var query = f.query.value;
-
- var newAction = existingAction + "?query=" + query + "&auths=" + authString;
- document.forms[0].action = newAction;
- }
- </script>
- </body>
-</html>
[02/39] git commit: ACCUMULO-1520 misc functional test fixes
Posted by ec...@apache.org.
ACCUMULO-1520 misc functional test fixes
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.5@1494361 13f79535-47bb-0310-9956-ffa450edef68
(cherry picked from commit 9ac99ea8b402792b36c494c02ac3103e7a061e2a)
Reason: Test
Author: Billie Rinaldi <bi...@apache.org>
Ref: ACCUMULO-1792
Hadoop2 by default emits extra logging information to stdout which causes accumulo's ListTables to emit
extra data to stdout. This filters out these invalid lines ListTables.
Author: Jon Hsieh <jm...@apache.org>
Included the presplit multiplier portion because the LongRowTest has been failing for me regularly on VMs on 1.4.5-SNAPSHOT and this change makes 1.4.x consistent with the other active development branches.
Included the search example change for consistency since the rest of hte patch was applicable.
Author: Sean Busbey <bu...@clouderagovt.com>
Signed-off-by: Eric Newton <er...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/aff0d641
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/aff0d641
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/aff0d641
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: aff0d641274d78a5682defbd5e6d0717aac68546
Parents: 1fe5f7f
Author: Jonathan M Hsieh <jo...@cloudera.com>
Authored: Mon Jun 24 00:36:19 2013 -0700
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Nov 25 16:06:41 2013 -0500
----------------------------------------------------------------------
.../apache/accumulo/server/test/functional/LargeRowTest.java | 2 +-
test/system/auto/TestUtils.py | 5 +++++
test/system/auto/simple/examples.py | 6 +++---
3 files changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/aff0d641/src/server/src/main/java/org/apache/accumulo/server/test/functional/LargeRowTest.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/test/functional/LargeRowTest.java b/src/server/src/main/java/org/apache/accumulo/server/test/functional/LargeRowTest.java
index bc02929..df98f6e 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/test/functional/LargeRowTest.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/test/functional/LargeRowTest.java
@@ -95,7 +95,7 @@ public class LargeRowTest extends FunctionalTest {
UtilWaitThread.sleep(10000);
- checkSplits(REG_TABLE_NAME, NUM_PRE_SPLITS / 2, NUM_PRE_SPLITS);
+ checkSplits(REG_TABLE_NAME, NUM_PRE_SPLITS / 2, NUM_PRE_SPLITS * 4);
verify(REG_TABLE_NAME);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/aff0d641/test/system/auto/TestUtils.py
----------------------------------------------------------------------
diff --git a/test/system/auto/TestUtils.py b/test/system/auto/TestUtils.py
index b70677a..b7659a2 100755
--- a/test/system/auto/TestUtils.py
+++ b/test/system/auto/TestUtils.py
@@ -458,6 +458,11 @@ class TestUtilsMixin:
out,err = handle.communicate()
self.assert_(handle.returncode==0)
for line in out.split('\n') :
+
+ # filter out unwanted extra hadoop2 logging
+ if line.find("=>") < 0:
+ continue
+
left, right = line.split("=>")
left = left.strip()
right = right.strip()
http://git-wip-us.apache.org/repos/asf/accumulo/blob/aff0d641/test/system/auto/simple/examples.py
----------------------------------------------------------------------
diff --git a/test/system/auto/simple/examples.py b/test/system/auto/simple/examples.py
index b22411a..bed321f 100755
--- a/test/system/auto/simple/examples.py
+++ b/test/system/auto/simple/examples.py
@@ -80,14 +80,14 @@ class Examples(TestUtilsMixin, unittest.TestCase):
'dataTable',
visibility,
100000,
- ACCUMULO_HOME+"/src")
+ ACCUMULO_HOME+"/test")
self.comment(" searching for a file")
handle = self.runOn('localhost', [self.accumulo_sh(), 'org.apache.accumulo.examples.simple.dirlist.QueryUtil',
INSTANCE_NAME, ZOOKEEPERS, ROOT, ROOT_PASSWORD,
- 'indexTable', auths, 'Fate.java', '-search'])
+ 'indexTable', auths, 'examples.py', '-search'])
out, err = handle.communicate()
self.assert_(handle.returncode == 0)
- self.assert_(out.find('accumulo/server/fate/Fate.java') >= 0)
+ self.assert_(out.find('test/system/auto/simple/examples.py') >= 0)
self.comment(" found file at " + out)
[18/39] git commit: ACCUMULO-804 applied fixes to changes that broke
compilation against hadoop 2.0
Posted by ec...@apache.org.
ACCUMULO-804 applied fixes to changes that broke compilation against hadoop 2.0
git-svn-id: https://svn.apache.org/repos/asf/accumulo/trunk@1431056 13f79535-47bb-0310-9956-ffa450edef68
(cherry picked from commit 42cca8a3c910ec4d813b7611b2e65e70e57c6d66)
Reason: Hadoop2 compat
Author: Billie Rinaldi <bi...@apache.org>
Ref: ACCUMULO-1792
Signed-off-by: Eric Newton <er...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ec2aaa6a
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ec2aaa6a
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ec2aaa6a
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: ec2aaa6aff59fa65593641bf03b719a3147f3791
Parents: fc3fdf3
Author: Jonathan M Hsieh <jo...@cloudera.com>
Authored: Wed May 29 13:16:18 2013 -0700
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Nov 25 16:06:42 2013 -0500
----------------------------------------------------------------------
.../test/java/org/apache/accumulo/core/conf/PropertyTest.java | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ec2aaa6a/src/core/src/test/java/org/apache/accumulo/core/conf/PropertyTest.java
----------------------------------------------------------------------
diff --git a/src/core/src/test/java/org/apache/accumulo/core/conf/PropertyTest.java b/src/core/src/test/java/org/apache/accumulo/core/conf/PropertyTest.java
index f929921..551d042 100644
--- a/src/core/src/test/java/org/apache/accumulo/core/conf/PropertyTest.java
+++ b/src/core/src/test/java/org/apache/accumulo/core/conf/PropertyTest.java
@@ -92,5 +92,11 @@ public class PropertyTest {
typeCheckValidFormat(PropertyType.HOSTLIST, "localhost", "server1,server2,server3", "server1:1111,server2:3333", "localhost:1111", "server2:1111",
"www.server", "www.server:1111", "www.server.com", "www.server.com:111");
typeCheckInvalidFormat(PropertyType.HOSTLIST, ":111", "local host");
+
+ typeCheckValidFormat(PropertyType.ABSOLUTEPATH, "/foo", "/foo/c", "/");
+ // in hadoop 2.0 Path only normalizes Windows paths properly when run on a Windows system
+ // this makes the following checks fail
+ // typeCheckValidFormat(PropertyType.ABSOLUTEPATH, "d:\\foo12", "c:\\foo\\g", "c:\\foo\\c", "c:\\");
+ typeCheckInvalidFormat(PropertyType.ABSOLUTEPATH, "foo12", "foo/g", "foo\\c");
}
}
[31/39] git commit: ACCUMULO-804 fix the functional tests for
hadoop-2.0
Posted by ec...@apache.org.
ACCUMULO-804 fix the functional tests for hadoop-2.0
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.5@1458388 13f79535-47bb-0310-9956-ffa450edef68
(cherry picked from commit 17ef9c25ef2970b02b4771217f4bdf06fa0517a8)
Reason: Hadoop2 Compat
Author: Eric C. Newton
Ref: ACCUMULO-1792
Signed-off-by: Eric Newton <er...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d5f16575
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d5f16575
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d5f16575
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: d5f16575f721d4fa7977dd1d1e8a9d9e7481b1ab
Parents: 4f7fb04
Author: Sean Busbey <bu...@cloudera.com>
Authored: Thu Oct 10 13:19:16 2013 -0500
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Nov 25 16:06:44 2013 -0500
----------------------------------------------------------------------
test/system/auto/TestUtils.py | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d5f16575/test/system/auto/TestUtils.py
----------------------------------------------------------------------
diff --git a/test/system/auto/TestUtils.py b/test/system/auto/TestUtils.py
index 0e1bf1a..f6eca67 100755
--- a/test/system/auto/TestUtils.py
+++ b/test/system/auto/TestUtils.py
@@ -67,6 +67,10 @@ $ACCUMULO_HOME/server/target/classes/,
$HADOOP_CONF_DIR,
$HADOOP_PREFIX/[^.].*.jar,
$HADOOP_PREFIX/lib/[^.].*.jar,
+ $HADOOP_PREFIX/share/hadoop/common/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,
"""
log = logging.getLogger('test.auto')
[30/39] git commit: ACCUMULO-645 switch to using HADOOP_PREFIX and
HADOOP_CONF_DIR
Posted by ec...@apache.org.
ACCUMULO-645 switch to using HADOOP_PREFIX and HADOOP_CONF_DIR
git-svn-id: https://svn.apache.org/repos/asf/accumulo/trunk@1431051 13f79535-47bb-0310-9956-ffa450edef68
(cherry picked from commit b8fdefd91927f82809dfdc4004569f2e223ebdeb)
Reason: Hadoop2 compat
Author: Eric C. Newton <ec...@apache.org>
Ref: ACCUMULO-1792
Signed-off-by: Eric Newton <er...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/344e7254
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/344e7254
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/344e7254
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 344e725426e76b952d0f390d65ef6062c35fdc03
Parents: 651033a
Author: Eric C. Newton <ec...@apache.org>
Authored: Wed Jan 9 20:43:35 2013 +0000
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Nov 25 16:06:44 2013 -0500
----------------------------------------------------------------------
bin/config.sh | 16 ++++++++--------
conf/examples/1GB/native-standalone/accumulo-env.sh | 10 +++++++++-
conf/examples/1GB/standalone/accumulo-env.sh | 10 +++++++++-
conf/examples/2GB/native-standalone/accumulo-env.sh | 9 ++++++++-
conf/examples/2GB/standalone/accumulo-env.sh | 10 +++++++++-
conf/examples/3GB/native-standalone/accumulo-env.sh | 9 ++++++++-
conf/examples/3GB/standalone/accumulo-env.sh | 9 ++++++++-
.../512MB/native-standalone/accumulo-env.sh | 9 ++++++++-
conf/examples/512MB/standalone/accumulo-env.sh | 9 ++++++++-
9 files changed, 75 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/344e7254/bin/config.sh
----------------------------------------------------------------------
diff --git a/bin/config.sh b/bin/config.sh
index 5836b46..d9300d5 100755
--- a/bin/config.sh
+++ b/bin/config.sh
@@ -72,23 +72,23 @@ if [ -z ${ACCUMULO_VERSION} ]; then
ACCUMULO_VERSION=1.4.5-SNAPSHOT
fi
-if [ -z "$HADOOP_HOME" ]
+if [ -z "$HADOOP_PREFIX" ]
then
HADOOP_HOME="`which hadoop`"
- if [ -z "$HADOOP_HOME" ]
+ if [ -z "$HADOOP_PREFIX" ]
then
- echo "You must set HADOOP_HOME"
+ echo "You must set HADOOP_PREFIX"
exit 1
fi
- HADOOP_HOME=`dirname $HADOOP_HOME`
- HADOOP_HOME=`dirname $HADOOP_HOME`
+ HADOOP_HOME=`dirname $HADOOP_PREFIX`
+ HADOOP_HOME=`dirname $HADOOP_PREFIX`
fi
-if [ ! -d "$HADOOP_HOME" ]
+if [ ! -d "$HADOOP_PREFIX" ]
then
- echo "$HADOOP_HOME is not a directory"
+ echo "$HADOOP_PREFIX is not a directory"
exit 1
fi
-export HADOOP_HOME
+export HADOOP_PREFIX
if [ ! -f "$ACCUMULO_CONF_DIR/masters" -o ! -f "$ACCUMULO_CONF_DIR/slaves" ]
then
http://git-wip-us.apache.org/repos/asf/accumulo/blob/344e7254/conf/examples/1GB/native-standalone/accumulo-env.sh
----------------------------------------------------------------------
diff --git a/conf/examples/1GB/native-standalone/accumulo-env.sh b/conf/examples/1GB/native-standalone/accumulo-env.sh
index 4033152..15caef6 100755
--- a/conf/examples/1GB/native-standalone/accumulo-env.sh
+++ b/conf/examples/1GB/native-standalone/accumulo-env.sh
@@ -27,9 +27,17 @@
### you may want to use smaller values, especially when running everything
### on a single machine.
###
+if [ -z "$HADOOP_HOME" ]
+then
+ test -z "$HADOOP_PREFIX" && export HADOOP_PREFIX=/path/to/hadoop
+else
+ HADOOP_PREFIX="$HADOOP_HOME"
+ unset HADOOP_HOME
+fi
+test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/conf"
+
test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java
-test -z "$HADOOP_HOME" && export HADOOP_HOME=/path/to/hadoop
test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/path/to/zookeeper
test -z "$ACCUMULO_LOG_DIR" && export ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs
if [ -f ${ACCUMULO_CONF_DIR}/accumulo.policy ]
http://git-wip-us.apache.org/repos/asf/accumulo/blob/344e7254/conf/examples/1GB/standalone/accumulo-env.sh
----------------------------------------------------------------------
diff --git a/conf/examples/1GB/standalone/accumulo-env.sh b/conf/examples/1GB/standalone/accumulo-env.sh
index 4d09467..5c54a39 100755
--- a/conf/examples/1GB/standalone/accumulo-env.sh
+++ b/conf/examples/1GB/standalone/accumulo-env.sh
@@ -27,9 +27,17 @@
### you may want to use smaller values, especially when running everything
### on a single machine.
###
+if [ -z "$HADOOP_HOME" ]
+then
+ test -z "$HADOOP_PREFIX" && export HADOOP_PREFIX=/path/to/hadoop
+else
+ HADOOP_PREFIX="$HADOOP_HOME"
+ unset HADOOP_HOME
+fi
+test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/conf"
+
test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java
-test -z "$HADOOP_HOME" && export HADOOP_HOME=/path/to/hadoop
test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/path/to/zookeeper
test -z "$ACCUMULO_LOG_DIR" && export ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs
if [ -f ${ACCUMULO_CONF_DIR}/accumulo.policy ]
http://git-wip-us.apache.org/repos/asf/accumulo/blob/344e7254/conf/examples/2GB/native-standalone/accumulo-env.sh
----------------------------------------------------------------------
diff --git a/conf/examples/2GB/native-standalone/accumulo-env.sh b/conf/examples/2GB/native-standalone/accumulo-env.sh
index c2346ff..4f93bb4 100755
--- a/conf/examples/2GB/native-standalone/accumulo-env.sh
+++ b/conf/examples/2GB/native-standalone/accumulo-env.sh
@@ -27,9 +27,16 @@
### you may want to use smaller values, especially when running everything
### on a single machine.
###
+if [ -z "$HADOOP_HOME" ]
+then
+ test -z "$HADOOP_PREFIX" && export HADOOP_PREFIX=/path/to/hadoop
+else
+ HADOOP_PREFIX="$HADOOP_HOME"
+ unset HADOOP_HOME
+fi
+test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/conf"
test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java
-test -z "$HADOOP_HOME" && export HADOOP_HOME=/path/to/hadoop
test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/path/to/zookeeper
test -z "$ACCUMULO_LOG_DIR" && export ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs
if [ -f ${ACCUMULO_CONF_DIR}/accumulo.policy ]
http://git-wip-us.apache.org/repos/asf/accumulo/blob/344e7254/conf/examples/2GB/standalone/accumulo-env.sh
----------------------------------------------------------------------
diff --git a/conf/examples/2GB/standalone/accumulo-env.sh b/conf/examples/2GB/standalone/accumulo-env.sh
index 3f6e272..4595783 100755
--- a/conf/examples/2GB/standalone/accumulo-env.sh
+++ b/conf/examples/2GB/standalone/accumulo-env.sh
@@ -27,9 +27,17 @@
### you may want to use smaller values, especially when running everything
### on a single machine.
###
+if [ -z "$HADOOP_HOME" ]
+then
+ test -z "$HADOOP_PREFIX" && export HADOOP_PREFIX=/path/to/hadoop
+else
+ HADOOP_PREFIX="$HADOOP_HOME"
+ unset HADOOP_HOME
+fi
+test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/conf"
+
test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java
-test -z "$HADOOP_HOME" && export HADOOP_HOME=/path/to/hadoop
test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/path/to/zookeeper
test -z "$ACCUMULO_LOG_DIR" && export ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs
if [ -f ${ACCUMULO_CONF_DIR}/accumulo.policy ]
http://git-wip-us.apache.org/repos/asf/accumulo/blob/344e7254/conf/examples/3GB/native-standalone/accumulo-env.sh
----------------------------------------------------------------------
diff --git a/conf/examples/3GB/native-standalone/accumulo-env.sh b/conf/examples/3GB/native-standalone/accumulo-env.sh
index 6fe3f3f..d5484be 100755
--- a/conf/examples/3GB/native-standalone/accumulo-env.sh
+++ b/conf/examples/3GB/native-standalone/accumulo-env.sh
@@ -27,9 +27,16 @@
### you may want to use smaller values, especially when running everything
### on a single machine.
###
+if [ -z "$HADOOP_HOME" ]
+then
+ test -z "$HADOOP_PREFIX" && export HADOOP_PREFIX=/path/to/hadoop
+else
+ HADOOP_PREFIX="$HADOOP_HOME"
+ unset HADOOP_HOME
+fi
+test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/conf"
test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java
-test -z "$HADOOP_HOME" && export HADOOP_HOME=/path/to/hadoop
test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/path/to/zookeeper
test -z "$ACCUMULO_LOG_DIR" && export ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs
if [ -f ${ACCUMULO_CONF_DIR}/accumulo.policy ]
http://git-wip-us.apache.org/repos/asf/accumulo/blob/344e7254/conf/examples/3GB/standalone/accumulo-env.sh
----------------------------------------------------------------------
diff --git a/conf/examples/3GB/standalone/accumulo-env.sh b/conf/examples/3GB/standalone/accumulo-env.sh
index c554f61..10b1803 100755
--- a/conf/examples/3GB/standalone/accumulo-env.sh
+++ b/conf/examples/3GB/standalone/accumulo-env.sh
@@ -27,9 +27,16 @@
### you may want to use smaller values, especially when running everything
### on a single machine.
###
+if [ -z "$HADOOP_HOME" ]
+then
+ test -z "$HADOOP_PREFIX" && export HADOOP_PREFIX=/path/to/hadoop
+else
+ HADOOP_PREFIX="$HADOOP_HOME"
+ unset HADOOP_HOME
+fi
+test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/conf"
test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java
-test -z "$HADOOP_HOME" && export HADOOP_HOME=/path/to/hadoop
test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/path/to/zookeeper
test -z "$ACCUMULO_LOG_DIR" && export ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs
if [ -f ${ACCUMULO_CONF_DIR}/accumulo.policy ]
http://git-wip-us.apache.org/repos/asf/accumulo/blob/344e7254/conf/examples/512MB/native-standalone/accumulo-env.sh
----------------------------------------------------------------------
diff --git a/conf/examples/512MB/native-standalone/accumulo-env.sh b/conf/examples/512MB/native-standalone/accumulo-env.sh
index 23c77c4..750da89 100755
--- a/conf/examples/512MB/native-standalone/accumulo-env.sh
+++ b/conf/examples/512MB/native-standalone/accumulo-env.sh
@@ -27,9 +27,16 @@
### you may want to use smaller values, especially when running everything
### on a single machine.
###
+if [ -z "$HADOOP_HOME" ]
+then
+ test -z "$HADOOP_PREFIX" && export HADOOP_PREFIX=/path/to/hadoop
+else
+ HADOOP_PREFIX="$HADOOP_HOME"
+ unset HADOOP_HOME
+fi
+test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/conf"
test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java
-test -z "$HADOOP_HOME" && export HADOOP_HOME=/path/to/hadoop
test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/path/to/zookeeper
test -z "$ACCUMULO_LOG_DIR" && export ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs
if [ -f ${ACCUMULO_CONF_DIR}/accumulo.policy ]
http://git-wip-us.apache.org/repos/asf/accumulo/blob/344e7254/conf/examples/512MB/standalone/accumulo-env.sh
----------------------------------------------------------------------
diff --git a/conf/examples/512MB/standalone/accumulo-env.sh b/conf/examples/512MB/standalone/accumulo-env.sh
index 85a6784..bf21762 100755
--- a/conf/examples/512MB/standalone/accumulo-env.sh
+++ b/conf/examples/512MB/standalone/accumulo-env.sh
@@ -27,9 +27,16 @@
### you may want to use smaller values, especially when running everything
### on a single machine.
###
+if [ -z "$HADOOP_HOME" ]
+then
+ test -z "$HADOOP_PREFIX" && export HADOOP_PREFIX=/path/to/hadoop
+else
+ HADOOP_PREFIX="$HADOOP_HOME"
+ unset HADOOP_HOME
+fi
+test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/conf"
test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java
-test -z "$HADOOP_HOME" && export HADOOP_HOME=/path/to/hadoop
test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/path/to/zookeeper
test -z "$ACCUMULO_LOG_DIR" && export ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs
if [ -f ${ACCUMULO_CONF_DIR}/accumulo.policy ]
[15/39] git commit: ACCUMULO-600 removed wikisearch from trunk
Posted by ec...@apache.org.
ACCUMULO-600 removed wikisearch from trunk
git-svn-id: https://svn.apache.org/repos/asf/accumulo/trunk@1370489 13f79535-47bb-0310-9956-ffa450edef68
(cherry picked from commit d1e5c6ebe2796154b514ec8f147720d70b3800b5)
Reason: Maintainability
Author: Billie Rinaldi <bi...@apache.org>
Ref: ACCUMULO-1792
Differs from upstream by leaving a tombstone marker pointing to the contrib project.
Author: Sean Busbey <bu...@cloudera.com>
Signed-off-by: Eric Newton <er...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/8db62992
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/8db62992
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/8db62992
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 8db629923cf4f89b5055f80aa1f39251fd63b25c
Parents: 7fa0085
Author: Sean Busbey <bu...@cloudera.com>
Authored: Tue Oct 22 13:21:55 2013 -0500
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Nov 25 16:06:42 2013 -0500
----------------------------------------------------------------------
.../1GB/native-standalone/generic_logger.xml | 4 -
conf/examples/1GB/standalone/generic_logger.xml | 4 -
.../2GB/native-standalone/generic_logger.xml | 4 -
conf/examples/2GB/standalone/generic_logger.xml | 4 -
.../3GB/native-standalone/generic_logger.xml | 4 -
conf/examples/3GB/standalone/generic_logger.xml | 4 -
.../512MB/native-standalone/generic_logger.xml | 4 -
.../512MB/standalone/generic_logger.xml | 4 -
src/examples/pom.xml | 1 -
src/examples/wikisearch/README | 69 +-
src/examples/wikisearch/README.parallel | 65 -
src/examples/wikisearch/ingest/bin/ingest.sh | 46 -
.../wikisearch/ingest/bin/ingest_parallel.sh | 46 -
.../ingest/conf/wikipedia.xml.example | 43 -
.../ingest/conf/wikipedia_parallel.xml.example | 75 -
src/examples/wikisearch/ingest/pom.xml | 160 --
.../wikisearch/ingest/src/assembly/dist.xml | 38 -
.../wikisearch/ingest/ArticleExtractor.java | 207 --
.../wikisearch/ingest/LRUOutputCombiner.java | 75 -
.../ingest/WikipediaConfiguration.java | 198 --
.../wikisearch/ingest/WikipediaIngester.java | 206 --
.../wikisearch/ingest/WikipediaInputFormat.java | 136 --
.../wikisearch/ingest/WikipediaMapper.java | 245 ---
.../ingest/WikipediaPartitionedIngester.java | 310 ---
.../ingest/WikipediaPartitionedMapper.java | 310 ---
.../wikisearch/ingest/WikipediaPartitioner.java | 89 -
.../iterator/GlobalIndexUidCombiner.java | 94 -
.../wikisearch/iterator/TextIndexCombiner.java | 102 -
.../normalizer/LcNoDiacriticsNormalizer.java | 49 -
.../wikisearch/normalizer/NoOpNormalizer.java | 23 -
.../wikisearch/normalizer/Normalizer.java | 32 -
.../wikisearch/normalizer/NumberNormalizer.java | 42 -
.../output/BufferingRFileRecordWriter.java | 140 --
.../output/SortingRFileOutputFormat.java | 121 --
.../wikisearch/protobuf/TermWeight.java | 424 ----
.../examples/wikisearch/protobuf/Uid.java | 470 -----
.../reader/AggregatingRecordReader.java | 171 --
.../wikisearch/reader/LfLineReader.java | 173 --
.../wikisearch/reader/LongLineRecordReader.java | 136 --
.../examples/wikisearch/util/TextUtil.java | 109 -
.../ingest/src/main/protobuf/TermWeight.proto | 28 -
.../ingest/src/main/protobuf/Uid.proto | 29 -
.../ingest/src/main/protobuf/compile_protos.sh | 19 -
.../ingest/StandaloneStatusReporter.java | 70 -
.../ingest/WikipediaInputSplitTest.java | 69 -
.../wikisearch/ingest/WikipediaMapperTest.java | 163 --
.../wikisearch/iterator/GlobalIndexUidTest.java | 192 --
.../wikisearch/iterator/TextIndexTest.java | 185 --
.../normalizer/testNumberNormalizer.java | 90 -
.../reader/AggregatingRecordReaderTest.java | 287 ---
.../src/test/resources/enwiki-20110901-001.xml | 153 --
src/examples/wikisearch/pom.xml | 253 ---
src/examples/wikisearch/query-war/pom.xml | 66 -
.../src/main/webapp/WEB-INF/jboss-web.xml | 20 -
.../query-war/src/main/webapp/WEB-INF/web.xml | 57 -
.../query-war/src/main/webapp/style.xsl | 47 -
.../wikisearch/query-war/src/main/webapp/ui.jsp | 131 --
.../query-war/src/test/resources/test.xml | 1651 ---------------
src/examples/wikisearch/query/pom.xml | 180 --
.../wikisearch/query/src/assembly/dist.xml | 40 -
.../wikisearch/function/QueryFunctions.java | 68 -
.../iterator/AbstractEvaluatingIterator.java | 323 ---
.../wikisearch/iterator/AndIterator.java | 921 ---------
.../iterator/BooleanLogicIterator.java | 1949 ------------------
.../iterator/BooleanLogicTreeNode.java | 523 -----
.../iterator/DefaultIteratorEnvironment.java | 58 -
.../wikisearch/iterator/EvaluatingIterator.java | 115 --
.../wikisearch/iterator/FieldIndexIterator.java | 736 -------
.../iterator/OptimizedQueryIterator.java | 205 --
.../wikisearch/iterator/OrIterator.java | 822 --------
.../wikisearch/iterator/ReadAheadIterator.java | 297 ---
.../iterator/UniqFieldNameValueIterator.java | 342 ---
.../examples/wikisearch/jexl/Arithmetic.java | 126 --
.../wikisearch/logic/AbstractQueryLogic.java | 883 --------
.../examples/wikisearch/logic/ContentLogic.java | 109 -
.../examples/wikisearch/logic/QueryLogic.java | 195 --
.../examples/wikisearch/parser/EventFields.java | 227 --
.../parser/FieldIndexQueryReWriter.java | 1139 ----------
.../parser/JexlOperatorConstants.java | 105 -
.../wikisearch/parser/QueryEvaluator.java | 291 ---
.../examples/wikisearch/parser/QueryParser.java | 845 --------
.../wikisearch/parser/RangeCalculator.java | 1199 -----------
.../examples/wikisearch/parser/TreeBuilder.java | 675 ------
.../examples/wikisearch/parser/TreeNode.java | 235 ---
.../examples/wikisearch/query/IQuery.java | 66 -
.../examples/wikisearch/query/Query.java | 239 ---
.../examples/wikisearch/sample/Document.java | 61 -
.../examples/wikisearch/sample/Field.java | 58 -
.../examples/wikisearch/sample/Results.java | 53 -
.../examples/wikisearch/util/BaseKeyParser.java | 77 -
.../wikisearch/util/FieldIndexKeyParser.java | 71 -
.../examples/wikisearch/util/KeyParser.java | 70 -
.../src/main/resources/META-INF/MANIFEST.MF | 2 -
.../main/resources/META-INF/ejb-jar.xml.example | 62 -
.../logic/StandaloneStatusReporter.java | 70 -
.../wikisearch/logic/TestQueryLogic.java | 186 --
.../src/test/resources/enwiki-20110901-001.xml | 153 --
97 files changed, 1 insertion(+), 21722 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/conf/examples/1GB/native-standalone/generic_logger.xml
----------------------------------------------------------------------
diff --git a/conf/examples/1GB/native-standalone/generic_logger.xml b/conf/examples/1GB/native-standalone/generic_logger.xml
index dc45710..5dc38ac 100644
--- a/conf/examples/1GB/native-standalone/generic_logger.xml
+++ b/conf/examples/1GB/native-standalone/generic_logger.xml
@@ -69,10 +69,6 @@
<level value="INFO"/>
</logger>
- <logger name="org.apache.accumulo.examples.wikisearch">
- <level value="INFO"/>
- </logger>
-
<logger name="org.mortbay.log">
<level value="WARN"/>
</logger>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/conf/examples/1GB/standalone/generic_logger.xml
----------------------------------------------------------------------
diff --git a/conf/examples/1GB/standalone/generic_logger.xml b/conf/examples/1GB/standalone/generic_logger.xml
index dc45710..5dc38ac 100644
--- a/conf/examples/1GB/standalone/generic_logger.xml
+++ b/conf/examples/1GB/standalone/generic_logger.xml
@@ -69,10 +69,6 @@
<level value="INFO"/>
</logger>
- <logger name="org.apache.accumulo.examples.wikisearch">
- <level value="INFO"/>
- </logger>
-
<logger name="org.mortbay.log">
<level value="WARN"/>
</logger>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/conf/examples/2GB/native-standalone/generic_logger.xml
----------------------------------------------------------------------
diff --git a/conf/examples/2GB/native-standalone/generic_logger.xml b/conf/examples/2GB/native-standalone/generic_logger.xml
index dc45710..5dc38ac 100644
--- a/conf/examples/2GB/native-standalone/generic_logger.xml
+++ b/conf/examples/2GB/native-standalone/generic_logger.xml
@@ -69,10 +69,6 @@
<level value="INFO"/>
</logger>
- <logger name="org.apache.accumulo.examples.wikisearch">
- <level value="INFO"/>
- </logger>
-
<logger name="org.mortbay.log">
<level value="WARN"/>
</logger>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/conf/examples/2GB/standalone/generic_logger.xml
----------------------------------------------------------------------
diff --git a/conf/examples/2GB/standalone/generic_logger.xml b/conf/examples/2GB/standalone/generic_logger.xml
index dc45710..5dc38ac 100644
--- a/conf/examples/2GB/standalone/generic_logger.xml
+++ b/conf/examples/2GB/standalone/generic_logger.xml
@@ -69,10 +69,6 @@
<level value="INFO"/>
</logger>
- <logger name="org.apache.accumulo.examples.wikisearch">
- <level value="INFO"/>
- </logger>
-
<logger name="org.mortbay.log">
<level value="WARN"/>
</logger>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/conf/examples/3GB/native-standalone/generic_logger.xml
----------------------------------------------------------------------
diff --git a/conf/examples/3GB/native-standalone/generic_logger.xml b/conf/examples/3GB/native-standalone/generic_logger.xml
index dc45710..5dc38ac 100644
--- a/conf/examples/3GB/native-standalone/generic_logger.xml
+++ b/conf/examples/3GB/native-standalone/generic_logger.xml
@@ -69,10 +69,6 @@
<level value="INFO"/>
</logger>
- <logger name="org.apache.accumulo.examples.wikisearch">
- <level value="INFO"/>
- </logger>
-
<logger name="org.mortbay.log">
<level value="WARN"/>
</logger>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/conf/examples/3GB/standalone/generic_logger.xml
----------------------------------------------------------------------
diff --git a/conf/examples/3GB/standalone/generic_logger.xml b/conf/examples/3GB/standalone/generic_logger.xml
index dc45710..5dc38ac 100644
--- a/conf/examples/3GB/standalone/generic_logger.xml
+++ b/conf/examples/3GB/standalone/generic_logger.xml
@@ -69,10 +69,6 @@
<level value="INFO"/>
</logger>
- <logger name="org.apache.accumulo.examples.wikisearch">
- <level value="INFO"/>
- </logger>
-
<logger name="org.mortbay.log">
<level value="WARN"/>
</logger>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/conf/examples/512MB/native-standalone/generic_logger.xml
----------------------------------------------------------------------
diff --git a/conf/examples/512MB/native-standalone/generic_logger.xml b/conf/examples/512MB/native-standalone/generic_logger.xml
index dc45710..5dc38ac 100644
--- a/conf/examples/512MB/native-standalone/generic_logger.xml
+++ b/conf/examples/512MB/native-standalone/generic_logger.xml
@@ -69,10 +69,6 @@
<level value="INFO"/>
</logger>
- <logger name="org.apache.accumulo.examples.wikisearch">
- <level value="INFO"/>
- </logger>
-
<logger name="org.mortbay.log">
<level value="WARN"/>
</logger>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/conf/examples/512MB/standalone/generic_logger.xml
----------------------------------------------------------------------
diff --git a/conf/examples/512MB/standalone/generic_logger.xml b/conf/examples/512MB/standalone/generic_logger.xml
index dc45710..5dc38ac 100644
--- a/conf/examples/512MB/standalone/generic_logger.xml
+++ b/conf/examples/512MB/standalone/generic_logger.xml
@@ -69,10 +69,6 @@
<level value="INFO"/>
</logger>
- <logger name="org.apache.accumulo.examples.wikisearch">
- <level value="INFO"/>
- </logger>
-
<logger name="org.mortbay.log">
<level value="WARN"/>
</logger>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/pom.xml
----------------------------------------------------------------------
diff --git a/src/examples/pom.xml b/src/examples/pom.xml
index 2d56be6..0ec2184 100644
--- a/src/examples/pom.xml
+++ b/src/examples/pom.xml
@@ -29,7 +29,6 @@
<modules>
<module>simple</module>
- <module>wikisearch</module>
</modules>
<repositories>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/README
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/README b/src/examples/wikisearch/README
index 041490f..806de3c 100644
--- a/src/examples/wikisearch/README
+++ b/src/examples/wikisearch/README
@@ -1,68 +1 @@
- Apache Accumulo Wikipedia Search Example
-
- This project contains a sample application for ingesting and querying wikipedia data.
-
-
- Ingest
- ------
-
- Prerequisites
- -------------
- 1. Accumulo, Hadoop, and ZooKeeper must be installed and running
- 2. One or more wikipedia dump files (http://dumps.wikimedia.org/backup-index.html) placed in an HDFS directory.
- You will want to grab the files with the link name of pages-articles.xml.bz2
- 3. Though not strictly required, the ingest will go more quickly if the files are decompressed:
-
- $ bunzip2 < enwiki-*-pages-articles.xml.bz2 | hadoop fs -put - /wikipedia/enwiki-pages-articles.xml
-
-
- INSTRUCTIONS
- ------------
- 1. Copy the ingest/conf/wikipedia.xml.example to ingest/conf/wikipedia.xml and change it to specify Accumulo information.
- 2. Copy the ingest/lib/wikisearch-*.jar and ingest/lib/protobuf*.jar to $ACCUMULO_HOME/lib/ext
- 3. Then run ingest/bin/ingest.sh with one argument (the name of the directory in HDFS where the wikipedia XML
- files reside) and this will kick off a MapReduce job to ingest the data into Accumulo.
-
- Query
- -----
-
- Prerequisites
- -------------
- 1. The query software was tested using JBoss AS 6. Install this unless you feel like messing with the installation.
-
- NOTE: Ran into a bug (https://issues.jboss.org/browse/RESTEASY-531) that did not allow an EJB3.1 war file. The
- workaround is to separate the RESTEasy servlet from the EJBs by creating an EJB jar and a WAR file.
-
- INSTRUCTIONS
- -------------
- 1. Copy the query/src/main/resources/META-INF/ejb-jar.xml.example file to
- query/src/main/resources/META-INF/ejb-jar.xml. Modify to the file to contain the same
- information that you put into the wikipedia.xml file from the Ingest step above.
- 2. Re-build the query distribution by running 'mvn package assembly:single' in the top-level directory.
- 3. Untar the resulting file in the $JBOSS_HOME/server/default directory.
-
- $ cd $JBOSS_HOME/server/default
- $ tar -xzf $ACCUMULO_HOME/src/examples/wikisearch/query/target/wikisearch-query*.tar.gz
-
- This will place the dependent jars in the lib directory and the EJB jar into the deploy directory.
- 4. Next, copy the wikisearch*.war file in the query-war/target directory to $JBOSS_HOME/server/default/deploy.
- 5. Start JBoss ($JBOSS_HOME/bin/run.sh)
- 6. Use the Accumulo shell and give the user permissions for the wikis that you loaded, for example:
- setauths -u <user> -s all,enwiki,eswiki,frwiki,fawiki
- 7. Copy the following jars to the $ACCUMULO_HOME/lib/ext directory from the $JBOSS_HOME/server/default/lib directory:
-
- commons-lang*.jar
- kryo*.jar
- minlog*.jar
- commons-jexl*.jar
- google-collections*.jar
-
- 8. Copy the $JBOSS_HOME/server/default/deploy/wikisearch-query*.jar to $ACCUMULO_HOME/lib/ext.
-
-
- 9. At this point you should be able to open a browser and view the page: http://localhost:8080/accumulo-wikisearch/ui/ui.jsp.
- You can issue the queries using this user interface or via the following REST urls: <host>/accumulo-wikisearch/rest/Query/xml,
- <host>/accumulo-wikisearch/rest/Query/html, <host>/accumulo-wikisearch/rest/Query/yaml, or <host>/accumulo-wikisearch/rest/Query/json.
- There are two parameters to the REST service, query and auths. The query parameter is the same string that you would type
- into the search box at ui.jsp, and the auths parameter is a comma-separated list of wikis that you want to search (i.e.
- enwiki,frwiki,dewiki, etc. Or you can use all)
+The Accumulo Wikipedia Search Example has moved to [a contrib project](http://accumulo.apache.org/contrib.html). For more information, see [ACCUMULO-600](https://issues.apache.org/jira/browse/ACCUMULO-600) and the [wikisearch contrib repository](https://git-wip-us.apache.org/repos/asf?p=accumulo-wikisearch.git;a=summary).
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/README.parallel
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/README.parallel b/src/examples/wikisearch/README.parallel
deleted file mode 100644
index 477556b..0000000
--- a/src/examples/wikisearch/README.parallel
+++ /dev/null
@@ -1,65 +0,0 @@
- Apache Accumulo Wikipedia Search Example (parallel version)
-
- This project contains a sample application for ingesting and querying wikipedia data.
-
-
- Ingest
- ------
-
- Prerequisites
- -------------
- 1. Accumulo, Hadoop, and ZooKeeper must be installed and running
- 2. One or more wikipedia dump files (http://dumps.wikimedia.org/backup-index.html) placed in an HDFS directory.
- You will want to grab the files with the link name of pages-articles.xml.bz2
-
-
- INSTRUCTIONS
- ------------
- 1. Copy the ingest/conf/wikipedia_parallel.xml.example to ingest/conf/wikipedia.xml and change it to specify Accumulo information.
- 2. Copy the ingest/lib/wikisearch-*.jar and ingest/lib/protobuf*.jar to $ACCUMULO_HOME/lib/ext
- 3. Then run ingest/bin/ingest_parallel.sh with one argument (the name of the directory in HDFS where the wikipedia XML
- files reside) and this will kick off a MapReduce job to ingest the data into Accumulo.
-
- Query
- -----
-
- Prerequisites
- -------------
- 1. The query software was tested using JBoss AS 6. Install this unless you feel like messing with the installation.
-
- NOTE: Ran into a bug (https://issues.jboss.org/browse/RESTEASY-531) that did not allow an EJB3.1 war file. The
- workaround is to separate the RESTEasy servlet from the EJBs by creating an EJB jar and a WAR file.
-
- INSTRUCTIONS
- -------------
- 1. Copy the query/src/main/resources/META-INF/ejb-jar.xml.example file to
- query/src/main/resources/META-INF/ejb-jar.xml. Modify to the file to contain the same
- information that you put into the wikipedia.xml file from the Ingest step above.
- 2. Re-build the query distribution by running 'mvn package assembly:single' in the top-level directory.
- 3. Untar the resulting file in the $JBOSS_HOME/server/default directory.
-
- $ cd $JBOSS_HOME/server/default
- $ tar -xzf $ACCUMULO_HOME/src/examples/wikisearch/query/target/wikisearch-query*.tar.gz
-
- This will place the dependent jars in the lib directory and the EJB jar into the deploy directory.
- 4. Next, copy the wikisearch*.war file in the query-war/target directory to $JBOSS_HOME/server/default/deploy.
- 5. Start JBoss ($JBOSS_HOME/bin/run.sh)
- 6. Use the Accumulo shell and give the user permissions for the wikis that you loaded, for example:
- setauths -u <user> -s all,enwiki,eswiki,frwiki,fawiki
- 7. Copy the following jars to the $ACCUMULO_HOME/lib/ext directory from the $JBOSS_HOME/server/default/lib directory:
-
- commons-lang*.jar
- kryo*.jar
- minlog*.jar
- commons-jexl*.jar
- google-collections*.jar
-
- 8. Copy the $JBOSS_HOME/server/default/deploy/wikisearch-query*.jar to $ACCUMULO_HOME/lib/ext.
-
-
- 9. At this point you should be able to open a browser and view the page: http://localhost:8080/accumulo-wikisearch/ui/ui.jsp.
- You can issue the queries using this user interface or via the following REST urls: <host>/accumulo-wikisearch/rest/Query/xml,
- <host>/accumulo-wikisearch/rest/Query/html, <host>/accumulo-wikisearch/rest/Query/yaml, or <host>/accumulo-wikisearch/rest/Query/json.
- There are two parameters to the REST service, query and auths. The query parameter is the same string that you would type
- into the search box at ui.jsp, and the auths parameter is a comma-separated list of wikis that you want to search (i.e.
- enwiki,frwiki,dewiki, etc. Or you can use all)
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/bin/ingest.sh
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/bin/ingest.sh b/src/examples/wikisearch/ingest/bin/ingest.sh
deleted file mode 100755
index acdcbf8..0000000
--- a/src/examples/wikisearch/ingest/bin/ingest.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-
-THIS_SCRIPT="$0"
-SCRIPT_DIR="${THIS_SCRIPT%/*}"
-SCRIPT_DIR=`cd $SCRIPT_DIR ; pwd`
-echo $SCRIPT_DIR
-
-#
-# Add our jars
-#
-for f in $SCRIPT_DIR/../lib/*.jar; do
- CLASSPATH=${CLASSPATH}:$f
-done
-
-#
-# Transform the classpath into a comma-separated list also
-#
-LIBJARS=`echo $CLASSPATH | sed 's/^://' | sed 's/:/,/g'`
-
-
-#
-# Map/Reduce job
-#
-JAR=$SCRIPT_DIR/../lib/wikisearch-ingest-1.4.5-SNAPSHOT.jar
-CONF=$SCRIPT_DIR/../conf/wikipedia.xml
-HDFS_DATA_DIR=$1
-export HADOOP_CLASSPATH=$CLASSPATH
-echo "hadoop jar $JAR org.apache.accumulo.examples.wikisearch.ingest.WikipediaIngester -libjars $LIBJARS -conf $CONF -Dwikipedia.input=${HDFS_DATA_DIR}"
-hadoop jar $JAR org.apache.accumulo.examples.wikisearch.ingest.WikipediaIngester -libjars $LIBJARS -conf $CONF -Dwikipedia.input=${HDFS_DATA_DIR}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/bin/ingest_parallel.sh
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/bin/ingest_parallel.sh b/src/examples/wikisearch/ingest/bin/ingest_parallel.sh
deleted file mode 100755
index 8c63ac0..0000000
--- a/src/examples/wikisearch/ingest/bin/ingest_parallel.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-
-THIS_SCRIPT="$0"
-SCRIPT_DIR="${THIS_SCRIPT%/*}"
-SCRIPT_DIR=`cd $SCRIPT_DIR ; pwd`
-echo $SCRIPT_DIR
-
-#
-# Add our jars
-#
-for f in $SCRIPT_DIR/../lib/*.jar; do
- CLASSPATH=${CLASSPATH}:$f
-done
-
-#
-# Transform the classpath into a comma-separated list also
-#
-LIBJARS=`echo $CLASSPATH | sed 's/^://' | sed 's/:/,/g'`
-
-
-#
-# Map/Reduce job
-#
-JAR=$SCRIPT_DIR/../lib/wikisearch-ingest-1.4.5-SNAPSHOT.jar
-CONF=$SCRIPT_DIR/../conf/wikipedia.xml
-HDFS_DATA_DIR=$1
-export HADOOP_CLASSPATH=$CLASSPATH
-echo "hadoop jar $JAR org.apache.accumulo.examples.wikisearch.ingest.WikipediaPartitionedIngester -libjars $LIBJARS -conf $CONF -Dwikipedia.input=${HDFS_DATA_DIR}"
-hadoop jar $JAR org.apache.accumulo.examples.wikisearch.ingest.WikipediaPartitionedIngester -libjars $LIBJARS -conf $CONF -Dwikipedia.input=${HDFS_DATA_DIR}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/conf/wikipedia.xml.example
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/conf/wikipedia.xml.example b/src/examples/wikisearch/ingest/conf/wikipedia.xml.example
deleted file mode 100644
index b08742e..0000000
--- a/src/examples/wikisearch/ingest/conf/wikipedia.xml.example
+++ /dev/null
@@ -1,43 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration>
- <property>
- <name>wikipedia.accumulo.zookeepers</name>
- <value><!--zookeeper servers --></value>
- </property>
- <property>
- <name>wikipedia.accumulo.instance_name</name>
- <value><!--instance name --></value>
- </property>
- <property>
- <name>wikipedia.accumulo.user</name>
- <value><!--user name --></value>
- </property>
- <property>
- <name>wikipedia.accumulo.password</name>
- <value><!-- password --></value>
- </property>
- <property>
- <name>wikipedia.accumulo.table</name>
- <value><!--table name --></value>
- </property>
- <property>
- <name>wikipedia.ingest.partitions</name>
- <value><!--number of partitions --></value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/conf/wikipedia_parallel.xml.example
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/conf/wikipedia_parallel.xml.example b/src/examples/wikisearch/ingest/conf/wikipedia_parallel.xml.example
deleted file mode 100644
index 53220f0..0000000
--- a/src/examples/wikisearch/ingest/conf/wikipedia_parallel.xml.example
+++ /dev/null
@@ -1,75 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration>
- <property>
- <name>wikipedia.accumulo.zookeepers</name>
- <value><!--zookeeper servers --></value>
- </property>
- <property>
- <name>wikipedia.accumulo.instance_name</name>
- <value><!--instance name --></value>
- </property>
- <property>
- <name>wikipedia.accumulo.user</name>
- <value><!--user name --></value>
- </property>
- <property>
- <name>wikipedia.accumulo.password</name>
- <value><!-- password --></value>
- </property>
- <property>
- <name>wikipedia.accumulo.table</name>
- <value><!--table name --></value>
- </property>
- <property>
- <name>wikipedia.ingest.partitions</name>
- <value><!--number of partitions --></value>
- </property>
- <property>
- <name>wikipedia.partitioned.directory</name>
- <value><!--hdfs directory for intemediate partitioned storage --></value>
- </property>
- <property>
- <name>wikipedia.ingest.groups</name>
- <value><!--the number of intermediate partition groups to generate --></value>
- </property>
- <property>
- <name>wikipedia.run.partitioner</name>
- <value><!--whether to run the partitioner step --></value>
- </property>
- <property>
- <name>wikipedia.run.ingest</name>
- <value><!--whether to run the ingest step --></value>
- </property>
- <property>
- <name>wikipedia.bulk.ingest</name>
- <value><!--whether to use bulk ingest vice streaming ingest --></value>
- </property>
- <property>
- <name>wikipedia.bulk.ingest.dir</name>
- <value><!--the directory to store rfiles for bulk ingest --></value>
- </property>
- <property>
- <name>wikipedia.bulk.ingest.failure.dir</name>
- <value><!--the directory to store failed rfiles after bulk ingest --></value>
- </property>
- <property>
- <name>wikipedia.bulk.ingest.buffer.size</name>
- <value><!--the ammount of memory to use for buffering and sorting key/value pairs in each mapper before writing rfiles --></value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/pom.xml
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/pom.xml b/src/examples/wikisearch/ingest/pom.xml
deleted file mode 100644
index 31d7110..0000000
--- a/src/examples/wikisearch/ingest/pom.xml
+++ /dev/null
@@ -1,160 +0,0 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <artifactId>accumulo-wikisearch</artifactId>
- <groupId>org.apache.accumulo</groupId>
- <version>1.4.5-SNAPSHOT</version>
- <relativePath>../</relativePath>
- </parent>
-
- <artifactId>wikisearch-ingest</artifactId>
- <name>wikisearch-ingest</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-start</artifactId>
- </dependency>
- <dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.collections</groupId>
- <artifactId>google-collections</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-analyzers</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-wikipedia</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </dependency>
- <dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-server</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>cloudtrace</artifactId>
- <scope>runtime</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libthrift</artifactId>
- <scope>runtime</scope>
- </dependency>
- <dependency>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>copy-dependencies</id>
- <phase>process-resources</phase>
- <goals>
- <goal>copy-dependencies</goal>
- </goals>
- <configuration>
- <outputDirectory>lib</outputDirectory>
- <!-- just grab the non-provided runtime dependencies -->
- <includeArtifactIds>commons-lang,google-collections,lucene-core,lucene-analyzers,lucene-wikipedia,protobuf-java,accumulo-core,hadoop-core,libthrift,cloudtrace,zookeeper,commons-codec</includeArtifactIds>
- <excludeTransitive>false</excludeTransitive>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <configuration>
- <descriptors>
- <descriptor>src/assembly/dist.xml</descriptor>
- </descriptors>
- <tarLongFileMode>gnu</tarLongFileMode>
- </configuration>
- </plugin>
- </plugins>
- </build>
-
- <profiles>
- <!-- profile for building against Hadoop 1.0.x
- Activate by not specifying hadoop.profile -->
- <profile>
- <id>hadoop-1.0</id>
- <activation>
- <property>
- <name>!hadoop.profile</name>
- </property>
- </activation>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- </dependency>
- </dependencies>
- </profile>
- <!-- profile for building against Hadoop 2.0.x
- Activate using: mvn -Dhadoop.profile=2.0 -->
- <profile>
- <id>hadoop-2.0</id>
- <activation>
- <property>
- <name>hadoop.profile</name>
- <value>2.0</value>
- </property>
- </activation>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- </dependency>
- </dependencies>
- </profile>
- </profiles>
-
-</project>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/assembly/dist.xml
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/assembly/dist.xml b/src/examples/wikisearch/ingest/src/assembly/dist.xml
deleted file mode 100644
index e3e59c1..0000000
--- a/src/examples/wikisearch/ingest/src/assembly/dist.xml
+++ /dev/null
@@ -1,38 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<assembly>
- <id>dist</id>
- <formats>
- <format>tar.gz</format>
- </formats>
- <baseDirectory></baseDirectory>
- <fileSets>
- <fileSet>
- <directory>lib</directory>
- <fileMode>0644</fileMode>
- </fileSet>
- <fileSet>
- <directory>bin</directory>
- <fileMode>0744</fileMode>
- </fileSet>
- <fileSet>
- <directory>conf</directory>
- <fileMode>0644</fileMode>
- </fileSet>
- </fileSets>
-</assembly>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/ArticleExtractor.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/ArticleExtractor.java b/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/ArticleExtractor.java
deleted file mode 100644
index 0699cfa..0000000
--- a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/ArticleExtractor.java
+++ /dev/null
@@ -1,207 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.ingest;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.io.Reader;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.HashMap;
-import java.util.Map;
-
-import javax.xml.namespace.QName;
-import javax.xml.stream.XMLInputFactory;
-import javax.xml.stream.XMLStreamException;
-import javax.xml.stream.XMLStreamReader;
-
-import org.apache.accumulo.examples.wikisearch.normalizer.LcNoDiacriticsNormalizer;
-import org.apache.accumulo.examples.wikisearch.normalizer.NumberNormalizer;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-
-
-public class ArticleExtractor {
-
- public final static SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'Z");
- private static NumberNormalizer nn = new NumberNormalizer();
- private static LcNoDiacriticsNormalizer lcdn = new LcNoDiacriticsNormalizer();
-
- public static class Article implements Writable {
- int id;
- String title;
- long timestamp;
- String comments;
- String text;
-
- public Article(){}
-
- private Article(int id, String title, long timestamp, String comments, String text) {
- super();
- this.id = id;
- this.title = title;
- this.timestamp = timestamp;
- this.comments = comments;
- this.text = text;
- }
-
- public int getId() {
- return id;
- }
-
- public String getTitle() {
- return title;
- }
-
- public String getComments() {
- return comments;
- }
-
- public String getText() {
- return text;
- }
-
- public long getTimestamp() {
- return timestamp;
- }
-
- public Map<String,Object> getFieldValues() {
- Map<String,Object> fields = new HashMap<String,Object>();
- fields.put("ID", this.id);
- fields.put("TITLE", this.title);
- fields.put("TIMESTAMP", this.timestamp);
- fields.put("COMMENTS", this.comments);
- return fields;
- }
-
- public Map<String,String> getNormalizedFieldValues() {
- Map<String,String> fields = new HashMap<String,String>();
- fields.put("ID", nn.normalizeFieldValue("ID", this.id));
- fields.put("TITLE", lcdn.normalizeFieldValue("TITLE", this.title));
- fields.put("TIMESTAMP", nn.normalizeFieldValue("TIMESTAMP", this.timestamp));
- fields.put("COMMENTS", lcdn.normalizeFieldValue("COMMENTS", this.comments));
- return fields;
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- id = in.readInt();
- Text foo = new Text();
- foo.readFields(in);
- title = foo.toString();
- timestamp = in.readLong();
- foo.readFields(in);
- comments = foo.toString();
- foo.readFields(in);
- text = foo.toString();
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeInt(id);
- (new Text(title)).write(out);
- out.writeLong(timestamp);
- (new Text(comments)).write(out);
- (new Text(text)).write(out);
- }
-
- }
-
- public ArticleExtractor() {}
-
- private static XMLInputFactory xmlif = XMLInputFactory.newInstance();
-
- static
- {
- xmlif.setProperty(XMLInputFactory.IS_REPLACING_ENTITY_REFERENCES, Boolean.TRUE);
- }
-
- public Article extract(Reader reader) {
-
- XMLStreamReader xmlr = null;
-
- try {
- xmlr = xmlif.createXMLStreamReader(reader);
- } catch (XMLStreamException e1) {
- throw new RuntimeException(e1);
- }
-
- QName titleName = QName.valueOf("title");
- QName textName = QName.valueOf("text");
- QName revisionName = QName.valueOf("revision");
- QName timestampName = QName.valueOf("timestamp");
- QName commentName = QName.valueOf("comment");
- QName idName = QName.valueOf("id");
-
- Map<QName,StringBuilder> tags = new HashMap<QName,StringBuilder>();
- for (QName tag : new QName[] {titleName, textName, timestampName, commentName, idName}) {
- tags.put(tag, new StringBuilder());
- }
-
- StringBuilder articleText = tags.get(textName);
- StringBuilder titleText = tags.get(titleName);
- StringBuilder timestampText = tags.get(timestampName);
- StringBuilder commentText = tags.get(commentName);
- StringBuilder idText = tags.get(idName);
-
- StringBuilder current = null;
- boolean inRevision = false;
- while (true) {
- try {
- if (!xmlr.hasNext())
- break;
- xmlr.next();
- } catch (XMLStreamException e) {
- throw new RuntimeException(e);
- }
- QName currentName = null;
- if (xmlr.hasName()) {
- currentName = xmlr.getName();
- }
- if (xmlr.isStartElement() && tags.containsKey(currentName)) {
- if (!inRevision || (!currentName.equals(revisionName) && !currentName.equals(idName))) {
- current = tags.get(currentName);
- current.setLength(0);
- }
- } else if (xmlr.isStartElement() && currentName.equals(revisionName)) {
- inRevision = true;
- } else if (xmlr.isEndElement() && currentName.equals(revisionName)) {
- inRevision = false;
- } else if (xmlr.isEndElement() && current != null) {
- if (textName.equals(currentName)) {
-
- String title = titleText.toString();
- String text = articleText.toString();
- String comment = commentText.toString();
- int id = Integer.parseInt(idText.toString());
- long timestamp;
- try {
- timestamp = dateFormat.parse(timestampText.append("+0000").toString()).getTime();
- return new Article(id, title, timestamp, comment, text);
- } catch (ParseException e) {
- return null;
- }
- }
- current = null;
- } else if (current != null && xmlr.hasText()) {
- current.append(xmlr.getText());
- }
- }
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/LRUOutputCombiner.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/LRUOutputCombiner.java b/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/LRUOutputCombiner.java
deleted file mode 100644
index 7d7b6dc..0000000
--- a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/LRUOutputCombiner.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.ingest;
-
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-public class LRUOutputCombiner<Key,Value> extends LinkedHashMap<Key,Value> {
-
- private static final long serialVersionUID = 1L;
-
- public static abstract class Fold<Value> {
- public abstract Value fold(Value oldValue, Value newValue);
- }
-
- public static abstract class Output<Key,Value> {
- public abstract void output(Key key, Value value);
- }
-
- private final int capacity;
- private final Fold<Value> fold;
- private final Output<Key,Value> output;
-
- private long cacheHits = 0;
- private long cacheMisses = 0;
-
- public LRUOutputCombiner(int capacity, Fold<Value> fold, Output<Key,Value> output) {
- super(capacity + 1, 1.1f, true);
- this.capacity = capacity;
- this.fold = fold;
- this.output = output;
- }
-
- protected boolean removeEldestEntry(Map.Entry<Key,Value> eldest) {
- if (size() > capacity) {
- output.output(eldest.getKey(), eldest.getValue());
- return true;
- }
- return false;
- }
-
- @Override
- public Value put(Key key, Value value) {
- Value val = get(key);
- if (val != null) {
- value = fold.fold(val, value);
- cacheHits++;
- } else {
- cacheMisses++;
- }
- super.put(key, value);
- return null;
- }
-
- public void flush() {
- for (Map.Entry<Key,Value> e : entrySet()) {
- output.output(e.getKey(), e.getValue());
- }
- clear();
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaConfiguration.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaConfiguration.java b/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaConfiguration.java
deleted file mode 100644
index 27a28a1..0000000
--- a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaConfiguration.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.ingest;
-
-import java.io.IOException;
-
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.SimpleAnalyzer;
-
-public class WikipediaConfiguration {
- public final static String INSTANCE_NAME = "wikipedia.accumulo.instance_name";
- public final static String USER = "wikipedia.accumulo.user";
- public final static String PASSWORD = "wikipedia.accumulo.password";
- public final static String TABLE_NAME = "wikipedia.accumulo.table";
-
- public final static String ZOOKEEPERS = "wikipedia.accumulo.zookeepers";
-
- public final static String NAMESPACES_FILENAME = "wikipedia.namespaces.filename";
- public final static String LANGUAGES_FILENAME = "wikipedia.languages.filename";
- public final static String WORKING_DIRECTORY = "wikipedia.ingest.working";
-
- public final static String ANALYZER = "wikipedia.index.analyzer";
-
- public final static String NUM_PARTITIONS = "wikipedia.ingest.partitions";
-
- public final static String NUM_GROUPS = "wikipedia.ingest.groups";
-
- public final static String PARTITIONED_ARTICLES_DIRECTORY = "wikipedia.partitioned.directory";
-
- public final static String RUN_PARTITIONER = "wikipedia.run.partitioner";
- public final static String RUN_INGEST = "wikipedia.run.ingest";
- public final static String BULK_INGEST = "wikipedia.bulk.ingest";
- public final static String BULK_INGEST_DIR = "wikipedia.bulk.ingest.dir";
- public final static String BULK_INGEST_FAILURE_DIR = "wikipedia.bulk.ingest.failure.dir";
- public final static String BULK_INGEST_BUFFER_SIZE = "wikipedia.bulk.ingest.buffer.size";
- public final static String PARTITIONED_INPUT_MIN_SPLIT_SIZE = "wikipedia.min.input.split.size";
-
-
- public static String getUser(Configuration conf) {
- return conf.get(USER);
- };
-
- public static byte[] getPassword(Configuration conf) {
- String pass = conf.get(PASSWORD);
- if (pass == null) {
- return null;
- }
- return pass.getBytes();
- }
-
- public static String getTableName(Configuration conf) {
- String tablename = conf.get(TABLE_NAME);
- if (tablename == null) {
- throw new RuntimeException("No data table name specified in " + TABLE_NAME);
- }
- return tablename;
- }
-
- public static String getInstanceName(Configuration conf) {
- return conf.get(INSTANCE_NAME);
- }
-
- public static String getZookeepers(Configuration conf) {
- String zookeepers = conf.get(ZOOKEEPERS);
- if (zookeepers == null) {
- throw new RuntimeException("No zookeepers specified in " + ZOOKEEPERS);
- }
- return zookeepers;
- }
-
- public static Path getNamespacesFile(Configuration conf) {
- String filename = conf.get(NAMESPACES_FILENAME, new Path(getWorkingDirectory(conf), "namespaces.dat").toString());
- return new Path(filename);
- }
-
- public static Path getLanguagesFile(Configuration conf) {
- String filename = conf.get(LANGUAGES_FILENAME, new Path(getWorkingDirectory(conf), "languages.txt").toString());
- return new Path(filename);
- }
-
- public static Path getWorkingDirectory(Configuration conf) {
- String filename = conf.get(WORKING_DIRECTORY);
- return new Path(filename);
- }
-
- public static Analyzer getAnalyzer(Configuration conf) throws IOException {
- Class<? extends Analyzer> analyzerClass = conf.getClass(ANALYZER, SimpleAnalyzer.class, Analyzer.class);
- return ReflectionUtils.newInstance(analyzerClass, conf);
- }
-
- public static Connector getConnector(Configuration conf) throws AccumuloException, AccumuloSecurityException {
- return getInstance(conf).getConnector(getUser(conf), getPassword(conf));
- }
-
- public static Instance getInstance(Configuration conf) {
- return new ZooKeeperInstance(getInstanceName(conf), getZookeepers(conf));
- }
-
- public static int getNumPartitions(Configuration conf) {
- return conf.getInt(NUM_PARTITIONS, 25);
- }
-
- public static int getNumGroups(Configuration conf) {
- return conf.getInt(NUM_GROUPS, 1);
- }
-
- public static Path getPartitionedArticlesPath(Configuration conf) {
- return new Path(conf.get(PARTITIONED_ARTICLES_DIRECTORY));
- }
-
- public static long getMinInputSplitSize(Configuration conf) {
- return conf.getLong(PARTITIONED_INPUT_MIN_SPLIT_SIZE, 1l << 27);
- }
-
- public static boolean runPartitioner(Configuration conf) {
- return conf.getBoolean(RUN_PARTITIONER, false);
- }
-
- public static boolean runIngest(Configuration conf) {
- return conf.getBoolean(RUN_INGEST, true);
- }
-
- public static boolean bulkIngest(Configuration conf) {
- return conf.getBoolean(BULK_INGEST, true);
- }
-
- public static String bulkIngestDir(Configuration conf) {
- return conf.get(BULK_INGEST_DIR);
- }
-
- public static String bulkIngestFailureDir(Configuration conf) {
- return conf.get(BULK_INGEST_FAILURE_DIR);
- }
-
- public static long bulkIngestBufferSize(Configuration conf) {
- return conf.getLong(BULK_INGEST_BUFFER_SIZE,1l<<28);
- }
-
- /**
- * Helper method to get properties from Hadoop configuration
- *
- * @param <T>
- * @param conf
- * @param propertyName
- * @param resultClass
- * @throws IllegalArgumentException
- * if property is not defined, null, or empty. Or if resultClass is not handled.
- * @return value of property
- */
- @SuppressWarnings("unchecked")
- public static <T> T isNull(Configuration conf, String propertyName, Class<T> resultClass) {
- String p = conf.get(propertyName);
- if (StringUtils.isEmpty(p))
- throw new IllegalArgumentException(propertyName + " must be specified");
-
- if (resultClass.equals(String.class))
- return (T) p;
- else if (resultClass.equals(String[].class))
- return (T) conf.getStrings(propertyName);
- else if (resultClass.equals(Boolean.class))
- return (T) Boolean.valueOf(p);
- else if (resultClass.equals(Long.class))
- return (T) Long.valueOf(p);
- else if (resultClass.equals(Integer.class))
- return (T) Integer.valueOf(p);
- else if (resultClass.equals(Float.class))
- return (T) Float.valueOf(p);
- else if (resultClass.equals(Double.class))
- return (T) Double.valueOf(p);
- else
- throw new IllegalArgumentException(resultClass.getSimpleName() + " is unhandled.");
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java b/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java
deleted file mode 100644
index 50415a7..0000000
--- a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java
+++ /dev/null
@@ -1,206 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.ingest;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.IteratorSetting;
-import org.apache.accumulo.core.client.IteratorSetting.Column;
-import org.apache.accumulo.core.client.TableExistsException;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.client.admin.TableOperations;
-import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
-import org.apache.accumulo.core.iterators.user.SummingCombiner;
-import org.apache.accumulo.examples.wikisearch.iterator.GlobalIndexUidCombiner;
-import org.apache.accumulo.examples.wikisearch.iterator.TextIndexCombiner;
-import org.apache.accumulo.examples.wikisearch.reader.AggregatingRecordReader;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-
-public class WikipediaIngester extends Configured implements Tool {
-
- public final static String INGEST_LANGUAGE = "wikipedia.ingest_language";
- public final static String SPLIT_FILE = "wikipedia.split_file";
- public final static String TABLE_NAME = "wikipedia.table";
-
- public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new Configuration(), new WikipediaIngester(), args);
- System.exit(res);
- }
-
- private void createTables(TableOperations tops, String tableName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException,
- TableExistsException {
- // Create the shard table
- String indexTableName = tableName + "Index";
- String reverseIndexTableName = tableName + "ReverseIndex";
- String metadataTableName = tableName + "Metadata";
-
- // create the shard table
- if (!tops.exists(tableName)) {
- // Set a text index combiner on the given field names. No combiner is set if the option is not supplied
- String textIndexFamilies = WikipediaMapper.TOKENS_FIELD_NAME;
-
- tops.create(tableName);
- if (textIndexFamilies.length() > 0) {
- System.out.println("Adding content combiner on the fields: " + textIndexFamilies);
-
- IteratorSetting setting = new IteratorSetting(10, TextIndexCombiner.class);
- List<Column> columns = new ArrayList<Column>();
- for (String family : StringUtils.split(textIndexFamilies, ',')) {
- columns.add(new Column("fi\0" + family));
- }
- TextIndexCombiner.setColumns(setting, columns);
- TextIndexCombiner.setLossyness(setting, true);
-
- tops.attachIterator(tableName, setting, EnumSet.allOf(IteratorScope.class));
- }
-
- // Set the locality group for the full content column family
- tops.setLocalityGroups(tableName, Collections.singletonMap("WikipediaDocuments", Collections.singleton(new Text(WikipediaMapper.DOCUMENT_COLUMN_FAMILY))));
-
- }
-
- if (!tops.exists(indexTableName)) {
- tops.create(indexTableName);
- // Add the UID combiner
- IteratorSetting setting = new IteratorSetting(19, "UIDAggregator", GlobalIndexUidCombiner.class);
- GlobalIndexUidCombiner.setCombineAllColumns(setting, true);
- GlobalIndexUidCombiner.setLossyness(setting, true);
- tops.attachIterator(indexTableName, setting, EnumSet.allOf(IteratorScope.class));
- }
-
- if (!tops.exists(reverseIndexTableName)) {
- tops.create(reverseIndexTableName);
- // Add the UID combiner
- IteratorSetting setting = new IteratorSetting(19, "UIDAggregator", GlobalIndexUidCombiner.class);
- GlobalIndexUidCombiner.setCombineAllColumns(setting, true);
- GlobalIndexUidCombiner.setLossyness(setting, true);
- tops.attachIterator(reverseIndexTableName, setting, EnumSet.allOf(IteratorScope.class));
- }
-
- if (!tops.exists(metadataTableName)) {
- // Add the SummingCombiner with VARLEN encoding for the frequency column
- tops.create(metadataTableName);
- IteratorSetting setting = new IteratorSetting(10, SummingCombiner.class);
- SummingCombiner.setColumns(setting, Collections.singletonList(new Column("f")));
- SummingCombiner.setEncodingType(setting, SummingCombiner.Type.VARLEN);
- tops.attachIterator(metadataTableName, setting, EnumSet.allOf(IteratorScope.class));
- }
- }
-
- @Override
- public int run(String[] args) throws Exception {
- Job job = new Job(getConf(), "Ingest Wikipedia");
- Configuration conf = job.getConfiguration();
- conf.set("mapred.map.tasks.speculative.execution", "false");
-
- String tablename = WikipediaConfiguration.getTableName(conf);
-
- String zookeepers = WikipediaConfiguration.getZookeepers(conf);
- String instanceName = WikipediaConfiguration.getInstanceName(conf);
-
- String user = WikipediaConfiguration.getUser(conf);
- byte[] password = WikipediaConfiguration.getPassword(conf);
- Connector connector = WikipediaConfiguration.getConnector(conf);
-
- TableOperations tops = connector.tableOperations();
-
- createTables(tops, tablename);
-
- configureJob(job);
-
- List<Path> inputPaths = new ArrayList<Path>();
- SortedSet<String> languages = new TreeSet<String>();
- FileSystem fs = FileSystem.get(conf);
- Path parent = new Path(conf.get("wikipedia.input"));
- listFiles(parent, fs, inputPaths, languages);
-
- System.out.println("Input files in " + parent + ":" + inputPaths.size());
- Path[] inputPathsArray = new Path[inputPaths.size()];
- inputPaths.toArray(inputPathsArray);
-
- System.out.println("Languages:" + languages.size());
-
- FileInputFormat.setInputPaths(job, inputPathsArray);
-
- job.setMapperClass(WikipediaMapper.class);
- job.setNumReduceTasks(0);
- job.setMapOutputKeyClass(Text.class);
- job.setMapOutputValueClass(Mutation.class);
- job.setOutputFormatClass(AccumuloOutputFormat.class);
- AccumuloOutputFormat.setOutputInfo(job.getConfiguration(), user, password, true, tablename);
- AccumuloOutputFormat.setZooKeeperInstance(job.getConfiguration(), instanceName, zookeepers);
-
- return job.waitForCompletion(true) ? 0 : 1;
- }
-
- public final static PathFilter partFilter = new PathFilter() {
- @Override
- public boolean accept(Path path) {
- return path.getName().startsWith("part");
- };
- };
-
- protected void configureJob(Job job) {
- Configuration conf = job.getConfiguration();
- job.setJarByClass(WikipediaIngester.class);
- job.setInputFormatClass(WikipediaInputFormat.class);
- conf.set(AggregatingRecordReader.START_TOKEN, "<page>");
- conf.set(AggregatingRecordReader.END_TOKEN, "</page>");
- }
-
- protected static final Pattern filePattern = Pattern.compile("([a-z_]+).*.xml(.bz2)?");
-
- protected void listFiles(Path path, FileSystem fs, List<Path> files, Set<String> languages) throws IOException {
- for (FileStatus status : fs.listStatus(path)) {
- if (status.isDir()) {
- listFiles(status.getPath(), fs, files, languages);
- } else {
- Path p = status.getPath();
- Matcher matcher = filePattern.matcher(p.getName());
- if (matcher.matches()) {
- languages.add(matcher.group(1));
- files.add(p);
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaInputFormat.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaInputFormat.java b/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaInputFormat.java
deleted file mode 100644
index c582cbf..0000000
--- a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaInputFormat.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.ingest;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.accumulo.examples.wikisearch.reader.AggregatingRecordReader;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.lib.input.FileSplit;
-import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
-
-
-public class WikipediaInputFormat extends TextInputFormat {
-
- public static class WikipediaInputSplit extends InputSplit implements Writable {
-
- public WikipediaInputSplit(){}
-
- public WikipediaInputSplit(FileSplit fileSplit, int partition)
- {
- this.fileSplit = fileSplit;
- this.partition = partition;
- }
-
- private FileSplit fileSplit = null;
- private int partition = -1;
-
- public int getPartition()
- {
- return partition;
- }
-
- public FileSplit getFileSplit()
- {
- return fileSplit;
- }
-
- @Override
- public long getLength() throws IOException, InterruptedException {
- return fileSplit.getLength();
- }
-
- @Override
- public String[] getLocations() throws IOException, InterruptedException {
- // for highly replicated files, returning all of the locations can lead to bunching
- // TODO replace this with a subset of the locations
- return fileSplit.getLocations();
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- Path file = new Path(in.readUTF());
- long start = in.readLong();
- long length = in.readLong();
- String [] hosts = null;
- if(in.readBoolean())
- {
- int numHosts = in.readInt();
- hosts = new String[numHosts];
- for(int i = 0; i < numHosts; i++)
- hosts[i] = in.readUTF();
- }
- fileSplit = new FileSplit(file, start, length, hosts);
- partition = in.readInt();
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeUTF(fileSplit.getPath().toString());
- out.writeLong(fileSplit.getStart());
- out.writeLong(fileSplit.getLength());
- String [] hosts = fileSplit.getLocations();
- if(hosts == null)
- {
- out.writeBoolean(false);
- }
- else
- {
- out.writeBoolean(true);
- out.writeInt(hosts.length);
- for(String host:hosts)
- out.writeUTF(host);
- }
- out.writeInt(partition);
- }
-
- }
-
- @Override
- public List<InputSplit> getSplits(JobContext job) throws IOException {
- List<InputSplit> superSplits = super.getSplits(job);
- List<InputSplit> splits = new ArrayList<InputSplit>();
-
- int numGroups = WikipediaConfiguration.getNumGroups(job.getConfiguration());
-
- for(int group = 0; group < numGroups; group++)
- {
- for(InputSplit split:superSplits)
- {
- FileSplit fileSplit = (FileSplit)split;
- splits.add(new WikipediaInputSplit(fileSplit,group));
- }
- }
- return splits;
- }
-
- @Override
- public RecordReader<LongWritable,Text> createRecordReader(InputSplit split, TaskAttemptContext context) {
- return new AggregatingRecordReader();
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaMapper.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaMapper.java b/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaMapper.java
deleted file mode 100644
index 8565b09..0000000
--- a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaMapper.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- *
- */
-package org.apache.accumulo.examples.wikisearch.ingest;
-
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.StringReader;
-import java.nio.charset.Charset;
-import java.util.HashSet;
-import java.util.IllegalFormatException;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.accumulo.examples.wikisearch.ingest.ArticleExtractor.Article;
-import org.apache.accumulo.examples.wikisearch.ingest.WikipediaInputFormat.WikipediaInputSplit;
-import org.apache.accumulo.examples.wikisearch.normalizer.LcNoDiacriticsNormalizer;
-import org.apache.accumulo.examples.wikisearch.protobuf.Uid;
-import org.apache.accumulo.examples.wikisearch.protobuf.Uid.List.Builder;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.lib.input.FileSplit;
-import org.apache.log4j.Logger;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
-import org.apache.lucene.wikipedia.analysis.WikipediaTokenizer;
-
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Multimap;
-
-public class WikipediaMapper extends Mapper<LongWritable,Text,Text,Mutation> {
-
- private static final Logger log = Logger.getLogger(WikipediaMapper.class);
-
- public final static Charset UTF8 = Charset.forName("UTF-8");
- public static final String DOCUMENT_COLUMN_FAMILY = "d";
- public static final String METADATA_EVENT_COLUMN_FAMILY = "e";
- public static final String METADATA_INDEX_COLUMN_FAMILY = "i";
- public static final String TOKENS_FIELD_NAME = "TEXT";
-
- private final static Pattern languagePattern = Pattern.compile("([a-z_]+).*.xml(.bz2)?");
- private static final Value NULL_VALUE = new Value(new byte[0]);
- private static final String cvPrefix = "all|";
-
- private ArticleExtractor extractor;
- private String language;
- private int numPartitions = 0;
- private ColumnVisibility cv = null;
-
- private int myGroup = -1;
- private int numGroups = -1;
-
- private Text tablename = null;
- private Text indexTableName = null;
- private Text reverseIndexTableName = null;
- private Text metadataTableName = null;
-
- @Override
- public void setup(Context context) {
- Configuration conf = context.getConfiguration();
- tablename = new Text(WikipediaConfiguration.getTableName(conf));
- indexTableName = new Text(tablename + "Index");
- reverseIndexTableName = new Text(tablename + "ReverseIndex");
- metadataTableName = new Text(tablename + "Metadata");
-
- WikipediaInputSplit wiSplit = (WikipediaInputSplit)context.getInputSplit();
- myGroup = wiSplit.getPartition();
- numGroups = WikipediaConfiguration.getNumGroups(conf);
-
- FileSplit split = wiSplit.getFileSplit();
- String fileName = split.getPath().getName();
- Matcher matcher = languagePattern.matcher(fileName);
- if (matcher.matches()) {
- language = matcher.group(1).replace('_', '-').toLowerCase();
- } else {
- throw new RuntimeException("Unknown ingest language! " + fileName);
- }
- extractor = new ArticleExtractor();
- numPartitions = WikipediaConfiguration.getNumPartitions(conf);
- cv = new ColumnVisibility(cvPrefix + language);
-
- }
-
- /**
- * We will partition the documents based on the document id
- *
- * @param article
- * @param numPartitions
- * @return The number of the partition for a given article.
- * @throws IllegalFormatException
- */
- public static int getPartitionId(Article article, int numPartitions) throws IllegalFormatException {
- return article.getId() % numPartitions;
- }
-
- static HashSet<String> metadataSent = new HashSet<String>();
-
- @Override
- protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
- Article article = extractor.extract(new InputStreamReader(new ByteArrayInputStream(value.getBytes()), UTF8));
- String NULL_BYTE = "\u0000";
- String colfPrefix = language + NULL_BYTE;
- String indexPrefix = "fi" + NULL_BYTE;
- if (article != null) {
- int groupId = WikipediaMapper.getPartitionId(article, numGroups);
- if(groupId != myGroup)
- return;
- Text partitionId = new Text(Integer.toString(WikipediaMapper.getPartitionId(article, numPartitions)));
-
- // Create the mutations for the document.
- // Row is partition id, colf is language0articleid, colq is fieldName\0fieldValue
- Mutation m = new Mutation(partitionId);
- for (Entry<String,Object> entry : article.getFieldValues().entrySet()) {
- m.put(colfPrefix + article.getId(), entry.getKey() + NULL_BYTE + entry.getValue().toString(), cv, article.getTimestamp(), NULL_VALUE);
- // Create mutations for the metadata table.
- String metadataKey = entry.getKey() + METADATA_EVENT_COLUMN_FAMILY + language;
- if (!metadataSent.contains(metadataKey)) {
- Mutation mm = new Mutation(entry.getKey());
- mm.put(METADATA_EVENT_COLUMN_FAMILY, language, cv, article.getTimestamp(), NULL_VALUE);
- context.write(metadataTableName, mm);
- metadataSent.add(metadataKey);
- }
- }
-
- // Tokenize the content
- Set<String> tokens = getTokens(article);
-
- // We are going to put the fields to be indexed into a multimap. This allows us to iterate
- // over the entire set once.
- Multimap<String,String> indexFields = HashMultimap.create();
- // Add the normalized field values
- LcNoDiacriticsNormalizer normalizer = new LcNoDiacriticsNormalizer();
- for (Entry<String,String> index : article.getNormalizedFieldValues().entrySet())
- indexFields.put(index.getKey(), index.getValue());
- // Add the tokens
- for (String token : tokens)
- indexFields.put(TOKENS_FIELD_NAME, normalizer.normalizeFieldValue("", token));
-
- for (Entry<String,String> index : indexFields.entries()) {
- // Create mutations for the in partition index
- // Row is partition id, colf is 'fi'\0fieldName, colq is fieldValue\0language\0article id
- m.put(indexPrefix + index.getKey(), index.getValue() + NULL_BYTE + colfPrefix + article.getId(), cv, article.getTimestamp(), NULL_VALUE);
-
- // Create mutations for the global index
- // Create a UID object for the Value
- Builder uidBuilder = Uid.List.newBuilder();
- uidBuilder.setIGNORE(false);
- uidBuilder.setCOUNT(1);
- uidBuilder.addUID(Integer.toString(article.getId()));
- Uid.List uidList = uidBuilder.build();
- Value val = new Value(uidList.toByteArray());
-
- // Create mutations for the global index
- // Row is field value, colf is field name, colq is partitionid\0language, value is Uid.List object
- Mutation gm = new Mutation(index.getValue());
- gm.put(index.getKey(), partitionId + NULL_BYTE + language, cv, article.getTimestamp(), val);
- context.write(indexTableName, gm);
-
- // Create mutations for the global reverse index
- Mutation grm = new Mutation(StringUtils.reverse(index.getValue()));
- grm.put(index.getKey(), partitionId + NULL_BYTE + language, cv, article.getTimestamp(), val);
- context.write(reverseIndexTableName, grm);
-
- // Create mutations for the metadata table.
- String metadataKey = index.getKey() + METADATA_INDEX_COLUMN_FAMILY + language;
- if (!metadataSent.contains(metadataKey)) {
- Mutation mm = new Mutation(index.getKey());
- mm.put(METADATA_INDEX_COLUMN_FAMILY, language + NULL_BYTE + LcNoDiacriticsNormalizer.class.getName(), cv, article.getTimestamp(), NULL_VALUE);
- context.write(metadataTableName, mm);
- metadataSent.add(metadataKey);
- }
- }
- // Add the entire text to the document section of the table.
- // row is the partition, colf is 'd', colq is language\0articleid, value is Base64 encoded GZIP'd document
- m.put(DOCUMENT_COLUMN_FAMILY, colfPrefix + article.getId(), cv, article.getTimestamp(), new Value(Base64.encodeBase64(article.getText().getBytes())));
- context.write(tablename, m);
-
- } else {
- context.getCounter("wikipedia", "invalid articles").increment(1);
- }
- context.progress();
- }
-
- /**
- * Tokenize the wikipedia content
- *
- * @param article
- * @return
- * @throws IOException
- */
- static Set<String> getTokens(Article article) throws IOException {
- Set<String> tokenList = new HashSet<String>();
- WikipediaTokenizer tok = new WikipediaTokenizer(new StringReader(article.getText()));
- TermAttribute term = tok.addAttribute(TermAttribute.class);
- try {
- while (tok.incrementToken()) {
- String token = term.term();
- if (!StringUtils.isEmpty(token))
- tokenList.add(token);
- }
- } catch (IOException e) {
- log.error("Error tokenizing text", e);
- } finally {
- try {
- tok.end();
- } catch (IOException e) {
- log.error("Error calling end()", e);
- } finally {
- try {
- tok.close();
- } catch (IOException e) {
- log.error("Error closing tokenizer", e);
- }
- }
- }
- return tokenList;
- }
-
-}
[14/39] ACCUMULO-600 removed wikisearch from trunk
Posted by ec...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaPartitionedIngester.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaPartitionedIngester.java b/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaPartitionedIngester.java
deleted file mode 100644
index 90b8308..0000000
--- a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaPartitionedIngester.java
+++ /dev/null
@@ -1,310 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.ingest;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.IteratorSetting;
-import org.apache.accumulo.core.client.IteratorSetting.Column;
-import org.apache.accumulo.core.client.TableExistsException;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.client.admin.TableOperations;
-import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
-import org.apache.accumulo.core.iterators.user.SummingCombiner;
-import org.apache.accumulo.core.tabletserver.thrift.MutationLogger.log_args;
-import org.apache.accumulo.examples.wikisearch.ingest.ArticleExtractor.Article;
-import org.apache.accumulo.examples.wikisearch.iterator.GlobalIndexUidCombiner;
-import org.apache.accumulo.examples.wikisearch.iterator.TextIndexCombiner;
-import org.apache.accumulo.examples.wikisearch.output.SortingRFileOutputFormat;
-import org.apache.accumulo.examples.wikisearch.reader.AggregatingRecordReader;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.log4j.Logger;
-
-public class WikipediaPartitionedIngester extends Configured implements Tool {
-
- private static final Logger log = Logger.getLogger(WikipediaPartitionedIngester.class);
-
- public final static String INGEST_LANGUAGE = "wikipedia.ingest_language";
- public final static String SPLIT_FILE = "wikipedia.split_file";
- public final static String TABLE_NAME = "wikipedia.table";
-
- public static void main(String[] args) throws Exception {
- int res = ToolRunner.run(new Configuration(), new WikipediaPartitionedIngester(), args);
- System.exit(res);
- }
-
- private void createTables(TableOperations tops, String tableName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException,
- TableExistsException {
- // Create the shard table
- String indexTableName = tableName + "Index";
- String reverseIndexTableName = tableName + "ReverseIndex";
- String metadataTableName = tableName + "Metadata";
-
- // create the shard table
- if (!tops.exists(tableName)) {
- // Set a text index combiner on the given field names. No combiner is set if the option is not supplied
- String textIndexFamilies = WikipediaMapper.TOKENS_FIELD_NAME;
-
- tops.create(tableName);
- if (textIndexFamilies.length() > 0) {
- System.out.println("Adding content combiner on the fields: " + textIndexFamilies);
-
- IteratorSetting setting = new IteratorSetting(10, TextIndexCombiner.class);
- List<Column> columns = new ArrayList<Column>();
- for (String family : StringUtils.split(textIndexFamilies, ',')) {
- columns.add(new Column("fi\0" + family));
- }
- TextIndexCombiner.setColumns(setting, columns);
- TextIndexCombiner.setLossyness(setting, true);
-
- tops.attachIterator(tableName, setting, EnumSet.allOf(IteratorScope.class));
- }
-
- // Set the locality group for the full content column family
- tops.setLocalityGroups(tableName, Collections.singletonMap("WikipediaDocuments", Collections.singleton(new Text(WikipediaMapper.DOCUMENT_COLUMN_FAMILY))));
-
- }
-
- if (!tops.exists(indexTableName)) {
- tops.create(indexTableName);
- // Add the UID combiner
- IteratorSetting setting = new IteratorSetting(19, "UIDAggregator", GlobalIndexUidCombiner.class);
- GlobalIndexUidCombiner.setCombineAllColumns(setting, true);
- GlobalIndexUidCombiner.setLossyness(setting, true);
- tops.attachIterator(indexTableName, setting, EnumSet.allOf(IteratorScope.class));
- }
-
- if (!tops.exists(reverseIndexTableName)) {
- tops.create(reverseIndexTableName);
- // Add the UID combiner
- IteratorSetting setting = new IteratorSetting(19, "UIDAggregator", GlobalIndexUidCombiner.class);
- GlobalIndexUidCombiner.setCombineAllColumns(setting, true);
- GlobalIndexUidCombiner.setLossyness(setting, true);
- tops.attachIterator(reverseIndexTableName, setting, EnumSet.allOf(IteratorScope.class));
- }
-
- if (!tops.exists(metadataTableName)) {
- // Add the SummingCombiner with VARLEN encoding for the frequency column
- tops.create(metadataTableName);
- IteratorSetting setting = new IteratorSetting(10, SummingCombiner.class);
- SummingCombiner.setColumns(setting, Collections.singletonList(new Column("f")));
- SummingCombiner.setEncodingType(setting, SummingCombiner.Type.VARLEN);
- tops.attachIterator(metadataTableName, setting, EnumSet.allOf(IteratorScope.class));
- }
- }
-
- @Override
- public int run(String[] args) throws Exception {
- Configuration conf = getConf();
- if(WikipediaConfiguration.runPartitioner(conf))
- {
- int result = runPartitionerJob();
- if(result != 0)
- return result;
- }
- if(WikipediaConfiguration.runIngest(conf))
- {
- int result = runIngestJob();
- if(result != 0)
- return result;
- if(WikipediaConfiguration.bulkIngest(conf))
- return loadBulkFiles();
- }
- return 0;
- }
-
- private int runPartitionerJob() throws Exception
- {
- Job partitionerJob = new Job(getConf(), "Partition Wikipedia");
- Configuration partitionerConf = partitionerJob.getConfiguration();
- partitionerConf.set("mapred.map.tasks.speculative.execution", "false");
-
- configurePartitionerJob(partitionerJob);
-
- List<Path> inputPaths = new ArrayList<Path>();
- SortedSet<String> languages = new TreeSet<String>();
- FileSystem fs = FileSystem.get(partitionerConf);
- Path parent = new Path(partitionerConf.get("wikipedia.input"));
- listFiles(parent, fs, inputPaths, languages);
-
- System.out.println("Input files in " + parent + ":" + inputPaths.size());
- Path[] inputPathsArray = new Path[inputPaths.size()];
- inputPaths.toArray(inputPathsArray);
-
- System.out.println("Languages:" + languages.size());
-
- // setup input format
-
- WikipediaInputFormat.setInputPaths(partitionerJob, inputPathsArray);
-
- partitionerJob.setMapperClass(WikipediaPartitioner.class);
- partitionerJob.setNumReduceTasks(0);
-
- // setup output format
- partitionerJob.setMapOutputKeyClass(Text.class);
- partitionerJob.setMapOutputValueClass(Article.class);
- partitionerJob.setOutputKeyClass(Text.class);
- partitionerJob.setOutputValueClass(Article.class);
- partitionerJob.setOutputFormatClass(SequenceFileOutputFormat.class);
- Path outputDir = WikipediaConfiguration.getPartitionedArticlesPath(partitionerConf);
- SequenceFileOutputFormat.setOutputPath(partitionerJob, outputDir);
- SequenceFileOutputFormat.setCompressOutput(partitionerJob, true);
- SequenceFileOutputFormat.setOutputCompressionType(partitionerJob, CompressionType.RECORD);
-
- return partitionerJob.waitForCompletion(true) ? 0 : 1;
- }
-
- private int runIngestJob() throws Exception
- {
- Job ingestJob = new Job(getConf(), "Ingest Partitioned Wikipedia");
- Configuration ingestConf = ingestJob.getConfiguration();
- ingestConf.set("mapred.map.tasks.speculative.execution", "false");
-
- configureIngestJob(ingestJob);
-
- String tablename = WikipediaConfiguration.getTableName(ingestConf);
-
- Connector connector = WikipediaConfiguration.getConnector(ingestConf);
-
- TableOperations tops = connector.tableOperations();
-
- createTables(tops, tablename);
-
- ingestJob.setMapperClass(WikipediaPartitionedMapper.class);
- ingestJob.setNumReduceTasks(0);
-
- // setup input format
- ingestJob.setInputFormatClass(SequenceFileInputFormat.class);
- SequenceFileInputFormat.setInputPaths(ingestJob, WikipediaConfiguration.getPartitionedArticlesPath(ingestConf));
- // TODO make split size configurable
- SequenceFileInputFormat.setMinInputSplitSize(ingestJob, WikipediaConfiguration.getMinInputSplitSize(ingestConf));
-
- // setup output format
- ingestJob.setMapOutputKeyClass(Text.class);
- ingestJob.setMapOutputValueClass(Mutation.class);
-
- if(WikipediaConfiguration.bulkIngest(ingestConf))
- {
- ingestJob.setOutputFormatClass(SortingRFileOutputFormat.class);
- SortingRFileOutputFormat.setMaxBufferSize(ingestConf, WikipediaConfiguration.bulkIngestBufferSize(ingestConf));
- String bulkIngestDir = WikipediaConfiguration.bulkIngestDir(ingestConf);
- if(bulkIngestDir == null)
- {
- log.error("Bulk ingest dir not set");
- return 1;
- }
- SortingRFileOutputFormat.setPathName(ingestConf, WikipediaConfiguration.bulkIngestDir(ingestConf));
- } else {
- ingestJob.setOutputFormatClass(AccumuloOutputFormat.class);
- String zookeepers = WikipediaConfiguration.getZookeepers(ingestConf);
- String instanceName = WikipediaConfiguration.getInstanceName(ingestConf);
- String user = WikipediaConfiguration.getUser(ingestConf);
- byte[] password = WikipediaConfiguration.getPassword(ingestConf);
- AccumuloOutputFormat.setOutputInfo(ingestJob.getConfiguration(), user, password, true, tablename);
- AccumuloOutputFormat.setZooKeeperInstance(ingestJob.getConfiguration(), instanceName, zookeepers);
- }
-
- return ingestJob.waitForCompletion(true) ? 0 : 1;
- }
-
- private int loadBulkFiles() throws IOException, AccumuloException, AccumuloSecurityException, TableNotFoundException
- {
- Configuration conf = getConf();
-
- Connector connector = WikipediaConfiguration.getConnector(conf);
-
- FileSystem fs = FileSystem.get(conf);
- String directory = WikipediaConfiguration.bulkIngestDir(conf);
-
- String failureDirectory = WikipediaConfiguration.bulkIngestFailureDir(conf);
-
- for(FileStatus status: fs.listStatus(new Path(directory)))
- {
- if(status.isDir() == false)
- continue;
- Path dir = status.getPath();
- Path failPath = new Path(failureDirectory+"/"+dir.getName());
- fs.mkdirs(failPath);
- connector.tableOperations().importDirectory(dir.getName(), dir.toString(), failPath.toString(), true);
- }
-
- return 0;
- }
-
- public final static PathFilter partFilter = new PathFilter() {
- @Override
- public boolean accept(Path path) {
- return path.getName().startsWith("part");
- };
- };
-
- protected void configurePartitionerJob(Job job) {
- Configuration conf = job.getConfiguration();
- job.setJarByClass(WikipediaPartitionedIngester.class);
- job.setInputFormatClass(WikipediaInputFormat.class);
- conf.set(AggregatingRecordReader.START_TOKEN, "<page>");
- conf.set(AggregatingRecordReader.END_TOKEN, "</page>");
- }
-
- protected void configureIngestJob(Job job) {
- job.setJarByClass(WikipediaPartitionedIngester.class);
- }
-
- protected static final Pattern filePattern = Pattern.compile("([a-z_]+).*.xml(.bz2)?");
-
- protected void listFiles(Path path, FileSystem fs, List<Path> files, Set<String> languages) throws IOException {
- for (FileStatus status : fs.listStatus(path)) {
- if (status.isDir()) {
- listFiles(status.getPath(), fs, files, languages);
- } else {
- Path p = status.getPath();
- Matcher matcher = filePattern.matcher(p.getName());
- if (matcher.matches()) {
- languages.add(matcher.group(1));
- files.add(p);
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaPartitionedMapper.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaPartitionedMapper.java b/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaPartitionedMapper.java
deleted file mode 100644
index bb4ae64..0000000
--- a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaPartitionedMapper.java
+++ /dev/null
@@ -1,310 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- *
- */
-package org.apache.accumulo.examples.wikisearch.ingest;
-
-
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.util.HashSet;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.MultiTableBatchWriter;
-import org.apache.accumulo.core.client.MutationsRejectedException;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.accumulo.examples.wikisearch.ingest.ArticleExtractor.Article;
-import org.apache.accumulo.examples.wikisearch.iterator.GlobalIndexUidCombiner;
-import org.apache.accumulo.examples.wikisearch.normalizer.LcNoDiacriticsNormalizer;
-import org.apache.accumulo.examples.wikisearch.protobuf.Uid;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Mapper;
-
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Multimap;
-
-public class WikipediaPartitionedMapper extends Mapper<Text,Article,Text,Mutation> {
-
- // private static final Logger log = Logger.getLogger(WikipediaPartitionedMapper.class);
-
- public final static Charset UTF8 = Charset.forName("UTF-8");
- public static final String DOCUMENT_COLUMN_FAMILY = "d";
- public static final String METADATA_EVENT_COLUMN_FAMILY = "e";
- public static final String METADATA_INDEX_COLUMN_FAMILY = "i";
- public static final String TOKENS_FIELD_NAME = "TEXT";
-
- private static final Value NULL_VALUE = new Value(new byte[0]);
- private static final String cvPrefix = "all|";
-
- private int numPartitions = 0;
-
- private Text tablename = null;
- private Text indexTableName = null;
- private Text reverseIndexTableName = null;
- private Text metadataTableName = null;
-
- private static class MutationInfo {
- final String row;
- final String colfam;
- final String colqual;
- final ColumnVisibility cv;
- final long timestamp;
-
- public MutationInfo(String row, String colfam, String colqual, ColumnVisibility cv, long timestamp) {
- super();
- this.row = row;
- this.colfam = colfam;
- this.colqual = colqual;
- this.cv = cv;
- this.timestamp = timestamp;
- }
-
- @Override
- public boolean equals(Object obj) {
- MutationInfo other = (MutationInfo)obj;
- return (row == other.row || row.equals(other.row)) &&
- (colfam == other.colfam || colfam.equals(other.colfam)) &&
- colqual.equals(other.colqual) &&
- (cv == other.cv || cv.equals(other.cv)) &&
- timestamp == other.timestamp;
- }
-
- @Override
- public int hashCode() {
- return row.hashCode() ^ colfam.hashCode() ^ colqual.hashCode() ^ cv.hashCode() ^ (int)timestamp;
- }
- }
-
- private LRUOutputCombiner<MutationInfo,CountAndSet> wikiIndexOutput;
- private LRUOutputCombiner<MutationInfo,CountAndSet> wikiReverseIndexOutput;
- private LRUOutputCombiner<MutationInfo,Value> wikiMetadataOutput;
-
- private static class CountAndSet
- {
- public int count;
- public HashSet<String> set;
-
- public CountAndSet(String entry)
- {
- set = new HashSet<String>();
- set.add(entry);
- count = 1;
- }
- }
-
- MultiTableBatchWriter mtbw;
-
- @Override
- public void setup(final Context context) {
- Configuration conf = context.getConfiguration();
- tablename = new Text(WikipediaConfiguration.getTableName(conf));
- indexTableName = new Text(tablename + "Index");
- reverseIndexTableName = new Text(tablename + "ReverseIndex");
- metadataTableName = new Text(tablename + "Metadata");
-
- try {
- mtbw = WikipediaConfiguration.getConnector(conf).createMultiTableBatchWriter(10000000, 1000, 10);
- } catch (AccumuloException e) {
- throw new RuntimeException(e);
- } catch (AccumuloSecurityException e) {
- throw new RuntimeException(e);
- }
-
- final Text metadataTableNameFinal = metadataTableName;
- final Text indexTableNameFinal = indexTableName;
- final Text reverseIndexTableNameFinal = reverseIndexTableName;
-
- numPartitions = WikipediaConfiguration.getNumPartitions(conf);
-
- LRUOutputCombiner.Fold<CountAndSet> indexFold =
- new LRUOutputCombiner.Fold<CountAndSet>() {
- @Override
- public CountAndSet fold(CountAndSet oldValue, CountAndSet newValue) {
- oldValue.count += newValue.count;
- if(oldValue.set == null || newValue.set == null)
- {
- oldValue.set = null;
- return oldValue;
- }
- oldValue.set.addAll(newValue.set);
- if(oldValue.set.size() > GlobalIndexUidCombiner.MAX)
- oldValue.set = null;
- return oldValue;
- }
- };
- LRUOutputCombiner.Output<MutationInfo,CountAndSet> indexOutput =
- new LRUOutputCombiner.Output<WikipediaPartitionedMapper.MutationInfo,CountAndSet>() {
-
- @Override
- public void output(MutationInfo key, CountAndSet value)
- {
- Uid.List.Builder builder = Uid.List.newBuilder();
- builder.setCOUNT(value.count);
- if (value.set == null) {
- builder.setIGNORE(true);
- builder.clearUID();
- } else {
- builder.setIGNORE(false);
- builder.addAllUID(value.set);
- }
- Uid.List list = builder.build();
- Value val = new Value(list.toByteArray());
- Mutation m = new Mutation(key.row);
- m.put(key.colfam, key.colqual, key.cv, key.timestamp, val);
- try {
- mtbw.getBatchWriter(indexTableNameFinal.toString()).addMutation(m);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
- };
- LRUOutputCombiner.Output<MutationInfo,CountAndSet> reverseIndexOutput =
- new LRUOutputCombiner.Output<WikipediaPartitionedMapper.MutationInfo,CountAndSet>() {
-
- @Override
- public void output(MutationInfo key, CountAndSet value)
- {
- Uid.List.Builder builder = Uid.List.newBuilder();
- builder.setCOUNT(value.count);
- if (value.set == null) {
- builder.setIGNORE(true);
- builder.clearUID();
- } else {
- builder.setIGNORE(false);
- builder.addAllUID(value.set);
- }
- Uid.List list = builder.build();
- Value val = new Value(list.toByteArray());
- Mutation m = new Mutation(key.row);
- m.put(key.colfam, key.colqual, key.cv, key.timestamp, val);
- try {
- mtbw.getBatchWriter(reverseIndexTableNameFinal.toString()).addMutation(m);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
- };
-
- wikiIndexOutput = new LRUOutputCombiner<WikipediaPartitionedMapper.MutationInfo,CountAndSet>(10000,indexFold,indexOutput);
- wikiReverseIndexOutput = new LRUOutputCombiner<WikipediaPartitionedMapper.MutationInfo,CountAndSet>(10000, indexFold,reverseIndexOutput);
- wikiMetadataOutput = new LRUOutputCombiner<WikipediaPartitionedMapper.MutationInfo,Value>(10000,
- new LRUOutputCombiner.Fold<Value>() {
- @Override
- public Value fold(Value oldValue, Value newValue) {
- return oldValue;
- }},
- new LRUOutputCombiner.Output<MutationInfo,Value>() {
- @Override
- public void output(MutationInfo key, Value value) {
- Mutation m = new Mutation(key.row);
- m.put(key.colfam, key.colqual, key.cv, key.timestamp, value);
- try {
- mtbw.getBatchWriter(metadataTableNameFinal.toString()).addMutation(m);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }});
- }
-
-
-
- @Override
- protected void cleanup(Context context) throws IOException, InterruptedException {
- wikiIndexOutput.flush();
- wikiMetadataOutput.flush();
- wikiReverseIndexOutput.flush();
- try {
- mtbw.close();
- } catch (MutationsRejectedException e) {
- throw new RuntimeException(e);
- }
- }
-
-
-
- @Override
- protected void map(Text language, Article article, Context context) throws IOException, InterruptedException {
- String NULL_BYTE = "\u0000";
- String colfPrefix = language.toString() + NULL_BYTE;
- String indexPrefix = "fi" + NULL_BYTE;
- ColumnVisibility cv = new ColumnVisibility(cvPrefix + language);
-
- if (article != null) {
- Text partitionId = new Text(Integer.toString(WikipediaMapper.getPartitionId(article, numPartitions)));
-
- // Create the mutations for the document.
- // Row is partition id, colf is language0articleid, colq is fieldName\0fieldValue
- Mutation m = new Mutation(partitionId);
- for (Entry<String,Object> entry : article.getFieldValues().entrySet()) {
- m.put(colfPrefix + article.getId(), entry.getKey() + NULL_BYTE + entry.getValue().toString(), cv, article.getTimestamp(), NULL_VALUE);
- // Create mutations for the metadata table.
- MutationInfo mm = new MutationInfo(entry.getKey(), METADATA_EVENT_COLUMN_FAMILY, language.toString(), cv, article.getTimestamp());
- wikiMetadataOutput.put(mm, NULL_VALUE);
- }
-
- // Tokenize the content
- Set<String> tokens = WikipediaMapper.getTokens(article);
-
- // We are going to put the fields to be indexed into a multimap. This allows us to iterate
- // over the entire set once.
- Multimap<String,String> indexFields = HashMultimap.create();
- // Add the normalized field values
- LcNoDiacriticsNormalizer normalizer = new LcNoDiacriticsNormalizer();
- for (Entry<String,String> index : article.getNormalizedFieldValues().entrySet())
- indexFields.put(index.getKey(), index.getValue());
- // Add the tokens
- for (String token : tokens)
- indexFields.put(TOKENS_FIELD_NAME, normalizer.normalizeFieldValue("", token));
-
- for (Entry<String,String> index : indexFields.entries()) {
- // Create mutations for the in partition index
- // Row is partition id, colf is 'fi'\0fieldName, colq is fieldValue\0language\0article id
- m.put(indexPrefix + index.getKey(), index.getValue() + NULL_BYTE + colfPrefix + article.getId(), cv, article.getTimestamp(), NULL_VALUE);
-
- // Create mutations for the global index
- // Row is field value, colf is field name, colq is partitionid\0language, value is Uid.List object
- MutationInfo gm = new MutationInfo(index.getValue(),index.getKey(),partitionId + NULL_BYTE + language, cv, article.getTimestamp());
- wikiIndexOutput.put(gm, new CountAndSet(Integer.toString(article.getId())));
-
- // Create mutations for the global reverse index
- MutationInfo grm = new MutationInfo(StringUtils.reverse(index.getValue()),index.getKey(),partitionId + NULL_BYTE + language, cv, article.getTimestamp());
- wikiReverseIndexOutput.put(grm, new CountAndSet(Integer.toString(article.getId())));
-
- // Create mutations for the metadata table.
- MutationInfo mm = new MutationInfo(index.getKey(),METADATA_INDEX_COLUMN_FAMILY, language + NULL_BYTE + LcNoDiacriticsNormalizer.class.getName(), cv, article.getTimestamp());
- wikiMetadataOutput.put(mm, NULL_VALUE);
- }
- // Add the entire text to the document section of the table.
- // row is the partition, colf is 'd', colq is language\0articleid, value is Base64 encoded GZIP'd document
- m.put(DOCUMENT_COLUMN_FAMILY, colfPrefix + article.getId(), cv, article.getTimestamp(), new Value(Base64.encodeBase64(article.getText().getBytes())));
- context.write(tablename, m);
-
- } else {
- context.getCounter("wikipedia", "invalid articles").increment(1);
- }
- context.progress();
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaPartitioner.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaPartitioner.java b/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaPartitioner.java
deleted file mode 100644
index 3507108..0000000
--- a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaPartitioner.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- *
- */
-package org.apache.accumulo.examples.wikisearch.ingest;
-
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.nio.charset.Charset;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.accumulo.examples.wikisearch.ingest.ArticleExtractor.Article;
-import org.apache.accumulo.examples.wikisearch.ingest.WikipediaInputFormat.WikipediaInputSplit;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.lib.input.FileSplit;
-
-public class WikipediaPartitioner extends Mapper<LongWritable,Text,Text,Article> {
-
- // private static final Logger log = Logger.getLogger(WikipediaPartitioner.class);
-
- public final static Charset UTF8 = Charset.forName("UTF-8");
- public static final String DOCUMENT_COLUMN_FAMILY = "d";
- public static final String METADATA_EVENT_COLUMN_FAMILY = "e";
- public static final String METADATA_INDEX_COLUMN_FAMILY = "i";
- public static final String TOKENS_FIELD_NAME = "TEXT";
-
- private final static Pattern languagePattern = Pattern.compile("([a-z_]+).*.xml(.bz2)?");
-
- private ArticleExtractor extractor;
- private String language;
-
- private int myGroup = -1;
- private int numGroups = -1;
-
- @Override
- public void setup(Context context) {
- Configuration conf = context.getConfiguration();
-
- WikipediaInputSplit wiSplit = (WikipediaInputSplit)context.getInputSplit();
- myGroup = wiSplit.getPartition();
- numGroups = WikipediaConfiguration.getNumGroups(conf);
-
- FileSplit split = wiSplit.getFileSplit();
- String fileName = split.getPath().getName();
- Matcher matcher = languagePattern.matcher(fileName);
- if (matcher.matches()) {
- language = matcher.group(1).replace('_', '-').toLowerCase();
- } else {
- throw new RuntimeException("Unknown ingest language! " + fileName);
- }
- extractor = new ArticleExtractor();
- }
-
- @Override
- protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
- Article article = extractor.extract(new InputStreamReader(new ByteArrayInputStream(value.getBytes()), UTF8));
- if (article != null) {
- int groupId = WikipediaMapper.getPartitionId(article, numGroups);
- if(groupId != myGroup)
- return;
- context.write(new Text(language), article);
- } else {
- context.getCounter("wikipedia", "invalid articles").increment(1);
- context.progress();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/GlobalIndexUidCombiner.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/GlobalIndexUidCombiner.java b/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/GlobalIndexUidCombiner.java
deleted file mode 100644
index 4702521..0000000
--- a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/GlobalIndexUidCombiner.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.iterator;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Map;
-
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.IteratorEnvironment;
-import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
-import org.apache.accumulo.core.iterators.TypedValueCombiner;
-import org.apache.accumulo.core.iterators.ValueFormatException;
-import org.apache.accumulo.examples.wikisearch.protobuf.Uid;
-
-import com.google.protobuf.InvalidProtocolBufferException;
-
-/**
- *
- */
-public class GlobalIndexUidCombiner extends TypedValueCombiner<Uid.List> {
- public static final Encoder<Uid.List> UID_LIST_ENCODER = new UidListEncoder();
- public static final int MAX = 20;
-
- @Override
- public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
- super.init(source, options, env);
- setEncoder(UID_LIST_ENCODER);
- }
-
- @Override
- public Uid.List typedReduce(Key key, Iterator<Uid.List> iter) {
- Uid.List.Builder builder = Uid.List.newBuilder();
- HashSet<String> uids = new HashSet<String>();
- boolean seenIgnore = false;
- long count = 0;
- while (iter.hasNext()) {
- Uid.List v = iter.next();
- if (null == v)
- continue;
- count = count + v.getCOUNT();
- if (v.getIGNORE()) {
- seenIgnore = true;
- }
- uids.addAll(v.getUIDList());
- }
- // Special case logic
- // If we have aggregated more than MAX UIDs, then null out the UID list and set IGNORE to true
- // However, always maintain the count
- builder.setCOUNT(count);
- if (uids.size() > MAX || seenIgnore) {
- builder.setIGNORE(true);
- builder.clearUID();
- } else {
- builder.setIGNORE(false);
- builder.addAllUID(uids);
- }
- return builder.build();
- }
-
- public static class UidListEncoder implements Encoder<Uid.List> {
- @Override
- public byte[] encode(Uid.List v) {
- return v.toByteArray();
- }
-
- @Override
- public Uid.List decode(byte[] b) {
- if (b.length == 0)
- return null;
- try {
- return Uid.List.parseFrom(b);
- } catch (InvalidProtocolBufferException e) {
- throw new ValueFormatException("Value passed to aggregator was not of type Uid.List");
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/TextIndexCombiner.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/TextIndexCombiner.java b/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/TextIndexCombiner.java
deleted file mode 100644
index 85f3e1e..0000000
--- a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/TextIndexCombiner.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.iterator;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.IteratorEnvironment;
-import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
-import org.apache.accumulo.core.iterators.TypedValueCombiner;
-import org.apache.accumulo.core.iterators.ValueFormatException;
-import org.apache.accumulo.examples.wikisearch.protobuf.TermWeight;
-
-import com.google.protobuf.InvalidProtocolBufferException;
-
-/**
- *
- */
-public class TextIndexCombiner extends TypedValueCombiner<TermWeight.Info> {
- public static final Encoder<TermWeight.Info> TERMWEIGHT_INFO_ENCODER = new TermWeightInfoEncoder();
-
- @Override
- public TermWeight.Info typedReduce(Key key, Iterator<TermWeight.Info> iter) {
- TermWeight.Info.Builder builder = TermWeight.Info.newBuilder();
- List<Integer> offsets = new ArrayList<Integer>();
- float normalizedTermFrequency = 0f;
-
- while (iter.hasNext()) {
- TermWeight.Info info = iter.next();
- if (null == info)
- continue;
-
- // Add each offset into the list maintaining sorted order
- for (int offset : info.getWordOffsetList()) {
- int pos = Collections.binarySearch(offsets, offset);
-
- if (pos < 0) {
- // Undo the transform on the insertion point
- offsets.add((-1 * pos) - 1, offset);
- } else {
- offsets.add(pos, offset);
- }
- }
-
- if (info.getNormalizedTermFrequency() > 0) {
- normalizedTermFrequency += info.getNormalizedTermFrequency();
- }
- }
-
- // Keep the sorted order we tried to maintain
- for (int i = 0; i < offsets.size(); ++i) {
- builder.addWordOffset(offsets.get(i));
- }
-
- builder.setNormalizedTermFrequency(normalizedTermFrequency);
- return builder.build();
- }
-
- @Override
- public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
- super.init(source, options, env);
- setEncoder(TERMWEIGHT_INFO_ENCODER);
- }
-
- public static class TermWeightInfoEncoder implements Encoder<TermWeight.Info> {
- @Override
- public byte[] encode(TermWeight.Info v) {
- return v.toByteArray();
- }
-
- @Override
- public TermWeight.Info decode(byte[] b) {
- if (b.length == 0)
- return null;
- try {
- return TermWeight.Info.parseFrom(b);
- } catch (InvalidProtocolBufferException e) {
- throw new ValueFormatException("Value passed to aggregator was not of type TermWeight.Info");
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/normalizer/LcNoDiacriticsNormalizer.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/normalizer/LcNoDiacriticsNormalizer.java b/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/normalizer/LcNoDiacriticsNormalizer.java
deleted file mode 100644
index b15a5ee..0000000
--- a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/normalizer/LcNoDiacriticsNormalizer.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.normalizer;
-
-import java.text.Normalizer;
-import java.text.Normalizer.Form;
-import java.util.Locale;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * An {@link Normalizer} which performs the following steps:
- * <ol>
- * <li>Unicode canonical decomposition ({@link Form#NFD})</li>
- * <li>Removal of diacritical marks</li>
- * <li>Unicode canonical composition ({@link Form#NFC})</li>
- * <li>lower casing in the {@link Locale#ENGLISH English local}
- * </ol>
- */
-public class LcNoDiacriticsNormalizer implements org.apache.accumulo.examples.wikisearch.normalizer.Normalizer {
- private static final Pattern diacriticals = Pattern.compile("\\p{InCombiningDiacriticalMarks}");
-
- public String normalizeFieldValue(String fieldName, Object fieldValue) {
- String decomposed = Normalizer.normalize(fieldValue.toString(), Form.NFD);
- String noDiacriticals = removeDiacriticalMarks(decomposed);
- String recomposed = Normalizer.normalize(noDiacriticals, Form.NFC);
- return recomposed.toLowerCase(Locale.ENGLISH);
- }
-
- private String removeDiacriticalMarks(String str) {
- Matcher matcher = diacriticals.matcher(str);
- return matcher.replaceAll("");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/normalizer/NoOpNormalizer.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/normalizer/NoOpNormalizer.java b/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/normalizer/NoOpNormalizer.java
deleted file mode 100644
index 7498f76..0000000
--- a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/normalizer/NoOpNormalizer.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.normalizer;
-
-public class NoOpNormalizer implements Normalizer {
- public String normalizeFieldValue(String field, Object value) {
- return value.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/normalizer/Normalizer.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/normalizer/Normalizer.java b/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/normalizer/Normalizer.java
deleted file mode 100644
index ba3632a..0000000
--- a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/normalizer/Normalizer.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.normalizer;
-
-public interface Normalizer {
-
- /**
- * Creates normalized content for ingest based upon implemented logic.
- *
- * @param field
- * The field being normalized
- * @param value
- * The value to normalize
- * @return a normalized value
- */
- public String normalizeFieldValue(String field, Object value);
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/normalizer/NumberNormalizer.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/normalizer/NumberNormalizer.java b/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/normalizer/NumberNormalizer.java
deleted file mode 100644
index e0a5cc8..0000000
--- a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/normalizer/NumberNormalizer.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.normalizer;
-
-import org.apache.commons.lang.math.NumberUtils;
-import org.apache.lucene.util.NumericUtils;
-
-public class NumberNormalizer implements Normalizer {
-
- public String normalizeFieldValue(String field, Object value) {
- if (NumberUtils.isNumber(value.toString())) {
- Number n = NumberUtils.createNumber(value.toString());
- if (n instanceof Integer)
- return NumericUtils.intToPrefixCoded((Integer) n);
- else if (n instanceof Long)
- return NumericUtils.longToPrefixCoded((Long) n);
- else if (n instanceof Float)
- return NumericUtils.floatToPrefixCoded((Float) n);
- else if (n instanceof Double)
- return NumericUtils.doubleToPrefixCoded((Double) n);
- else
- throw new IllegalArgumentException("Unhandled numeric type: " + n.getClass());
- } else {
- throw new IllegalArgumentException("Value is not a number: " + value);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/output/BufferingRFileRecordWriter.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/output/BufferingRFileRecordWriter.java b/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/output/BufferingRFileRecordWriter.java
deleted file mode 100644
index 9b663de..0000000
--- a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/output/BufferingRFileRecordWriter.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.output;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.TreeMap;
-
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.data.ColumnUpdate;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.file.FileSKVWriter;
-import org.apache.accumulo.core.file.rfile.RFileOperations;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.RecordWriter;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-
-final class BufferingRFileRecordWriter extends RecordWriter<Text,Mutation> {
- private final long maxSize;
- private final AccumuloConfiguration acuconf;
- private final Configuration conf;
- private final String filenamePrefix;
- private final String taskID;
- private final FileSystem fs;
- private int fileCount = 0;
- private long size;
-
- private Map<Text,TreeMap<Key,Value>> buffers = new HashMap<Text,TreeMap<Key,Value>>();
- private Map<Text,Long> bufferSizes = new HashMap<Text,Long>();
-
- private TreeMap<Key,Value> getBuffer(Text tablename) {
- TreeMap<Key,Value> buffer = buffers.get(tablename);
- if (buffer == null) {
- buffer = new TreeMap<Key,Value>();
- buffers.put(tablename, buffer);
- bufferSizes.put(tablename, 0l);
- }
- return buffer;
- }
-
- private Text getLargestTablename() {
- long max = 0;
- Text table = null;
- for (Entry<Text,Long> e : bufferSizes.entrySet()) {
- if (e.getValue() > max) {
- max = e.getValue();
- table = e.getKey();
- }
- }
- return table;
- }
-
- private void flushLargestTable() throws IOException {
- Text tablename = getLargestTablename();
- if (tablename == null)
- return;
- long bufferSize = bufferSizes.get(tablename);
- TreeMap<Key,Value> buffer = buffers.get(tablename);
- if (buffer.size() == 0)
- return;
-
- String file = filenamePrefix + "/" + tablename + "/" + taskID + "_" + (fileCount++) + ".rf";
- // TODO get the table configuration for the given table?
- FileSKVWriter writer = RFileOperations.getInstance().openWriter(file, fs, conf, acuconf);
-
- // forget locality groups for now, just write everything to the default
- writer.startDefaultLocalityGroup();
-
- for (Entry<Key,Value> e : buffer.entrySet()) {
- writer.append(e.getKey(), e.getValue());
- }
-
- writer.close();
-
- size -= bufferSize;
- buffer.clear();
- bufferSizes.put(tablename, 0l);
- }
-
- BufferingRFileRecordWriter(long maxSize, AccumuloConfiguration acuconf, Configuration conf, String filenamePrefix, String taskID, FileSystem fs) {
- this.maxSize = maxSize;
- this.acuconf = acuconf;
- this.conf = conf;
- this.filenamePrefix = filenamePrefix;
- this.taskID = taskID;
- this.fs = fs;
- }
-
- @Override
- public void close(TaskAttemptContext arg0) throws IOException, InterruptedException {
- while (size > 0)
- flushLargestTable();
- }
-
- @Override
- public void write(Text table, Mutation mutation) throws IOException, InterruptedException {
- TreeMap<Key,Value> buffer = getBuffer(table);
- int mutationSize = 0;
- for (ColumnUpdate update : mutation.getUpdates()) {
- Key k = new Key(mutation.getRow(), update.getColumnFamily(), update.getColumnQualifier(), update.getColumnVisibility(), update.getTimestamp(),
- update.isDeleted());
- Value v = new Value(update.getValue());
- // TODO account for object overhead
- mutationSize += k.getSize();
- mutationSize += v.getSize();
- buffer.put(k, v);
- }
- size += mutationSize;
- long bufferSize = bufferSizes.get(table);
-
- // TODO use a MutableLong instead
- bufferSize += mutationSize;
- bufferSizes.put(table, bufferSize);
-
- while (size >= maxSize) {
- flushLargestTable();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/output/SortingRFileOutputFormat.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/output/SortingRFileOutputFormat.java b/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/output/SortingRFileOutputFormat.java
deleted file mode 100644
index 1fa8fdc..0000000
--- a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/output/SortingRFileOutputFormat.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.output;
-
-import java.io.IOException;
-
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.hadoop.mapreduce.RecordWriter;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-
-public class SortingRFileOutputFormat extends OutputFormat<Text,Mutation> {
-
- // private static final Logger log = Logger.getLogger(SortingRFileOutputFormat.class);
-
- public static final String PATH_NAME = "sortingrfileoutputformat.path";
- public static final String MAX_BUFFER_SIZE = "sortingrfileoutputformat.max.buffer.size";
-
- public static void setPathName(Configuration conf, String path) {
- conf.set(PATH_NAME, path);
- }
-
- public static String getPathName(Configuration conf) {
- return conf.get(PATH_NAME);
- }
-
- public static void setMaxBufferSize(Configuration conf, long maxBufferSize) {
- conf.setLong(MAX_BUFFER_SIZE, maxBufferSize);
- }
-
- public static long getMaxBufferSize(Configuration conf) {
- return conf.getLong(MAX_BUFFER_SIZE, -1);
- }
-
- @Override
- public void checkOutputSpecs(JobContext job) throws IOException, InterruptedException {
- // TODO make sure the path is writable?
- // TODO make sure the max buffer size is set and is reasonable
- }
-
- @Override
- public OutputCommitter getOutputCommitter(TaskAttemptContext arg0) throws IOException, InterruptedException {
- return new OutputCommitter() {
-
- @Override
- public void setupTask(TaskAttemptContext arg0) throws IOException {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public void setupJob(JobContext arg0) throws IOException {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public boolean needsTaskCommit(TaskAttemptContext arg0) throws IOException {
- // TODO Auto-generated method stub
- return false;
- }
-
- @Override
- public void commitTask(TaskAttemptContext arg0) throws IOException {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public void cleanupJob(JobContext arg0) throws IOException {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public void abortTask(TaskAttemptContext arg0) throws IOException {
- // TODO Auto-generated method stub
-
- }
- };
- }
-
- @Override
- public RecordWriter<Text,Mutation> getRecordWriter(TaskAttemptContext attempt) throws IOException, InterruptedException {
-
- // grab the configuration
- final Configuration conf = attempt.getConfiguration();
- // create a filename
- final String filenamePrefix = getPathName(conf);
- final String taskID = attempt.getTaskAttemptID().toString();
- // grab the max size
- final long maxSize = getMaxBufferSize(conf);
- // grab the FileSystem
- final FileSystem fs = FileSystem.get(conf);
- // create a default AccumuloConfiguration
- final AccumuloConfiguration acuconf = AccumuloConfiguration.getDefaultConfiguration();
-
- return new BufferingRFileRecordWriter(maxSize, acuconf, conf, filenamePrefix, taskID, fs);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/protobuf/TermWeight.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/protobuf/TermWeight.java b/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/protobuf/TermWeight.java
deleted file mode 100644
index bf5133f..0000000
--- a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/protobuf/TermWeight.java
+++ /dev/null
@@ -1,424 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-// Generated by the protocol buffer compiler. DO NOT EDIT!
-// source: TermWeight.proto
-
-package org.apache.accumulo.examples.wikisearch.protobuf;
-
-public final class TermWeight {
- private TermWeight() {}
-
- public static void registerAllExtensions(com.google.protobuf.ExtensionRegistry registry) {}
-
- public static final class Info extends com.google.protobuf.GeneratedMessage {
- // Use Info.newBuilder() to construct.
- private Info() {
- initFields();
- }
-
- private Info(boolean noInit) {}
-
- private static final Info defaultInstance;
-
- public static Info getDefaultInstance() {
- return defaultInstance;
- }
-
- public Info getDefaultInstanceForType() {
- return defaultInstance;
- }
-
- public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
- return org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.internal_static_protobuf_Info_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable internalGetFieldAccessorTable() {
- return org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.internal_static_protobuf_Info_fieldAccessorTable;
- }
-
- // required float normalizedTermFrequency = 1;
- public static final int NORMALIZEDTERMFREQUENCY_FIELD_NUMBER = 1;
- private boolean hasNormalizedTermFrequency;
- private float normalizedTermFrequency_ = 0F;
-
- public boolean hasNormalizedTermFrequency() {
- return hasNormalizedTermFrequency;
- }
-
- public float getNormalizedTermFrequency() {
- return normalizedTermFrequency_;
- }
-
- // repeated uint32 wordOffset = 2;
- public static final int WORDOFFSET_FIELD_NUMBER = 2;
- private java.util.List<java.lang.Integer> wordOffset_ = java.util.Collections.emptyList();
-
- public java.util.List<java.lang.Integer> getWordOffsetList() {
- return wordOffset_;
- }
-
- public int getWordOffsetCount() {
- return wordOffset_.size();
- }
-
- public int getWordOffset(int index) {
- return wordOffset_.get(index);
- }
-
- private void initFields() {}
-
- public final boolean isInitialized() {
- if (!hasNormalizedTermFrequency)
- return false;
- return true;
- }
-
- public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException {
- getSerializedSize();
- if (hasNormalizedTermFrequency()) {
- output.writeFloat(1, getNormalizedTermFrequency());
- }
- for (int element : getWordOffsetList()) {
- output.writeUInt32(2, element);
- }
- getUnknownFields().writeTo(output);
- }
-
- private int memoizedSerializedSize = -1;
-
- public int getSerializedSize() {
- int size = memoizedSerializedSize;
- if (size != -1)
- return size;
-
- size = 0;
- if (hasNormalizedTermFrequency()) {
- size += com.google.protobuf.CodedOutputStream.computeFloatSize(1, getNormalizedTermFrequency());
- }
- {
- int dataSize = 0;
- for (int element : getWordOffsetList()) {
- dataSize += com.google.protobuf.CodedOutputStream.computeUInt32SizeNoTag(element);
- }
- size += dataSize;
- size += 1 * getWordOffsetList().size();
- }
- size += getUnknownFields().getSerializedSize();
- memoizedSerializedSize = size;
- return size;
- }
-
- public static org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info parseFrom(com.google.protobuf.ByteString data) throws com.google.protobuf.InvalidProtocolBufferException {
- return newBuilder().mergeFrom(data).buildParsed();
- }
-
- public static org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info parseFrom(com.google.protobuf.ByteString data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return newBuilder().mergeFrom(data, extensionRegistry).buildParsed();
- }
-
- public static org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info parseFrom(byte[] data) throws com.google.protobuf.InvalidProtocolBufferException {
- return newBuilder().mergeFrom(data).buildParsed();
- }
-
- public static org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info parseFrom(byte[] data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return newBuilder().mergeFrom(data, extensionRegistry).buildParsed();
- }
-
- public static org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info parseFrom(java.io.InputStream input) throws java.io.IOException {
- return newBuilder().mergeFrom(input).buildParsed();
- }
-
- public static org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info parseFrom(java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return newBuilder().mergeFrom(input, extensionRegistry).buildParsed();
- }
-
- public static org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info parseDelimitedFrom(java.io.InputStream input) throws java.io.IOException {
- Builder builder = newBuilder();
- if (builder.mergeDelimitedFrom(input)) {
- return builder.buildParsed();
- } else {
- return null;
- }
- }
-
- public static org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info parseDelimitedFrom(java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- Builder builder = newBuilder();
- if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
- return builder.buildParsed();
- } else {
- return null;
- }
- }
-
- public static org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info parseFrom(com.google.protobuf.CodedInputStream input) throws java.io.IOException {
- return newBuilder().mergeFrom(input).buildParsed();
- }
-
- public static org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info parseFrom(com.google.protobuf.CodedInputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return newBuilder().mergeFrom(input, extensionRegistry).buildParsed();
- }
-
- public static Builder newBuilder() {
- return Builder.create();
- }
-
- public Builder newBuilderForType() {
- return newBuilder();
- }
-
- public static Builder newBuilder(org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info prototype) {
- return newBuilder().mergeFrom(prototype);
- }
-
- public Builder toBuilder() {
- return newBuilder(this);
- }
-
- public static final class Builder extends com.google.protobuf.GeneratedMessage.Builder<Builder> {
- private org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info result;
-
- // Construct using protobuf.TermWeight.Info.newBuilder()
- private Builder() {}
-
- private static Builder create() {
- Builder builder = new Builder();
- builder.result = new org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info();
- return builder;
- }
-
- protected org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info internalGetResult() {
- return result;
- }
-
- public Builder clear() {
- if (result == null) {
- throw new IllegalStateException("Cannot call clear() after build().");
- }
- result = new org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info();
- return this;
- }
-
- public Builder clone() {
- return create().mergeFrom(result);
- }
-
- public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() {
- return org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info.getDescriptor();
- }
-
- public org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info getDefaultInstanceForType() {
- return org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info.getDefaultInstance();
- }
-
- public boolean isInitialized() {
- return result.isInitialized();
- }
-
- public org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info build() {
- if (result != null && !isInitialized()) {
- throw newUninitializedMessageException(result);
- }
- return buildPartial();
- }
-
- private org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info buildParsed() throws com.google.protobuf.InvalidProtocolBufferException {
- if (!isInitialized()) {
- throw newUninitializedMessageException(result).asInvalidProtocolBufferException();
- }
- return buildPartial();
- }
-
- public org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info buildPartial() {
- if (result == null) {
- throw new IllegalStateException("build() has already been called on this Builder.");
- }
- if (result.wordOffset_ != java.util.Collections.EMPTY_LIST) {
- result.wordOffset_ = java.util.Collections.unmodifiableList(result.wordOffset_);
- }
- org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info returnMe = result;
- result = null;
- return returnMe;
- }
-
- public Builder mergeFrom(com.google.protobuf.Message other) {
- if (other instanceof org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info) {
- return mergeFrom((org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info) other);
- } else {
- super.mergeFrom(other);
- return this;
- }
- }
-
- public Builder mergeFrom(org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info other) {
- if (other == org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info.getDefaultInstance())
- return this;
- if (other.hasNormalizedTermFrequency()) {
- setNormalizedTermFrequency(other.getNormalizedTermFrequency());
- }
- if (!other.wordOffset_.isEmpty()) {
- if (result.wordOffset_.isEmpty()) {
- result.wordOffset_ = new java.util.ArrayList<java.lang.Integer>();
- }
- result.wordOffset_.addAll(other.wordOffset_);
- }
- this.mergeUnknownFields(other.getUnknownFields());
- return this;
- }
-
- public Builder mergeFrom(com.google.protobuf.CodedInputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet.newBuilder(this.getUnknownFields());
- while (true) {
- int tag = input.readTag();
- switch (tag) {
- case 0:
- this.setUnknownFields(unknownFields.build());
- return this;
- default: {
- if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) {
- this.setUnknownFields(unknownFields.build());
- return this;
- }
- break;
- }
- case 13: {
- setNormalizedTermFrequency(input.readFloat());
- break;
- }
- case 16: {
- addWordOffset(input.readUInt32());
- break;
- }
- case 18: {
- int length = input.readRawVarint32();
- int limit = input.pushLimit(length);
- while (input.getBytesUntilLimit() > 0) {
- addWordOffset(input.readUInt32());
- }
- input.popLimit(limit);
- break;
- }
- }
- }
- }
-
- // required float normalizedTermFrequency = 1;
- public boolean hasNormalizedTermFrequency() {
- return result.hasNormalizedTermFrequency();
- }
-
- public float getNormalizedTermFrequency() {
- return result.getNormalizedTermFrequency();
- }
-
- public Builder setNormalizedTermFrequency(float value) {
- result.hasNormalizedTermFrequency = true;
- result.normalizedTermFrequency_ = value;
- return this;
- }
-
- public Builder clearNormalizedTermFrequency() {
- result.hasNormalizedTermFrequency = false;
- result.normalizedTermFrequency_ = 0F;
- return this;
- }
-
- // repeated uint32 wordOffset = 2;
- public java.util.List<java.lang.Integer> getWordOffsetList() {
- return java.util.Collections.unmodifiableList(result.wordOffset_);
- }
-
- public int getWordOffsetCount() {
- return result.getWordOffsetCount();
- }
-
- public int getWordOffset(int index) {
- return result.getWordOffset(index);
- }
-
- public Builder setWordOffset(int index, int value) {
- result.wordOffset_.set(index, value);
- return this;
- }
-
- public Builder addWordOffset(int value) {
- if (result.wordOffset_.isEmpty()) {
- result.wordOffset_ = new java.util.ArrayList<java.lang.Integer>();
- }
- result.wordOffset_.add(value);
- return this;
- }
-
- public Builder addAllWordOffset(java.lang.Iterable<? extends java.lang.Integer> values) {
- if (result.wordOffset_.isEmpty()) {
- result.wordOffset_ = new java.util.ArrayList<java.lang.Integer>();
- }
- super.addAll(values, result.wordOffset_);
- return this;
- }
-
- public Builder clearWordOffset() {
- result.wordOffset_ = java.util.Collections.emptyList();
- return this;
- }
-
- // @@protoc_insertion_point(builder_scope:protobuf.Info)
- }
-
- static {
- defaultInstance = new Info(true);
- org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.internalForceInit();
- defaultInstance.initFields();
- }
-
- // @@protoc_insertion_point(class_scope:protobuf.Info)
- }
-
- private static com.google.protobuf.Descriptors.Descriptor internal_static_protobuf_Info_descriptor;
- private static com.google.protobuf.GeneratedMessage.FieldAccessorTable internal_static_protobuf_Info_fieldAccessorTable;
-
- public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() {
- return descriptor;
- }
-
- private static com.google.protobuf.Descriptors.FileDescriptor descriptor;
- static {
- java.lang.String[] descriptorData = {"\n\020TermWeight.proto\022\010protobuf\";\n\004Info\022\037\n\027"
- + "normalizedTermFrequency\030\001 \002(\002\022\022\n\nwordOff" + "set\030\002 \003(\rB\014\n\010protobufH\001"};
- com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
- public com.google.protobuf.ExtensionRegistry assignDescriptors(com.google.protobuf.Descriptors.FileDescriptor root) {
- descriptor = root;
- internal_static_protobuf_Info_descriptor = getDescriptor().getMessageTypes().get(0);
- internal_static_protobuf_Info_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable(
- internal_static_protobuf_Info_descriptor, new java.lang.String[] {"NormalizedTermFrequency", "WordOffset",}, org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info.class,
- org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info.Builder.class);
- return null;
- }
- };
- com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom(descriptorData, new com.google.protobuf.Descriptors.FileDescriptor[] {},
- assigner);
- }
-
- public static void internalForceInit() {}
-
- // @@protoc_insertion_point(outer_class_scope)
-}
[34/39] git commit: ACCUMULO-1876 default for slf4j.version
Posted by ec...@apache.org.
ACCUMULO-1876 default for slf4j.version
Sets default to 1.4.3 as hadoop-1.0 is default in parent pom for 1.5.1-SNAPSHOT
(cherry picked from commit c93872afd3e191ea81e353e609749128d3b5c417)
Reason: bugfix
Author: Michael Wall <mj...@gmail.com>
Ref: ACCUMULO-1792
Signed-off-by: Eric Newton <er...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c0939746
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c0939746
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c0939746
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: c093974647a93f70320c5d62eeff00e27fa3149d
Parents: 5e3967f
Author: Michael Wall <mj...@gmail.com>
Authored: Fri Nov 15 16:29:46 2013 -0600
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Nov 25 16:06:44 2013 -0500
----------------------------------------------------------------------
pom.xml | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c0939746/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ca550d5..eda8390 100644
--- a/pom.xml
+++ b/pom.xml
@@ -868,6 +868,8 @@
<properties>
<targetJdk>1.6</targetJdk>
+ <!-- overwritten in profiles hadoop-1.0 or hadoop-2.0 -->
+ <slf4j.version>1.4.3</slf4j.version>
<zookeeper.version>3.3.1</zookeeper.version>
</properties>
[23/39] git commit: ACCUMULO-876 added maven profiles to assist in
compiling against different versions of hadoop
Posted by ec...@apache.org.
ACCUMULO-876 added maven profiles to assist in compiling against different versions of hadoop
git-svn-id: https://svn.apache.org/repos/asf/accumulo/trunk@1419924 13f79535-47bb-0310-9956-ffa450edef68
(cherry picked from commit 65c271c2a46b87eb0e4991d7fd7b6fbd7456c0a8)
Reason: Hadoop2 Compat
Author: Billie Rinaldi <bi...@apache.org>
Ref: ACCUMULO-1792
This patch differs from upstream by fixing poms not in the accumulo 1.5 release and keeping the 1.4 specific build instructions in the README.
Author: Jonathan Hsieh <jo...@cloudera.com>
Author: Sean Busbey <bu...@cloudera.com>
Signed-off-by: Eric Newton <er...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0f97284d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0f97284d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0f97284d
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 0f97284da4c1b33eccccf1b423003d490e2f81a8
Parents: 31e4dd1
Author: Billie Rinaldi <bi...@apache.org>
Authored: Tue Dec 11 00:01:39 2012 +0000
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Nov 25 16:06:42 2013 -0500
----------------------------------------------------------------------
README | 8 +-
pom.xml | 78 +++++++++++++++++---
src/core/pom.xml | 56 ++++++++++++--
src/examples/simple/pom.xml | 50 +++++++++++--
src/examples/wikisearch/ingest/pom.xml | 42 +++++++++--
src/examples/wikisearch/query/pom.xml | 43 +++++++++--
src/minicluster/pom.xml | 41 ++++++++--
src/proxy/pom.xml | 2 +-
src/server/pom.xml | 50 ++++++++++++-
.../server/monitor/servlets/DefaultServlet.java | 64 ++++++----------
src/start/pom.xml | 38 +++++++++-
src/trace/pom.xml | 2 +-
12 files changed, 385 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0f97284d/README
----------------------------------------------------------------------
diff --git a/README b/README
index ebe89e4..674769d 100644
--- a/README
+++ b/README
@@ -17,7 +17,13 @@ However, if you only have source code, or you wish to make changes, you need to
have maven configured to get Accumulo prerequisites from repositories. See
the pom.xml file for the necessary components.
-Run "mvn package && mvn assembly:single -N"
+Run "mvn package && mvn assembly:single -N". By default,
+Accumulo compiles against Hadoop 1.0.4. To compile against a different version
+that is compatible with Hadoop 1.0, specify hadoop.version on the command line,
+e.g. "-Dhadoop.version=0.20.205.0" or "-Dhadoop.version=1.1.0". To compile
+against Hadoop 2.0, specify "-Dhadoop.profile=2.0". By default this uses
+2.0.2-alpha. To compile against a different 2.0-compatible version, specify
+the profile and version, e.g. "-Dhadoop.profile=2.0 -Dhadoop.version=0.23.5".
If you are running on another Unix-like operating system (OSX, etc) then
you may wish to build the native libraries. They are not strictly necessary
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0f97284d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 9ed2fdf..e4c06fa 100644
--- a/pom.xml
+++ b/pom.xml
@@ -640,6 +640,62 @@
</plugins>
</build>
</profile>
+ <!-- profile for building against Hadoop 1.0.x
+ Activate by not specifying hadoop.profile -->
+ <profile>
+ <id>hadoop-1.0</id>
+ <activation>
+ <property>
+ <name>!hadoop.profile</name>
+ </property>
+ </activation>
+ <properties>
+ <hadoop.version>1.0.4</hadoop.version>
+ <slf4j.version>1.4.3</slf4j.version>
+ </properties>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+ </profile>
+ <!-- profile for building against Hadoop 2.0.x
+ Activate using: mvn -Dhadoop.profile=2.0 -->
+ <profile>
+ <id>hadoop-2.0</id>
+ <activation>
+ <property>
+ <name>hadoop.profile</name>
+ <value>2.0</value>
+ </property>
+ </activation>
+ <properties>
+ <slf4j.version>1.6.1</slf4j.version>
+ <hadoop.version>2.0.2-alpha</hadoop.version>
+ <avro.version>1.5.3</avro.version>
+ </properties>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ <version>${avro.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+ </profile>
</profiles>
<dependencyManagement>
@@ -660,15 +716,9 @@
<!-- provided dependencies needed at runtime -->
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- <version>0.20.203.0</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
+ <groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
- <version>3.3.1</version>
+ <version>${zookeeper.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
@@ -680,7 +730,7 @@
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
- <version>[5.1,)</version>
+ <version>[6.1,7.0)</version>
<scope>provided</scope>
</dependency>
@@ -771,12 +821,12 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
- <version>1.4.3</version>
+ <version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
- <version>1.4.3</version>
+ <version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
@@ -815,4 +865,10 @@
<system>Apache's JIRA issue tracker</system>
<url>https://issues.apache.org/jira/browse/ACCUMULO</url>
</issueManagement>
+
+ <properties>
+ <targetJdk>1.6</targetJdk>
+ <zookeeper.version>3.3.1</zookeeper.version>
+ </properties>
+
</project>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0f97284d/src/core/pom.xml
----------------------------------------------------------------------
diff --git a/src/core/pom.xml b/src/core/pom.xml
index d7aa1fc..77a4a72 100644
--- a/src/core/pom.xml
+++ b/src/core/pom.xml
@@ -76,14 +76,58 @@
</plugin>
</plugins>
</build>
-
+
+ <profiles>
+ <!-- profile for building against Hadoop 1.0.x
+ Activate by not specifying hadoop.profile -->
+ <profile>
+ <id>hadoop-1.0</id>
+ <activation>
+ <property>
+ <name>!hadoop.profile</name>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+ <!-- profile for building against Hadoop 2.0.x
+ Activate using: mvn -Dhadoop.profile=2.0 -->
+ <profile>
+ <id>hadoop-2.0</id>
+ <activation>
+ <property>
+ <name>hadoop.profile</name>
+ <value>2.0</value>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+
<dependencies>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-start</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>cloudtrace</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</dependency>
<dependency>
@@ -108,10 +152,6 @@
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-start</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.accumulo</groupId>
<artifactId>cloudtrace</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0f97284d/src/examples/simple/pom.xml
----------------------------------------------------------------------
diff --git a/src/examples/simple/pom.xml b/src/examples/simple/pom.xml
index 1df99a1..6834b87 100644
--- a/src/examples/simple/pom.xml
+++ b/src/examples/simple/pom.xml
@@ -28,16 +28,56 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>examples-simple</artifactId>
<name>examples-simple</name>
-
+
+ <profiles>
+ <!-- profile for building against Hadoop 1.0.x
+ Activate by not specifying hadoop.profile -->
+ <profile>
+ <id>hadoop-1.0</id>
+ <activation>
+ <property>
+ <name>!hadoop.profile</name>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+ <!-- profile for building against Hadoop 2.0.x
+ Activate using: mvn -Dhadoop.profile=2.0 -->
+ <profile>
+ <id>hadoop-2.0</id>
+ <activation>
+ <property>
+ <name>hadoop.profile</name>
+ <value>2.0</value>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+
<dependencies>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- </dependency>
- <dependency>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-core</artifactId>
</dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0f97284d/src/examples/wikisearch/ingest/pom.xml
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/pom.xml b/src/examples/wikisearch/ingest/pom.xml
index 29b2047..31d7110 100644
--- a/src/examples/wikisearch/ingest/pom.xml
+++ b/src/examples/wikisearch/ingest/pom.xml
@@ -29,11 +29,7 @@
<dependencies>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
+ <groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</dependency>
<dependency>
@@ -125,4 +121,40 @@
</plugins>
</build>
+ <profiles>
+ <!-- profile for building against Hadoop 1.0.x
+ Activate by not specifying hadoop.profile -->
+ <profile>
+ <id>hadoop-1.0</id>
+ <activation>
+ <property>
+ <name>!hadoop.profile</name>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+ <!-- profile for building against Hadoop 2.0.x
+ Activate using: mvn -Dhadoop.profile=2.0 -->
+ <profile>
+ <id>hadoop-2.0</id>
+ <activation>
+ <property>
+ <name>hadoop.profile</name>
+ <value>2.0</value>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+
</project>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0f97284d/src/examples/wikisearch/query/pom.xml
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/pom.xml b/src/examples/wikisearch/query/pom.xml
index 05b1e46..8c6e6d3 100644
--- a/src/examples/wikisearch/query/pom.xml
+++ b/src/examples/wikisearch/query/pom.xml
@@ -38,11 +38,7 @@
<artifactId>accumulo-core</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
+ <groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<scope>runtime</scope>
</dependency>
@@ -144,4 +140,41 @@
</plugins>
</build>
<modelVersion>4.0.0</modelVersion>
+
+ <profiles>
+ <!-- profile for building against Hadoop 1.0.x
+ Activate by not specifying hadoop.profile -->
+ <profile>
+ <id>hadoop-1.0</id>
+ <activation>
+ <property>
+ <name>!hadoop.profile</name>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+ <!-- profile for building against Hadoop 2.0.x
+ Activate using: mvn -Dhadoop.profile=2.0 -->
+ <profile>
+ <id>hadoop-2.0</id>
+ <activation>
+ <property>
+ <name>hadoop.profile</name>
+ <value>2.0</value>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+
</project>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0f97284d/src/minicluster/pom.xml
----------------------------------------------------------------------
diff --git a/src/minicluster/pom.xml b/src/minicluster/pom.xml
index 3e23b0c..3c8843d 100644
--- a/src/minicluster/pom.xml
+++ b/src/minicluster/pom.xml
@@ -50,14 +50,45 @@
</plugins>
</build>
+ <profiles>
+ <!-- profile for building against Hadoop 1.0.x
+ Activate by not specifying hadoop.profile -->
+ <profile>
+ <id>hadoop-1.0</id>
+ <activation>
+ <property>
+ <name>!hadoop.profile</name>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+ <!-- profile for building against Hadoop 2.0.x
+ Activate using: mvn -Dhadoop.profile=2.0 -->
+ <profile>
+ <id>hadoop-2.0</id>
+ <activation>
+ <property>
+ <name>hadoop.profile</name>
+ <value>2.0</value>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
<dependencies>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
+ <groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0f97284d/src/proxy/pom.xml
----------------------------------------------------------------------
diff --git a/src/proxy/pom.xml b/src/proxy/pom.xml
index 5f2afc2..5d0414d 100644
--- a/src/proxy/pom.xml
+++ b/src/proxy/pom.xml
@@ -87,7 +87,7 @@
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
+ <groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<scope>compile</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0f97284d/src/server/pom.xml
----------------------------------------------------------------------
diff --git a/src/server/pom.xml b/src/server/pom.xml
index 6421bc6..cc55e90 100644
--- a/src/server/pom.xml
+++ b/src/server/pom.xml
@@ -53,11 +53,11 @@
<dependencies>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libthrift</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
+ <groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</dependency>
<dependency>
@@ -100,6 +100,50 @@
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </dependency>
</dependencies>
+ <profiles>
+ <!-- profile for building against Hadoop 1.0.x
+ Activate by not specifying hadoop.profile -->
+ <profile>
+ <id>hadoop-1.0</id>
+ <activation>
+ <property>
+ <name>!hadoop.profile</name>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+ <!-- profile for building against Hadoop 2.0.x
+ Activate using: mvn -Dhadoop.profile=2.0 -->
+ <profile>
+ <id>hadoop-2.0</id>
+ <activation>
+ <property>
+ <name>hadoop.profile</name>
+ <value>2.0</value>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+
</project>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0f97284d/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java b/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java
index 465b253..9591d4a 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java
@@ -21,7 +21,6 @@ import java.io.FileInputStream;
import java.io.FilePermission;
import java.io.IOException;
import java.io.InputStream;
-import java.net.InetSocketAddress;
import java.security.AccessControlContext;
import java.security.AccessController;
import java.security.PermissionCollection;
@@ -55,9 +54,6 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.mapred.ClusterStatus;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobTracker;
public class DefaultServlet extends BasicServlet {
@@ -81,13 +77,17 @@ public class DefaultServlet extends BasicServlet {
path = path.substring(1);
InputStream data = BasicServlet.class.getClassLoader().getResourceAsStream(path);
ServletOutputStream out = resp.getOutputStream();
- if (data != null) {
- byte[] buffer = new byte[1024];
- int n;
- while ((n = data.read(buffer)) > 0)
- out.write(buffer, 0, n);
- } else {
- out.write(("could not get resource " + path + "").getBytes());
+ try {
+ if (data != null) {
+ byte[] buffer = new byte[1024];
+ int n;
+ while ((n = data.read(buffer)) > 0)
+ out.write(buffer, 0, n);
+ } else {
+ out.write(("could not get resource " + path + "").getBytes());
+ }
+ } finally {
+ data.close();
}
} catch (Throwable t) {
log.error(t, t);
@@ -111,9 +111,10 @@ public class DefaultServlet extends BasicServlet {
@Override
public IOException run() {
+ InputStream data = null;
try {
File file = new File(aHome + path);
- InputStream data = new FileInputStream(file.getAbsolutePath());
+ data = new FileInputStream(file.getAbsolutePath());
byte[] buffer = new byte[1024];
int n;
ServletOutputStream out = resp.getOutputStream();
@@ -122,6 +123,14 @@ public class DefaultServlet extends BasicServlet {
return null;
} catch (IOException e) {
return e;
+ } finally {
+ if (data != null) {
+ try {
+ data.close();
+ } catch (IOException ex) {
+ log.error(ex, ex);
+ }
+ }
}
}
}, acc);
@@ -208,10 +217,6 @@ public class DefaultServlet extends BasicServlet {
sb.append("</td>\n");
sb.append("<td class='noborder'>\n");
- doJobTrackerTable(sb);
- sb.append("</td>\n");
-
- sb.append("<td class='noborder'>\n");
doZooKeeperTable(sb);
sb.append("</td>\n");
@@ -323,33 +328,6 @@ public class DefaultServlet extends BasicServlet {
sb.append("</table>\n");
}
- private void doJobTrackerTable(StringBuilder sb) {
- // Job Tracker
- Configuration conf = CachedConfiguration.getInstance();
- sb.append("<table>\n");
- try {
- InetSocketAddress address = JobTracker.getAddress(conf);
- @SuppressWarnings("deprecation")
- JobClient jc = new JobClient(new org.apache.hadoop.mapred.JobConf(conf));
- String httpAddress = conf.get("mapred.job.tracker.http.address");
- String port = httpAddress.split(":")[1];
- String href = "http://" + address.getHostName() + ":" + port;
- String activeUrl = href + "/machines.jsp?type=active";
- String blacklistUrl = href + "/machines.jsp?type=blacklisted";
- sb.append("<tr><th colspan='2'><a href='" + href + "'>JobTracker</a></th></tr>\n");
- boolean highlight = false;
- tableRow(sb, (highlight = !highlight), "Running Jobs", jc.jobsToComplete().length);
- ClusterStatus status = jc.getClusterStatus();
- tableRow(sb, (highlight = !highlight), "Map Tasks", status.getMapTasks() + "/" + status.getMaxMapTasks());
- tableRow(sb, (highlight = !highlight), "Reduce Tasks", status.getReduceTasks() + "/" + status.getMaxReduceTasks());
- tableRow(sb, (highlight = !highlight), "<a href='" + activeUrl + "'>Trackers</a>", status.getTaskTrackers());
- tableRow(sb, (highlight = !highlight), "<a href='" + blacklistUrl + "'>Blacklisted</a>", status.getBlacklistedTrackers());
- } catch (Exception ex) {
- sb.append("<tr><td colspan='2'><span class='error'>Job Tracker is Down</span></td></tr>\n");
- }
- sb.append("</table>\n");
- }
-
private void doZooKeeperTable(StringBuilder sb) throws IOException {
// Zookeepers
sb.append("<table>\n");
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0f97284d/src/start/pom.xml
----------------------------------------------------------------------
diff --git a/src/start/pom.xml b/src/start/pom.xml
index 98d9d47..de3be18 100644
--- a/src/start/pom.xml
+++ b/src/start/pom.xml
@@ -49,7 +49,43 @@
</plugin>
</plugins>
</build>
-
+
+ <profiles>
+ <!-- profile for building against Hadoop 1.0.x
+ Activate by not specifying hadoop.profile -->
+ <profile>
+ <id>hadoop-1.0</id>
+ <activation>
+ <property>
+ <name>!hadoop.profile</name>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+ <!-- profile for building against Hadoop 2.0.x
+ Activate using: mvn -Dhadoop.profile=2.0 -->
+ <profile>
+ <id>hadoop-2.0</id>
+ <activation>
+ <property>
+ <name>hadoop.profile</name>
+ <value>2.0</value>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+
<dependencies>
<dependency>
<groupId>log4j</groupId>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0f97284d/src/trace/pom.xml
----------------------------------------------------------------------
diff --git a/src/trace/pom.xml b/src/trace/pom.xml
index 1ee00a3..941742c 100644
--- a/src/trace/pom.xml
+++ b/src/trace/pom.xml
@@ -46,7 +46,7 @@
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
+ <groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</dependency>
</dependencies>
[36/39] git commit: ACCUMULO-645 found another HADOOP_HOME reference
Posted by ec...@apache.org.
ACCUMULO-645 found another HADOOP_HOME reference
git-svn-id: https://svn.apache.org/repos/asf/accumulo/trunk@1433058 13f79535-47bb-0310-9956-ffa450edef68
(cherry picked from commit e7f6e5ff4f80d1f2bfe3f6daf0889d51b5593c6a)
Reason: Hadoop2 Compat
Author: Eric C. Newton <ec...@apache.org>
Ref: ACCUMULO-1792
Signed-off-by: Eric Newton <er...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5dd9016a
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5dd9016a
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5dd9016a
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 5dd9016a1d406618531725e6ce111b59acae72ec
Parents: d16fd77
Author: Eric C. Newton <ec...@apache.org>
Authored: Mon Jan 14 19:11:44 2013 +0000
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Nov 25 16:06:44 2013 -0500
----------------------------------------------------------------------
.../org/apache/accumulo/start/classloader/AccumuloClassLoader.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5dd9016a/src/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
----------------------------------------------------------------------
diff --git a/src/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java b/src/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
index fd41938..ff02d13 100644
--- a/src/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
+++ b/src/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
@@ -111,7 +111,7 @@ public class AccumuloClassLoader {
+ "$ACCUMULO_HOME/lib/[^.].*.jar,\n"
+ "$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n"
+ "$HADOOP_PREFIX/[^.].*.jar,\n" + "$HADOOP_CONF_DIR,\n"
- + "$HADOOP_HOME/lib/[^.].*.jar,\n";
+ + "$HADOOP_PREFIX/lib/[^.].*.jar,\n";
/**
* Dynamic classpath. These locations will be monitored for changes.
[09/39] ACCUMULO-600 removed wikisearch from trunk
Posted by ec...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/BooleanLogicIterator.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/BooleanLogicIterator.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/BooleanLogicIterator.java
deleted file mode 100644
index 09ad8d3..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/BooleanLogicIterator.java
+++ /dev/null
@@ -1,1949 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.iterator;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Enumeration;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.PriorityQueue;
-import java.util.Set;
-
-import org.apache.accumulo.core.data.ByteSequence;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.IteratorEnvironment;
-import org.apache.accumulo.core.iterators.OptionDescriber;
-import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
-import org.apache.accumulo.examples.wikisearch.parser.JexlOperatorConstants;
-import org.apache.accumulo.examples.wikisearch.parser.QueryParser;
-import org.apache.accumulo.examples.wikisearch.parser.QueryParser.QueryTerm;
-import org.apache.accumulo.examples.wikisearch.parser.RangeCalculator.RangeBounds;
-import org.apache.accumulo.examples.wikisearch.parser.TreeNode;
-import org.apache.accumulo.examples.wikisearch.util.FieldIndexKeyParser;
-import org.apache.commons.jexl2.parser.ASTAndNode;
-import org.apache.commons.jexl2.parser.ASTEQNode;
-import org.apache.commons.jexl2.parser.ASTERNode;
-import org.apache.commons.jexl2.parser.ASTGENode;
-import org.apache.commons.jexl2.parser.ASTGTNode;
-import org.apache.commons.jexl2.parser.ASTJexlScript;
-import org.apache.commons.jexl2.parser.ASTLENode;
-import org.apache.commons.jexl2.parser.ASTLTNode;
-import org.apache.commons.jexl2.parser.ASTNENode;
-import org.apache.commons.jexl2.parser.ASTNRNode;
-import org.apache.commons.jexl2.parser.ASTNotNode;
-import org.apache.commons.jexl2.parser.ASTOrNode;
-import org.apache.commons.jexl2.parser.ParseException;
-import org.apache.commons.jexl2.parser.ParserTreeConstants;
-import org.apache.hadoop.io.Text;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-
-import com.google.common.collect.Multimap;
-
-public class BooleanLogicIterator implements SortedKeyValueIterator<Key,Value>, OptionDescriber {
-
- private static final Collection<ByteSequence> EMPTY_COL_FAMS = new ArrayList<ByteSequence>();
- protected static final Logger log = Logger.getLogger(BooleanLogicIterator.class);
- public static final String QUERY_OPTION = "expr";
- public static final String TERM_CARDINALITIES = "TERM_CARDINALITIES"; // comma separated list of term : count
- public static final String FIELD_INDEX_QUERY = "FIELD_INDEX_QUERY";
- public static final String FIELD_NAME_PREFIX = "fi\0";
- // --------------------------------------------------------------------------
- private static IteratorEnvironment env = new DefaultIteratorEnvironment();
- protected Text nullText = new Text();
- private Key topKey = null;
- private Value topValue = null;
- private SortedKeyValueIterator<Key,Value> sourceIterator;
- private BooleanLogicTreeNode root;
- private PriorityQueue<BooleanLogicTreeNode> positives;
- private ArrayList<BooleanLogicTreeNode> negatives = new ArrayList<BooleanLogicTreeNode>();
- private ArrayList<BooleanLogicTreeNode> rangerators;
- private String updatedQuery;
- private Map<String,Long> termCardinalities = new HashMap<String,Long>();
- private Range overallRange = null;
- private FieldIndexKeyParser keyParser;
-
- public BooleanLogicIterator() {
- keyParser = new FieldIndexKeyParser();
- rangerators = new ArrayList<BooleanLogicTreeNode>();
- }
-
- public BooleanLogicIterator(BooleanLogicIterator other, IteratorEnvironment env) {
- if (other.sourceIterator != null) {
- this.sourceIterator = other.sourceIterator.deepCopy(env);
- }
- keyParser = new FieldIndexKeyParser();
- rangerators = new ArrayList<BooleanLogicTreeNode>();
- log.debug("Congratulations, you've reached the BooleanLogicIterator");
- }
-
- public static void setLogLevel(Level lev) {
- log.setLevel(lev);
- }
-
- public void setDebug(Level lev) {
- log.setLevel(lev);
- }
-
- public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
- return new BooleanLogicIterator(this, env);
- }
-
- /**
- * <b>init</b> is responsible for setting up the iterator. It will pull the serialized boolean parse tree from the options mapping and construct the
- * appropriate sub-iterators
- *
- * Once initialized, this iterator will automatically seek to the first matching instance. If no top key exists, that means an event matching the boolean
- * logic did not exist in the partition. Subsequent calls to next will move the iterator and all sub-iterators to the next match.
- *
- * @param source
- * The underlying SortedkeyValueIterator.
- * @param options
- * A Map<String, String> of options.
- * @param env
- * The iterator environment
- * @throws IOException
- */
- public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
- validateOptions(options);
- try {
- if (log.isDebugEnabled()) {
- log.debug("Congratulations, you've reached the BooleanLogicIterator.init method");
- }
- // Copy the source iterator
- sourceIterator = source.deepCopy(env);
-
- // Potentially take advantage of term cardinalities
- String[] terms = null;
- if (null != options.get(TERM_CARDINALITIES)) {
- terms = options.get(TERM_CARDINALITIES).split(",");
- for (String term : terms) {
- int idx = term.indexOf(":");
- if (-1 != idx) {
- termCardinalities.put(term.substring(0, idx), Long.parseLong(term.substring(idx + 1)));
- }
- }
- }
-
- // Step 1: Parse the query
- if (log.isDebugEnabled()) {
- log.debug("QueryParser");
- }
- QueryParser qp = new QueryParser();
- qp.execute(this.updatedQuery); // validateOptions updates the updatedQuery
-
- // need to build the query tree based on jexl parsing.
- // Step 2: refactor QueryTree - inplace modification
- if (log.isDebugEnabled()) {
- log.debug("transformTreeNode");
- }
- TreeNode tree = qp.getIteratorTree();
- this.root = transformTreeNode(tree);
-
- if (log.isDebugEnabled()) {
- log.debug("refactorTree");
- }
- this.root = refactorTree(this.root);
-
- if (log.isDebugEnabled()) {
- log.debug("collapseBranches");
- }
- collapseBranches(root);
-
- // Step 3: create iterators where we need them.
- createIteratorTree(this.root);
- if (log.isDebugEnabled()) {
- log.debug("Query tree after iterator creation:\n\t" + this.root.getContents());
- }
- // Step 4: split the positive and negative leaves
- splitLeaves(this.root);
-
- } catch (ParseException ex) {
- log.error("ParseException in init: " + ex);
- throw new IllegalArgumentException("Failed to parse query", ex);
- } catch (Exception ex) {
- throw new IllegalArgumentException("probably had no indexed terms", ex);
- }
-
- }
-
- /* *************************************************************************
- * Methods for sub iterator creation.
- */
- private void createIteratorTree(BooleanLogicTreeNode root) throws IOException {
- if (log.isDebugEnabled()) {
- log.debug("BoolLogic createIteratorTree()");
- }
- // Walk the tree, if all of your children are leaves, roll you into the
- // appropriate iterator.
- Enumeration<?> dfe = root.depthFirstEnumeration();
-
- while (dfe.hasMoreElements()) {
- BooleanLogicTreeNode node = (BooleanLogicTreeNode) dfe.nextElement();
- if (!node.isLeaf() && node.getType() != ParserTreeConstants.JJTJEXLSCRIPT) {
- // try to roll up.
- if (canRollUp(node)) {
- node.setRollUp(true);
- if (node.getType() == ParserTreeConstants.JJTANDNODE) {
- if (log.isDebugEnabled()) {
- log.debug("creating IntersectingIterator");
- }
- node.setUserObject(createIntersectingIterator(node));
- } else if (node.getType() == ParserTreeConstants.JJTORNODE) {
- node.setUserObject(createOrIterator(node));
- } else {
- // throw an error.
- log.debug("createIteratorTree, encounterd a node type I do not know about: " + node.getType());
- log.debug("createIteratorTree, node contents: " + node.getContents());
- }
- node.removeAllChildren();
- }
- }
- }
-
- // now for remaining leaves, create basic iterators.
- // you can add in specialized iterator mappings here if necessary.
- dfe = root.depthFirstEnumeration();
- while (dfe.hasMoreElements()) {
- BooleanLogicTreeNode node = (BooleanLogicTreeNode) dfe.nextElement();
- if (node.isLeaf() && node.getType() != ParserTreeConstants.JJTANDNODE && node.getType() != ParserTreeConstants.JJTORNODE) {
- node.setUserObject(createFieldIndexIterator(node));
- }
- }
- }
-
- private AndIterator createIntersectingIterator(BooleanLogicTreeNode node) throws IOException {
- if (log.isDebugEnabled()) {
- log.debug("createIntersectingIterator(node)");
- log.debug("fName: " + node.getFieldName() + " , fValue: " + node.getFieldValue() + " , operator: " + node.getFieldOperator());
- }
- Text[] columnFamilies = new Text[node.getChildCount()];
- Text[] termValues = new Text[node.getChildCount()];
- boolean[] negationMask = new boolean[node.getChildCount()];
- Enumeration<?> children = node.children();
- int i = 0;
- while (children.hasMoreElements()) {
- BooleanLogicTreeNode child = (BooleanLogicTreeNode) children.nextElement();
- columnFamilies[i] = child.getFieldName();
- termValues[i] = child.getFieldValue();
- negationMask[i] = child.isNegated();
- i++;
- }
-
- AndIterator ii = new AndIterator();
- Map<String,String> options = new HashMap<String,String>();
- options.put(AndIterator.columnFamiliesOptionName, AndIterator.encodeColumns(columnFamilies));
- options.put(AndIterator.termValuesOptionName, AndIterator.encodeTermValues(termValues));
- options.put(AndIterator.notFlagsOptionName, AndIterator.encodeBooleans(negationMask));
-
- ii.init(sourceIterator.deepCopy(env), options, env);
- return ii;
- }
-
- private OrIterator createOrIterator(BooleanLogicTreeNode node) throws IOException {
- if (log.isDebugEnabled()) {
- log.debug("createOrIterator(node)");
- log.debug("fName: " + node.getFieldName() + " , fValue: " + node.getFieldValue() + " , operator: " + node.getFieldOperator());
- }
-
- Enumeration<?> children = node.children();
- ArrayList<Text> fams = new ArrayList<Text>();
- ArrayList<Text> quals = new ArrayList<Text>();
- while (children.hasMoreElements()) {
- BooleanLogicTreeNode child = (BooleanLogicTreeNode) children.nextElement();
- fams.add(child.getFieldName());
- quals.add(child.getFieldValue());
- }
-
- OrIterator iter = new OrIterator();
- SortedKeyValueIterator<Key,Value> source = sourceIterator.deepCopy(env);
- for (int i = 0; i < fams.size(); i++) {
- iter.addTerm(source, fams.get(i), quals.get(i), env);
- }
-
- return iter;
- }
-
- /*
- * This takes the place of the SortedKeyIterator used previously. This iterator is bound to the partitioned table structure. When next is called it will jump
- * rows as necessary internally versus needing to do it externally as was the case with the SortedKeyIterator.
- */
- private FieldIndexIterator createFieldIndexIterator(BooleanLogicTreeNode node) throws IOException {
- if (log.isDebugEnabled()) {
- log.debug("BoolLogic.createFieldIndexIterator()");
- log.debug("fName: " + node.getFieldName() + " , fValue: " + node.getFieldValue() + " , operator: " + node.getFieldOperator());
- }
- Text rowId = null;
- sourceIterator.seek(new Range(), EMPTY_COL_FAMS, false);
- if (sourceIterator.hasTop()) {
- rowId = sourceIterator.getTopKey().getRow();
- }
-
- FieldIndexIterator iter = new FieldIndexIterator(node.getType(), rowId, node.getFieldName(), node.getFieldValue(), node.isNegated(),
- node.getFieldOperator());
-
- Map<String,String> options = new HashMap<String,String>();
- iter.init(sourceIterator.deepCopy(env), options, env);
- if (log.isDebugEnabled()) {
- FieldIndexIterator.setLogLevel(Level.DEBUG);
- } else {
- FieldIndexIterator.setLogLevel(Level.OFF);
- }
-
- return iter;
- }
-
- /* *************************************************************************
- * Methods for testing the tree WRT boolean logic.
- */
- // After all iterator pointers have been advanced, test if the current
- // record passes the boolean logic.
- private boolean testTreeState() {
- if (log.isDebugEnabled()) {
- log.debug("BoolLogic testTreeState() begin");
- }
- Enumeration<?> dfe = this.root.depthFirstEnumeration();
- while (dfe.hasMoreElements()) {
- BooleanLogicTreeNode node = (BooleanLogicTreeNode) dfe.nextElement();
- if (!node.isLeaf()) {
-
- int type = node.getType();
- if (type == ParserTreeConstants.JJTANDNODE) { // BooleanLogicTreeNode.NodeType.AND) {
- handleAND(node);
- } else if (type == ParserTreeConstants.JJTORNODE) {// BooleanLogicTreeNode.NodeType.OR) {
- handleOR(node);
- } else if (type == ParserTreeConstants.JJTJEXLSCRIPT) {// BooleanLogicTreeNode.NodeType.HEAD) {
- handleHEAD(node);
- } else if (type == ParserTreeConstants.JJTNOTNODE) { // BooleanLogicTreeNode.NodeType.NOT) {
- // there should not be any "NOT"s.
- // throw new Exception();
- }
- } else {
- // it is a leaf, if it is an AND or OR do something
- if (node.getType() == ParserTreeConstants.JJTORNODE) {// BooleanLogicTreeNode.NodeType.OR) { //OrIterator
- node.setValid(node.hasTop());
- node.reSet();
- node.addToSet(node.getTopKey());
-
- } else if (node.getType() == ParserTreeConstants.JJTANDNODE || node.getType() == ParserTreeConstants.JJTEQNODE
- || node.getType() == ParserTreeConstants.JJTERNODE || node.getType() == ParserTreeConstants.JJTLENODE
- || node.getType() == ParserTreeConstants.JJTLTNODE || node.getType() == ParserTreeConstants.JJTGENODE
- || node.getType() == ParserTreeConstants.JJTGTNODE) {
- // sub iterator guarantees it is in its internal range,
- // otherwise, no top.
- node.setValid(node.hasTop());
- }
- }
- }
-
- if (log.isDebugEnabled()) {
- log.debug("BoolLogic.testTreeState end, treeState:: " + this.root.getContents() + " , valid: " + root.isValid());
- }
- return this.root.isValid();
- }
-
- private void handleHEAD(BooleanLogicTreeNode node) {
- Enumeration<?> children = node.children();
- while (children.hasMoreElements()) {
- BooleanLogicTreeNode child = (BooleanLogicTreeNode) children.nextElement();
-
- if (child.getType() == ParserTreeConstants.JJTANDNODE) {// BooleanLogicTreeNode.NodeType.AND) {
- node.setValid(child.isValid());
- node.setTopKey(child.getTopKey());
- } else if (child.getType() == ParserTreeConstants.JJTORNODE) {// BooleanLogicTreeNode.NodeType.OR) {
- node.setValid(child.isValid());
- node.setTopKey(child.getTopKey());
- } else if (child.getType() == ParserTreeConstants.JJTEQNODE || child.getType() == ParserTreeConstants.JJTERNODE
- || child.getType() == ParserTreeConstants.JJTGTNODE || child.getType() == ParserTreeConstants.JJTGENODE
- || child.getType() == ParserTreeConstants.JJTLTNODE || child.getType() == ParserTreeConstants.JJTLENODE) {// BooleanLogicTreeNode.NodeType.SEL) {
- node.setValid(true);
- node.setTopKey(child.getTopKey());
- if (child.getTopKey() == null) {
- node.setValid(false);
- }
- }
- }// end while
-
- // I have to be valid AND have a top key
- if (node.isValid() && !node.hasTop()) {
- node.setValid(false);
- }
- }
-
- private void handleAND(BooleanLogicTreeNode me) {
- if (log.isDebugEnabled()) {
- log.debug("handleAND::" + me.getContents());
- }
- Enumeration<?> children = me.children();
- me.setValid(true); // it's easier to prove false than true
-
- HashSet<Key> goodSet = new HashSet<Key>();
- HashSet<Key> badSet = new HashSet<Key>();
- while (children.hasMoreElements()) {
- BooleanLogicTreeNode child = (BooleanLogicTreeNode) children.nextElement();
-
- if (child.getType() == ParserTreeConstants.JJTEQNODE || child.getType() == ParserTreeConstants.JJTANDNODE
- || child.getType() == ParserTreeConstants.JJTERNODE || child.getType() == ParserTreeConstants.JJTNENODE
- || child.getType() == ParserTreeConstants.JJTGENODE || child.getType() == ParserTreeConstants.JJTLENODE
- || child.getType() == ParserTreeConstants.JJTGTNODE || child.getType() == ParserTreeConstants.JJTLTNODE) {
-
- if (child.isNegated()) {
- if (child.hasTop()) {
- badSet.add(child.getTopKey());
- if (goodSet.contains(child.getTopKey())) {
- me.setValid(false);
- return;
- }
- if (child.isValid()) {
- me.setValid(false);
- return;
- }
- }
- } else {
- if (child.hasTop()) {
- if (log.isDebugEnabled()) {
- log.debug("handleAND, child node: " + child.getContents());
- }
- // if you're in the bad set, you're done.
- if (badSet.contains(child.getTopKey())) {
- if (log.isDebugEnabled()) {
- log.debug("handleAND, child is in bad set, setting parent false");
- }
- me.setValid(false);
- return;
- }
-
- // if good set is empty, add it.
- if (goodSet.isEmpty()) {
- if (log.isDebugEnabled()) {
- log.debug("handleAND, goodSet is empty, adding child: " + child.getContents());
- }
- goodSet.add(child.getTopKey());
- } else {
- // must be in the good set & not in the bad set
- // if either fails, I'm false.
- if (!goodSet.contains(child.getTopKey())) {
- if (log.isDebugEnabled()) {
- log.debug("handleAND, goodSet is not empty, and does NOT contain child, setting false. child: " + child.getContents());
- }
- me.setValid(false);
- return;
- } else {
- // trim the good set to this one value
- // (handles the case were the initial encounters were ORs)
- goodSet = new HashSet<Key>();
- goodSet.add(child.getTopKey());
- if (log.isDebugEnabled()) {
- log.debug("handleAND, child in goodset, trim to this value: " + child.getContents());
- }
- }
- }
- } else {
- // test if its children are all false
- if (child.getChildCount() > 0) {
- Enumeration<?> subchildren = child.children();
- boolean allFalse = true;
- while (subchildren.hasMoreElements()) {
- BooleanLogicTreeNode subchild = (BooleanLogicTreeNode) subchildren.nextElement();
- if (!subchild.isNegated()) {
- allFalse = false;
- break;
- } else if (subchild.isNegated() && subchild.hasTop()) {
- allFalse = false;
- break;
- }
- }
- if (!allFalse) {
- me.setValid(false);
- return;
- }
- } else {
- // child returned a null value and is not a negation, this in turn makes me false.
- me.setValid(false);
- return;
- }
- }
- }
-
- } else if (child.getType() == ParserTreeConstants.JJTORNODE) {// BooleanLogicTreeNode.NodeType.OR) {
-
- // NOTE: The OR may be an OrIterator in which case it will only produce
- // a single unique identifier, or it may be a pure logical construct and
- // be capable of producing multiple unique identifiers.
- // This should handle all cases.
- Iterator<?> iter = child.getSetIterator();
- boolean goodSetEmpty = goodSet.isEmpty();
- boolean matchedOne = false;
- boolean pureNegations = true;
- if (!child.isValid()) {
- if (log.isDebugEnabled()) {
- log.debug("handleAND, child is an OR and it is not valid, setting false, ALL NEGATED?: " + child.isChildrenAllNegated());
- }
- me.setValid(false); // I'm an AND if one of my children is false, I'm false.
- return;
- } else if (child.isValid() && !child.hasTop()) {
- // pure negation, do nothing
- } else if (child.isValid() && child.hasTop()) { // I need to match one
- if (log.isDebugEnabled()) {
- log.debug("handleAND, child OR, valid and has top, means not pureNegations");
- }
- pureNegations = false;
- while (iter.hasNext()) {
- Key i = (Key) iter.next();
- if (child.isNegated()) {
- badSet.add(i);
- if (goodSet.contains(i)) {
- if (log.isDebugEnabled()) {
- log.debug("handleAND, child OR, goodSet contains bad value: " + i);
- }
- me.setValid(false);
- return;
- }
- } else {
- // if the good set is empty, then push all of my ids.
- if (goodSetEmpty && !badSet.contains(i)) {
- goodSet.add(i);
- matchedOne = true;
- } else {
- // I need at least one to match
- if (goodSet.contains(i)) {
- matchedOne = true;
- }
- }
- }
- }
- }
-
- // is the goodSet still empty? that means were were only negations
- // otherwise, if it's not empty and we didn't match one, false
- if (child.isNegated()) {
- // we're ok
- } else {
- if (goodSet.isEmpty() && !pureNegations) {
- if (log.isDebugEnabled()) {
- log.debug("handleAND, child OR, empty goodset && !pureNegations, set false");
- }
- // that's bad, we weren't negated, should've pushed something in there.
- me.setValid(false);
- return;
- } else if (!goodSet.isEmpty() && !pureNegations) { // goodSet contains values.
- if (!matchedOne) { // but we didn't match any.
- if (log.isDebugEnabled()) {
- log.debug("handleAND, child OR, goodSet had values but I didn't match any, false");
- }
- me.setValid(false);
- return;
- }
-
- // we matched something, trim the set.
- // i.e. two child ORs
- goodSet = child.getIntersection(goodSet);
- }
- }
-
- }
- }// end while
-
- if (goodSet.isEmpty()) { // && log.isDebugEnabled()) {
- if (log.isDebugEnabled()) {
- log.debug("handleAND-> goodSet is empty, pure negations?");
- }
- } else {
- me.setTopKey(Collections.min(goodSet));
- if (log.isDebugEnabled()) {
- log.debug("End of handleAND, this node's topKey: " + me.getTopKey());
- }
- }
- }
-
- private void handleOR(BooleanLogicTreeNode me) {
- Enumeration<?> children = me.children();
- // I'm an OR node, need at least one positive.
- me.setValid(false);
- me.reSet();
- me.setTopKey(null);
- boolean allNegated = true;
- while (children.hasMoreElements()) {
- // 3 cases for child: SEL, AND, OR
- // and negation
- BooleanLogicTreeNode child = (BooleanLogicTreeNode) children.nextElement();
- if (child.getType() == ParserTreeConstants.JJTEQNODE || child.getType() == ParserTreeConstants.JJTNENODE
- || child.getType() == ParserTreeConstants.JJTANDNODE || child.getType() == ParserTreeConstants.JJTERNODE
- || child.getType() == ParserTreeConstants.JJTNRNODE || child.getType() == ParserTreeConstants.JJTLENODE
- || child.getType() == ParserTreeConstants.JJTLTNODE || child.getType() == ParserTreeConstants.JJTGENODE
- || child.getType() == ParserTreeConstants.JJTGTNODE) {
-
- if (child.hasTop()) {
- if (child.isNegated()) {
- // do nothing.
- } else {
- allNegated = false;
- // I have something add it to my set.
- if (child.isValid()) {
- me.addToSet(child.getTopKey());
- }
- }
- } else if (!child.isNegated()) { // I have a non-negated child
- allNegated = false;
- // that child could be pure negations in which case I'm true
- me.setValid(child.isValid());
- }
-
- } else if (child.getType() == ParserTreeConstants.JJTORNODE) {// BooleanLogicTreeNode.NodeType.OR) {
- if (child.hasTop()) {
- if (!child.isNegated()) {
- allNegated = false;
- // add its rowIds to my rowIds
- Iterator<?> iter = child.getSetIterator();
- while (iter.hasNext()) {
- Key i = (Key) iter.next();
- if (i != null) {
- me.addToSet(i);
- }
- }
- }
- } else {
- // Or node that doesn't have a top, check if it's valid or not
- // because it could be pure negations itself.
- if (child.isValid()) {
- me.setValid(true);
- }
- }
- }
- }// end while
-
- if (allNegated) {
- // do all my children have top?
- children = me.children();
- while (children.hasMoreElements()) {
- BooleanLogicTreeNode child = (BooleanLogicTreeNode) children.nextElement();
- if (!child.hasTop()) {
- me.setValid(true);
- me.setTopKey(null);
- return;
- }
- }
- me.setValid(false);
-
- } else {
- Key k = me.getMinUniqueID();
- if (k == null) {
- me.setValid(false);
- } else {
- me.setValid(true);
- me.setTopKey(k);
- }
- }
- }
-
- /* *************************************************************************
- * Utility methods.
- */
- // Transforms the TreeNode tree of query.parser into the
- // BooleanLogicTreeNodeJexl form.
- public BooleanLogicTreeNode transformTreeNode(TreeNode node) throws ParseException {
- if (node.getType().equals(ASTEQNode.class) || node.getType().equals(ASTNENode.class)) {
- if (log.isDebugEnabled()) {
- log.debug("Equals Node");
- }
-
- Multimap<String,QueryTerm> terms = node.getTerms();
- for (String fName : terms.keySet()) {
- Collection<QueryTerm> values = terms.get(fName);
-
- for (QueryTerm t : values) {
- if (null == t || null == t.getValue()) {
- continue;
- }
- String fValue = t.getValue().toString();
- fValue = fValue.replaceAll("'", "");
- boolean negated = t.getOperator().equals("!=");
-
- if (!fName.startsWith(FIELD_NAME_PREFIX)) {
- fName = FIELD_NAME_PREFIX + fName;
- }
- BooleanLogicTreeNode child = new BooleanLogicTreeNode(ParserTreeConstants.JJTEQNODE, fName, fValue, negated);
- return child;
- }
- }
- }
-
- if (node.getType().equals(ASTERNode.class) || node.getType().equals(ASTNRNode.class)) {
- if (log.isDebugEnabled()) {
- log.debug("Regex Node");
- }
-
- Multimap<String,QueryTerm> terms = node.getTerms();
- for (String fName : terms.keySet()) {
- Collection<QueryTerm> values = terms.get(fName);
- for (QueryTerm t : values) {
- if (null == t || null == t.getValue()) {
- continue;
- }
- String fValue = t.getValue().toString();
- fValue = fValue.replaceAll("'", "");
- boolean negated = node.getType().equals(ASTNRNode.class);
-
- if (!fName.startsWith(FIELD_NAME_PREFIX)) {
- fName = FIELD_NAME_PREFIX + fName;
- }
-
- BooleanLogicTreeNode child = new BooleanLogicTreeNode(ParserTreeConstants.JJTERNODE, fName, fValue, negated);
- return child;
- }
- }
- }
-
- if (node.getType().equals(ASTLTNode.class) || node.getType().equals(ASTLENode.class) || node.getType().equals(ASTGTNode.class)
- || node.getType().equals(ASTGENode.class)) {
- Multimap<String,QueryTerm> terms = node.getTerms();
- for (String fName : terms.keySet()) {
- Collection<QueryTerm> values = terms.get(fName);
-
- if (!fName.startsWith(FIELD_NAME_PREFIX)) {
- fName = FIELD_NAME_PREFIX + fName;
- }
- for (QueryTerm t : values) {
- if (null == t || null == t.getValue()) {
- continue;
- }
- String fValue = t.getValue().toString();
- fValue = fValue.replaceAll("'", "").toLowerCase();
- boolean negated = false; // to be negated, must be child of Not, which is handled elsewhere.
- int mytype = JexlOperatorConstants.getJJTNodeType(t.getOperator());
-
- BooleanLogicTreeNode child = new BooleanLogicTreeNode(mytype, fName, fValue, negated);
- if (log.isDebugEnabled()) {
- log.debug("adding child node: " + child.getContents());
- }
- return child;
- }
- }
- }
-
- BooleanLogicTreeNode returnNode = null;
-
- if (node.getType().equals(ASTAndNode.class) || node.getType().equals(ASTOrNode.class)) {
- int parentType = node.getType().equals(ASTAndNode.class) ? ParserTreeConstants.JJTANDNODE : ParserTreeConstants.JJTORNODE;
- if (log.isDebugEnabled()) {
- log.debug("AND/OR node: " + parentType);
- }
- if (node.isLeaf() || !node.getTerms().isEmpty()) {
- returnNode = new BooleanLogicTreeNode(parentType);
- Multimap<String,QueryTerm> terms = node.getTerms();
- for (String fName : terms.keySet()) {
- Collection<QueryTerm> values = terms.get(fName);
- if (!fName.startsWith(FIELD_NAME_PREFIX)) {
- fName = FIELD_NAME_PREFIX + fName;
- }
- for (QueryTerm t : values) {
- if (null == t || null == t.getValue()) {
- continue;
- }
- String fValue = t.getValue().toString();
- fValue = fValue.replaceAll("'", "");
- boolean negated = t.getOperator().equals("!=");
- int mytype = JexlOperatorConstants.getJJTNodeType(t.getOperator());
- BooleanLogicTreeNode child = new BooleanLogicTreeNode(mytype, fName, fValue, negated);
- if (log.isDebugEnabled()) {
- log.debug("adding child node: " + child.getContents());
- }
- returnNode.add(child);
- }
- }
- } else {
- returnNode = new BooleanLogicTreeNode(parentType);
-
- }
- } else if (node.getType().equals(ASTNotNode.class)) {
- if (log.isDebugEnabled()) {
- log.debug("NOT node");
- }
- if (node.isLeaf()) {
- // NOTE: this should be cleaned up a bit.
- Multimap<String,QueryTerm> terms = node.getTerms();
- for (String fName : terms.keySet()) {
- Collection<QueryTerm> values = terms.get(fName);
-
- if (!fName.startsWith(FIELD_NAME_PREFIX)) {
- fName = FIELD_NAME_PREFIX + fName;
- }
- for (QueryTerm t : values) {
- if (null == t || null == t.getValue()) {
- continue;
- }
- String fValue = t.getValue().toString();
- fValue = fValue.replaceAll("'", "").toLowerCase();
- boolean negated = !t.getOperator().equals("!=");
- int mytype = JexlOperatorConstants.getJJTNodeType(t.getOperator());
-
- if (!fName.startsWith(FIELD_NAME_PREFIX)) {
- fName = FIELD_NAME_PREFIX + fName;
- }
- return new BooleanLogicTreeNode(mytype, fName, fValue, negated);
- }
- }
- } else {
- returnNode = new BooleanLogicTreeNode(ParserTreeConstants.JJTNOTNODE);
- }
- } else if (node.getType().equals(ASTJexlScript.class) || node.getType().getSimpleName().equals("RootNode")) {
-
- if (log.isDebugEnabled()) {
- log.debug("ROOT/JexlScript node");
- }
- if (node.isLeaf()) {
- returnNode = new BooleanLogicTreeNode(ParserTreeConstants.JJTJEXLSCRIPT);
- // NOTE: this should be cleaned up a bit.
- Multimap<String,QueryTerm> terms = node.getTerms();
- for (String fName : terms.keySet()) {
- Collection<QueryTerm> values = terms.get(fName);
-
- if (!fName.startsWith(FIELD_NAME_PREFIX)) {
- fName = FIELD_NAME_PREFIX + fName;
- }
- for (QueryTerm t : values) {
- if (null == t || null == t.getValue()) {
- continue;
- }
- String fValue = t.getValue().toString();
- fValue = fValue.replaceAll("'", "").toLowerCase();
- boolean negated = t.getOperator().equals("!=");
- int mytype = JexlOperatorConstants.getJJTNodeType(t.getOperator());
-
- BooleanLogicTreeNode child = new BooleanLogicTreeNode(mytype, fName, fValue, negated);
- returnNode.add(child);
- return returnNode;
- }
- }
- } else {
- returnNode = new BooleanLogicTreeNode(ParserTreeConstants.JJTJEXLSCRIPT);
- }
- } else {
- log.error("Currently Unsupported Node type: " + node.getClass().getName() + " \t" + node.getType());
- }
- for (TreeNode child : node.getChildren()) {
- returnNode.add(transformTreeNode(child));
- }
-
- return returnNode;
- }
-
- // After tree conflicts have been resolve, we can collapse branches where
- // leaves have been pruned.
- public static void collapseBranches(BooleanLogicTreeNode myroot) throws Exception {
-
- // NOTE: doing a depth first enumeration didn't wory when I started
- // removing nodes halfway through. The following method does work,
- // it's essentially a reverse breadth first traversal.
- List<BooleanLogicTreeNode> nodes = new ArrayList<BooleanLogicTreeNode>();
- Enumeration<?> bfe = myroot.breadthFirstEnumeration();
-
- while (bfe.hasMoreElements()) {
- BooleanLogicTreeNode node = (BooleanLogicTreeNode) bfe.nextElement();
- nodes.add(node);
- }
-
- // walk backwards
- for (int i = nodes.size() - 1; i >= 0; i--) {
- BooleanLogicTreeNode node = nodes.get(i);
- if (log.isDebugEnabled()) {
- log.debug("collapseBranches, inspecting node: " + node.toString() + " " + node.printNode());
- }
-
- if (node.getType() == ParserTreeConstants.JJTANDNODE || node.getType() == ParserTreeConstants.JJTORNODE) {
- if (node.getChildCount() == 0 && !node.isRangeNode()) {
- node.removeFromParent();
- } else if (node.getChildCount() == 1) {
- BooleanLogicTreeNode p = (BooleanLogicTreeNode) node.getParent();
- BooleanLogicTreeNode c = (BooleanLogicTreeNode) node.getFirstChild();
- node.removeFromParent();
- p.add(c);
-
- }
- } else if (node.getType() == ParserTreeConstants.JJTJEXLSCRIPT) {
- if (node.getChildCount() == 0) {
- if (log.isDebugEnabled()) {
- log.debug("collapseBranches, headNode has no children");
- }
- throw new Exception("Head node has no children.");
- }
- }
- }
-
- }
-
- public BooleanLogicTreeNode refactorTree(BooleanLogicTreeNode myroot) {
- List<BooleanLogicTreeNode> nodes = new ArrayList<BooleanLogicTreeNode>();
- Enumeration<?> bfe = myroot.breadthFirstEnumeration();
-
- while (bfe.hasMoreElements()) {
- BooleanLogicTreeNode node = (BooleanLogicTreeNode) bfe.nextElement();
- nodes.add(node);
- }
-
- // walk backwards
- for (int i = nodes.size() - 1; i >= 0; i--) {
- BooleanLogicTreeNode node = nodes.get(i);
- if (node.getType() == ParserTreeConstants.JJTANDNODE || node.getType() == ParserTreeConstants.JJTORNODE) {
- // 1. check to see if all children are negated
- // 2. check to see if we have to handle ranges.
-
- Map<Text,RangeBounds> ranges = new HashMap<Text,RangeBounds>();
- Enumeration<?> children = node.children();
- boolean allNegated = true;
- while (children.hasMoreElements()) {
- BooleanLogicTreeNode child = (BooleanLogicTreeNode) children.nextElement();
- if (!child.isNegated()) {
- allNegated = false;
- // break;
- }
-
- // currently we are not allowing unbounded ranges, so they must sit under an AND node.
- if (node.getType() == ParserTreeConstants.JJTANDNODE) {
- // check for ranges
- if (child.getType() == JexlOperatorConstants.JJTGTNODE) {
- if (log.isDebugEnabled()) {
- log.debug("refactor: GT " + child.getContents());
- }
- if (ranges.containsKey(child.getFieldName())) {
- RangeBounds rb = ranges.get(child.getFieldName());
- rb.setLower(child.getFieldValue());
- } else {
- RangeBounds rb = new RangeBounds();
- rb.setLower(child.getFieldValue());
- ranges.put(child.getFieldName(), rb);
- }
- } else if (child.getType() == JexlOperatorConstants.JJTGENODE) {
- if (log.isDebugEnabled()) {
- log.debug("refactor: GE " + child.getContents());
- }
- if (ranges.containsKey(child.getFieldName())) {
- RangeBounds rb = ranges.get(child.getFieldName());
- rb.setLower(child.getFieldValue());
- } else {
- RangeBounds rb = new RangeBounds();
- rb.setLower(child.getFieldValue());
- ranges.put(child.getFieldName(), rb);
- }
- } else if (child.getType() == JexlOperatorConstants.JJTLTNODE) {
- if (log.isDebugEnabled()) {
- log.debug("refactor: LT " + child.getContents());
- }
- if (ranges.containsKey(child.getFieldName())) {
- RangeBounds rb = ranges.get(child.getFieldName());
- rb.setUpper(child.getFieldValue());
- } else {
- RangeBounds rb = new RangeBounds();
- rb.setUpper(child.getFieldValue());
- ranges.put(child.getFieldName(), rb);
- }
- } else if (child.getType() == JexlOperatorConstants.JJTLENODE) {
- if (log.isDebugEnabled()) {
- log.debug("refactor: LE " + child.getContents());
- }
- if (ranges.containsKey(child.getFieldName())) {
- RangeBounds rb = ranges.get(child.getFieldName());
- rb.setUpper(child.getFieldValue());
- } else {
- RangeBounds rb = new RangeBounds();
- rb.setUpper(child.getFieldValue());
- ranges.put(child.getFieldName(), rb);
- }
- }
- }
- }
- if (allNegated) {
- node.setChildrenAllNegated(true);
- }
-
- // see if the AND node had a range.
- if (node.getType() == ParserTreeConstants.JJTANDNODE) {
-
- // if(ranges.containsKey(node.getFieldName())){
- if (!ranges.isEmpty()) {
- // we have a range, process it
- if (node.getChildCount() <= 2 && ranges.size() == 1) {
- if (log.isDebugEnabled()) {
- log.debug("AND range 2 children or less");
- }
- // only has a range, modify the node
- node.setType(ParserTreeConstants.JJTORNODE);
- node.removeAllChildren();
- // RangeBounds rb = ranges.get(node.getFieldName());
-
- for (Entry<Text,RangeBounds> entry : ranges.entrySet()) {
- Text fName = entry.getKey();
- RangeBounds rb = entry.getValue();
- node.setFieldName(fName);
- node.setFieldValue(new Text(""));
- node.setLowerBound(rb.getLower());
- node.setUpperBound(rb.getUpper());
- node.setRangeNode(true);
- }
-
- rangerators.add(node);
-
- if (log.isDebugEnabled()) {
- log.debug("refactor: " + node.getContents());
- log.debug("refactor: " + node.getLowerBound() + " " + node.getUpperBound());
- }
-
- } else {
- if (log.isDebugEnabled()) {
- log.debug("AND range more than 2 children");
- }
- // node has range plus other children, create another node from the range
- // remove lt,le,gt,ge from parent and push in a single node
-
- // removing nodes via enumeration doesn't work, push into a list
- // and walk backwards
- List<BooleanLogicTreeNode> temp = new ArrayList<BooleanLogicTreeNode>();
- Enumeration<?> e = node.children();
- while (e.hasMoreElements()) {
- BooleanLogicTreeNode c = (BooleanLogicTreeNode) e.nextElement();
- temp.add(c);
- }
-
- for (int j = temp.size() - 1; j >= 0; j--) {
- BooleanLogicTreeNode c = temp.get(j);
- if (c.getType() == JexlOperatorConstants.JJTLENODE || c.getType() == JexlOperatorConstants.JJTLTNODE
- || c.getType() == JexlOperatorConstants.JJTGENODE || c.getType() == JexlOperatorConstants.JJTGTNODE) {
- c.removeFromParent();
- }
- }
-
- for (Entry<Text,RangeBounds> entry : ranges.entrySet()) {
- Text fName = entry.getKey();
- BooleanLogicTreeNode nchild = new BooleanLogicTreeNode(ParserTreeConstants.JJTORNODE, fName.toString(), "");
- RangeBounds rb = entry.getValue();
- nchild.setFieldValue(new Text(""));
- nchild.setLowerBound(rb.getLower());
- nchild.setUpperBound(rb.getUpper());
- nchild.setRangeNode(true);
- node.add(nchild);
- rangerators.add(nchild);
- }
-
- if (log.isDebugEnabled()) {
- log.debug("refactor: " + node.getContents());
- }
- }
- }
- }
-
- }
- }
-
- return myroot;
-
- }
-
- // If all children are of type SEL, roll this up into an AND or OR node.
- private static boolean canRollUp(BooleanLogicTreeNode parent) {
- if (log.isDebugEnabled()) {
- log.debug("canRollUp: testing " + parent.getContents());
- }
- if (parent.getChildCount() < 1) {
- if (log.isDebugEnabled()) {
- log.debug("canRollUp: child count < 1, return false");
- }
- return false;
- }
- Enumeration<?> e = parent.children();
- while (e.hasMoreElements()) {
- BooleanLogicTreeNode child = (BooleanLogicTreeNode) e.nextElement();
-
- if (child.getType() != ParserTreeConstants.JJTEQNODE) {// BooleanLogicTreeNode.NodeType.SEL) {
- if (log.isDebugEnabled()) {
- log.debug("canRollUp: child.getType -> " + ParserTreeConstants.jjtNodeName[child.getType()] + " int: " + child.getType() + " return false");
- }
- return false;
- }
-
- if (child.isNegated()) {
- if (log.isDebugEnabled()) {
- log.debug("canRollUp: child.isNegated, return false");
- }
- return false;
- }
-
- if (child.getFieldValue().toString().contains("*")) {
- if (log.isDebugEnabled()) {
- log.debug("canRollUp: child has wildcard: " + child.getFieldValue());
- }
- return false;
- }
- }
- return true;
- }
-
- /**
- * Small utility function to print out the depth-first enumeration of the tree. Specify the root or sub root of the tree you wish to view.
- *
- * @param root
- * The root node of the tree or sub-tree.
- */
- public static void showDepthFirstTraversal(BooleanLogicTreeNode root) {
- System.out.println("DepthFirstTraversal");
- Enumeration<?> e = root.depthFirstEnumeration();
- int i = -1;
- while (e.hasMoreElements()) {
- i += 1;
- BooleanLogicTreeNode n = (BooleanLogicTreeNode) e.nextElement();
- System.out.println(i + " : " + n);
- }
- }
-
- public static void showBreadthFirstTraversal(BooleanLogicTreeNode root) {
- System.out.println("BreadthFirstTraversal");
- log.debug("BooleanLogicIterator.showBreadthFirstTraversal()");
- Enumeration<?> e = root.breadthFirstEnumeration();
- int i = -1;
- while (e.hasMoreElements()) {
- i += 1;
- BooleanLogicTreeNode n = (BooleanLogicTreeNode) e.nextElement();
- System.out.println(i + " : " + n);
- log.debug(i + " : " + n);
- }
- }
-
- private void splitLeaves(BooleanLogicTreeNode node) {
- if (log.isDebugEnabled()) {
- log.debug("BoolLogic: splitLeaves()");
- }
- positives = new PriorityQueue<BooleanLogicTreeNode>(10, new BooleanLogicTreeNodeComparator());
- // positives = new ArrayList<BooleanLogicTreeNodeJexl>();
- negatives.clear();
-
- Enumeration<?> dfe = node.depthFirstEnumeration();
- while (dfe.hasMoreElements()) {
- BooleanLogicTreeNode elem = (BooleanLogicTreeNode) dfe.nextElement();
-
- if (elem.isLeaf()) {
- if (elem.isNegated()) {
- negatives.add(elem);
- } else {
- positives.add(elem);
- }
- }
- }
- }
-
- private void reHeapPriorityQueue(BooleanLogicTreeNode node) {
- positives.clear();
- Enumeration<?> dfe = node.depthFirstEnumeration();
- BooleanLogicTreeNode elem;
- while (dfe.hasMoreElements()) {
- elem = (BooleanLogicTreeNode) dfe.nextElement();
- if (elem.isLeaf() && !elem.isNegated()) {
- positives.add(elem);
- }
- }
- }
-
- /* *************************************************************************
- * The iterator interface methods.
- */
- public boolean hasTop() {
- return (topKey != null);
- }
-
- public Key getTopKey() {
- if (log.isDebugEnabled()) {
- log.debug("getTopKey: " + topKey);
- }
- return topKey;
- }
-
- private void setTopKey(Key key) {
- if (this.overallRange != null && key != null) {
- if (overallRange.getEndKey() != null) { // if null end key, that means range is to the end of the tablet.
- if (!this.overallRange.contains(key)) {
- topKey = null;
- return;
- }
- }
- }
- topKey = key;
- }
-
- public Value getTopValue() {
- if (topValue == null) {
- topValue = new Value(new byte[0]);
- }
- return topValue;
- }
-
- private void resetNegatives() {
- for (BooleanLogicTreeNode neg : negatives) {
- neg.setTopKey(null);
- neg.setValid(true);
- }
- }
-
- private String getEventKeyUid(Key k) {
- if (k == null || k.getColumnFamily() == null) {
- return null;
- } else {
- return k.getColumnFamily().toString();
- }
- }
-
- private String getIndexKeyUid(Key k) {
- try {
- int idx = 0;
- String sKey = k.getColumnQualifier().toString();
- idx = sKey.indexOf("\0");
- return sKey.substring(idx + 1);
- } catch (Exception e) {
- return null;
- }
- }
-
- /*
- * Remember, the Key in the BooleanLogicTreeNode is different structurally than the Key in its sub iterator because the key BooleanLogic needs to return is an
- * event key created from the index key (which is what the sub iterators are looking at!)
- */
- private Key getOptimizedAdvanceKey() throws IOException {
- if (log.isDebugEnabled()) {
- log.debug("getOptimizedAdvanceKey() called");
- }
- Enumeration<?> bfe = root.breadthFirstEnumeration();
- ArrayList<BooleanLogicTreeNode> bfl = new ArrayList<BooleanLogicTreeNode>();
- while (bfe.hasMoreElements()) {
- BooleanLogicTreeNode node = (BooleanLogicTreeNode) bfe.nextElement();
- if (!node.isNegated()) {
- node.setAdvanceKey(node.getTopKey());
- node.setDone(false);
- bfl.add(node);
- }
- }
-
- // walk the tree backwards
- for (int i = bfl.size() - 1; i >= 0; i--) {
- if (bfl.get(i).isLeaf() || bfl.get(i).isNegated()) {
- if (log.isDebugEnabled()) {
- log.debug("leaf, isDone?: " + bfl.get(i).isDone());
- }
- continue;
- }
-
- BooleanLogicTreeNode node = bfl.get(i);
- node.setDone(false);
- if (log.isDebugEnabled()) {
- log.debug("for loop, node: " + node + " isDone? " + node.isDone());
- }
- if (node.getType() == ParserTreeConstants.JJTANDNODE) {
- // get max
- BooleanLogicTreeNode max = null;
- Enumeration<?> children = node.children();
- boolean firstTime = true;
- while (children.hasMoreElements()) {
- BooleanLogicTreeNode child = (BooleanLogicTreeNode) children.nextElement();
-
- if (child.isNegated() || child.isChildrenAllNegated()) {
- continue;
- }
-
- // all advance keys were initially set from topkey for the leaves.
- if (child.getAdvanceKey() == null) {
- log.debug("\tchild does not advance key: " + child.printNode());
- // i'm an and, i have a child that's done, mark me as done.
- node.setDone(true);
- break;
- } else {
- log.debug("\tchild advanceKey: " + child.getAdvanceKey());
- }
-
- if (firstTime) {
- firstTime = false;
- max = child;
- if (log.isDebugEnabled()) {
- log.debug("\tAND block, first valid child: " + child);
- }
- continue;
- }
-
- log.debug("\tAND block, max: " + max);
- log.debug("\tAND block, child: " + child);
-
- // first test row
- if (max.getAdvanceKey().getRow().compareTo(child.getAdvanceKey().getRow()) < 0) {
- max = child;
- if (log.isDebugEnabled()) {
- log.debug("\tAND block, child row greater, new max.");
- }
- continue;
- }
-
- // if rows are equal, test uids
- String uid_max = getEventKeyUid(max.getAdvanceKey());
- String uid_child = getEventKeyUid(child.getAdvanceKey());
- if (log.isDebugEnabled()) {
- if (uid_max == null) {
- log.debug("\tuid_max is currently null");
- } else {
- log.debug("\tuid_max: " + uid_max);
- }
- if (uid_child == null) {
- log.debug("\tuid_child is null");
- } else {
- log.debug("\tuid_child: " + uid_child);
- }
- }
-
- if (uid_max != null && uid_child != null) {
- if (uid_max.compareTo(uid_child) < 0) {
- max = child;
- }
- } else if (uid_child == null) { // one or the other is null so we want the next row
- max = child;
- log.debug("uid_child is null, we need to grab the next row.");
- break;
- } else {
- log.debug("max is null and child is not, who should we keep? child: " + child);
- break;
- }
- } // end while
- if (log.isDebugEnabled()) {
- log.debug("attemptOptimization: AND with children, max: " + max);
- }
- if (max != null) {
- node.setAdvanceKey(max.getAdvanceKey());
- } else {
- if (log.isDebugEnabled()) {
- log.debug("AND block finished, max is null");
- }
- node.setDone(true);
- }
-
- } else if (node.getType() == ParserTreeConstants.JJTORNODE) {
- // get min
- BooleanLogicTreeNode min = null;
- Enumeration<?> children = node.children();
- boolean firstTime = true;
- int numChildren = node.getChildCount();
- int allChildrenDone = 0;
-
- while (children.hasMoreElements()) {
- BooleanLogicTreeNode child = (BooleanLogicTreeNode) children.nextElement();
-
- if (log.isDebugEnabled()) {
- log.debug("\tOR block start, child: " + child);
- }
- if (child.isNegated() || child.isChildrenAllNegated()) {
- if (log.isDebugEnabled()) {
- log.debug("\tskip negated child: " + child);
- }
- numChildren -= 1;
- continue;
- }
- if (child.isDone()) {
- if (log.isDebugEnabled()) {
- log.debug("\tchild is done: " + child);
- }
- allChildrenDone += 1;
- if (numChildren == allChildrenDone) {
- if (log.isDebugEnabled()) {
- log.debug("\tnumChildren==allChildrenDone, setDone & break");
- }
- // we're done here
- node.setDone(true);
- break;
- }
- }
-
- if (child.getAdvanceKey() == null) {
- log.debug("\tOR child doesn't have top or an AdvanceKey");
- continue;
- }
- if (firstTime) {
- if (log.isDebugEnabled()) {
- log.debug("\tOR block, first valid node, min=child: " + child + " advanceKey: " + child.getAdvanceKey());
- }
-
- firstTime = false;
- min = child;
- continue;
- }
- if (log.isDebugEnabled()) {
- log.debug("\tOR block, min: " + min);
- log.debug("\tOR block, child: " + child);
- }
- if (min.getAdvanceKey().getRow().toString().compareTo(child.getAdvanceKey().getRow().toString()) > 0) {
- // child row is less than min, set min to child
- min = child;
- if (log.isDebugEnabled()) {
- log.debug("\tmin row was greater than child, min=child: " + min);
- }
- continue;
-
- } else if (min.getAdvanceKey().getRow().compareTo(child.getAdvanceKey().getRow()) < 0) {
- // min row is less child, skip
- if (log.isDebugEnabled()) {
- log.debug("\tmin row less than childs, keep min: " + min);
- }
- continue;
-
- } else { // they're equal, test uids
- String uid_min = getEventKeyUid(min.getAdvanceKey());
- String uid_child = getEventKeyUid(child.getAdvanceKey());
- if (log.isDebugEnabled()) {
- log.debug("\ttesting uids, uid_min: " + uid_min + " uid_child: " + uid_child);
- }
- if (uid_min != null && uid_child != null) {
- if (uid_min.compareTo(uid_child) > 0) {
-
- min = child;
- if (log.isDebugEnabled()) {
- log.debug("\tuid_min > uid_child, set min to child: " + min);
- }
- }
- } else if (uid_min == null) {
- if (log.isDebugEnabled()) {
- log.debug("\tuid_min is null, take childs: " + uid_child);
- }
- min = child;
- }
- }
- }// end while
- if (log.isDebugEnabled()) {
- log.debug("attemptOptimization: OR with children, min: " + min);
- }
-
- if (min != null) {
- if (log.isDebugEnabled()) {
- log.debug("OR block, min != null, advanceKey? " + min.getAdvanceKey());
- }
- node.setAdvanceKey(min.getAdvanceKey());
- } else {
- log.debug("OR block, min is null..." + min);
- node.setAdvanceKey(null);
- node.setDone(true);
- }
-
- } else if (node.getType() == ParserTreeConstants.JJTJEXLSCRIPT) { // HEAD node
- if (log.isDebugEnabled()) {
- log.debug("getOptimizedAdvanceKey, HEAD node");
- }
- BooleanLogicTreeNode child = (BooleanLogicTreeNode) node.getFirstChild();
-
- if (child.isDone()) {
- if (log.isDebugEnabled()) {
- log.debug("Head node's child is done, need to move to the next row");
- }
- Key k = child.getAdvanceKey();
- if (k == null) {
- if (log.isDebugEnabled()) {
- log.debug("HEAD node, advance key is null, try to grab next row from topKey");
- }
- if (hasTop()) {
- k = this.getTopKey();
- child.setAdvanceKey(new Key(new Text(k.getRow().toString() + "\1")));
- } else {
- return null;
- }
- } else {
- Text row = new Text(k.getRow().toString() + "\1");
- k = new Key(row);
- child.setAdvanceKey(k);
- }
-
- }
- if (log.isDebugEnabled()) {
- log.debug("advance Key: " + child.getAdvanceKey());
- }
- Key key = new Key(child.getAdvanceKey().getRow(), child.getAdvanceKey().getColumnFamily(), child.getAdvanceKey().getColumnFamily());
- return key;
-
- }// end else
- }// end for
- return null;
- }
-
- /*
- * The incoming jump key has been formatted into the structure of an index key, but the leaves are eventkeys
- */
- private boolean jump(Key jumpKey) throws IOException {
- if (log.isDebugEnabled()) {
- log.debug("JUMP!");
- }
- Enumeration<?> bfe = root.breadthFirstEnumeration();
- while (bfe.hasMoreElements()) {
- BooleanLogicTreeNode n = (BooleanLogicTreeNode) bfe.nextElement();
- n.setAdvanceKey(null);
- } // now advance all nodes to the advance key
-
- if (log.isDebugEnabled()) {
- log.debug("jump, All leaves need to advance to: " + jumpKey);
- }
-
- String advanceUid = getIndexKeyUid(jumpKey);
- if (log.isDebugEnabled()) {
- log.debug("advanceUid => " + advanceUid);
- }
- boolean ok = true;
- for (BooleanLogicTreeNode leaf : positives) {
- leaf.jump(jumpKey);
- }
- return ok;
- }
-
- @SuppressWarnings("unused")
- public void next() throws IOException {
- if (log.isDebugEnabled()) {
- log.debug("next() method called");
- }
- boolean finished = false;
- boolean ok = true;
- if (positives.isEmpty()) {
- setTopKey(null);
- return;
- }
-
- Key previousJumpKey = null;
- while (!finished) {
-
- Key jumpKey = this.getOptimizedAdvanceKey();
-
- if (jumpKey == null) { // stop?
- if (log.isDebugEnabled()) {
- log.debug("next(), jump key is null, stopping");
- }
- setTopKey(null);
- return;
- }
-
- if (log.isDebugEnabled()) {
- if (jumpKey != null) {
- log.debug("next(), jumpKey: " + jumpKey);
- } else {
- log.debug("jumpKey is null");
- }
- }
-
- boolean same = false;
- if (jumpKey != null && topKey != null) {
- // check that the uid's are not the same
- same = getIndexKeyUid(jumpKey).equals(getEventKeyUid(topKey));
- if (log.isDebugEnabled()) {
- log.debug("jumpKeyUid: " + getIndexKeyUid(jumpKey) + " topKeyUid: " + getEventKeyUid(topKey));
- }
- }
-
- if (log.isDebugEnabled()) {
- log.debug("previousJumpKey: " + previousJumpKey);
- log.debug("current JumpKey: " + jumpKey);
- }
-
- if (jumpKey != null && !this.overallRange.contains(jumpKey)) {
- if (log.isDebugEnabled()) {
- log.debug("jumpKey is outside of range, that means the next key is out of range, stopping");
- log.debug("jumpKey: " + jumpKey + " overallRange.endKey: " + overallRange.getEndKey());
- }
- // stop
- setTopKey(null);
- return;
- }
-
- boolean previousSame = false;
- if (previousJumpKey != null && jumpKey != null) {
- previousSame = previousJumpKey.equals(jumpKey);
- }
- // -----------------------------------
- // OPTIMIZED block
- if (jumpKey != null && !same && !previousSame && ok) {
- previousJumpKey = jumpKey;
- ok = jump(jumpKey); // attempt to jump everybody forward to this row and uid.
- // tryJump = false;
-
- // now test the tree state.
- if (testTreeState()) {
- Key tempKey = root.getTopKey();
- // it is potentially valid, now we need to seek all of the negatives
- if (!negatives.isEmpty()) {
- advanceNegatives(this.root.getTopKey());
- if (!testTreeState()) {
- continue;
- }
- }
-
- if (root.getTopKey().equals(tempKey)) {
- // it's valid set nextKey and make sure it's not the same as topKey.
- if (log.isDebugEnabled()) {
- if (this.root.hasTop()) {
- log.debug("this.root.getTopKey()->" + this.root.getTopKey());
- } else {
- log.debug("next, this.root.getTopKey() is null");
- }
-
- if (topKey != null) {
- log.debug("topKey->" + topKey);
-
- } else {
- log.debug("topKey is null");
- }
- }
- if (compare(topKey, this.root.getTopKey()) != 0) {
- // topKey = this.root.getTopKey();
- setTopKey(this.root.getTopKey());
- return;
- }
- }
- }
-
- // --------------------------------------
- // Regular next block
- } else {
-
- reHeapPriorityQueue(this.root);
- BooleanLogicTreeNode node;
-
- while (true) {
- node = positives.poll();
- if (!node.isDone() && node.hasTop()) {
- break;
- }
-
- if (positives.isEmpty()) {
- setTopKey(null);
- return;
- }
- }
-
- if (log.isDebugEnabled()) {
- if (jumpKey == null) {
- log.debug("no jump, jumpKey is null");
- } else if (topKey == null) {
- log.debug("no jump, jumpKey: " + jumpKey + " topKey: null");
- } else {
- log.debug("no jump, jumpKey: " + jumpKey + " topKey: " + topKey);
- }
- log.debug("next, (no jump) min node: " + node);
- log.debug(node);
- }
- node.next();
- resetNegatives();
-
- if (!node.hasTop()) {
- // it may be part of an or, so it could be ok.
- node.setValid(false);
- if (testTreeState()) {
- // it's valid set nextKey and make sure it's not the same as topKey.
- if (topKey.compareTo(this.root.getTopKey()) != 0) {
- // topKey = this.root.getTopKey();
- if (this.overallRange != null) {
- if (this.overallRange.contains(root.getTopKey())) {
- setTopKey(this.root.getTopKey());
- return;
- } else {
- setTopKey(null);
- finished = true;
- return;
- }
-
- } else {
- setTopKey(this.root.getTopKey());
- return;
- }
- }
- }
- } else {
-
- if (overallRange.contains(node.getTopKey())) {
- // the node had something so push it back into priority queue
- positives.add(node);
- }
-
- // now test the tree state.
- if (testTreeState()) {
- Key tempKey = root.getTopKey();
- // it is potentially valid, now we need to seek all of the negatives
- if (!negatives.isEmpty()) {
- advanceNegatives(this.root.getTopKey());
- if (!testTreeState()) {
- continue;
- }
- }
-
- if (root.getTopKey().equals(tempKey)) {
- // it's valid set nextKey and make sure it's not the same as topKey.
- if (log.isDebugEnabled()) {
- if (this.root.hasTop()) {
- log.debug("this.root.getTopKey()->" + this.root.getTopKey());
- } else {
- log.debug("next, this.root.getTopKey() is null");
- }
-
- if (topKey != null) {
- log.debug("topKey->" + topKey);
-
- } else {
- log.debug("topKey is null");
- }
- }
- if (compare(topKey, this.root.getTopKey()) != 0) {
- // topKey = this.root.getTopKey();
- if (this.overallRange != null) {
- if (overallRange.contains(this.root.getTopKey())) {
- setTopKey(this.root.getTopKey());
- return;
- } else {
- topKey = null;
- finished = true;
- return;
- }
- } else {
- setTopKey(this.root.getTopKey());
- return;
- }
- }
- }
- }
-
- }
-
- // is the priority queue empty?
- if (positives.isEmpty()) {
- finished = true;
- topKey = null;
- }
- }
- }
- }
-
- /*
- * create a range for the given row of the
- */
- private void advanceNegatives(Key k) throws IOException {
- if (log.isDebugEnabled()) {
- log.debug("advancingNegatives for Key: " + k);
- }
- Text rowID = k.getRow();
- Text colFam = k.getColumnFamily();
-
- for (BooleanLogicTreeNode neg : negatives) {
- Key startKey = new Key(rowID, neg.getFieldName(), new Text(neg.getFieldValue() + "\0" + colFam));
- Key endKey = new Key(rowID, neg.getFieldName(), new Text(neg.getFieldValue() + "\0" + colFam + "\1"));
- Range range = new Range(startKey, true, endKey, false);
-
- if (log.isDebugEnabled()) {
- log.debug("range: " + range);
- }
- neg.seek(range, EMPTY_COL_FAMS, false);
-
- if (neg.hasTop()) {
- neg.setValid(false);
- }
- if (log.isDebugEnabled()) {
- if (neg.hasTop()) {
- log.debug("neg top key: " + neg.getTopKey());
- } else {
- log.debug("neg has no top");
- }
- }
- }
- }
-
- public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
- this.overallRange = range;
- if (log.isDebugEnabled()) {
- log.debug("seek, overallRange: " + overallRange);
- }
- // Given some criteria, advance all iterators to that position.
- // NOTE: All of our iterators exist in the leaves.
- topKey = null;
- root.setTopKey(null);
-
- // set up the range iterators for the given seek range.
- // these should exist in the positives as OR iterators, but need special setup.
- setupRangerators(range);
-
- // don't take this out, if you jump rows on the tablet you could have
- // pulled nodes out of the positives priority queue. On a call to seek
- // it is usually jumping rows, so everything needs to become possibly
- // valid again.
- reHeapPriorityQueue(this.root);
- for (BooleanLogicTreeNode node : positives) {
- node.setDone(false);
- node.seek(range, columnFamilies, inclusive);
- if (log.isDebugEnabled()) {
- String tk = "empty";
- if (node.hasTop()) {
- tk = node.getTopKey().toString();
- }
- log.debug("leaf: " + node.getContents() + " topKey: " + tk);
- }
- }
-
- // Now that all nodes have been seek'd recreate the priorityQueue to sort them properly.
- splitLeaves(this.root);
- resetNegatives();
-
- // test Tree, if it's not valid, call next
- if (testTreeState() && overallRange.contains(root.getTopKey())) {
- if (!negatives.isEmpty()) {
- // now advance negatives
- advanceNegatives(this.root.getTopKey());
- if (!testTreeState()) {
- next();
- }
- }
-
- if (log.isDebugEnabled()) {
- log.debug("overallRange " + overallRange + " topKey " + this.root.getTopKey() + " contains " + overallRange.contains(this.root.getTopKey()));
- }
-
- if (overallRange.contains(this.root.getTopKey()) && this.root.isValid()) {
- setTopKey(this.root.getTopKey());
- } else {
- setTopKey(null);
- return;
- }
- } else {
- // seek failed in the logic test, but there may be other possible
- // values which satisfy the logic tree. Make sure our iterators aren't
- // all null, and then call next.
-
- // if(!root.hasTop()){
- if (log.isDebugEnabled()) {
- log.debug("seek, testTreeState is false, HEAD(root) does not have top");
- }
- // check nodes in positives to see if they're all null/outside range
- // or if nothing percolated up to root yet.
- List<BooleanLogicTreeNode> removals = new ArrayList<BooleanLogicTreeNode>();
- for (BooleanLogicTreeNode node : positives) {
- if (!node.hasTop() || !overallRange.contains(node.getTopKey())) {
- removals.add(node);
- }
- }
- for (BooleanLogicTreeNode node : removals) {
- positives.remove(node);
- }
- next();
- return;
- }
- }
-
- private int compare(Key k1, Key k2) {
- if (k1 != null && k2 != null) {
- return k1.compareTo(k2);
- } else if (k1 == null && k2 == null) {
- return 0;
- } else if (k1 == null) { // in this case, null is considered bigger b/c it's closer to the end of the table.
- return 1;
- } else {
- return -1;
- }
- }
-
- private void setupRangerators(Range range) throws IOException {
- if (rangerators == null || rangerators.isEmpty()) {
- return;
- }
- for (BooleanLogicTreeNode node : rangerators) {
- Set<String> fValues = new HashSet<String>();
- OrIterator orIter = new OrIterator();
- SortedKeyValueIterator<Key,Value> siter = sourceIterator.deepCopy(env);
- // create UniqFieldNameValueIterator to find uniq field names values
- UniqFieldNameValueIterator uniq = new UniqFieldNameValueIterator(node.getFieldName(), node.getLowerBound(), node.getUpperBound());
- uniq.setSource(siter);
- uniq.seek(range, EMPTY_COL_FAMS, false);
- while (uniq.hasTop()) {
- // log.debug("uniq.top: "+uniq.getTopKey());
- Key k = uniq.getTopKey();
- keyParser.parse(k);
- String val = keyParser.getFieldValue();
- if (!fValues.contains(val)) {
- fValues.add(val);
- orIter.addTerm(siter, node.getFieldName(), new Text(val), env);
- if (log.isDebugEnabled()) {
- log.debug("setupRangerators, adding to OR: " + node.getFieldName() + ":" + val);
- }
- } else {
- log.debug("already have this one: " + val);
- }
- uniq.next();
- }
- node.setUserObject(orIter);
- }
-
- }
-
- /* *************************************************************************
- * Inner classes
- */
- public class BooleanLogicTreeNodeComparator implements Comparator<Object> {
-
- public int compare(Object o1, Object o2) {
- BooleanLogicTreeNode n1 = (BooleanLogicTreeNode) o1;
- BooleanLogicTreeNode n2 = (BooleanLogicTreeNode) o2;
-
- Key k1 = n1.getTopKey();
- Key k2 = n2.getTopKey();
- if (log.isDebugEnabled()) {
- String t1 = "null";
- String t2 = "null";
- if (k1 != null) {
- t1 = k1.getRow().toString() + "\0" + k1.getColumnFamily().toString();
- }
- if (k2 != null) {
- t2 = k2.getRow().toString() + "\0" + k2.getColumnFamily().toString();
- }
- log.debug("BooleanLogicTreeNodeComparator \tt1: " + t1 + " t2: " + t2);
- }
- // return t1.compareTo(t2);
-
- if (k1 != null && k2 != null) {
- return k1.compareTo(k2);
- } else if (k1 == null && k2 == null) {
- return 0;
- } else if (k1 == null) {
- return 1;
- } else {
- return -1;
- }
-
- }
- }
-
- public IteratorOptions describeOptions() {
- return new IteratorOptions(getClass().getSimpleName(), "evaluates event objects against an expression", Collections.singletonMap(QUERY_OPTION,
- "query expression"), null);
- }
-
- public boolean validateOptions(Map<String,String> options) {
- if (!options.containsKey(QUERY_OPTION)) {
- return false;
- }
- if (!options.containsKey(FIELD_INDEX_QUERY)) {
- return false;
- }
- this.updatedQuery = options.get(FIELD_INDEX_QUERY);
- return true;
- }
-}
[37/39] git commit: ACCUMULO-645 found another HADOOP_HOME reference
Posted by ec...@apache.org.
ACCUMULO-645 found another HADOOP_HOME reference
git-svn-id: https://svn.apache.org/repos/asf/accumulo/trunk@1433102 13f79535-47bb-0310-9956-ffa450edef68
(cherry picked from commit 16c9a002e2d2c1556d8f836217042ebcf008da5a)
Reason: Hadoop2 Compat
Author: Eric C. Newton <ec...@apache.org>
Ref: ACCUMULO-1792
Signed-off-by: Eric Newton <er...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3a350960
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3a350960
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3a350960
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 3a3509604a71a549c4b1f07d1411b3e25bf3f7bf
Parents: 5dd9016
Author: Eric C. Newton <ec...@apache.org>
Authored: Mon Jan 14 20:26:20 2013 +0000
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Nov 25 16:06:44 2013 -0500
----------------------------------------------------------------------
conf/examples/1GB/native-standalone/accumulo-site.xml | 6 +++---
conf/examples/1GB/standalone/accumulo-site.xml | 6 +++---
conf/examples/2GB/native-standalone/accumulo-site.xml | 6 +++---
conf/examples/2GB/standalone/accumulo-site.xml | 6 +++---
conf/examples/3GB/native-standalone/accumulo-site.xml | 6 +++---
conf/examples/3GB/standalone/accumulo-site.xml | 6 +++---
conf/examples/512MB/native-standalone/accumulo-site.xml | 6 +++---
conf/examples/512MB/standalone/accumulo-site.xml | 6 +++---
8 files changed, 24 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3a350960/conf/examples/1GB/native-standalone/accumulo-site.xml
----------------------------------------------------------------------
diff --git a/conf/examples/1GB/native-standalone/accumulo-site.xml b/conf/examples/1GB/native-standalone/accumulo-site.xml
index 3f4705d..97f70d5 100644
--- a/conf/examples/1GB/native-standalone/accumulo-site.xml
+++ b/conf/examples/1GB/native-standalone/accumulo-site.xml
@@ -102,9 +102,9 @@
$ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar,
$ACCUMULO_HOME/lib/[^.].*.jar,
$ZOOKEEPER_HOME/zookeeper[^.].*.jar,
- $HADOOP_HOME/conf,
- $HADOOP_HOME/[^.].*.jar,
- $HADOOP_HOME/lib/[^.].*.jar,
+ $HADOOP_CONF_DIR,
+ $HADOOP_PREFIX/[^.].*.jar,
+ $HADOOP_PREFIX/lib/[^.].*.jar,
</value>
<description>Classpaths that accumulo checks for updates and class files.
When using the Security Manager, please remove the ".../target/classes/" values.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3a350960/conf/examples/1GB/standalone/accumulo-site.xml
----------------------------------------------------------------------
diff --git a/conf/examples/1GB/standalone/accumulo-site.xml b/conf/examples/1GB/standalone/accumulo-site.xml
index 5c45e21..f857fd0 100644
--- a/conf/examples/1GB/standalone/accumulo-site.xml
+++ b/conf/examples/1GB/standalone/accumulo-site.xml
@@ -97,9 +97,9 @@
$ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar,
$ACCUMULO_HOME/lib/[^.].*.jar,
$ZOOKEEPER_HOME/zookeeper[^.].*.jar,
- $HADOOP_HOME/conf,
- $HADOOP_HOME/[^.].*.jar,
- $HADOOP_HOME/lib/[^.].*.jar,
+ $HADOOP_CONF_DIR,
+ $HADOOP_PREFIX/[^.].*.jar,
+ $HADOOP_PREFIX/lib/[^.].*.jar,
</value>
<description>Classpaths that accumulo checks for updates and class files.
When using the Security Manager, please remove the ".../target/classes/" values.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3a350960/conf/examples/2GB/native-standalone/accumulo-site.xml
----------------------------------------------------------------------
diff --git a/conf/examples/2GB/native-standalone/accumulo-site.xml b/conf/examples/2GB/native-standalone/accumulo-site.xml
index f39b367..3988e66 100644
--- a/conf/examples/2GB/native-standalone/accumulo-site.xml
+++ b/conf/examples/2GB/native-standalone/accumulo-site.xml
@@ -92,9 +92,9 @@
$ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar,
$ACCUMULO_HOME/lib/[^.].*.jar,
$ZOOKEEPER_HOME/zookeeper[^.].*.jar,
- $HADOOP_HOME/conf,
- $HADOOP_HOME/[^.].*.jar,
- $HADOOP_HOME/lib/[^.].*.jar,
+ $HADOOP_CONF_DIR,
+ $HADOOP_PREFIX/[^.].*.jar,
+ $HADOOP_PREFIX/lib/[^.].*.jar,
</value>
<description>Classpaths that accumulo checks for updates and class files.
When using the Security Manager, please remove the ".../target/classes/" values.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3a350960/conf/examples/2GB/standalone/accumulo-site.xml
----------------------------------------------------------------------
diff --git a/conf/examples/2GB/standalone/accumulo-site.xml b/conf/examples/2GB/standalone/accumulo-site.xml
index 653860b..9f5d7fd 100644
--- a/conf/examples/2GB/standalone/accumulo-site.xml
+++ b/conf/examples/2GB/standalone/accumulo-site.xml
@@ -92,9 +92,9 @@
$ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar,
$ACCUMULO_HOME/lib/[^.].*.jar,
$ZOOKEEPER_HOME/zookeeper[^.].*.jar,
- $HADOOP_HOME/conf,
- $HADOOP_HOME/[^.].*.jar,
- $HADOOP_HOME/lib/[^.].*.jar,
+ $HADOOP_CONF_DIR,
+ $HADOOP_PREFIX/[^.].*.jar,
+ $HADOOP_PREFIX/lib/[^.].*.jar,
</value>
<description>Classpaths that accumulo checks for updates and class files.
When using the Security Manager, please remove the ".../target/classes/" values.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3a350960/conf/examples/3GB/native-standalone/accumulo-site.xml
----------------------------------------------------------------------
diff --git a/conf/examples/3GB/native-standalone/accumulo-site.xml b/conf/examples/3GB/native-standalone/accumulo-site.xml
index 368cbfe..4509c96 100644
--- a/conf/examples/3GB/native-standalone/accumulo-site.xml
+++ b/conf/examples/3GB/native-standalone/accumulo-site.xml
@@ -87,9 +87,9 @@
$ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar,
$ACCUMULO_HOME/lib/[^.].*.jar,
$ZOOKEEPER_HOME/zookeeper[^.].*.jar,
- $HADOOP_HOME/conf,
- $HADOOP_HOME/[^.].*.jar,
- $HADOOP_HOME/lib/[^.].*.jar,
+ $HADOOP_CONF_DIR,
+ $HADOOP_PREFIX/[^.].*.jar,
+ $HADOOP_PREFIX/lib/[^.].*.jar,
</value>
<description>Classpaths that accumulo checks for updates and class files.
When using the Security Manager, please remove the ".../target/classes/" values.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3a350960/conf/examples/3GB/standalone/accumulo-site.xml
----------------------------------------------------------------------
diff --git a/conf/examples/3GB/standalone/accumulo-site.xml b/conf/examples/3GB/standalone/accumulo-site.xml
index 368cbfe..4509c96 100644
--- a/conf/examples/3GB/standalone/accumulo-site.xml
+++ b/conf/examples/3GB/standalone/accumulo-site.xml
@@ -87,9 +87,9 @@
$ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar,
$ACCUMULO_HOME/lib/[^.].*.jar,
$ZOOKEEPER_HOME/zookeeper[^.].*.jar,
- $HADOOP_HOME/conf,
- $HADOOP_HOME/[^.].*.jar,
- $HADOOP_HOME/lib/[^.].*.jar,
+ $HADOOP_CONF_DIR,
+ $HADOOP_PREFIX/[^.].*.jar,
+ $HADOOP_PREFIX/lib/[^.].*.jar,
</value>
<description>Classpaths that accumulo checks for updates and class files.
When using the Security Manager, please remove the ".../target/classes/" values.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3a350960/conf/examples/512MB/native-standalone/accumulo-site.xml
----------------------------------------------------------------------
diff --git a/conf/examples/512MB/native-standalone/accumulo-site.xml b/conf/examples/512MB/native-standalone/accumulo-site.xml
index 58ab600..7d22ab2 100644
--- a/conf/examples/512MB/native-standalone/accumulo-site.xml
+++ b/conf/examples/512MB/native-standalone/accumulo-site.xml
@@ -97,9 +97,9 @@
$ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar,
$ACCUMULO_HOME/lib/[^.].*.jar,
$ZOOKEEPER_HOME/zookeeper[^.].*.jar,
- $HADOOP_HOME/conf,
- $HADOOP_HOME/[^.].*.jar,
- $HADOOP_HOME/lib/[^.].*.jar,
+ $HADOOP_CONF_DIR,
+ $HADOOP_PREFIX/[^.].*.jar,
+ $HADOOP_PREFIX/lib/[^.].*.jar,
</value>
<description>Classpaths that accumulo checks for updates and class files.
When using the Security Manager, please remove the ".../target/classes/" values.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3a350960/conf/examples/512MB/standalone/accumulo-site.xml
----------------------------------------------------------------------
diff --git a/conf/examples/512MB/standalone/accumulo-site.xml b/conf/examples/512MB/standalone/accumulo-site.xml
index 0044c8e..34868b5 100644
--- a/conf/examples/512MB/standalone/accumulo-site.xml
+++ b/conf/examples/512MB/standalone/accumulo-site.xml
@@ -97,9 +97,9 @@
$ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar,
$ACCUMULO_HOME/lib/[^.].*.jar,
$ZOOKEEPER_HOME/zookeeper[^.].*.jar,
- $HADOOP_HOME/conf,
- $HADOOP_HOME/[^.].*.jar,
- $HADOOP_HOME/lib/[^.].*.jar,
+ $HADOOP_CONF_DIR,
+ $HADOOP_PREFIX/[^.].*.jar,
+ $HADOOP_PREFIX/lib/[^.].*.jar,
</value>
<description>Classpaths that accumulo checks for updates and class files.
When using the Security Manager, please remove the ".../target/classes/" values.
[04/39] ACCUMULO-600 removed wikisearch from trunk
Posted by ec...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/TreeBuilder.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/TreeBuilder.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/TreeBuilder.java
deleted file mode 100644
index 58a3508..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/TreeBuilder.java
+++ /dev/null
@@ -1,675 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.parser;
-
-import java.io.StringReader;
-
-import org.apache.accumulo.examples.wikisearch.parser.QueryParser.EvaluationContext;
-import org.apache.accumulo.examples.wikisearch.parser.QueryParser.FunctionResult;
-import org.apache.accumulo.examples.wikisearch.parser.QueryParser.LiteralResult;
-import org.apache.accumulo.examples.wikisearch.parser.QueryParser.ObjectHolder;
-import org.apache.accumulo.examples.wikisearch.parser.QueryParser.QueryTerm;
-import org.apache.accumulo.examples.wikisearch.parser.QueryParser.TermResult;
-import org.apache.commons.jexl2.parser.ASTAdditiveNode;
-import org.apache.commons.jexl2.parser.ASTAdditiveOperator;
-import org.apache.commons.jexl2.parser.ASTAmbiguous;
-import org.apache.commons.jexl2.parser.ASTAndNode;
-import org.apache.commons.jexl2.parser.ASTArrayAccess;
-import org.apache.commons.jexl2.parser.ASTArrayLiteral;
-import org.apache.commons.jexl2.parser.ASTAssignment;
-import org.apache.commons.jexl2.parser.ASTBitwiseAndNode;
-import org.apache.commons.jexl2.parser.ASTBitwiseComplNode;
-import org.apache.commons.jexl2.parser.ASTBitwiseOrNode;
-import org.apache.commons.jexl2.parser.ASTBitwiseXorNode;
-import org.apache.commons.jexl2.parser.ASTBlock;
-import org.apache.commons.jexl2.parser.ASTConstructorNode;
-import org.apache.commons.jexl2.parser.ASTDivNode;
-import org.apache.commons.jexl2.parser.ASTEQNode;
-import org.apache.commons.jexl2.parser.ASTERNode;
-import org.apache.commons.jexl2.parser.ASTEmptyFunction;
-import org.apache.commons.jexl2.parser.ASTFalseNode;
-import org.apache.commons.jexl2.parser.ASTFloatLiteral;
-import org.apache.commons.jexl2.parser.ASTForeachStatement;
-import org.apache.commons.jexl2.parser.ASTFunctionNode;
-import org.apache.commons.jexl2.parser.ASTGENode;
-import org.apache.commons.jexl2.parser.ASTGTNode;
-import org.apache.commons.jexl2.parser.ASTIdentifier;
-import org.apache.commons.jexl2.parser.ASTIfStatement;
-import org.apache.commons.jexl2.parser.ASTIntegerLiteral;
-import org.apache.commons.jexl2.parser.ASTJexlScript;
-import org.apache.commons.jexl2.parser.ASTLENode;
-import org.apache.commons.jexl2.parser.ASTLTNode;
-import org.apache.commons.jexl2.parser.ASTMapEntry;
-import org.apache.commons.jexl2.parser.ASTMapLiteral;
-import org.apache.commons.jexl2.parser.ASTMethodNode;
-import org.apache.commons.jexl2.parser.ASTModNode;
-import org.apache.commons.jexl2.parser.ASTMulNode;
-import org.apache.commons.jexl2.parser.ASTNENode;
-import org.apache.commons.jexl2.parser.ASTNRNode;
-import org.apache.commons.jexl2.parser.ASTNotNode;
-import org.apache.commons.jexl2.parser.ASTNullLiteral;
-import org.apache.commons.jexl2.parser.ASTOrNode;
-import org.apache.commons.jexl2.parser.ASTReference;
-import org.apache.commons.jexl2.parser.ASTSizeFunction;
-import org.apache.commons.jexl2.parser.ASTSizeMethod;
-import org.apache.commons.jexl2.parser.ASTStringLiteral;
-import org.apache.commons.jexl2.parser.ASTTernaryNode;
-import org.apache.commons.jexl2.parser.ASTTrueNode;
-import org.apache.commons.jexl2.parser.ASTUnaryMinusNode;
-import org.apache.commons.jexl2.parser.ASTWhileStatement;
-import org.apache.commons.jexl2.parser.JexlNode;
-import org.apache.commons.jexl2.parser.ParseException;
-import org.apache.commons.jexl2.parser.Parser;
-import org.apache.commons.jexl2.parser.ParserVisitor;
-import org.apache.commons.jexl2.parser.SimpleNode;
-
-
-import com.google.common.collect.Multimap;
-
-/**
- * Class that parses the query and returns a tree of TreeNode's. This class rolls up clauses that are below like conjunctions (AND, OR) for the purposes of
- * creating intersecting iterators.
- *
- */
-public class TreeBuilder implements ParserVisitor {
-
- class RootNode extends JexlNode {
-
- public RootNode(int id) {
- super(id);
- }
-
- public RootNode(Parser p, int id) {
- super(p, id);
- }
-
- }
-
- private TreeNode rootNode = null;
- private TreeNode currentNode = null;
- private boolean currentlyInCheckChildren = false;
-
- public TreeBuilder(String query) throws ParseException {
- Parser p = new Parser(new StringReader(";"));
- ASTJexlScript script = p.parse(new StringReader(query), null);
- // Check to see if the child node is an AND or OR. If not, then
- // there must be just a single value in the query expression
- rootNode = new TreeNode();
- rootNode.setType(RootNode.class);
- currentNode = rootNode;
- EvaluationContext ctx = new EvaluationContext();
- script.childrenAccept(this, ctx);
- }
-
- public TreeBuilder(ASTJexlScript script) {
- // Check to see if the child node is an AND or OR. If not, then
- // there must be just a single value in the query expression
- rootNode = new TreeNode();
- rootNode.setType(RootNode.class);
- currentNode = rootNode;
- EvaluationContext ctx = new EvaluationContext();
- script.childrenAccept(this, ctx);
- }
-
- public TreeNode getRootNode() {
- return this.rootNode;
- }
-
- public Object visit(SimpleNode node, Object data) {
- return null;
- }
-
- public Object visit(ASTJexlScript node, Object data) {
- return null;
- }
-
- public Object visit(ASTBlock node, Object data) {
- return null;
- }
-
- public Object visit(ASTAmbiguous node, Object data) {
- return null;
- }
-
- public Object visit(ASTIfStatement node, Object data) {
- return null;
- }
-
- public Object visit(ASTWhileStatement node, Object data) {
- return null;
- }
-
- public Object visit(ASTForeachStatement node, Object data) {
- return null;
- }
-
- public Object visit(ASTAssignment node, Object data) {
- return null;
- }
-
- public Object visit(ASTTernaryNode node, Object data) {
- return null;
- }
-
- /**
- * @param node
- * @param failClass
- * @return false if any of the nodes equals the fail class or contain a NOT in the subtree
- */
- private boolean nodeCheck(JexlNode node, Class<?> failClass) {
- if (node.getClass().equals(failClass) || node.getClass().equals(ASTNotNode.class))
- return false;
- else {
- for (int i = 0; i < node.jjtGetNumChildren(); i++) {
- if (!nodeCheck(node.jjtGetChild(i), failClass))
- return false;
- }
- }
- return true;
- }
-
- /**
- * Checks to see if all of the child nodes are of the same type (AND/OR) and if so then aggregates all of the child terms. If not returns null.
- *
- * @param parent
- * @param parentNode
- * @return Map of field names to query terms or null
- */
- private Multimap<String,QueryTerm> checkChildren(JexlNode parent, EvaluationContext ctx) {
- // If the current node is an AND, then make sure that there is no
- // OR descendant node, and vice versa. If this is true, then we call
- // roll up all of the descendent values.
- this.currentlyInCheckChildren = true;
- Multimap<String,QueryTerm> rolledUpTerms = null;
- boolean result = false;
- if (parent.getClass().equals(ASTOrNode.class)) {
- for (int i = 0; i < parent.jjtGetNumChildren(); i++) {
- result = nodeCheck(parent.jjtGetChild(i), ASTAndNode.class);
- if (!result)
- break;
- }
- } else {
- for (int i = 0; i < parent.jjtGetNumChildren(); i++) {
- result = nodeCheck(parent.jjtGetChild(i), ASTOrNode.class);
- if (!result)
- break;
- }
- }
- if (result) {
- // Set current node to a fake node and
- // roll up the children from this node using the visitor pattern.
- TreeNode rollupFakeNode = new TreeNode();
- TreeNode previous = this.currentNode;
- this.currentNode = rollupFakeNode;
- // Run the visitor with the fake node.
- parent.childrenAccept(this, ctx);
- // Get the terms from the fake node
- rolledUpTerms = this.currentNode.getTerms();
- // Reset the current node pointer
- this.currentNode = previous;
- }
- this.currentlyInCheckChildren = false;
- return rolledUpTerms;
- }
-
- public Object visit(ASTOrNode node, Object data) {
- boolean previouslyInOrContext = false;
- EvaluationContext ctx = null;
- if (null != data && data instanceof EvaluationContext) {
- ctx = (EvaluationContext) data;
- previouslyInOrContext = ctx.inOrContext;
- } else {
- ctx = new EvaluationContext();
- }
- ctx.inOrContext = true;
- // Are we being called from the checkChildren method? If so, then we
- // are rolling up terms. If not, then we need to call check children.
- if (currentlyInCheckChildren) {
- // Process both sides of this node.
- node.jjtGetChild(0).jjtAccept(this, data);
- node.jjtGetChild(1).jjtAccept(this, data);
- } else {
- // Create a new OR node under the current node.
- TreeNode orNode = new TreeNode();
- orNode.setType(ASTOrNode.class);
- orNode.setParent(this.currentNode);
- this.currentNode.getChildren().add(orNode);
- Multimap<String,QueryTerm> terms = checkChildren(node, ctx);
- if (terms == null) {
- // Then there was no rollup, set the current node to the orNode
- // and process the children. Be sure to set the current Node to
- // the or node in between calls because we could be processing
- // an AND node below and the current node will have been switched.
- // Process both sides of this node.
- currentNode = orNode;
- node.jjtGetChild(0).jjtAccept(this, data);
- currentNode = orNode;
- node.jjtGetChild(1).jjtAccept(this, data);
- } else {
- // There was a rollup, don't process the children and set the terms
- // on the or node.
- orNode.setTerms(terms);
- }
- }
- // reset the state
- if (null != data && !previouslyInOrContext)
- ctx.inOrContext = false;
- return null;
- }
-
- public Object visit(ASTAndNode node, Object data) {
- boolean previouslyInAndContext = false;
- EvaluationContext ctx = null;
- if (null != data && data instanceof EvaluationContext) {
- ctx = (EvaluationContext) data;
- previouslyInAndContext = ctx.inAndContext;
- } else {
- ctx = new EvaluationContext();
- }
- ctx.inAndContext = true;
- // Are we being called from the checkChildren method? If so, then we
- // are rolling up terms. If not, then we need to call check children.
- if (currentlyInCheckChildren) {
- // Process both sides of this node.
- node.jjtGetChild(0).jjtAccept(this, data);
- node.jjtGetChild(1).jjtAccept(this, data);
- } else {
- // Create a new And node under the current node.
- TreeNode andNode = new TreeNode();
- andNode.setType(ASTAndNode.class);
- andNode.setParent(this.currentNode);
- this.currentNode.getChildren().add(andNode);
- Multimap<String,QueryTerm> terms = checkChildren(node, ctx);
- if (terms == null) {
- // Then there was no rollup, set the current node to the orNode
- // and process the children. Be sure to set the current Node to
- // the and node in between calls because we could be processing
- // an OR node below and the current node will have been switched.
- // Process both sides of this node.
- currentNode = andNode;
- node.jjtGetChild(0).jjtAccept(this, data);
- currentNode = andNode;
- node.jjtGetChild(1).jjtAccept(this, data);
- } else {
- // There was a rollup, don't process the children and set the terms
- // on the or node.
- andNode.setTerms(terms);
- }
- }
- if (null != data && !previouslyInAndContext)
- ctx.inAndContext = false;
- return null;
- }
-
- public Object visit(ASTBitwiseOrNode node, Object data) {
- return null;
- }
-
- public Object visit(ASTBitwiseXorNode node, Object data) {
- return null;
- }
-
- public Object visit(ASTBitwiseAndNode node, Object data) {
- return null;
- }
-
- public Object visit(ASTEQNode node, Object data) {
- StringBuilder fieldName = new StringBuilder();
- ObjectHolder value = new ObjectHolder();
- // Process both sides of this node.
- Object left = node.jjtGetChild(0).jjtAccept(this, data);
- Object right = node.jjtGetChild(1).jjtAccept(this, data);
- // Ignore functions in the query
- if (left instanceof FunctionResult || right instanceof FunctionResult)
- return null;
- decodeResults(left, right, fieldName, value);
- // We need to check to see if we are in a NOT context. If so,
- // then we need to reverse the negation.
- boolean negated = false;
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- if (ctx.inNotContext)
- negated = !negated;
- }
- QueryTerm term = new QueryTerm(negated, JexlOperatorConstants.getOperator(node.getClass()), value.getObject());
- this.currentNode.getTerms().put(fieldName.toString(), term);
- return null;
- }
-
- public Object visit(ASTNENode node, Object data) {
- StringBuilder fieldName = new StringBuilder();
- ObjectHolder value = new ObjectHolder();
- // Process both sides of this node.
- Object left = node.jjtGetChild(0).jjtAccept(this, data);
- Object right = node.jjtGetChild(1).jjtAccept(this, data);
- // Ignore functions in the query
- if (left instanceof FunctionResult || right instanceof FunctionResult)
- return null;
- decodeResults(left, right, fieldName, value);
- // We need to check to see if we are in a NOT context. If so,
- // then we need to reverse the negation.
- boolean negated = true;
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- if (ctx.inNotContext)
- negated = !negated;
- }
- QueryTerm term = new QueryTerm(negated, JexlOperatorConstants.getOperator(node.getClass()), value.getObject());
- this.currentNode.getTerms().put(fieldName.toString(), term);
- return null;
- }
-
- public Object visit(ASTLTNode node, Object data) {
- StringBuilder fieldName = new StringBuilder();
- ObjectHolder value = new ObjectHolder();
- // Process both sides of this node.
- Object left = node.jjtGetChild(0).jjtAccept(this, data);
- Object right = node.jjtGetChild(1).jjtAccept(this, data);
- // Ignore functions in the query
- if (left instanceof FunctionResult || right instanceof FunctionResult)
- return null;
- decodeResults(left, right, fieldName, value);
- // We need to check to see if we are in a NOT context. If so,
- // then we need to reverse the negation.
- boolean negated = false;
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- if (ctx.inNotContext)
- negated = !negated;
- }
- QueryTerm term = new QueryTerm(negated, JexlOperatorConstants.getOperator(node.getClass()), value.getObject());
- this.currentNode.getTerms().put(fieldName.toString(), term);
- return null;
- }
-
- public Object visit(ASTGTNode node, Object data) {
- StringBuilder fieldName = new StringBuilder();
- ObjectHolder value = new ObjectHolder();
- // Process both sides of this node.
- Object left = node.jjtGetChild(0).jjtAccept(this, data);
- Object right = node.jjtGetChild(1).jjtAccept(this, data);
- // Ignore functions in the query
- if (left instanceof FunctionResult || right instanceof FunctionResult)
- return null;
- decodeResults(left, right, fieldName, value);
- // We need to check to see if we are in a NOT context. If so,
- // then we need to reverse the negation.
- boolean negated = false;
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- if (ctx.inNotContext)
- negated = !negated;
- }
- QueryTerm term = new QueryTerm(negated, JexlOperatorConstants.getOperator(node.getClass()), value.getObject());
- this.currentNode.getTerms().put(fieldName.toString(), term);
- return null;
- }
-
- public Object visit(ASTLENode node, Object data) {
- StringBuilder fieldName = new StringBuilder();
- ObjectHolder value = new ObjectHolder();
- // Process both sides of this node.
- Object left = node.jjtGetChild(0).jjtAccept(this, data);
- Object right = node.jjtGetChild(1).jjtAccept(this, data);
- // Ignore functions in the query
- if (left instanceof FunctionResult || right instanceof FunctionResult)
- return null;
- decodeResults(left, right, fieldName, value);
- // We need to check to see if we are in a NOT context. If so,
- // then we need to reverse the negation.
- boolean negated = false;
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- if (ctx.inNotContext)
- negated = !negated;
- }
- QueryTerm term = new QueryTerm(negated, JexlOperatorConstants.getOperator(node.getClass()), value.getObject());
- this.currentNode.getTerms().put(fieldName.toString(), term);
- return null;
- }
-
- public Object visit(ASTGENode node, Object data) {
- StringBuilder fieldName = new StringBuilder();
- ObjectHolder value = new ObjectHolder();
- // Process both sides of this node.
- Object left = node.jjtGetChild(0).jjtAccept(this, data);
- Object right = node.jjtGetChild(1).jjtAccept(this, data);
- // Ignore functions in the query
- if (left instanceof FunctionResult || right instanceof FunctionResult)
- return null;
- decodeResults(left, right, fieldName, value);
- // We need to check to see if we are in a NOT context. If so,
- // then we need to reverse the negation.
- boolean negated = false;
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- if (ctx.inNotContext)
- negated = !negated;
- }
- QueryTerm term = new QueryTerm(negated, JexlOperatorConstants.getOperator(node.getClass()), value.getObject());
- this.currentNode.getTerms().put(fieldName.toString(), term);
- return null;
- }
-
- public Object visit(ASTERNode node, Object data) {
- StringBuilder fieldName = new StringBuilder();
- ObjectHolder value = new ObjectHolder();
- // Process both sides of this node.
- Object left = node.jjtGetChild(0).jjtAccept(this, data);
- Object right = node.jjtGetChild(1).jjtAccept(this, data);
- // Ignore functions in the query
- if (left instanceof FunctionResult || right instanceof FunctionResult)
- return null;
- decodeResults(left, right, fieldName, value);
- // We need to check to see if we are in a NOT context. If so,
- // then we need to reverse the negation.
- boolean negated = false;
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- if (ctx.inNotContext)
- negated = !negated;
- }
- QueryTerm term = new QueryTerm(negated, JexlOperatorConstants.getOperator(node.getClass()), value.getObject());
- this.currentNode.getTerms().put(fieldName.toString(), term);
- return null;
- }
-
- public Object visit(ASTNRNode node, Object data) {
- StringBuilder fieldName = new StringBuilder();
- ObjectHolder value = new ObjectHolder();
- // Process both sides of this node.
- Object left = node.jjtGetChild(0).jjtAccept(this, data);
- Object right = node.jjtGetChild(1).jjtAccept(this, data);
- // Ignore functions in the query
- if (left instanceof FunctionResult || right instanceof FunctionResult)
- return null;
- decodeResults(left, right, fieldName, value);
- // We need to check to see if we are in a NOT context. If so,
- // then we need to reverse the negation.
- boolean negated = true;
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- if (ctx.inNotContext)
- negated = !negated;
- }
- QueryTerm term = new QueryTerm(negated, "!~", value.getObject());
- this.currentNode.getTerms().put(fieldName.toString(), term);
- return null;
- }
-
- public Object visit(ASTAdditiveNode node, Object data) {
- return null;
- }
-
- public Object visit(ASTAdditiveOperator node, Object data) {
- return null;
- }
-
- public Object visit(ASTMulNode node, Object data) {
- return null;
- }
-
- public Object visit(ASTDivNode node, Object data) {
- return null;
- }
-
- public Object visit(ASTModNode node, Object data) {
- return null;
- }
-
- public Object visit(ASTUnaryMinusNode node, Object data) {
- return null;
- }
-
- public Object visit(ASTBitwiseComplNode node, Object data) {
- return null;
- }
-
- public Object visit(ASTNotNode node, Object data) {
- boolean previouslyInNotContext = false;
- EvaluationContext ctx = null;
- if (null != data && data instanceof EvaluationContext) {
- ctx = (EvaluationContext) data;
- previouslyInNotContext = ctx.inNotContext;
- } else {
- ctx = new EvaluationContext();
- }
- ctx.inNotContext = true;
- // Create a new node in the tree to represent the NOT
- // Create a new And node under the current node.
- TreeNode notNode = new TreeNode();
- notNode.setType(ASTNotNode.class);
- notNode.setParent(this.currentNode);
- this.currentNode.getChildren().add(notNode);
- this.currentNode = notNode;
- // Process both sides of this node.
- node.jjtGetChild(0).jjtAccept(this, ctx);
- // reset the state
- if (null != data && !previouslyInNotContext)
- ctx.inNotContext = false;
- return null;
- }
-
- public Object visit(ASTIdentifier node, Object data) {
- return new TermResult(node.image);
- }
-
- public Object visit(ASTNullLiteral node, Object data) {
- return new LiteralResult(node.image);
- }
-
- public Object visit(ASTTrueNode node, Object data) {
- return new LiteralResult(node.image);
- }
-
- public Object visit(ASTFalseNode node, Object data) {
- return new LiteralResult(node.image);
- }
-
- public Object visit(ASTIntegerLiteral node, Object data) {
- return new LiteralResult(node.image);
- }
-
- public Object visit(ASTFloatLiteral node, Object data) {
- return new LiteralResult(node.image);
- }
-
- public Object visit(ASTStringLiteral node, Object data) {
- return new LiteralResult("'" + node.image + "'");
- }
-
- public Object visit(ASTArrayLiteral node, Object data) {
- return null;
- }
-
- public Object visit(ASTMapLiteral node, Object data) {
- return null;
- }
-
- public Object visit(ASTMapEntry node, Object data) {
- return null;
- }
-
- public Object visit(ASTEmptyFunction node, Object data) {
- return null;
- }
-
- public Object visit(ASTSizeFunction node, Object data) {
- return null;
- }
-
- public Object visit(ASTFunctionNode node, Object data) {
- // objectNode 0 is the prefix
- // objectNode 1 is the identifier , the others are parameters.
- // process the remaining arguments
- FunctionResult fr = new FunctionResult();
- int argc = node.jjtGetNumChildren() - 2;
- for (int i = 0; i < argc; i++) {
- // Process both sides of this node.
- Object result = node.jjtGetChild(i + 2).jjtAccept(this, data);
- if (result instanceof TermResult) {
- TermResult tr = (TermResult) result;
- fr.getTerms().add(tr);
- }
- }
- return fr;
- }
-
- public Object visit(ASTMethodNode node, Object data) {
- return null;
- }
-
- public Object visit(ASTSizeMethod node, Object data) {
- return null;
- }
-
- public Object visit(ASTConstructorNode node, Object data) {
- return null;
- }
-
- public Object visit(ASTArrayAccess node, Object data) {
- return null;
- }
-
- public Object visit(ASTReference node, Object data) {
- return node.jjtGetChild(0).jjtAccept(this, data);
- }
-
- private void decodeResults(Object left, Object right, StringBuilder fieldName, ObjectHolder holder) {
- if (left instanceof TermResult) {
- TermResult tr = (TermResult) left;
- fieldName.append((String) tr.value);
- // Then the right has to be the value
- if (right instanceof LiteralResult) {
- holder.setObject(((LiteralResult) right).value);
- } else {
- throw new IllegalArgumentException("Object mismatch");
- }
- } else if (right instanceof TermResult) {
- TermResult tr = (TermResult) right;
- fieldName.append((String) tr.value);
- if (left instanceof LiteralResult) {
- holder.setObject(((LiteralResult) left).value);
- } else {
- throw new IllegalArgumentException("Object mismatch");
- }
-
- } else {
- throw new IllegalArgumentException("No Term specified in query");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/TreeNode.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/TreeNode.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/TreeNode.java
deleted file mode 100644
index 57ea369..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/TreeNode.java
+++ /dev/null
@@ -1,235 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.parser;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Enumeration;
-import java.util.List;
-import java.util.NoSuchElementException;
-import java.util.Vector;
-
-import org.apache.accumulo.examples.wikisearch.parser.QueryParser.QueryTerm;
-import org.apache.commons.jexl2.parser.JexlNode;
-
-
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Multimap;
-
-public class TreeNode {
-
- private Class<? extends JexlNode> type = null;
- /* navigation elements */
- private TreeNode parent = null;
- private List<TreeNode> children = new ArrayList<TreeNode>();
- private Multimap<String,QueryTerm> terms = HashMultimap.create();
-
- public TreeNode() {
- super();
- }
-
- public Class<? extends JexlNode> getType() {
- return type;
- }
-
- public TreeNode getParent() {
- return parent;
- }
-
- public List<TreeNode> getChildren() {
- return children;
- }
-
- public Enumeration<TreeNode> getChildrenAsEnumeration() {
- return Collections.enumeration(children);
- }
-
- public Multimap<String,QueryTerm> getTerms() {
- return terms;
- }
-
- public void setType(Class<? extends JexlNode> type) {
- this.type = type;
- }
-
- public void setParent(TreeNode parent) {
- this.parent = parent;
- }
-
- public void setChildren(List<TreeNode> children) {
- this.children = children;
- }
-
- public void setTerms(Multimap<String,QueryTerm> terms) {
- this.terms = terms;
- }
-
- public boolean isLeaf() {
- return children.isEmpty();
- }
-
- @Override
- public String toString() {
- StringBuilder buf = new StringBuilder();
- buf.append("Type: ").append(type.getSimpleName());
- buf.append(" Terms: ");
- if (null == terms) {
- buf.append("null");
- } else {
- buf.append(terms.toString());
- }
- return buf.toString();
- }
-
- public final Enumeration<?> depthFirstEnumeration() {
- return new PostorderEnumeration(this);
- }
-
- public Enumeration<?> breadthFirstEnumeration() {
- return new BreadthFirstEnumeration(this);
- }
-
- public final class PostorderEnumeration implements Enumeration<TreeNode> {
-
- protected TreeNode root;
- protected Enumeration<TreeNode> children;
- protected Enumeration<TreeNode> subtree;
-
- public PostorderEnumeration(TreeNode rootNode) {
- super();
- root = rootNode;
- children = root.getChildrenAsEnumeration();
- subtree = EMPTY_ENUMERATION;
- }
-
- public boolean hasMoreElements() {
- return root != null;
- }
-
- public TreeNode nextElement() {
- TreeNode retval;
-
- if (subtree.hasMoreElements()) {
- retval = subtree.nextElement();
- } else if (children.hasMoreElements()) {
- subtree = new PostorderEnumeration((TreeNode) children.nextElement());
- retval = subtree.nextElement();
- } else {
- retval = root;
- root = null;
- }
-
- return retval;
- }
- } // End of class PostorderEnumeration
-
- static public final Enumeration<TreeNode> EMPTY_ENUMERATION = new Enumeration<TreeNode>() {
-
- public boolean hasMoreElements() {
- return false;
- }
-
- public TreeNode nextElement() {
- throw new NoSuchElementException("No more elements");
- }
- };
-
- final class BreadthFirstEnumeration implements Enumeration<TreeNode> {
- protected Queue queue;
-
- public BreadthFirstEnumeration(TreeNode rootNode) {
- super();
- Vector<TreeNode> v = new Vector<TreeNode>(1);
- v.addElement(rootNode); // PENDING: don't really need a vector
- queue = new Queue();
- queue.enqueue(v.elements());
- }
-
- public boolean hasMoreElements() {
- return (!queue.isEmpty() && ((Enumeration<?>) queue.firstObject()).hasMoreElements());
- }
-
- public TreeNode nextElement() {
- Enumeration<?> enumer = (Enumeration<?>) queue.firstObject();
- TreeNode node = (TreeNode) enumer.nextElement();
- Enumeration<?> children = node.getChildrenAsEnumeration();
-
- if (!enumer.hasMoreElements()) {
- queue.dequeue();
- }
- if (children.hasMoreElements()) {
- queue.enqueue(children);
- }
- return node;
- }
-
- // A simple queue with a linked list data structure.
- final class Queue {
- QNode head; // null if empty
- QNode tail;
-
- final class QNode {
- public Object object;
- public QNode next; // null if end
-
- public QNode(Object object, QNode next) {
- this.object = object;
- this.next = next;
- }
- }
-
- public void enqueue(Object anObject) {
- if (head == null) {
- head = tail = new QNode(anObject, null);
- } else {
- tail.next = new QNode(anObject, null);
- tail = tail.next;
- }
- }
-
- public Object dequeue() {
- if (head == null) {
- throw new NoSuchElementException("No more elements");
- }
-
- Object retval = head.object;
- QNode oldHead = head;
- head = head.next;
- if (head == null) {
- tail = null;
- } else {
- oldHead.next = null;
- }
- return retval;
- }
-
- public Object firstObject() {
- if (head == null) {
- throw new NoSuchElementException("No more elements");
- }
-
- return head.object;
- }
-
- public boolean isEmpty() {
- return head == null;
- }
-
- } // End of class Queue
-
- } // End of class BreadthFirstEnumeration
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/query/IQuery.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/query/IQuery.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/query/IQuery.java
deleted file mode 100644
index 9f1a8ed..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/query/IQuery.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.query;
-
-import javax.ws.rs.Consumes;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-
-import org.apache.accumulo.examples.wikisearch.sample.Results;
-
-
-@Path("/Query")
-public interface IQuery {
-
- @GET
- @POST
- @Path("/html")
- @Consumes("*/*")
- public String html(@QueryParam("query") String query, @QueryParam("auths") String auths);
-
- @GET
- @POST
- @Path("/xml")
- @Consumes("*/*")
- @Produces("application/xml")
- public Results xml(@QueryParam("query") String query, @QueryParam("auths") String auths);
-
- @GET
- @POST
- @Path("/json")
- @Consumes("*/*")
- @Produces("application/json")
- public Results json(@QueryParam("query") String query, @QueryParam("auths") String auths);
-
- @GET
- @POST
- @Path("/yaml")
- @Consumes("*/*")
- @Produces("text/x-yaml")
- public Results yaml(@QueryParam("query") String query, @QueryParam("auths") String auths);
-
- @GET
- @POST
- @Path("/content")
- @Consumes("*/*")
- @Produces("application/xml")
- public Results content(@QueryParam("query") String query, @QueryParam("auths") String auths);
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/query/Query.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/query/Query.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/query/Query.java
deleted file mode 100644
index d7dab3a..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/query/Query.java
+++ /dev/null
@@ -1,239 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.query;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.StringReader;
-import java.io.StringWriter;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.text.ParseException;
-import java.util.ArrayList;
-import java.util.List;
-
-import javax.annotation.PostConstruct;
-import javax.annotation.PreDestroy;
-import javax.annotation.Resource;
-import javax.ejb.EJBException;
-import javax.ejb.Local;
-import javax.ejb.Stateless;
-import javax.xml.bind.JAXBContext;
-import javax.xml.bind.Marshaller;
-import javax.xml.transform.Templates;
-import javax.xml.transform.Transformer;
-import javax.xml.transform.TransformerFactory;
-import javax.xml.transform.stream.StreamResult;
-import javax.xml.transform.stream.StreamSource;
-
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
-import org.apache.accumulo.examples.wikisearch.logic.ContentLogic;
-import org.apache.accumulo.examples.wikisearch.logic.QueryLogic;
-import org.apache.accumulo.examples.wikisearch.sample.Results;
-import org.apache.log4j.Logger;
-
-@Stateless
-@Local(IQuery.class)
-public class Query implements IQuery {
-
- private static final Logger log = Logger.getLogger(Query.class);
-
- // Inject values from XML configuration file
- @Resource(name = "instanceName")
- private String instanceName;
-
- @Resource(name = "zooKeepers")
- private String zooKeepers;
-
- @Resource(name = "username")
- private String username;
-
- @Resource(name = "password")
- private String password;
-
- @Resource(name = "tableName")
- private String tableName;
-
- @Resource(name = "threads")
- private int threads;
-
- private static final String XSL = "/accumulo-wikisearch/style.xsl";
-
- @PostConstruct
- public void init() {
- log.info("Post Construct");
- }
-
- @PreDestroy
- public void close() {
- log.info("Close called.");
- }
-
- /*
- * (non-Javadoc)
- *
- * @see sample.query.IQuery#html(java.lang.String, java.lang.String)
- */
- public String html(String query, String auths) {
- log.info("HTML query: " + query);
- URL u;
- try {
- u = new URL("http://" + System.getProperty("jboss.bind.address") + ":" + System.getProperty("jboss.web.http.port") + XSL);
- } catch (MalformedURLException e1) {
- throw new EJBException("Unable to load XSL stylesheet", e1);
- }
- InputStream xslContent;
- try {
- xslContent = u.openStream();
- } catch (IOException e1) {
- throw new EJBException("Unable to get xsl content", e1);
- }
-
- StringWriter xml = new StringWriter();
- StringWriter html = new StringWriter();
-
- Results results = query(query, auths);
- try {
- // Marshall the query results object
- JAXBContext ctx = JAXBContext.newInstance(Results.class);
- Marshaller m = ctx.createMarshaller();
- m.marshal(results, xml);
-
- // Perform XSL transform on the xml.
- StringReader reader = new StringReader(xml.toString());
- TransformerFactory tf = TransformerFactory.newInstance();
- // Create the transformer from the xsl
- Templates xsl = tf.newTemplates(new StreamSource(xslContent));
- Transformer t = xsl.newTransformer();
- t.transform(new StreamSource(reader), new StreamResult(html));
-
- } catch (Exception e) {
- throw new EJBException("Error processing query results", e);
- } finally {
- try {
- xslContent.close();
- } catch (IOException e) {
- throw new EJBException("Unable to close input stream", e);
- }
- }
- return html.toString();
- }
-
- /*
- * (non-Javadoc)
- *
- * @see sample.query.IQuery#xml(java.lang.String, java.lang.String)
- */
- public Results xml(String query, String auths) {
- log.info("XML query: " + query);
- return query(query, auths);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see sample.query.IQuery#json(java.lang.String, java.lang.String)
- */
- public Results json(String query, String auths) {
- log.info("JSON query: " + query);
- return query(query, auths);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see sample.query.IQuery#yaml(java.lang.String, java.lang.String)
- */
- public Results yaml(String query, String auths) {
- log.info("YAML query: " + query);
- return query(query, auths);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see sample.query.IQuery#content(java.lang.String, java.lang.String)
- */
- public Results content(String query, String auths) {
- log.info("Content query: " + query);
- Connector connector = null;
- if (null == instanceName || null == zooKeepers || null == username || null == password)
- throw new EJBException("Required parameters not set. [instanceName = " + this.instanceName + ", zookeepers = " + this.zooKeepers + ", username = "
- + this.username + ", password = " + this.password + "]. Check values in ejb-jar.xml");
- Instance instance = new ZooKeeperInstance(this.instanceName, this.zooKeepers);
- try {
- log.info("Connecting to [instanceName = " + this.instanceName + ", zookeepers = " + this.zooKeepers + ", username = " + this.username + ", password = "
- + this.password + "].");
- connector = instance.getConnector(this.username, this.password.getBytes());
- } catch (Exception e) {
- throw new EJBException("Error getting connector from instance", e);
- }
-
- // Create list of auths
- List<String> authorizations = new ArrayList<String>();
- if (auths != null && auths.length() > 0)
- for (String a : auths.split(","))
- authorizations.add(a);
- ContentLogic table = new ContentLogic();
- table.setTableName(tableName);
- return table.runQuery(connector, query, authorizations);
-
- }
-
- /**
- * calls the query logic with the parameters, returns results
- *
- * @param query
- * @param auths
- * @return The results of a query
- * @throws ParseException
- */
- public Results query(String query, String auths) {
-
- Connector connector = null;
- if (null == instanceName || null == zooKeepers || null == username || null == password)
- throw new EJBException("Required parameters not set. [instanceName = " + this.instanceName + ", zookeepers = " + this.zooKeepers + ", username = "
- + this.username + ", password = " + this.password + "]. Check values in ejb-jar.xml");
- Instance instance = new ZooKeeperInstance(this.instanceName, this.zooKeepers);
- try {
- log.info("Connecting to [instanceName = " + this.instanceName + ", zookeepers = " + this.zooKeepers + ", username = " + this.username + ", password = "
- + this.password + "].");
- connector = instance.getConnector(this.username, this.password.getBytes());
- } catch (Exception e) {
- throw new EJBException("Error getting connector from instance", e);
- }
-
- // Create list of auths
- List<String> authorizations = new ArrayList<String>();
- if (auths != null && auths.length() > 0)
- for (String a : auths.split(","))
- authorizations.add(a);
-
- QueryLogic table = new QueryLogic();
- table.setTableName(tableName);
- table.setMetadataTableName(tableName + "Metadata");
- table.setIndexTableName(tableName + "Index");
- table.setReverseIndexTableName(tableName + "ReverseIndex");
- table.setQueryThreads(threads);
- table.setUnevaluatedFields("TEXT");
- table.setUseReadAheadIterator(false);
- return table.runQuery(connector, authorizations, query, null, null, null);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/sample/Document.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/sample/Document.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/sample/Document.java
deleted file mode 100644
index a5b6ccf..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/sample/Document.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.sample;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-
-@XmlAccessorType(XmlAccessType.FIELD)
-public class Document {
-
- @XmlElement
- private String id = null;
-
- @XmlElement
- private List<Field> field = new ArrayList<Field>();
-
- public Document() {
- super();
- }
-
- public Document(String id, List<Field> fields) {
- super();
- this.id = id;
- this.field = fields;
- }
-
- public String getId() {
- return id;
- }
-
- public List<Field> getFields() {
- return field;
- }
-
- public void setId(String id) {
- this.id = id;
- }
-
- public void setFields(List<Field> fields) {
- this.field = fields;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/sample/Field.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/sample/Field.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/sample/Field.java
deleted file mode 100644
index 9f904a6..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/sample/Field.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.sample;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlAttribute;
-import javax.xml.bind.annotation.XmlValue;
-
-@XmlAccessorType(XmlAccessType.FIELD)
-public class Field {
-
- @XmlAttribute
- private String name = null;
- @XmlValue
- private String value = null;
-
- public Field() {
- super();
- }
-
- public Field(String fieldName, String fieldValue) {
- super();
- this.name = fieldName;
- this.value = fieldValue;
- }
-
- public String getFieldName() {
- return name;
- }
-
- public String getFieldValue() {
- return value;
- }
-
- public void setFieldName(String fieldName) {
- this.name = fieldName;
- }
-
- public void setFieldValue(String fieldValue) {
- this.value = fieldValue;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/sample/Results.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/sample/Results.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/sample/Results.java
deleted file mode 100644
index fa804b4..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/sample/Results.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.sample;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-
-@XmlRootElement
-@XmlAccessorType(XmlAccessType.FIELD)
-public class Results {
-
- @XmlElement
- private List<Document> document = new ArrayList<Document>();
-
- public Results() {
- super();
- }
-
- public List<Document> getResults() {
- return document;
- }
-
- public void setResults(List<Document> results) {
- this.document = results;
- }
-
- public int size() {
- if (null == document)
- return 0;
- else
- return document.size();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/util/BaseKeyParser.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/util/BaseKeyParser.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/util/BaseKeyParser.java
deleted file mode 100644
index a4b2115..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/util/BaseKeyParser.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.util;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.accumulo.core.data.Key;
-
-public class BaseKeyParser {
- public static final String ROW_FIELD = "row";
- public static final String COLUMN_FAMILY_FIELD = "columnFamily";
- public static final String COLUMN_QUALIFIER_FIELD = "columnQualifier";
-
- protected Map<String,String> keyFields = new HashMap<String,String>();
- protected Key key = null;
-
- /**
- * Parses a Key object into its constituent fields. This method clears any prior values, so the object can be reused without requiring a new instantiation.
- * This default implementation makes the row, columnFamily, and columnQualifier available.
- *
- * @param key
- */
- public void parse(Key key) {
- this.key = key;
-
- keyFields.clear();
-
- keyFields.put(ROW_FIELD, key.getRow().toString());
- keyFields.put(COLUMN_FAMILY_FIELD, key.getColumnFamily().toString());
- keyFields.put(COLUMN_QUALIFIER_FIELD, key.getColumnQualifier().toString());
- }
-
- public String getFieldValue(String fieldName) {
- return keyFields.get(fieldName);
- }
-
- public String[] getFieldNames() {
- String[] fieldNames = new String[keyFields.size()];
- return keyFields.keySet().toArray(fieldNames);
- }
-
- public BaseKeyParser duplicate() {
- return new BaseKeyParser();
- }
-
- public String getRow() {
- return keyFields.get(ROW_FIELD);
- }
-
- public String getColumnFamily() {
- return keyFields.get(COLUMN_FAMILY_FIELD);
- }
-
- public String getColumnQualifier() {
- return keyFields.get(COLUMN_QUALIFIER_FIELD);
- }
-
- public Key getKey() {
- return this.key;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/util/FieldIndexKeyParser.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/util/FieldIndexKeyParser.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/util/FieldIndexKeyParser.java
deleted file mode 100644
index 6fc48cd..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/util/FieldIndexKeyParser.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.util;
-
-import org.apache.accumulo.core.data.Key;
-
-public class FieldIndexKeyParser extends KeyParser {
-
- public static final String DELIMITER = "\0";
-
- @Override
- public void parse(Key key) {
- super.parse(key);
-
- String[] colFamParts = this.keyFields.get(BaseKeyParser.COLUMN_FAMILY_FIELD).split(DELIMITER);
- this.keyFields.put(FIELDNAME_FIELD, colFamParts.length >= 2 ? colFamParts[1] : "");
-
- String[] colQualParts = this.keyFields.get(BaseKeyParser.COLUMN_QUALIFIER_FIELD).split(DELIMITER);
- this.keyFields.put(SELECTOR_FIELD, colQualParts.length >= 1 ? colQualParts[0] : "");
- this.keyFields.put(DATATYPE_FIELD, colQualParts.length >= 2 ? colQualParts[1] : "");
- this.keyFields.put(UID_FIELD, colQualParts.length >= 3 ? colQualParts[2] : "");
- }
-
- @Override
- public BaseKeyParser duplicate() {
- return new FieldIndexKeyParser();
- }
-
- @Override
- public String getSelector() {
- return keyFields.get(SELECTOR_FIELD);
- }
-
- @Override
- public String getDataType() {
- return keyFields.get(DATATYPE_FIELD);
- }
-
- @Override
- public String getFieldName() {
- return keyFields.get(FIELDNAME_FIELD);
- }
-
- @Override
- public String getUid() {
- return keyFields.get(UID_FIELD);
- }
-
- public String getDataTypeUid() {
- return getDataType() + DELIMITER + getUid();
- }
-
- // An alias for getSelector
- public String getFieldValue() {
- return getSelector();
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/util/KeyParser.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/util/KeyParser.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/util/KeyParser.java
deleted file mode 100644
index 5648e0e..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/util/KeyParser.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.util;
-
-import org.apache.accumulo.core.data.Key;
-
-public class KeyParser extends BaseKeyParser {
- public static final String SELECTOR_FIELD = "selector";
- public static final String DATATYPE_FIELD = "dataType";
- public static final String FIELDNAME_FIELD = "fieldName";
- public static final String UID_FIELD = "uid";
- public static final String DELIMITER = "\0";
-
- @Override
- public void parse(Key key) {
- super.parse(key);
-
- String[] colFamParts = this.keyFields.get(BaseKeyParser.COLUMN_FAMILY_FIELD).split(DELIMITER);
- this.keyFields.put(FIELDNAME_FIELD, colFamParts.length >= 2 ? colFamParts[1] : "");
-
- String[] colQualParts = this.keyFields.get(BaseKeyParser.COLUMN_QUALIFIER_FIELD).split(DELIMITER);
- this.keyFields.put(SELECTOR_FIELD, colQualParts.length >= 1 ? colQualParts[0] : "");
- this.keyFields.put(DATATYPE_FIELD, colQualParts.length >= 2 ? colQualParts[1] : "");
- this.keyFields.put(UID_FIELD, colQualParts.length >= 3 ? colQualParts[2] : "");
- }
-
- @Override
- public BaseKeyParser duplicate() {
- return new KeyParser();
- }
-
- public String getSelector() {
- return keyFields.get(SELECTOR_FIELD);
- }
-
- public String getDataType() {
- return keyFields.get(DATATYPE_FIELD);
- }
-
- public String getFieldName() {
- return keyFields.get(FIELDNAME_FIELD);
- }
-
- public String getUid() {
- return keyFields.get(UID_FIELD);
- }
-
- public String getDataTypeUid() {
- return getDataType() + DELIMITER + getUid();
- }
-
- // An alias for getSelector
- public String getFieldValue() {
- return getSelector();
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/resources/META-INF/MANIFEST.MF
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/resources/META-INF/MANIFEST.MF b/src/examples/wikisearch/query/src/main/resources/META-INF/MANIFEST.MF
deleted file mode 100644
index 59499bc..0000000
--- a/src/examples/wikisearch/query/src/main/resources/META-INF/MANIFEST.MF
+++ /dev/null
@@ -1,2 +0,0 @@
-Manifest-Version: 1.0
-
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/resources/META-INF/ejb-jar.xml.example
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/resources/META-INF/ejb-jar.xml.example b/src/examples/wikisearch/query/src/main/resources/META-INF/ejb-jar.xml.example
deleted file mode 100644
index dbfabae..0000000
--- a/src/examples/wikisearch/query/src/main/resources/META-INF/ejb-jar.xml.example
+++ /dev/null
@@ -1,62 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<ejb-jar xmlns="http://java.sun.com/xml/ns/javaee"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://java.sun.com/xml/ns/javaee
- http://java.sun.com/xml/ns/javaee/ejb-jar_3_1.xsd" version="3.1">
- <enterprise-beans>
- <session>
- <ejb-name>Query</ejb-name>
- <env-entry>
- <env-entry-name>instanceName</env-entry-name>
- <env-entry-type>java.lang.String</env-entry-type>
- <env-entry-value><!-- replace me --></env-entry-value>
- </env-entry>
- <env-entry>
- <env-entry-name>zooKeepers</env-entry-name>
- <env-entry-type>java.lang.String</env-entry-type>
- <env-entry-value><!-- replace me --></env-entry-value>
- </env-entry>
- <env-entry>
- <env-entry-name>username</env-entry-name>
- <env-entry-type>java.lang.String</env-entry-type>
- <env-entry-value><!-- replace me --></env-entry-value>
- </env-entry>
- <env-entry>
- <env-entry-name>password</env-entry-name>
- <env-entry-type>java.lang.String</env-entry-type>
- <env-entry-value><!-- replace me --></env-entry-value>
- </env-entry>
- <env-entry>
- <env-entry-name>tableName</env-entry-name>
- <env-entry-type>java.lang.String</env-entry-type>
- <env-entry-value>wiki</env-entry-value>
- </env-entry>
- <env-entry>
- <env-entry-name>partitions</env-entry-name>
- <env-entry-type>java.lang.Integer</env-entry-type>
- <env-entry-value>100</env-entry-value>
- </env-entry>
- <env-entry>
- <env-entry-name>threads</env-entry-name>
- <env-entry-type>java.lang.Integer</env-entry-type>
- <env-entry-value>8</env-entry-value>
- </env-entry>
- </session>
- </enterprise-beans>
-</ejb-jar>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/StandaloneStatusReporter.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/StandaloneStatusReporter.java b/src/examples/wikisearch/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/StandaloneStatusReporter.java
deleted file mode 100644
index 35743b3..0000000
--- a/src/examples/wikisearch/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/StandaloneStatusReporter.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.logic;
-
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.mapreduce.Counters;
-import org.apache.hadoop.mapreduce.StatusReporter;
-
-public class StandaloneStatusReporter extends StatusReporter {
-
- private Counters c = new Counters();
-
- private long filesProcessed = 0;
- private long recordsProcessed = 0;
-
- public Counters getCounters() {
- return c;
- }
-
- @Override
- public Counter getCounter(Enum<?> name) {
- return c.findCounter(name);
- }
-
- @Override
- public Counter getCounter(String group, String name) {
- return c.findCounter(group, name);
- }
-
- @Override
- public void progress() {
- // do nothing
- }
-
- @Override
- public void setStatus(String status) {
- // do nothing
- }
-
- public long getFilesProcessed() {
- return filesProcessed;
- }
-
- public long getRecordsProcessed() {
- return recordsProcessed;
- }
-
- public void incrementFilesProcessed() {
- filesProcessed++;
- recordsProcessed = 0;
- }
-
- public void incrementRecordsProcessed() {
- recordsProcessed++;
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/TestQueryLogic.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/TestQueryLogic.java b/src/examples/wikisearch/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/TestQueryLogic.java
deleted file mode 100644
index 938f01b..0000000
--- a/src/examples/wikisearch/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/TestQueryLogic.java
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.logic;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map.Entry;
-
-import junit.framework.Assert;
-
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.MutationsRejectedException;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.mock.MockInstance;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.util.ContextFactory;
-import org.apache.accumulo.examples.wikisearch.ingest.WikipediaConfiguration;
-import org.apache.accumulo.examples.wikisearch.ingest.WikipediaInputFormat.WikipediaInputSplit;
-import org.apache.accumulo.examples.wikisearch.ingest.WikipediaMapper;
-import org.apache.accumulo.examples.wikisearch.parser.RangeCalculator;
-import org.apache.accumulo.examples.wikisearch.reader.AggregatingRecordReader;
-import org.apache.accumulo.examples.wikisearch.sample.Document;
-import org.apache.accumulo.examples.wikisearch.sample.Field;
-import org.apache.accumulo.examples.wikisearch.sample.Results;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.RecordWriter;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.lib.input.FileSplit;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TestQueryLogic {
-
- private static final String METADATA_TABLE_NAME = "wikiMetadata";
-
- private static final String TABLE_NAME = "wiki";
-
- private static final String INDEX_TABLE_NAME = "wikiIndex";
-
- private static final String RINDEX_TABLE_NAME = "wikiReverseIndex";
-
- private static final String TABLE_NAMES[] = {METADATA_TABLE_NAME, TABLE_NAME, RINDEX_TABLE_NAME, INDEX_TABLE_NAME};
-
- private class MockAccumuloRecordWriter extends RecordWriter<Text,Mutation> {
- @Override
- public void write(Text key, Mutation value) throws IOException, InterruptedException {
- try {
- writerMap.get(key).addMutation(value);
- } catch (MutationsRejectedException e) {
- throw new IOException("Error adding mutation", e);
- }
- }
-
- @Override
- public void close(TaskAttemptContext context) throws IOException, InterruptedException {
- try {
- for (BatchWriter w : writerMap.values()) {
- w.flush();
- w.close();
- }
- } catch (MutationsRejectedException e) {
- throw new IOException("Error closing Batch Writer", e);
- }
- }
-
- }
-
- private Connector c = null;
- private Configuration conf = new Configuration();
- private HashMap<Text,BatchWriter> writerMap = new HashMap<Text,BatchWriter>();
- private QueryLogic table = null;
-
- @Before
- public void setup() throws Exception {
-
- Logger.getLogger(AbstractQueryLogic.class).setLevel(Level.DEBUG);
- Logger.getLogger(QueryLogic.class).setLevel(Level.DEBUG);
- Logger.getLogger(RangeCalculator.class).setLevel(Level.DEBUG);
-
- conf.set(AggregatingRecordReader.START_TOKEN, "<page>");
- conf.set(AggregatingRecordReader.END_TOKEN, "</page>");
- conf.set(WikipediaConfiguration.TABLE_NAME, TABLE_NAME);
- conf.set(WikipediaConfiguration.NUM_PARTITIONS, "1");
- conf.set(WikipediaConfiguration.NUM_GROUPS, "1");
-
- MockInstance i = new MockInstance();
- c = i.getConnector("root", "");
- for (String table : TABLE_NAMES) {
- try {
- c.tableOperations().delete(table);
- } catch (Exception ex) {}
- c.tableOperations().create(table);
- writerMap.put(new Text(table), c.createBatchWriter(table, 1000L, 1000L, 1));
- }
-
- TaskAttemptContext context = ContextFactory.createTaskAttemptContext(conf);
-
- RawLocalFileSystem fs = new RawLocalFileSystem();
- fs.setConf(conf);
-
- URL url = ClassLoader.getSystemResource("enwiki-20110901-001.xml");
- Assert.assertNotNull(url);
- File data = new File(url.toURI());
- Path tmpFile = new Path(data.getAbsolutePath());
-
- // Setup the Mapper
- WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(tmpFile, 0, fs.pathToFile(tmpFile).length(), null), 0);
- AggregatingRecordReader rr = new AggregatingRecordReader();
- Path ocPath = new Path(tmpFile, "oc");
- OutputCommitter oc = new FileOutputCommitter(ocPath, context);
- fs.deleteOnExit(ocPath);
- StandaloneStatusReporter sr = new StandaloneStatusReporter();
- rr.initialize(split, context);
- MockAccumuloRecordWriter rw = new MockAccumuloRecordWriter();
- WikipediaMapper mapper = new WikipediaMapper();
-
- // Load data into Mock Accumulo
- Mapper<LongWritable,Text,Text,Mutation>.Context con = ContextFactory.createMapContext(mapper, context, rr, rw, oc, sr, split);
- mapper.run(con);
-
- // Flush and close record writers.
- rw.close(context);
-
- table = new QueryLogic();
- table.setMetadataTableName(METADATA_TABLE_NAME);
- table.setTableName(TABLE_NAME);
- table.setIndexTableName(INDEX_TABLE_NAME);
- table.setReverseIndexTableName(RINDEX_TABLE_NAME);
- table.setUseReadAheadIterator(false);
- }
-
- void debugQuery(String tableName) throws Exception {
- Scanner s = c.createScanner(tableName, new Authorizations());
- Range r = new Range();
- s.setRange(r);
- for (Entry<Key,Value> entry : s)
- System.out.println(entry.getKey().toString() + " " + entry.getValue().toString());
- }
-
- @Test
- public void testTitle() {
- Logger.getLogger(AbstractQueryLogic.class).setLevel(Level.OFF);
- Logger.getLogger(RangeCalculator.class).setLevel(Level.OFF);
- List<String> auths = new ArrayList<String>();
- auths.add("enwiki");
- Results results = table.runQuery(c, auths, "TITLE == 'afghanistanhistory'", null, null, null);
- for (Document doc : results.getResults()) {
- System.out.println("id: " + doc.getId());
- for (Field field : doc.getFields())
- System.out.println(field.getFieldName() + " -> " + field.getFieldValue());
- }
- }
-
-}
[27/39] git commit: ACCUMULO-1792 Cleanup test* system tests.
Posted by ec...@apache.org.
ACCUMULO-1792 Cleanup test* system tests.
This patch most resembles work done in ACCUMULO-745. 745 is a massive patch
applied to Accumulo 1.5 that changed all the command line handling of all of the
programs that had main method. When this update was done, several of the concerned
addressed in this patch were handled.
Signed-off-by: Eric Newton <er...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/651033a5
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/651033a5
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/651033a5
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 651033a522f9b1ac51508e60c3cdcd6da02643d8
Parents: 8db6299
Author: Jonathan Hsieh <jo...@cloudera.com>
Authored: Sun Jul 7 13:06:27 2013 -0700
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Nov 25 16:06:43 2013 -0500
----------------------------------------------------------------------
test/system/test1/README | 2 +-
test/system/test2/concurrent.sh | 90 +++++++++++++++---------------
test/system/test3/bigrow.sh | 12 ++--
test/system/test4/bulk_import_test.sh | 32 +++++------
test/system/test5/insert_test.sh | 18 ------
5 files changed, 68 insertions(+), 86 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/651033a5/test/system/test1/README
----------------------------------------------------------------------
diff --git a/test/system/test1/README b/test/system/test1/README
index 4c995fb..8003b3d 100644
--- a/test/system/test1/README
+++ b/test/system/test1/README
@@ -3,7 +3,7 @@ Command to run from command line
#Can run this test with pre-existing splits... use the following command to create the table with
#100 pre-existing splits
-#../../../bin/accumulo 'org.apache.accumulo.server.test.TestIngest$CreateTable' 0 5000000 100
+#../../../bin/accumulo 'org.apache.accumulo.server.test.TestIngest$CreateTable' 0 5000000 100 <user> <pw>
#could try running verify commands after stopping and restarting accumulo
http://git-wip-us.apache.org/repos/asf/accumulo/blob/651033a5/test/system/test2/concurrent.sh
----------------------------------------------------------------------
diff --git a/test/system/test2/concurrent.sh b/test/system/test2/concurrent.sh
index 69790a0..1ce9299 100755
--- a/test/system/test2/concurrent.sh
+++ b/test/system/test2/concurrent.sh
@@ -17,80 +17,80 @@
echo "ingesting first halves (0 to (500K - 1), 1M to (1.5M - 1), etc)"
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.TestIngest -timestamp 1 -size 50 -random 56 500000 0 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.TestIngest -timestamp 1 -size 50 -random 56 500000 1000000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.TestIngest -timestamp 1 -size 50 -random 56 500000 2000000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.TestIngest -timestamp 1 -size 50 -random 56 500000 3000000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.TestIngest -timestamp 1 -size 50 -random 56 500000 4000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -timestamp 1 -size 50 -random 56 500000 0 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -timestamp 1 -size 50 -random 56 500000 1000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -timestamp 1 -size 50 -random 56 500000 2000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -timestamp 1 -size 50 -random 56 500000 3000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -timestamp 1 -size 50 -random 56 500000 4000000 1 &
wait
echo "ingesting second halves (500K to (1M - 1), 1.5M to (2M - 1), etc) and verifying first halves"
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 500000 0 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 500000 1000000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 500000 2000000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 500000 3000000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 500000 4000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 500000 0 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 500000 1000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 500000 2000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 500000 3000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 500000 4000000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.TestIngest -timestamp 1 -size 50 -random 56 500000 500000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.TestIngest -timestamp 1 -size 50 -random 56 500000 1500000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.TestIngest -timestamp 1 -size 50 -random 56 500000 2500000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.TestIngest -timestamp 1 -size 50 -random 56 500000 3500000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.TestIngest -timestamp 1 -size 50 -random 56 500000 4500000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -timestamp 1 -size 50 -random 56 500000 500000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -timestamp 1 -size 50 -random 56 500000 1500000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -timestamp 1 -size 50 -random 56 500000 2500000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -timestamp 1 -size 50 -random 56 500000 3500000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -timestamp 1 -size 50 -random 56 500000 4500000 1 &
wait
echo "verifying complete range"
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 1000000 0 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 1000000 1000000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 1000000 2000000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 1000000 3000000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 1000000 4000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 1000000 0 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 1000000 1000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 1000000 2000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 1000000 3000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 1000000 4000000 1 &
wait
echo "ingesting first halves (0 to (500K - 1), 1M to (1.5M - 1), etc) w/ new timestamp AND verifying second half w/ old timestamp"
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.TestIngest -timestamp 2 -size 50 -random 57 500000 0 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.TestIngest -timestamp 2 -size 50 -random 57 500000 1000000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.TestIngest -timestamp 2 -size 50 -random 57 500000 2000000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.TestIngest -timestamp 2 -size 50 -random 57 500000 3000000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.TestIngest -timestamp 2 -size 50 -random 57 500000 4000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -timestamp 2 -size 50 -random 57 500000 0 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -timestamp 2 -size 50 -random 57 500000 1000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -timestamp 2 -size 50 -random 57 500000 2000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -timestamp 2 -size 50 -random 57 500000 3000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -timestamp 2 -size 50 -random 57 500000 4000000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 500000 500000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 500000 1500000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 500000 2500000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 500000 3500000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 500000 4500000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 500000 500000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 500000 1500000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 500000 2500000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 500000 3500000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 500000 4500000 1 &
wait
echo "ingesting second halves (500K to (1M - 1), 1.5M to (2M - 1), etc) and verifying first halves"
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 2 -random 57 500000 0 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 2 -random 57 500000 1000000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 2 -random 57 500000 2000000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 2 -random 57 500000 3000000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 2 -random 57 500000 4000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 2 -random 57 500000 0 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 2 -random 57 500000 1000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 2 -random 57 500000 2000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 2 -random 57 500000 3000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 2 -random 57 500000 4000000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.TestIngest -timestamp 2 -size 50 -random 57 500000 500000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.TestIngest -timestamp 2 -size 50 -random 57 500000 1500000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.TestIngest -timestamp 2 -size 50 -random 57 500000 2500000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.TestIngest -timestamp 2 -size 50 -random 57 500000 3500000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.TestIngest -timestamp 2 -size 50 -random 57 500000 4500000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -timestamp 2 -size 50 -random 57 500000 500000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -timestamp 2 -size 50 -random 57 500000 1500000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -timestamp 2 -size 50 -random 57 500000 2500000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -timestamp 2 -size 50 -random 57 500000 3500000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -timestamp 2 -size 50 -random 57 500000 4500000 1 &
wait
echo "verifying complete range"
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 2 -random 57 1000000 0 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 2 -random 57 1000000 1000000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 2 -random 57 1000000 2000000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 2 -random 57 1000000 3000000 1 &
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 2 -random 57 1000000 4000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 2 -random 57 1000000 0 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 2 -random 57 1000000 1000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 2 -random 57 1000000 2000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 2 -random 57 1000000 3000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 2 -random 57 1000000 4000000 1 &
wait
http://git-wip-us.apache.org/repos/asf/accumulo/blob/651033a5/test/system/test3/bigrow.sh
----------------------------------------------------------------------
diff --git a/test/system/test3/bigrow.sh b/test/system/test3/bigrow.sh
index f1388fe..30ded74 100755
--- a/test/system/test3/bigrow.sh
+++ b/test/system/test3/bigrow.sh
@@ -15,9 +15,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.TestIngest -timestamp 1 -size 50 -random 56 1 5000000 2000000;
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.TestIngest -timestamp 1 -size 50 -random 56 1000000 0 1;
-#../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.VerifyIngest -timestamp 1 -size 50 -random 56 1 5000000 2000000;
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 1000000 0 1;
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.TestIngest -timestamp 1 -size 50 -random 56 1000000 7000000 1;
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 1000000 7000000 1;
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -timestamp 1 -size 50 -random 56 1 5000000 2000000;
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -timestamp 1 -size 50 -random 56 1000000 0 1;
+#../../../bin/accumulo org.apache.accumulo.server.test.VerifyIngest -timestamp 1 -size 50 -random 56 1 5000000 2000000;
+../../../bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 1000000 0 1;
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -timestamp 1 -size 50 -random 56 1000000 7000000 1;
+../../../bin/accumulo org.apache.accumulo.server.test.VerifyIngest -size 50 -timestamp 1 -random 56 1000000 7000000 1;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/651033a5/test/system/test4/bulk_import_test.sh
----------------------------------------------------------------------
diff --git a/test/system/test4/bulk_import_test.sh b/test/system/test4/bulk_import_test.sh
index b8f3710..b71106b 100755
--- a/test/system/test4/bulk_import_test.sh
+++ b/test/system/test4/bulk_import_test.sh
@@ -15,22 +15,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-hadoop dfs -rmr /testmf
+hadoop dfs -rmr testmf
echo "creating first set of map files"
-../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -rfile /testmf/mf01 -timestamp 1 -size 50 -random 56 1000000 0 1 &
-../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -rfile /testmf/mf02 -timestamp 1 -size 50 -random 56 1000000 1000000 1 &
-../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -rfile /testmf/mf03 -timestamp 1 -size 50 -random 56 1000000 2000000 1 &
-../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -rfile /testmf/mf04 -timestamp 1 -size 50 -random 56 1000000 3000000 1 &
-../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -rfile /testmf/mf05 -timestamp 1 -size 50 -random 56 1000000 4000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -rfile testmf/mf01 -timestamp 1 -size 50 -random 56 1000000 0 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -rfile testmf/mf02 -timestamp 1 -size 50 -random 56 1000000 1000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -rfile testmf/mf03 -timestamp 1 -size 50 -random 56 1000000 2000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -rfile testmf/mf04 -timestamp 1 -size 50 -random 56 1000000 3000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -rfile testmf/mf05 -timestamp 1 -size 50 -random 56 1000000 4000000 1 &
wait
echo "bulk importing"
-hadoop dfs -rmr /testmfFail
-../../../bin/accumulo org.apache.accumulo.server.test.BulkImportDirectory root secret test_ingest /testmf /testmfFail
+hadoop dfs -rmr testmfFail
+../../../bin/accumulo org.apache.accumulo.server.test.BulkImportDirectory root secret test_ingest testmf testmfFail
echo "verifying"
@@ -42,22 +42,22 @@ echo "verifying"
wait
-hadoop dfs -rmr /testmf
+hadoop dfs -rmr testmf
echo "creating second set of map files"
-../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -rfile /testmf/mf01 -timestamp 2 -size 50 -random 57 1000000 0 1 &
-../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -rfile /testmf/mf02 -timestamp 2 -size 50 -random 57 1000000 1000000 1 &
-../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -rfile /testmf/mf03 -timestamp 2 -size 50 -random 57 1000000 2000000 1 &
-../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -rfile /testmf/mf04 -timestamp 2 -size 50 -random 57 1000000 3000000 1 &
-../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -rfile /testmf/mf05 -timestamp 2 -size 50 -random 57 1000000 4000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -rfile testmf/mf01 -timestamp 2 -size 50 -random 57 1000000 0 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -rfile testmf/mf02 -timestamp 2 -size 50 -random 57 1000000 1000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -rfile testmf/mf03 -timestamp 2 -size 50 -random 57 1000000 2000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -rfile testmf/mf04 -timestamp 2 -size 50 -random 57 1000000 3000000 1 &
+../../../bin/accumulo org.apache.accumulo.server.test.TestIngest -rfile testmf/mf05 -timestamp 2 -size 50 -random 57 1000000 4000000 1 &
wait
echo "bulk importing"
-hadoop dfs -rmr /testmfFail
-../../../bin/accumulo org.apache.accumulo.server.test.BulkImportDirectory root secret test_ingest /testmf /testmfFail
+hadoop dfs -rmr testmfFail
+../../../bin/accumulo org.apache.accumulo.server.test.BulkImportDirectory root secret test_ingest testmf testmfFail
echo "creating second set of map files"
http://git-wip-us.apache.org/repos/asf/accumulo/blob/651033a5/test/system/test5/insert_test.sh
----------------------------------------------------------------------
diff --git a/test/system/test5/insert_test.sh b/test/system/test5/insert_test.sh
deleted file mode 100755
index d23b303..0000000
--- a/test/system/test5/insert_test.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#! /usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-../../../bin/accumulo jar ../../../lib/accumulo.jar org.apache.accumulo.server.test.TestInsert
[08/39] ACCUMULO-600 removed wikisearch from trunk
Posted by ec...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/BooleanLogicTreeNode.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/BooleanLogicTreeNode.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/BooleanLogicTreeNode.java
deleted file mode 100644
index ccc5669..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/BooleanLogicTreeNode.java
+++ /dev/null
@@ -1,523 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.iterator;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Enumeration;
-import java.util.HashSet;
-import java.util.Iterator;
-
-import javax.swing.tree.DefaultMutableTreeNode;
-import javax.swing.tree.TreeNode;
-
-import org.apache.accumulo.core.data.ByteSequence;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
-import org.apache.accumulo.examples.wikisearch.parser.JexlOperatorConstants;
-import org.apache.commons.jexl2.parser.ParserTreeConstants;
-import org.apache.hadoop.io.Text;
-import org.apache.log4j.Logger;
-
-
-/**
- *
- *
- */
-public class BooleanLogicTreeNode extends DefaultMutableTreeNode {
-
- private static final long serialVersionUID = 1L;
- protected static final Logger log = Logger.getLogger(BooleanLogicTreeNode.class);
- private Key myTopKey = null;
- private Key advanceKey = null;
- private Text fValue = null;
- private Text fName = null;
- private boolean negated = false;
- private int type;
- private boolean done = false;
- private boolean valid = false;
- private boolean rollUp = false;
- private String fOperator = null;
- private boolean childrenAllNegated = false;
- private HashSet<Key> uids;
- private Text upperBound;
- private Text lowerBound;
- private boolean rangeNode;
-
- public BooleanLogicTreeNode() {
- super();
- uids = new HashSet<Key>();
- }
-
- public BooleanLogicTreeNode(int type) {
- super();
- this.type = type;
- uids = new HashSet<Key>();
- setOperator();
- }
-
- public BooleanLogicTreeNode(int type, boolean negate) {
- super();
- this.type = type;
- this.negated = negate;
- uids = new HashSet<Key>();
- setOperator();
- }
-
- public BooleanLogicTreeNode(int type, String fieldName, String fieldValue) {
- super();
- this.type = type;
- if (fieldValue != null) {
- this.fValue = new Text(fieldValue);
- }
- if (fieldName != null) {
- this.fName = new Text(fieldName);
- }
- uids = new HashSet<Key>();
- setOperator();
- }
-
- public BooleanLogicTreeNode(int type, String fieldName, String fieldValue, boolean negated) {
- super();
- this.type = type;
- if (fieldValue != null) {
- this.fValue = new Text(fieldValue);
- }
- if (fieldName != null) {
- this.fName = new Text(fieldName);
- }
- uids = new HashSet<Key>();
- this.negated = negated;
- setOperator();
- }
-
- public void setValid(boolean b) {
- this.valid = b;
- }
-
- public boolean isValid() {
- return this.valid;
- }
-
- public void setType(int t) {
- this.type = t;
- }
-
- public int getType() {
- return this.type;
- }
-
- public void setChildrenAllNegated(boolean childrenAllNegated) {
- this.childrenAllNegated = childrenAllNegated;
- }
-
- public boolean isChildrenAllNegated() {
- return childrenAllNegated;
- }
-
- public void setAdvanceKey(Key advanceKey) {
- this.advanceKey = advanceKey;
- }
-
- public Key getAdvanceKey() {
- return advanceKey;
- }
-
- public void setNegated(boolean b) {
- this.negated = b;
- }
-
- public boolean isNegated() {
- return negated;
- }
-
- public void setTopKey(Key id) {
- this.myTopKey = id;
- }
-
- public Key getTopKey() {
- return myTopKey;
- }
-
- public void setDone(boolean done) {
- this.done = done;
- }
-
- public boolean isDone() {
- return done;
- }
-
- public void setRollUp(boolean rollUp) {
- this.rollUp = rollUp;
- }
-
- public boolean isRollUp() {
- return rollUp;
- }
-
- public Text getFieldValue() {
- return fValue;
- }
-
- public void setFieldValue(Text term) {
- this.fValue = term;
- }
-
- public Text getFieldName() {
- return fName;
- }
-
- public void setFieldName(Text dataLocation) {
- this.fName = dataLocation;
- }
-
- public String getFieldOperator() {
- return fOperator;
- }
-
- private void setOperator() {
- this.fOperator = JexlOperatorConstants.getOperator(type);
- if (negated && this.fOperator.equals("!=")) {
- this.fOperator = JexlOperatorConstants.getOperator(JexlOperatorConstants.JJTEQNODE);
- }
- }
-
- public Text getLowerBound() {
- return lowerBound;
- }
-
- public void setLowerBound(Text lowerBound) {
- this.lowerBound = lowerBound;
- }
-
- public Text getUpperBound() {
- return upperBound;
- }
-
- public void setUpperBound(Text upperBound) {
- this.upperBound = upperBound;
- }
-
- public boolean isRangeNode() {
- return rangeNode;
- }
-
- public void setRangeNode(boolean rangeNode) {
- this.rangeNode = rangeNode;
- }
-
- public String getContents() {
- StringBuilder s = new StringBuilder("[");
- s.append(toString());
-
- if (children != null) {
- Enumeration<?> e = this.children();
- while (e.hasMoreElements()) {
- BooleanLogicTreeNode n = (BooleanLogicTreeNode) e.nextElement();
- s.append(",");
- s.append(n.getContents());
- }
- }
- s.append("]");
- return s.toString();
- }
-
- public String printNode() {
- StringBuilder s = new StringBuilder("[");
- s.append("Full Location & Term = ");
- if (this.fName != null) {
- s.append(this.fName.toString());
- } else {
- s.append("BlankDataLocation");
- }
- s.append(" ");
- if (this.fValue != null) {
- s.append(this.fValue.toString());
- } else {
- s.append("BlankTerm");
- }
- s.append("]");
- return s.toString();
- }
-
- @Override
- public String toString() {
- String uidStr = "none";
- if (myTopKey != null) {
- String cf = myTopKey.getColumnFamily().toString();
-
- uidStr = cf;
- }
- switch (type) {
- case ParserTreeConstants.JJTEQNODE:
- return fName.toString() + ":" + fValue.toString() + ", uid=" + uidStr + " , negation=" + this.isNegated();
- case ParserTreeConstants.JJTNENODE:
- return fName.toString() + ":" + fValue.toString() + ", uid=" + uidStr + " , negation=" + this.isNegated();
- case ParserTreeConstants.JJTERNODE:
- return fName.toString() + ":" + fValue.toString() + ", uid=" + uidStr + " , negation=" + this.isNegated();
- case ParserTreeConstants.JJTNRNODE:
- return fName.toString() + ":" + fValue.toString() + ", uid=" + uidStr + " , negation=" + this.isNegated();
- case ParserTreeConstants.JJTLENODE:
- return "<=:" + fName.toString() + ":" + fValue.toString() + ", uid=" + uidStr + " , negation=" + this.isNegated();
- case ParserTreeConstants.JJTLTNODE:
- return "<:" + fName.toString() + ":" + fValue.toString() + ", uid=" + uidStr + " , negation=" + this.isNegated();
- case ParserTreeConstants.JJTGENODE:
- return ">=:" + fName.toString() + ":" + fValue.toString() + ", uid=" + uidStr + " , negation=" + this.isNegated();
- case ParserTreeConstants.JJTGTNODE:
- return ">:" + fName.toString() + ":" + fValue.toString() + ", uid=" + uidStr + " , negation=" + this.isNegated();
- case ParserTreeConstants.JJTJEXLSCRIPT:
- return "HEAD" + ":" + uidStr + ":" + isValid();
- case ParserTreeConstants.JJTANDNODE:
- return "AND" + ":" + uidStr + ":" + isValid();
- case ParserTreeConstants.JJTNOTNODE:
- return "NOT";
- case ParserTreeConstants.JJTORNODE:
- return "OR" + ":" + uidStr + ":" + isValid();
- default:
- System.out.println("Problem in BLTNODE.toString()");
- return null;
- }
- }
-
- public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
-
- // always start fresh
- this.setTopKey(null);
- this.setDone(false);
-
- // get my user object which should be an iterator
- SortedKeyValueIterator<?,?> iter = (SortedKeyValueIterator<?,?>) this.getUserObject();
- if (iter != null) {
-
- iter.seek(range, columnFamilies, inclusive);
-
- if (iter.hasTop()) {
- Key key = (Key) iter.getTopKey();
- key = buildKey(key);
-
- this.setTopKey(key);
- if (log.isDebugEnabled()) {
- log.debug("BLTNODE.seek() -> found: " + this.getTopKey());
- }
- } else {
- if (log.isDebugEnabled()) {
- log.debug("BLTNODE.seek() -> hasTop::false");
- }
- this.setDone(true);
- }
- } else {
- if (log.isDebugEnabled()) {
- log.debug("BLTNODE.seek(), The iterator was null!");
- }
- this.setTopKey(null);
- }
- }
-
- public String buildTreePathString(TreeNode[] path) {
- StringBuilder s = new StringBuilder("[");
- for (TreeNode p : path) {
- s.append(p.toString());
- s.append(",");
- }
- s.deleteCharAt(s.length() - 1);
- s.append("]");
- return s.toString();
- }
-
- public void next() throws IOException {
-
- // always start fresh
- this.setTopKey(null);
-
- if (log.isDebugEnabled()) {
- TreeNode[] path = this.getPath();
- log.debug("BLTNODE.next() path-> " + this.buildTreePathString(path));
- }
-
- // have I been marked as done?
- if (this.isDone()) {
- if (log.isDebugEnabled()) {
- log.debug("I've been marked as done, returning");
- }
- return;
- }
-
- SortedKeyValueIterator<?,?> iter = (SortedKeyValueIterator<?,?>) this.getUserObject();
- iter.next();
-
- if (iter.hasTop()) {
- Key key = (Key) iter.getTopKey();
-
- // I have a valid topKey, pull out the piece I want
- key = buildKey(key);
- this.setTopKey(key);
-
- if (log.isDebugEnabled()) {
- log.debug("BLTNODE.next() -> found: " + this.getTopKey());
- }
- } else {
- // no top value has been returned, I'm done.
- if (log.isDebugEnabled()) {
- log.debug("BLTNODE.next() -> Nothing found");
- }
- this.setTopKey(null);
- this.setDone(true);
- }
-
- }
-
- public boolean jump(Key jumpKey) throws IOException {
- boolean ok = true;
- if (this.getType() == ParserTreeConstants.JJTEQNODE) {
- FieldIndexIterator iter = (FieldIndexIterator) this.getUserObject();
- ok = iter.jump(jumpKey);
- if (iter.hasTop()) {
- Key key = (Key) iter.getTopKey();
- key = buildKey(key);
-
- this.setTopKey(key);
- if (log.isDebugEnabled()) {
- log.debug("BLTNODE.jump() -> found: " + this.getTopKey());
- }
- } else {
- if (log.isDebugEnabled()) {
- log.debug("FieldIndexIteratorJexl does not have top after jump, marking done.");
- }
- this.setTopKey(null);
- this.setDone(true);
- }
-
- } else if (this.getType() == ParserTreeConstants.JJTANDNODE) {
- AndIterator iter = (AndIterator) this.getUserObject();
- ok = iter.jump(jumpKey);
- if (iter.hasTop()) {
- Key key = (Key) iter.getTopKey();
- key = buildKey(key);
-
- this.setTopKey(key);
- if (log.isDebugEnabled()) {
- log.debug("BLTNODE.jump() -> found: " + this.getTopKey());
- }
- } else {
- if (log.isDebugEnabled()) {
- log.debug("IntersectingIteratorJexl does not have top after jump, marking done.");
- }
- this.setTopKey(null);
- this.setDone(true);
- }
- } else if (this.getType() == ParserTreeConstants.JJTORNODE) {
- OrIterator iter = (OrIterator) this.getUserObject();
- ok = iter.jump(jumpKey);
- if (iter.hasTop()) {
- Key key = (Key) iter.getTopKey();
- key = buildKey(key);
-
- this.setTopKey(key);
- if (log.isDebugEnabled()) {
- log.debug("BLTNODE.jump() -> found: " + this.getTopKey());
- }
- } else {
- if (log.isDebugEnabled()) {
- log.debug("OrIteratorJexl does not have top after jump, marking done.");
- }
- this.setTopKey(null);
- this.setDone(true);
- }
- }
- return ok;
- }
-
- public void addToSet(Key i) {
- uids.add(i);
- }
-
- public void reSet() {
- uids = new HashSet<Key>();
- }
-
- public boolean inSet(Key t) {
- return uids.contains(t);
- }
-
- public Iterator<Key> getSetIterator() {
- return uids.iterator();
- }
-
- public HashSet<Key> getIntersection(HashSet<Key> h) {
- h.retainAll(uids);
- return h;
- }
-
- public Key getMinUniqueID() {
- Iterator<Key> iter = uids.iterator();
- Key min = null;
- while (iter.hasNext()) {
- Key t = (Key) iter.next();
- if (log.isDebugEnabled()) {
- log.debug("OR set member: " + t);
- }
- if (t != null) {
- if (min == null) {
- min = t;
- } else if (t.compareTo(min) < 0) {
- min = t;
- }
- }
- }
- return min;
- }
-
- public boolean hasTop() {
- // This part really needs to be cleaned up.
- // It was created before I knew what was being passed back.
- if (this.getType() == ParserTreeConstants.JJTORNODE) {
- // Are you a Logical OR or an OR Iterator
- if (!this.isLeaf()) { // logical construct
- // I have a set of keys
- return this.uids.size() > 0;
- } else { // or iterator, you only have possible key
- if (this.getTopKey() == null) {
- return false;
- } else {
- return true;
- }
- }
- } else {
- return this.getTopKey() != null;
- }
- }
-
- public static Key buildKey(Key key) {
- if (key == null) {
- log.error("Problem in BooleanLogicTreeNodeJexl.buildKey");
- return null;
- }
- // Build Key(Text row, Text colfam) where colFam is dataype\0uid
- String[] cq = key.getColumnQualifier().toString().split("\0");
- Text uuid = new Text(cq[cq.length - 2] + "\0" + cq[cq.length - 1]);
- Text row = key.getRow();
- if (log.isDebugEnabled()) {
- log.debug("Key-> r:" + row + " fam:" + uuid);
- }
- // System.out.println("Key-> r:"+row+" fam:"+uuid);
-
- Key k = new Key(row, uuid);
- // System.out.println("Key: "+k);
- return k;
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/DefaultIteratorEnvironment.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/DefaultIteratorEnvironment.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/DefaultIteratorEnvironment.java
deleted file mode 100644
index 353ce79..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/DefaultIteratorEnvironment.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.iterator;
-
-import java.io.IOException;
-
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.file.map.MyMapFile;
-import org.apache.accumulo.core.file.map.MyMapFile.Reader;
-import org.apache.accumulo.core.iterators.IteratorEnvironment;
-import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
-import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
-import org.apache.accumulo.core.util.CachedConfiguration;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-
-@SuppressWarnings("deprecation")
-public class DefaultIteratorEnvironment implements IteratorEnvironment {
-
- public Reader reserveMapFileReader(String mapFileName) throws IOException {
- Configuration conf = CachedConfiguration.getInstance();
- FileSystem fs = FileSystem.get(conf);
- return new MyMapFile.Reader(fs, mapFileName, conf);
- }
-
- public AccumuloConfiguration getConfig() {
- return AccumuloConfiguration.getDefaultConfiguration();
- }
-
- public IteratorScope getIteratorScope() {
- throw new UnsupportedOperationException();
- }
-
- public boolean isFullMajorCompaction() {
- throw new UnsupportedOperationException();
- }
-
- public void registerSideChannel(SortedKeyValueIterator<Key,Value> iter) {
- throw new UnsupportedOperationException();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/EvaluatingIterator.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/EvaluatingIterator.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/EvaluatingIterator.java
deleted file mode 100644
index d51023c..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/EvaluatingIterator.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.iterator;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-
-import org.apache.accumulo.core.data.ByteSequence;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.PartialKey;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.IteratorEnvironment;
-import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
-import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.accumulo.examples.wikisearch.parser.EventFields;
-import org.apache.accumulo.examples.wikisearch.parser.EventFields.FieldValue;
-import org.apache.commons.collections.map.LRUMap;
-import org.apache.hadoop.io.Text;
-
-
-public class EvaluatingIterator extends AbstractEvaluatingIterator {
-
- public static final String NULL_BYTE_STRING = "\u0000";
- LRUMap visibilityMap = new LRUMap();
-
- public EvaluatingIterator() {
- super();
- }
-
- public EvaluatingIterator(AbstractEvaluatingIterator other, IteratorEnvironment env) {
- super(other, env);
- }
-
- public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
- return new EvaluatingIterator(this, env);
- }
-
- @Override
- public PartialKey getKeyComparator() {
- return PartialKey.ROW_COLFAM;
- }
-
- @Override
- public Key getReturnKey(Key k) {
- // If we were using column visibility, then we would get the merged visibility here and use it in the key.
- // Remove the COLQ from the key and use the combined visibility
- Key r = new Key(k.getRowData().getBackingArray(), k.getColumnFamilyData().getBackingArray(), NULL_BYTE, k.getColumnVisibility().getBytes(),
- k.getTimestamp(), k.isDeleted(), false);
- return r;
- }
-
- @Override
- public void fillMap(EventFields event, Key key, Value value) {
- // If we were using column visibility, we would have to merge them here.
-
- // Pull the datatype from the colf in case we need to do anything datatype specific.
- // String colf = key.getColumnFamily().toString();
- // String datatype = colf.substring(0, colf.indexOf(NULL_BYTE_STRING));
-
- // For the partitioned table, the field name and field value are stored in the column qualifier
- // separated by a \0.
- String colq = key.getColumnQualifier().toString();// .toLowerCase();
- int idx = colq.indexOf(NULL_BYTE_STRING);
- String fieldName = colq.substring(0, idx);
- String fieldValue = colq.substring(idx + 1);
-
- event.put(fieldName, new FieldValue(getColumnVisibility(key), fieldValue.getBytes()));
- }
-
- /**
- * @param key
- * @return
- */
- public ColumnVisibility getColumnVisibility(Key key) {
- ColumnVisibility result = (ColumnVisibility) visibilityMap.get(key.getColumnVisibility());
- if (result != null)
- return result;
- result = new ColumnVisibility(key.getColumnVisibility().getBytes());
- visibilityMap.put(key.getColumnVisibility(), result);
- return result;
- }
-
- /**
- * Don't accept this key if the colf starts with 'fi'
- */
- @Override
- public boolean isKeyAccepted(Key key) throws IOException {
- if (key.getColumnFamily().toString().startsWith("fi")) {
- Key copy = new Key(key.getRow(), new Text("fi\01"));
- Collection<ByteSequence> columnFamilies = Collections.emptyList();
- this.iterator.seek(new Range(copy, copy), columnFamilies, true);
- if (this.iterator.hasTop())
- return isKeyAccepted(this.iterator.getTopKey());
- return true;
- }
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/FieldIndexIterator.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/FieldIndexIterator.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/FieldIndexIterator.java
deleted file mode 100644
index ad39ab3..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/FieldIndexIterator.java
+++ /dev/null
@@ -1,736 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.iterator;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.accumulo.core.data.ByteSequence;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.IteratorEnvironment;
-import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
-import org.apache.accumulo.core.iterators.WrappingIterator;
-import org.apache.accumulo.examples.wikisearch.function.QueryFunctions;
-import org.apache.accumulo.examples.wikisearch.util.FieldIndexKeyParser;
-import org.apache.commons.jexl2.Expression;
-import org.apache.commons.jexl2.JexlContext;
-import org.apache.commons.jexl2.JexlEngine;
-import org.apache.commons.jexl2.MapContext;
-import org.apache.commons.jexl2.parser.ParserTreeConstants;
-import org.apache.hadoop.io.Text;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-
-/**
- * This iterator should only return keys from the fi\0{fieldName}:{fieldValue} part of the shard table. Expect topKey to be CF, {datatype}\0{UID}
- */
-public class FieldIndexIterator extends WrappingIterator {
-
- private Key topKey = null;
- private Value topValue = null;
- private Range range = null;
- private Text currentRow;
- private Text fName = null;
- private String fNameString = null;
- private Text fValue = null;
- private String fOperator = null;
- private Expression expr = null;
- private static final Collection<ByteSequence> EMPTY_COL_FAMS = new ArrayList<ByteSequence>();
- protected static final Logger log = Logger.getLogger(FieldIndexIterator.class);
- private boolean negated = false;
- private int type;
- private static final String NULL_BYTE = "\0";
- private static final String ONE_BYTE = "\1";
- // According to the JEXL 2.0 docs, the engine is thread-safe. Let's create 1 engine per VM and
- // cache 128 expressions
- private static JexlEngine engine = new JexlEngine();
- private Range parentRange;
- private Text parentEndRow = null;
- private FieldIndexKeyParser keyParser;
-
- static {
- engine.setCache(128);
- Map<String,Object> functions = new HashMap<String,Object>();
- functions.put("f", QueryFunctions.class);
- engine.setFunctions(functions);
- }
-
- public static void setLogLevel(Level l) {
- log.setLevel(l);
- }
-
- // -------------------------------------------------------------------------
- // ------------- Constructors
- public FieldIndexIterator() {}
-
- public FieldIndexIterator(int type, Text rowId, Text fieldName, Text fieldValue, String operator) {
- this.fName = fieldName;
- this.fNameString = fName.toString().substring(3);
- this.fValue = fieldValue;
- this.fOperator = operator;
- this.range = buildRange(rowId);
- this.negated = false;
- this.type = type;
-
- // Create the Jexl expression, we need to add the ' around the field value
- StringBuilder buf = new StringBuilder();
- buf.append(fNameString).append(" ").append(this.fOperator).append(" ").append("'").append(fValue.toString()).append("'");
- this.expr = engine.createExpression(buf.toString());
-
- // Set a default KeyParser
- keyParser = createDefaultKeyParser();
- }
-
- public FieldIndexIterator(int type, Text rowId, Text fieldName, Text fieldValue, boolean neg, String operator) {
- this.fName = fieldName;
- this.fNameString = fName.toString().substring(3);
- this.fValue = fieldValue;
- this.fOperator = operator;
- this.range = buildRange(rowId);
- this.negated = neg;
- this.type = type;
-
- // Create the Jexl expression, we need to add the ' around the field value
- StringBuilder buf = new StringBuilder();
- buf.append(fNameString).append(" ").append(this.fOperator).append(" ").append("'").append(fValue.toString()).append("'");
- this.expr = engine.createExpression(buf.toString());
- // Set a default KeyParser
- keyParser = createDefaultKeyParser();
- }
-
- public FieldIndexIterator(FieldIndexIterator other, IteratorEnvironment env) {
- setSource(other.getSource().deepCopy(env));
- // Set a default KeyParser
- keyParser = createDefaultKeyParser();
- }
-
- private FieldIndexKeyParser createDefaultKeyParser() {
- FieldIndexKeyParser parser = new FieldIndexKeyParser();
- return parser;
- }
-
- @Override
- public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
- return new FieldIndexIterator(this, env);
- }
-
- @Override
- public Key getTopKey() {
- return topKey;
- }
-
- @Override
- public Value getTopValue() {
- return topValue;
- }
-
- @Override
- public boolean hasTop() {
- return (topKey != null);
- }
-
- @Override
- public void next() throws IOException {
- if (log.isDebugEnabled()) {
- log.debug("next()");
- }
- if (this.hasTop()) {
- currentRow = topKey.getRow();
- }
-
- getSource().next();
- while (true) {
- log.debug("next(), Range: " + range);
- if (getSource().hasTop()) {
- Key k = getSource().getTopKey();
- if (range.contains(k)) {
- if (matches(k)) {
- topKey = k;
- topValue = getSource().getTopValue();
- return;
- } else {
- getSource().next();
- }
-
- } else {
-
- if (parentEndRow != null) { // need to check it
- if (k.getRow().equals(currentRow)) {
- currentRow = getNextRow();
- } else if (currentRow == null || k.getRow().compareTo(currentRow) > 0) {
- currentRow = k.getRow();
- }
-
- if (currentRow == null || parentEndRow.compareTo(currentRow) < 0) {
- // you're done
- topKey = null;
- topValue = null;
- return;
- }
-
- } else { // we can go to end of the tablet
- if (k.getRow().equals(currentRow)) {
- currentRow = getNextRow();
- if (currentRow == null) {
- topKey = null;
- topValue = null;
- return;
- }
- } else if (currentRow == null || (k.getRow().compareTo(currentRow) > 0)) {
- currentRow = k.getRow();
- }
- }
-
- // construct new range and seek the source
- range = buildRange(currentRow);
- if (log.isDebugEnabled()) {
- log.debug("next, range: " + range);
- }
- getSource().seek(range, EMPTY_COL_FAMS, false);
- }
- } else {
- topKey = null;
- topValue = null;
- return;
- }
- }
- }
-
- /*
- * NOTE: there is some special magic here with range modification. If it's negated, assume the range is explicitly set and don't mess with it (this is how
- * it's called by the BooleanLogicIterator) Otherwise, modify the range to start at the beginning and set an explicit end point.
- *
- * In the future, maybe all we need to do is look for an endKey and modifying that.
- */
- @Override
- public void seek(Range r, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
- parentRange = r;
- if (log.isDebugEnabled()) {
- log.debug("begin seek, range: " + r);
- }
- if (parentRange.getEndKey() != null) {
- if (parentRange.getEndKey().getRow() != null) {
- parentEndRow = parentRange.getEndKey().getRow();
- if (log.isDebugEnabled()) {
- log.debug("begin seek, parentEndRow: " + parentEndRow);
- }
- }
- }
-
- try {
- if (isNegated()) {
- range = r;
- if (log.isDebugEnabled()) {
- log.debug("seek, negation, skipping range modification.");
- }
- } else {
- if (r.getStartKey() != null) {
- if (r.getStartKey().getRow() == null || r.getStartKey().getRow().toString().isEmpty()) {
- currentRow = getFirstRow();
- } else {
- currentRow = r.getStartKey().getRow();
- }
- this.range = buildRange(currentRow);
- } else {
- currentRow = getFirstRow();
- this.range = buildRange(currentRow);
- }
- }
-
- setTopKey(null);
- setTopValue(null);
-
- if (log.isDebugEnabled()) {
- log.debug("seek, incoming range: " + range);
- }
- getSource().seek(range, columnFamilies, inclusive);
-
- while (topKey == null) {
- if (getSource().hasTop()) {
- if (log.isDebugEnabled()) {
- log.debug("seek, source has top: " + getSource().getTopKey());
- }
- Key k = getSource().getTopKey();
- if (range.contains(k)) {
- if (matches(k)) {
- topKey = k;
- topValue = getSource().getTopValue();
- if (log.isDebugEnabled()) {
- log.debug("seek, source has top in valid range");
- }
- } else {
- getSource().next();
- }
- } else {
- if (log.isDebugEnabled()) {
- log.debug("seek, top out of range");
- String pEndRow = "empty";
- if (parentEndRow != null) {
- pEndRow = parentEndRow.toString();
- }
- log.debug("source.topKey.row: " + k.getRow() + "\t currentRow: " + currentRow + "\t parentEndRow: " + pEndRow);
- }
- if (isNegated()) {
- topKey = null;
- topValue = null;
- return;
- }
-
- if (parentEndRow != null) {
- // check it
- if (k.getRow().equals(currentRow)) {
- currentRow = getNextRow();
- }
-
- if (currentRow == null || parentEndRow.compareTo(currentRow) < 0) {
- // you're done
- topKey = null;
- topValue = null;
- return;
- }
-
- } else { // can go to end of the tablet
- if (k.getRow().equals(currentRow)) {
- currentRow = getNextRow();
- if (currentRow == null) {
- topKey = null;
- topValue = null;
- return;
- }
- }
- }
-
- // construct new range and seek the source
- range = buildRange(currentRow);
- if (log.isDebugEnabled()) {
- log.debug("currentRow: " + currentRow);
- log.debug("seek, range: " + range);
- }
- getSource().seek(range, columnFamilies, inclusive);
- }
- } else {
- if (log.isDebugEnabled()) {
- log.debug("seek, underlying source had no top key.");
- }
- topKey = null;
- topValue = null;
- return;
- }
- }
- if (log.isDebugEnabled()) {
- log.debug("seek, topKey found: " + topKey);
- }
- } catch (IOException e) {
- topKey = null;
- topValue = null;
- throw new IOException();
- }
- }
-
- // -------------------------------------------------------------------------
- // ------------- Public stuff
- public boolean isNegated() {
- return negated;
- }
-
- public Text getCurrentRow() {
- return currentRow;
- }
-
- public Text getfName() {
- return fName;
- }
-
- public Text getfValue() {
- return fValue;
- }
-
- // works like seek, but we need to avoid range issues.
- public boolean jump(Key jumpKey) throws IOException {
- if (log.isDebugEnabled()) {
- String pEndRow = "empty";
- if (parentEndRow != null) {
- pEndRow = parentEndRow.toString();
- }
- log.debug("jump, current range: " + range + " parentEndRow is: " + pEndRow);
-
- }
-
- if (parentEndRow != null && jumpKey.getRow().compareTo(parentEndRow) > 0) {
- // can't go there.
- if (log.isDebugEnabled()) {
- log.debug("jumpRow: " + jumpKey.getRow() + " is greater than my parentEndRow: " + parentEndRow);
- }
- return false;
- }
-
- int comp;
- if (!this.hasTop()) {
- if (log.isDebugEnabled()) {
- log.debug("current row: " + this.currentRow);
- }
-
- /*
- * if I don't have a top, then I should be out of my range for my current row. Need to check parent range to see if I'm supposed to continue to next row
- * or not. Current row can be null because maybe I never found anything in this row.
- */
-
- if (parentEndRow != null) {
- // if jumpKey row is greater than parentEndRow, stop
- if (jumpKey.getRow().compareTo(parentEndRow) > 0) {
- if (log.isDebugEnabled()) {
- log.debug("jumpKey row is greater than my parentEndRow, done");
- }
- return false;
- }
-
- // if my current row is null, I must have hit the end of the tablet
- if (currentRow == null) {
- if (log.isDebugEnabled()) {
- log.debug("I have parentEndRow, but no current row, must have hit end of tablet, done");
- }
- return false;
- }
-
- // if my current row is greater than jump row stop, a seek will be
- // called to get me going again. If my row is equal, but i don't
- // have a topkey, i'm done
- if (currentRow.compareTo(jumpKey.getRow()) >= 0) {
- if (log.isDebugEnabled()) {
- log.debug("I have parentEndRow, but topKey, and my currentRow is >= jumpRow, done");
- }
- return false;
- }
-
- } else { // we're allowed to go to the end of the tablet
- // if my current row is null, I must have hit the end of the tablet
- if (currentRow == null) {
- if (log.isDebugEnabled()) {
- log.debug("no parentEndRow and current Row is null, must have hit end of tablet, done");
- }
- return false;
- }
-
- if (currentRow.compareTo(jumpKey.getRow()) >= 0) {
- // i'm past or equal to the jump point and have no top,
- // jumping's not going to help
- if (log.isDebugEnabled()) {
- log.debug("no parentEndRow, no topKey, and currentRow is >= jumpRow, done");
- }
- return false;
- }
- }
-
- // ok, jumpKey is ahead of me I'll mark it and allow the normal
- // flow to jump there and see if I have top.
- if (log.isDebugEnabled()) {
- log.debug("no topKey, but jumpRow is ahead and I'm allowed to go to it, marking");
- }
- comp = -1;
-
- } else { // I have a topKey, I can do the normal comparisons
- if (log.isDebugEnabled()) {
- log.debug("have top, can do normal comparisons");
- }
- comp = this.topKey.getRow().compareTo(jumpKey.getRow());
- }
-
- // ------------------
- // compare rows
- if (comp > 0) { // my row is ahead of jump key
- if (canBeInNextRow()) {
- if (log.isDebugEnabled()) {
- log.debug("I'm ahead of jump row & it's ok.");
- log.debug("jumpRow: " + jumpKey.getRow() + " myRow: " + topKey.getRow() + " parentEndRow: " + parentEndRow);
- }
- return true;
- } else {
- if (log.isDebugEnabled()) {
- log.debug("I'm ahead of jump row & can't be here, or at end of tablet.");
- }
- topKey = null;
- topValue = null;
- return false;
- }
-
- } else if (comp < 0) { // a row behind jump key, need to move forward
- if (log.isDebugEnabled()) {
- String myRow = "";
- if (hasTop()) {
- myRow = topKey.getRow().toString();
- } else if (currentRow != null) {
- myRow = currentRow.toString();
- }
- log.debug("My row " + myRow + " is less than jump row: " + jumpKey.getRow() + " seeking");
- }
- range = buildRange(jumpKey.getRow());
- // this.seek(range, EMPTY_COL_FAMS, false);
-
- boolean success = jumpSeek(range);
- if (log.isDebugEnabled() && success) {
- log.debug("uid forced jump, found topKey: " + topKey);
- }
-
- if (!this.hasTop()) {
- log.debug("seeked with new row and had no top");
- topKey = null;
- topValue = null;
- return false;
- } else if (parentEndRow != null && currentRow.compareTo(parentEndRow) > 0) {
- if (log.isDebugEnabled()) {
- log.debug("myRow: " + getTopKey().getRow() + " is past parentEndRow: " + parentEndRow);
- }
- topKey = null;
- topValue = null;
- return false;
- }
- if (log.isDebugEnabled()) {
- log.debug("jumped, valid top: " + getTopKey());
- }
-
- return true;
-
- } else { // rows are equal, check the uid!
-
- keyParser.parse(topKey);
- String myUid = keyParser.getUid();
- keyParser.parse(jumpKey);
- String jumpUid = keyParser.getUid();
-
- int ucomp = myUid.compareTo(jumpUid);
- if (log.isDebugEnabled()) {
- log.debug("topKeyUid: " + myUid + " jumpUid: " + jumpUid + " myUid.compareTo(jumpUid)->" + ucomp);
- }
- if (ucomp < 0) { // need to move up
- log.debug("my uid is less than jumpUid, topUid: " + myUid + " jumpUid: " + jumpUid);
-
- Text cq = jumpKey.getColumnQualifier();
- int index = cq.find(NULL_BYTE);
- if (0 <= index) {
- cq.set(cq.getBytes(), index + 1, cq.getLength() - index - 1);
- } else {
- log.error("Expected a NULL separator in the column qualifier");
- this.topKey = null;
- this.topValue = null;
- return false;
- }
-
- // note my internal range stays the same, I just need to move forward
- Key startKey = new Key(topKey.getRow(), fName, new Text(fValue + NULL_BYTE + cq));
- Key endKey = new Key(topKey.getRow(), fName, new Text(fValue + ONE_BYTE));
- range = new Range(startKey, true, endKey, false);
- log.debug("Using range: " + range + " to seek");
- // source.seek(range, EMPTY_COL_FAMS, false);
- boolean success = jumpSeek(range);
- if (log.isDebugEnabled() && success) {
- log.debug("uid forced jump, found topKey: " + topKey);
- }
-
- return success;
-
- } else { // else do nothing
- log.debug("my uid is greater than jumpUid, topKey: " + topKey + " jumpKey: " + jumpKey);
- log.debug("doing nothing");
- }
- }
-
- return hasTop();
- }
-
- // -------------------------------------------------------------------------
- // ------------- Private stuff, KEEP OUT!!
- private void setTopKey(Key key) {
- topKey = key;
- }
-
- private void setTopValue(Value v) {
- this.topValue = v;
- }
-
- private boolean canBeInNextRow() {
- if (parentEndRow == null) {
- return true;
- } else if (currentRow == null) {
- return false;
- } else if (currentRow.compareTo(parentEndRow) <= 0) {
- return true;
- } else {
- return false;
- }
- }
-
- private Range buildRange(Text rowId) {
- if (type == ParserTreeConstants.JJTGTNODE || type == ParserTreeConstants.JJTGENODE || type == ParserTreeConstants.JJTLTNODE
- || type == ParserTreeConstants.JJTLENODE || type == ParserTreeConstants.JJTERNODE || type == ParserTreeConstants.JJTNRNODE) {
- Key startKey = new Key(rowId, fName);
- Key endKey = new Key(rowId, new Text(fName + NULL_BYTE));
- return (new Range(startKey, true, endKey, false));
- } else {
- // construct new range
- Key startKey = new Key(rowId, fName, new Text(fValue + NULL_BYTE));
- Key endKey = new Key(rowId, fName, new Text(fValue + ONE_BYTE));
- return (new Range(startKey, true, endKey, false));
- }
- }
-
- // need to build a range starting at the end of current row and seek the
- // source to it. If we get an IOException, that means we hit the end of the tablet.
- private Text getNextRow() throws IOException {
- if (log.isDebugEnabled()) {
- log.debug("getNextRow()");
- }
- Key fakeKey = new Key(new Text(currentRow + NULL_BYTE));
- Range fakeRange = new Range(fakeKey, fakeKey);
- getSource().seek(fakeRange, EMPTY_COL_FAMS, false);
- if (getSource().hasTop()) {
- return getSource().getTopKey().getRow();
- } else {
- return null;
- }
- }
-
- private Text getFirstRow() throws IOException {
- getSource().seek(new Range(), EMPTY_COL_FAMS, false);
- if (getSource().hasTop()) {
- return getSource().getTopKey().getRow();
- } else {
- throw new IOException();
- }
- }
-
- private boolean matches(Key k) {
- if (log.isDebugEnabled()) {
- log.debug("You've reached the match function!");
- }
- JexlContext ctx = new MapContext();
- // Add the field value from the key to the context
- // String fieldValue = k.getColumnQualifier().toString().split("\0")[0];
- // String fieldValue = getFieldValueFromKey(k);
- keyParser.parse(k);
- String fieldValue = keyParser.getFieldValue();
-
- ctx.set(fNameString, fieldValue);
- Object o = expr.evaluate(ctx);
- if (o instanceof Boolean && (((Boolean) o) == true)) {
- if (log.isDebugEnabled()) {
- log.debug("matches:: fName: " + fName + " , fValue: " + fieldValue + " , operator: " + fOperator + " , key: " + k);
- }
- return true;
- } else {
- if (log.isDebugEnabled()) {
- log.debug("NO MATCH:: fName: " + fName + " , fValue: " + fieldValue + " , operator: " + fOperator + " , key: " + k);
- }
- return false;
- }
- }
-
- private boolean jumpSeek(Range r) throws IOException {
- range = r;
- setTopKey(null);
- setTopValue(null);
- getSource().seek(range, EMPTY_COL_FAMS, false);
- while (topKey == null) {
- if (getSource().hasTop()) {
- if (log.isDebugEnabled()) {
- log.debug("jump, source has top: " + getSource().getTopKey());
- }
- Key k = getSource().getTopKey();
- if (range.contains(k)) {
- if (matches(k)) {
- topKey = k;
- topValue = getSource().getTopValue();
- if (log.isDebugEnabled()) {
- log.debug("jump, source has top in valid range");
- }
- } else {
- getSource().next();
- }
- } else {
- if (log.isDebugEnabled()) {
- log.debug("jump, top out of range");
- String pEndRow = "empty";
- if (parentEndRow != null) {
- pEndRow = parentEndRow.toString();
- }
- log.debug("source.topKey.row: " + k.getRow() + "\t currentRow: " + currentRow + "\t parentEndRow: " + pEndRow);
- }
-
- if (parentEndRow != null) {
-
- if (currentRow == null) {
- topKey = null;
- topValue = null;
- return false;
- }
- // check it
- if (k.getRow().equals(currentRow)) {
- currentRow = getNextRow();
- } else if (k.getRow().compareTo(currentRow) > 0) {
- currentRow = k.getRow();
- }
-
- if (currentRow == null || parentEndRow.compareTo(currentRow) < 0) {
- // you're done
- topKey = null;
- topValue = null;
- return false;
- }
-
- } else { // can go to end of the tablet
- if (currentRow == null || k.getRow() == null) {
- topKey = null;
- topValue = null;
- return false;
- }
-
- if (k.getRow().equals(currentRow)) {
- currentRow = getNextRow();
- if (currentRow == null) {
- topKey = null;
- topValue = null;
- return false;
- }
- } else if (k.getRow().compareTo(currentRow) > 0) {
- currentRow = k.getRow();
- }
- }
-
- // construct new range and seek the source
- range = buildRange(currentRow);
- if (log.isDebugEnabled()) {
- log.debug("jump, new build range: " + range);
- }
- getSource().seek(range, EMPTY_COL_FAMS, false);
- }
- } else {
- if (log.isDebugEnabled()) {
- log.debug("jump, underlying source had no top key.");
- }
- topKey = null;
- topValue = null;
- return false;
- }
- }// end while
-
- return hasTop();
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/OptimizedQueryIterator.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/OptimizedQueryIterator.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/OptimizedQueryIterator.java
deleted file mode 100644
index c8fa5ad..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/OptimizedQueryIterator.java
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.iterator;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-
-import org.apache.accumulo.core.data.ByteSequence;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.PartialKey;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.IteratorEnvironment;
-import org.apache.accumulo.core.iterators.OptionDescriber;
-import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
-import org.apache.log4j.Logger;
-
-/**
- * This iterator internally uses the BooleanLogicIterator to find event UIDs in the field index portion of the partition and uses the EvaluatingIterator to
- * evaluate the events against an expression. The key and value that are emitted from this iterator are the key and value that come from the EvaluatingIterator.
- */
-public class OptimizedQueryIterator implements SortedKeyValueIterator<Key,Value>, OptionDescriber {
-
- private static Logger log = Logger.getLogger(OptimizedQueryIterator.class);
- private EvaluatingIterator event = null;
- private SortedKeyValueIterator<Key,Value> index = null;
- private Key key = null;
- private Value value = null;
- private boolean eventSpecificRange = false;
-
- public IteratorOptions describeOptions() {
- Map<String,String> options = new HashMap<String,String>();
- options.put(EvaluatingIterator.QUERY_OPTION, "full query expression");
- options.put(BooleanLogicIterator.FIELD_INDEX_QUERY, "modified query for the field index query portion");
- options.put(ReadAheadIterator.QUEUE_SIZE, "parallel queue size");
- options.put(ReadAheadIterator.TIMEOUT, "parallel iterator timeout");
- return new IteratorOptions(getClass().getSimpleName(), "evaluates event objects against an expression using the field index", options, null);
- }
-
- public boolean validateOptions(Map<String,String> options) {
- if (options.containsKey(EvaluatingIterator.QUERY_OPTION) && options.containsKey(BooleanLogicIterator.FIELD_INDEX_QUERY)) {
- return true;
- }
- return false;
- }
-
- public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
- if (!validateOptions(options)) {
- throw new IllegalArgumentException("Invalid options");
- }
-
- // Setup the EvaluatingIterator
- event = new EvaluatingIterator();
- event.init(source.deepCopy(env), options, env);
-
- // if queue size and timeout are set, then use the read ahead iterator
- if (options.containsKey(ReadAheadIterator.QUEUE_SIZE) && options.containsKey(ReadAheadIterator.TIMEOUT)) {
- BooleanLogicIterator bli = new BooleanLogicIterator();
- bli.init(source, options, env);
- index = new ReadAheadIterator();
- index.init(bli, options, env);
- } else {
- index = new BooleanLogicIterator();
- // index.setDebug(Level.DEBUG);
- index.init(source, options, env);
- }
-
- }
-
- public OptimizedQueryIterator() {}
-
- public OptimizedQueryIterator(OptimizedQueryIterator other, IteratorEnvironment env) {
- this.event = other.event;
- this.index = other.index;
- }
-
- public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
- return new OptimizedQueryIterator(this, env);
- }
-
- public Key getTopKey() {
- if (log.isDebugEnabled()) {
- log.debug("getTopKey: " + key);
- }
- return key;
- }
-
- public Value getTopValue() {
- if (log.isDebugEnabled()) {
- log.debug("getTopValue: " + value);
- }
- return value;
- }
-
- public boolean hasTop() {
- if (log.isDebugEnabled()) {
- log.debug("hasTop: returned: " + (key != null));
- }
- return (key != null);
- }
-
- public void next() throws IOException {
- if (log.isDebugEnabled()) {
- log.debug("next");
- }
- if (key != null) {
- key = null;
- value = null;
- }
-
- if (eventSpecificRange) {
- // Then this will probably return nothing
- event.next();
- if (event.hasTop()) {
- key = event.getTopKey();
- value = event.getTopValue();
- }
- } else {
-
- do {
- index.next();
- // If the index has a match, then seek the event to the key
- if (index.hasTop()) {
- Key eventKey = index.getTopKey();
- Key endKey = eventKey.followingKey(PartialKey.ROW_COLFAM);
- Key startKey = new Key(eventKey.getRow(), eventKey.getColumnFamily());
- Range eventRange = new Range(startKey, endKey);
- HashSet<ByteSequence> cf = new HashSet<ByteSequence>();
- cf.add(eventKey.getColumnFamilyData());
- event.seek(eventRange, cf, true);
- if (event.hasTop()) {
- key = event.getTopKey();
- value = event.getTopValue();
- }
- }
- } while (key == null && index.hasTop());
- }
- // Sanity check. Make sure both returnValue and returnKey are null or both are not null
- if (!((key == null && value == null) || (key != null && value != null))) {
- log.warn("Key: " + ((key == null) ? "null" : key.toString()));
- log.warn("Value: " + ((value == null) ? "null" : value.toString()));
- throw new IOException("Return values are inconsistent");
- }
-
- }
-
- public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
- if (log.isDebugEnabled()) {
- log.debug("seek, range:" + range);
- }
- // Test the range to see if it is event specific.
- if (null != range.getEndKey() && range.getEndKey().getColumnFamily() != null && range.getEndKey().getColumnFamily().getLength() != 0) {
- if (log.isDebugEnabled()) {
- log.debug("Jumping straight to the event");
- }
- // Then this range is for a specific event. We don't need to use the index iterator to find it, we can just
- // seek to it with the event iterator and evaluate it.
- eventSpecificRange = true;
- event.seek(range, columnFamilies, inclusive);
- if (event.hasTop()) {
- key = event.getTopKey();
- value = event.getTopValue();
- }
- } else {
- if (log.isDebugEnabled()) {
- log.debug("Using BooleanLogicIteratorJexl");
- }
- // Seek the boolean logic iterator
- index.seek(range, columnFamilies, inclusive);
-
- // If the index has a match, then seek the event to the key
- if (index.hasTop()) {
- Key eventKey = index.getTopKey();
- // Range eventRange = new Range(eventKey, eventKey);
- Range eventRange = new Range(eventKey.getRow());
- HashSet<ByteSequence> cf = new HashSet<ByteSequence>();
- cf.add(eventKey.getColumnFamilyData());
- event.seek(eventRange, cf, true);
- if (event.hasTop()) {
- key = event.getTopKey();
- value = event.getTopValue();
- } else {
- next();
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/OrIterator.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/OrIterator.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/OrIterator.java
deleted file mode 100644
index 78c8576..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/OrIterator.java
+++ /dev/null
@@ -1,822 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.iterator;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.PriorityQueue;
-
-import org.apache.accumulo.core.data.ByteSequence;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.PartialKey;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.IteratorEnvironment;
-import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
-import org.apache.hadoop.io.Text;
-import org.apache.log4j.Logger;
-
-/**
- * An iterator that handles "OR" query constructs on the server side. This code has been adapted/merged from Heap and Multi Iterators.
- */
-public class OrIterator implements SortedKeyValueIterator<Key,Value> {
-
- private TermSource currentTerm;
- private ArrayList<TermSource> sources;
- private PriorityQueue<TermSource> sorted = new PriorityQueue<TermSource>(5);
- private static final Text nullText = new Text();
- private Key topKey = null;
- private Range overallRange;
- private Collection<ByteSequence> columnFamilies;
- private boolean inclusive;
- protected static final Logger log = Logger.getLogger(OrIterator.class);
- private Text parentEndRow;
-
- protected static class TermSource implements Comparable<TermSource> {
-
- public SortedKeyValueIterator<Key,Value> iter;
- public Text dataLocation;
- public Text term;
- public Text docid;
- public Text fieldTerm;
- public Key topKey;
- public boolean atEnd;
-
- public TermSource(TermSource other) {
- this.iter = other.iter;
- this.term = other.term;
- this.dataLocation = other.dataLocation;
- this.atEnd = other.atEnd;
- }
-
- public TermSource(SortedKeyValueIterator<Key,Value> iter, Text term) {
- this.iter = iter;
- this.term = term;
- this.atEnd = false;
- }
-
- public TermSource(SortedKeyValueIterator<Key,Value> iter, Text dataLocation, Text term) {
- this.iter = iter;
- this.dataLocation = dataLocation;
- this.term = term;
- this.atEnd = false;
- }
-
- public void setNew() {
- if (!this.atEnd && this.iter.hasTop()) {
- this.topKey = this.iter.getTopKey();
-
- if (log.isDebugEnabled()) {
- log.debug("OI.TermSource.setNew TS.iter.topKey >>" + topKey + "<<");
- }
-
- if (this.term == null) {
- this.docid = this.topKey.getColumnQualifier();
- } else {
- String cqString = this.topKey.getColumnQualifier().toString();
-
- int idx = cqString.indexOf("\0");
- this.fieldTerm = new Text(cqString.substring(0, idx));
- this.docid = new Text(cqString.substring(idx + 1));
- }
- } else {
- if (log.isDebugEnabled()) {
- log.debug("OI.TermSource.setNew Setting to null...");
- }
-
- // this.term = null;
- // this.dataLocation = null;
- this.topKey = null;
- this.fieldTerm = null;
- this.docid = null;
- }
- }
-
- public int compareTo(TermSource o) {
- // NOTE: If your implementation can have more than one row in a tablet,
- // you must compare row key here first, then column qualifier.
- // NOTE2: A null check is not needed because things are only added to the
- // sorted after they have been determined to be valid.
- // return this.docid.compareTo(o.docid);
- // return this.topKey.compareTo(o.topKey);
-
- // NOTE! We need to compare UID's, not Keys!
- Key k1 = topKey;
- Key k2 = o.topKey;
- // return t1.compareTo(t2);
- String uid1 = getUID(k1);
- String uid2 = getUID(k2);
-
- if (uid1 != null && uid2 != null) {
- return uid1.compareTo(uid2);
- } else if (uid1 == null && uid2 == null) {
- return 0;
- } else if (uid1 == null) {
- return 1;
- } else {
- return -1;
- }
-
- }
-
- @Override
- public String toString() {
- return "TermSource: " + this.dataLocation + " " + this.term;
- }
-
- public boolean hasTop() {
- return this.topKey != null;
- }
- }
-
- /**
- * Returns the given key's row
- *
- * @param key
- * @return The given key's row
- */
- protected Text getPartition(Key key) {
- return key.getRow();
- }
-
- /**
- * Returns the given key's dataLocation
- *
- * @param key
- * @return The given key's dataLocation
- */
- protected Text getDataLocation(Key key) {
- return key.getColumnFamily();
- }
-
- /**
- * Returns the given key's term
- *
- * @param key
- * @return The given key's term
- */
- protected Text getTerm(Key key) {
- int idx = 0;
- String sKey = key.getColumnQualifier().toString();
-
- idx = sKey.indexOf("\0");
- return new Text(sKey.substring(0, idx));
- }
-
- /**
- * Returns the given key's DocID
- *
- * @param key
- * @return The given key's DocID
- */
- protected Text getDocID(Key key) {
- int idx = 0;
- String sKey = key.getColumnQualifier().toString();
-
- idx = sKey.indexOf("\0");
- return new Text(sKey.substring(idx + 1));
- }
-
- /**
- * Returns the given key's UID
- *
- * @param key
- * @return The given key's UID
- */
- static protected String getUID(Key key) {
- try {
- int idx = 0;
- String sKey = key.getColumnQualifier().toString();
-
- idx = sKey.indexOf("\0");
- return sKey.substring(idx + 1);
- } catch (Exception e) {
- return null;
- }
- }
-
- public OrIterator() {
- this.sources = new ArrayList<TermSource>();
- }
-
- private OrIterator(OrIterator other, IteratorEnvironment env) {
- this.sources = new ArrayList<TermSource>();
-
- for (TermSource TS : other.sources) {
- this.sources.add(new TermSource(TS.iter.deepCopy(env), TS.dataLocation, TS.term));
- }
- }
-
- public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
- return new OrIterator(this, env);
- }
-
- public void addTerm(SortedKeyValueIterator<Key,Value> source, Text term, IteratorEnvironment env) {
- if (log.isDebugEnabled()) {
- log.debug("OI.addTerm Added source w/o family");
- log.debug("OI.addTerm term >>" + term + "<<");
- }
-
- // Don't deepcopy an iterator
- if (term == null) {
- this.sources.add(new TermSource(source, term));
- } else {
- this.sources.add(new TermSource(source.deepCopy(env), term));
- }
- }
-
- public void addTerm(SortedKeyValueIterator<Key,Value> source, Text dataLocation, Text term, IteratorEnvironment env) {
- if (log.isDebugEnabled()) {
- log.debug("OI.addTerm Added source ");
- log.debug("OI.addTerm family >>" + dataLocation + "<< term >>" + term + "<<");
- }
-
- // Don't deepcopy an iterator
- if (term == null) {
- this.sources.add(new TermSource(source, dataLocation, term));
- } else {
- this.sources.add(new TermSource(source.deepCopy(env), dataLocation, term));
- }
- }
-
- /**
- * Construct the topKey given the current <code>TermSource</code>
- *
- * @param TS
- * @return The top Key for a given TermSource
- */
- protected Key buildTopKey(TermSource TS) {
- if ((TS == null) || (TS.topKey == null)) {
- return null;
- }
-
- if (log.isDebugEnabled()) {
- log.debug("OI.buildTopKey New topKey >>" + new Key(TS.topKey.getRow(), TS.dataLocation, TS.docid) + "<<");
- }
-
- return new Key(TS.topKey.getRow(), TS.topKey.getColumnFamily(), TS.topKey.getColumnQualifier());
- }
-
- final public void next() throws IOException {
- if (log.isDebugEnabled()) {
- log.debug("OI.next Enter: sorted.size = " + sorted.size() + " currentTerm = " + ((currentTerm == null) ? "null" : "not null"));
- }
-
- if (currentTerm == null) {
- if (log.isDebugEnabled()) {
- log.debug("OI.next currentTerm is NULL... returning");
- }
-
- topKey = null;
- return;
- }
-
- // Advance currentTerm
- currentTerm.iter.next();
-
- advanceToMatch(currentTerm);
-
- currentTerm.setNew();
-
- // See if currentTerm is still valid, remove if not
- if (log.isDebugEnabled()) {
- log.debug("OI.next Checks (correct = 0,0,0): " + ((currentTerm.topKey != null) ? "0," : "1,") + ((currentTerm.dataLocation != null) ? "0," : "1,")
- + ((currentTerm.term != null && currentTerm.fieldTerm != null) ? (currentTerm.term.compareTo(currentTerm.fieldTerm)) : "0"));
- }
-
- if (currentTerm.topKey == null || ((currentTerm.dataLocation != null) && (currentTerm.term.compareTo(currentTerm.fieldTerm) != 0))) {
- if (log.isDebugEnabled()) {
- log.debug("OI.next removing entry:" + currentTerm.term);
- }
-
- currentTerm = null;
- }
-
- // optimization.
- // if size == 0, currentTerm is the only item left,
- // OR there are no items left.
- // In either case, we don't need to use the PriorityQueue
- if (sorted.size() > 0) {
- // sort the term back in
- if (currentTerm != null) {
- sorted.add(currentTerm);
- }
- // and get the current top item out.
- currentTerm = sorted.poll();
- }
-
- if (log.isDebugEnabled()) {
- log.debug("OI.next CurrentTerm is " + ((currentTerm == null) ? "null" : currentTerm));
- }
-
- topKey = buildTopKey(currentTerm);
-
- if (hasTop()) {
- if (overallRange != null && !overallRange.contains(topKey)) {
- topKey = null;
- }
- }
- }
-
- public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
-
- overallRange = new Range(range);
- if (log.isDebugEnabled()) {
- log.debug("seek, overallRange: " + overallRange);
- }
-
- if (range.getEndKey() != null && range.getEndKey().getRow() != null) {
- this.parentEndRow = range.getEndKey().getRow();
- }
-
- if (log.isDebugEnabled()) {
- log.debug("OI.seek Entry - sources.size = " + sources.size());
- log.debug("OI.seek Entry - currentTerm = " + ((currentTerm == null) ? "false" : currentTerm.iter.getTopKey()));
- log.debug("OI.seek Entry - Key from Range = " + ((range == null) ? "false" : range.getStartKey()));
- }
-
- // If sources.size is 0, there is nothing to process, so just return.
- if (sources.isEmpty()) {
- currentTerm = null;
- topKey = null;
- return;
- }
-
- this.columnFamilies = columnFamilies;
- this.inclusive = inclusive;
-
- Range newRange = range;
- Key sourceKey = null;
- Key startKey = null;
-
- if (range != null) {
- startKey = range.getStartKey();
- }
-
- // Clear the PriorityQueue so that we can re-populate it.
- sorted.clear();
-
- TermSource TS = null;
- Iterator<TermSource> iter = sources.iterator();
- // For each term, seek forward.
- // if a hit is not found, delete it from future searches.
- int counter = 1;
- while (iter.hasNext()) {
- TS = iter.next();
-
- TS.atEnd = false;
-
- if (sources.size() == 1) {
- currentTerm = TS;
- }
-
- if (log.isDebugEnabled()) {
- log.debug("OI.seek on TS >>" + TS + "<<");
- log.debug("OI.seek seeking source >>" + counter + "<< ");
- }
-
- counter++;
-
- newRange = range;
- sourceKey = null;
-
- if (startKey != null) {
- // Construct the new key for the range
- if (log.isDebugEnabled()) {
- log.debug("OI.seek startKey >>" + startKey + "<<");
- }
-
- if (startKey.getColumnQualifier() != null) {
- sourceKey = new Key(startKey.getRow(), (TS.dataLocation == null) ? nullText : TS.dataLocation, new Text(((TS.term == null) ? "" : TS.term + "\0")
- + range.getStartKey().getColumnQualifier()));
- } else {
- sourceKey = new Key(startKey.getRow(), (TS.dataLocation == null) ? nullText : TS.dataLocation, (TS.term == null) ? nullText : TS.term);
- }
-
- if (log.isDebugEnabled()) {
- log.debug("OI.seek Seeking to the key => " + sourceKey);
- }
-
- newRange = new Range(sourceKey, true, sourceKey.followingKey(PartialKey.ROW), false);
- } else {
- if (log.isDebugEnabled()) {
- log.debug("OI.seek Using the range Seek() argument to seek => " + newRange);
- }
- }
-
- TS.iter.seek(newRange, columnFamilies, inclusive);
-
- TS.setNew();
-
- // Make sure we're on a key with the correct dataLocation and term
- advanceToMatch(TS);
-
- TS.setNew();
-
- if (log.isDebugEnabled()) {
- log.debug("OI.seek sourceKey >>" + sourceKey + "<< ");
- log.debug("OI.seek topKey >>" + ((TS.topKey == null) ? "false" : TS.topKey) + "<< ");
- log.debug("OI.seek TS.fieldTerm == " + TS.fieldTerm);
-
- log.debug("OI.seek Checks (correct = 0,0,0 / 0,1,1): " + ((TS.topKey != null) ? "0," : "1,") + ((TS.dataLocation != null) ? "0," : "1,")
- + (((TS.term != null && TS.fieldTerm != null) && (TS.term.compareTo(TS.fieldTerm) != 0)) ? "0" : "1"));
- }
-
- if ((TS.topKey == null) || ((TS.dataLocation != null) && (TS.term.compareTo(TS.fieldTerm) != 0))) {
- // log.debug("OI.seek Removing " + TS.term);
- // iter.remove();
- } // Optimization if we only have one element
- else if (sources.size() > 0 || iter.hasNext()) {
- // We have more than one source to search for, use the priority queue
- sorted.add(TS);
- } else {
- // Don't need to continue, only had one item to search
- if (log.isDebugEnabled()) {
- log.debug("OI.seek new topKey >>" + ((topKey == null) ? "false" : topKey) + "<< ");
- }
-
- // make sure it is in the range if we have one.
- if (hasTop()) {
- if (overallRange != null && !overallRange.contains(topKey)) {
- if (log.isDebugEnabled()) {
- log.debug("seek, topKey: " + topKey + " is not in the overallRange: " + overallRange);
- }
- topKey = null;
- }
- }
- return;
- }
- }
-
- // And set currentTerm = the next valid key/term.
- currentTerm = sorted.poll();
-
- if (log.isDebugEnabled()) {
- log.debug("OI.seek currentTerm = " + currentTerm);
- }
-
- topKey = buildTopKey(currentTerm);
- if (topKey == null) {
- if (log.isDebugEnabled()) {
- log.debug("OI.seek() topKey is null");
- }
- }
-
- if (log.isDebugEnabled()) {
- log.debug("OI.seek new topKey >>" + ((topKey == null) ? "false" : topKey) + "<< ");
- }
-
- if (hasTop()) {
- if (overallRange != null && !overallRange.contains(topKey)) {
- if (log.isDebugEnabled()) {
- log.debug("seek, topKey: " + topKey + " is not in the overallRange: " + overallRange);
- }
- topKey = null;
- }
- }
-
- }
-
- final public Key getTopKey() {
- if (log.isDebugEnabled()) {
- log.debug("OI.getTopKey key >>" + topKey);
- }
-
- return topKey;
- }
-
- final public Value getTopValue() {
- if (log.isDebugEnabled()) {
- log.debug("OI.getTopValue key >>" + currentTerm.iter.getTopValue());
- }
-
- return currentTerm.iter.getTopValue();
- }
-
- final public boolean hasTop() {
- if (log.isDebugEnabled()) {
- log.debug("OI.hasTop = " + ((topKey == null) ? "false" : "true"));
- }
-
- return topKey != null;
- }
-
- public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- /**
- * Ensures that the current <code>TermSource</code> is pointing to a key with the correct <code>dataLocation</code> and <code>term</code> or sets
- * <code>topKey</code> to null if there is no such key remaining.
- *
- * @param TS
- * The <code>TermSource</code> to advance
- * @throws IOException
- */
- private void advanceToMatch(TermSource TS) throws IOException {
- boolean matched = false;
- while (!matched) {
- if (!TS.iter.hasTop()) {
- TS.topKey = null;
- return;
- }
-
- Key iterTopKey = TS.iter.getTopKey();
-
- if (log.isDebugEnabled()) {
- log.debug("OI.advanceToMatch current topKey = " + iterTopKey);
- }
-
- // we should compare the row to the end of the range
- if (overallRange.getEndKey() != null) {
-
- if (overallRange != null && !overallRange.contains(TS.iter.getTopKey())) {
- if (log.isDebugEnabled()) {
- log.debug("overallRange: " + overallRange + " does not contain TS.iter.topKey: " + TS.iter.getTopKey());
- log.debug("OI.advanceToMatch at the end, returning");
- }
-
- TS.atEnd = true;
- TS.topKey = null;
-
- return;
- } else {
- if (log.isDebugEnabled()) {
- log.debug("OI.advanceToMatch not at the end");
- }
- }
- } else {
- if (log.isDebugEnabled()) {
- log.debug("OI.advanceToMatch overallRange.getEndKey() == null");
- }
- }
-
- // Advance to the correct dataLocation
- if (log.isDebugEnabled()) {
- log.debug("Comparing dataLocations.");
- log.debug("OI.advanceToMatch dataLocationCompare: " + getDataLocation(iterTopKey) + " == " + TS.dataLocation);
- }
-
- int dataLocationCompare = getDataLocation(iterTopKey).compareTo(TS.dataLocation);
-
- if (log.isDebugEnabled()) {
- log.debug("OI.advanceToMatch dataLocationCompare = " + dataLocationCompare);
- }
-
- // Make sure we're at a row for this dataLocation
- if (dataLocationCompare < 0) {
- if (log.isDebugEnabled()) {
- log.debug("OI.advanceToMatch seek to desired dataLocation");
- }
-
- Key seekKey = new Key(iterTopKey.getRow(), TS.dataLocation, nullText);
-
- if (log.isDebugEnabled()) {
- log.debug("OI.advanceToMatch seeking to => " + seekKey);
- }
-
- TS.iter.seek(new Range(seekKey, true, null, false), columnFamilies, inclusive);
-
- continue;
- } else if (dataLocationCompare > 0) {
- if (log.isDebugEnabled()) {
- log.debug("OI.advanceToMatch advanced beyond desired dataLocation, seek to next row");
- }
-
- // Gone past the current dataLocation, seek to the next row
- Key seekKey = iterTopKey.followingKey(PartialKey.ROW);
-
- if (log.isDebugEnabled()) {
- log.debug("OI.advanceToMatch seeking to => " + seekKey);
- }
-
- TS.iter.seek(new Range(seekKey, true, null, false), columnFamilies, inclusive);
-
- continue;
- }
-
- // Advance to the correct term
- if (log.isDebugEnabled()) {
- log.debug("OI.advanceToMatch termCompare: " + getTerm(iterTopKey) + " == " + TS.term);
- }
-
- int termCompare = getTerm(iterTopKey).compareTo(TS.term);
-
- if (log.isDebugEnabled()) {
- log.debug("OI.advanceToMatch termCompare = " + termCompare);
- }
-
- // Make sure we're at a row for this term
- if (termCompare < 0) {
- if (log.isDebugEnabled()) {
- log.debug("OI.advanceToMatch seek to desired term");
- }
-
- Key seekKey = new Key(iterTopKey.getRow(), iterTopKey.getColumnFamily(), TS.term);
-
- if (log.isDebugEnabled()) {
- log.debug("OI.advanceToMatch seeking to => " + seekKey);
- }
-
- TS.iter.seek(new Range(seekKey, true, null, false), columnFamilies, inclusive);
-
- continue;
- } else if (termCompare > 0) {
- if (log.isDebugEnabled()) {
- log.debug("OI.advanceToMatch advanced beyond desired term, seek to next row");
- }
-
- // Gone past the current term, seek to the next row
- Key seekKey = iterTopKey.followingKey(PartialKey.ROW);
-
- if (log.isDebugEnabled()) {
- log.debug("OI.advanceToMatch seeking to => " + seekKey);
- }
-
- TS.iter.seek(new Range(seekKey, true, null, false), columnFamilies, inclusive);
- continue;
- }
-
- // If we made it here, we found a match
- matched = true;
- }
- }
-
- public boolean jump(Key jumpKey) throws IOException {
- if (log.isDebugEnabled()) {
- log.debug("OR jump: " + jumpKey);
- printTopKeysForTermSources();
- }
-
- // is the jumpKey outside my overall range?
- if (parentEndRow != null && parentEndRow.compareTo(jumpKey.getRow()) < 0) {
- // can't go there.
- if (log.isDebugEnabled()) {
- log.debug("jumpRow: " + jumpKey.getRow() + " is greater than my parentEndRow: " + parentEndRow);
- }
- return false;
- }
-
- // Clear the PriorityQueue so that we can re-populate it.
- sorted.clear();
-
- // check each term source and jump it if necessary.
- for (TermSource ts : sources) {
- int comp;
- if (!ts.hasTop()) {
- if (log.isDebugEnabled()) {
- log.debug("jump called, but ts.topKey is null, this one needs to move to next row.");
- }
- Key startKey = new Key(jumpKey.getRow(), ts.dataLocation, new Text(ts.term + "\0" + jumpKey.getColumnFamily()));
- Key endKey = null;
- if (parentEndRow != null) {
- endKey = new Key(parentEndRow);
- }
- Range newRange = new Range(startKey, true, endKey, false);
- ts.iter.seek(newRange, columnFamilies, inclusive);
- ts.setNew();
- advanceToMatch(ts);
- ts.setNew();
-
- } else {
- // check row, then uid
- comp = this.topKey.getRow().compareTo(jumpKey.getRow());
- if (comp > 0) {
- if (log.isDebugEnabled()) {
- log.debug("jump, our row is ahead of jumpKey.");
- log.debug("jumpRow: " + jumpKey.getRow() + " myRow: " + topKey.getRow() + " parentEndRow" + parentEndRow);
- }
- if (ts.hasTop()) {
- sorted.add(ts);
- }
- // do nothing, we're ahead of jumpKey row and have topkey
- } else if (comp < 0) { // a row behind jump key, need to move forward
- if (log.isDebugEnabled()) {
- log.debug("OR jump, row jump");
- }
- Key endKey = null;
- if (parentEndRow != null) {
- endKey = new Key(parentEndRow);
- }
- Key sKey = new Key(jumpKey.getRow());
- Range fake = new Range(sKey, true, endKey, false);
- ts.iter.seek(fake, columnFamilies, inclusive);
- ts.setNew();
- advanceToMatch(ts);
- ts.setNew();
- } else {
- // need to check uid
- String myUid = getUID(ts.topKey);
- String jumpUid = getUID(jumpKey);
-
- if (log.isDebugEnabled()) {
- if (myUid == null) {
- log.debug("myUid is null");
- } else {
- log.debug("myUid: " + myUid);
- }
-
- if (jumpUid == null) {
- log.debug("jumpUid is null");
- } else {
- log.debug("jumpUid: " + jumpUid);
- }
- }
-
- int ucomp = myUid.compareTo(jumpUid);
- if (ucomp < 0) {
- // need to move forward
- // create range and seek it.
- Text row = ts.topKey.getRow();
- Text cf = ts.topKey.getColumnFamily();
- String cq = ts.topKey.getColumnQualifier().toString().replaceAll(myUid, jumpUid);
- Text cq_text = new Text(cq);
- Key sKey = new Key(row, cf, cq_text);
- Key eKey = null;
- if (parentEndRow != null) {
- eKey = new Key(parentEndRow);
- }
- Range fake = new Range(sKey, true, eKey, false);
- if (log.isDebugEnabled()) {
- log.debug("uid jump, new ts.iter.seek range: " + fake);
- }
- ts.iter.seek(fake, columnFamilies, inclusive);
- ts.setNew();
- advanceToMatch(ts);
- ts.setNew();
-
- if (log.isDebugEnabled()) {
- if (ts.iter.hasTop()) {
- log.debug("ts.iter.topkey: " + ts.iter.getTopKey());
- } else {
- log.debug("ts.iter.topKey is null");
- }
- }
- }// else do nothing, we're ahead of jump key
- }
- }
-
- // ts should have moved, validate this particular ts.
- if (ts.hasTop()) {
- if (overallRange != null) {
- if (overallRange.contains(topKey)) {
- // if (topKey.getRow().compareTo(parentEndRow) < 0) {
- sorted.add(ts);
- }
- } else {
- sorted.add(ts);
- }
- }
- }
- // now get the top key from all TermSources.
- currentTerm = sorted.poll();
- if (log.isDebugEnabled()) {
- log.debug("OI.jump currentTerm = " + currentTerm);
- }
-
- topKey = buildTopKey(currentTerm);
- if (log.isDebugEnabled()) {
- log.debug("OI.jump new topKey >>" + ((topKey == null) ? "false" : topKey) + "<< ");
- }
- return hasTop();
- }
-
- private void printTopKeysForTermSources() {
- if (log.isDebugEnabled()) {
- for (TermSource ts : sources) {
- if (ts != null) {
- if (ts.topKey == null) {
- log.debug(ts.toString() + " topKey is null");
- } else {
- log.debug(ts.toString() + " topKey: " + ts.topKey);
- }
- } else {
- log.debug("ts is null");
- }
- }
-
- if (topKey != null) {
- log.debug("OrIterator current topKey: " + topKey);
- } else {
- log.debug("OrIterator current topKey is null");
- }
- }
- }
-}
[28/39] git commit: ACCUMULO-1876 Add default values for
hadoop.version and httpclient.version to make ivy work.
Posted by ec...@apache.org.
ACCUMULO-1876 Add default values for hadoop.version and httpclient.version to make ivy work.
(cherry picked from commit 20cbf248e27069c0c05a595342de73e9ecc43743)
Differs from original by changing Hadoop version to match current pom and leaving out dependencies not yet declared in profiles.
Reason: bugfix
Author: Josh Elser <el...@apache.org>
Ref: ACCUMULO-1792
Signed-off-by: Eric Newton <er...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6446ad5c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6446ad5c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6446ad5c
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 6446ad5c20fec4b4506a01ac259c26d08974faf4
Parents: c093974
Author: Josh Elser <el...@apache.org>
Authored: Fri Nov 15 16:39:11 2013 -0600
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Nov 25 16:06:44 2013 -0500
----------------------------------------------------------------------
pom.xml | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6446ad5c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index eda8390..d664a5b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -869,6 +869,8 @@
<properties>
<targetJdk>1.6</targetJdk>
<!-- overwritten in profiles hadoop-1.0 or hadoop-2.0 -->
+ <hadoop.version>1.0.4</hadoop.version>
+ <!-- overwritten in profiles hadoop-1.0 or hadoop-2.0 -->
<slf4j.version>1.4.3</slf4j.version>
<zookeeper.version>3.3.1</zookeeper.version>
</properties>
[06/39] ACCUMULO-600 removed wikisearch from trunk
Posted by ec...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/EventFields.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/EventFields.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/EventFields.java
deleted file mode 100644
index bd43088..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/EventFields.java
+++ /dev/null
@@ -1,227 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.parser;
-
-import java.nio.ByteBuffer;
-import java.util.Collection;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.accumulo.examples.wikisearch.parser.EventFields.FieldValue;
-
-import com.esotericsoftware.kryo.CustomSerialization;
-import com.esotericsoftware.kryo.Kryo;
-import com.esotericsoftware.kryo.serialize.ArraySerializer;
-import com.esotericsoftware.kryo.serialize.IntSerializer;
-import com.esotericsoftware.kryo.serialize.StringSerializer;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Multimap;
-import com.google.common.collect.Multiset;
-import com.google.common.collect.SetMultimap;
-
-/**
- * Object used to hold the fields in an event. This is a multimap because fields can be repeated.
- */
-public class EventFields implements SetMultimap<String,FieldValue>, CustomSerialization {
-
- private static boolean kryoInitialized = false;
- private static ArraySerializer valueSerializer = null;
-
- private Multimap<String,FieldValue> map = null;
-
- public static class FieldValue {
- ColumnVisibility visibility;
- byte[] value;
-
- public FieldValue(ColumnVisibility visibility, byte[] value) {
- super();
- this.visibility = visibility;
- this.value = value;
- }
-
- public ColumnVisibility getVisibility() {
- return visibility;
- }
-
- public byte[] getValue() {
- return value;
- }
-
- public void setVisibility(ColumnVisibility visibility) {
- this.visibility = visibility;
- }
-
- public void setValue(byte[] value) {
- this.value = value;
- }
-
- public int size() {
- return visibility.flatten().length + value.length;
- }
-
- @Override
- public String toString() {
- StringBuilder buf = new StringBuilder();
- if (null != visibility)
- buf.append(" visibility: ").append(new String(visibility.flatten()));
- if (null != value)
- buf.append(" value size: ").append(value.length);
- if (null != value)
- buf.append(" value: ").append(new String(value));
- return buf.toString();
- }
-
- }
-
- public EventFields() {
- map = HashMultimap.create();
- }
-
- public int size() {
- return map.size();
- }
-
- public boolean isEmpty() {
- return map.isEmpty();
- }
-
- public boolean containsKey(Object key) {
- return map.containsKey(key);
- }
-
- public boolean containsValue(Object value) {
- return map.containsValue(value);
- }
-
- public boolean containsEntry(Object key, Object value) {
- return map.containsEntry(key, value);
- }
-
- public boolean put(String key, FieldValue value) {
- return map.put(key, value);
- }
-
- public boolean remove(Object key, Object value) {
- return map.remove(key, value);
- }
-
- public boolean putAll(String key, Iterable<? extends FieldValue> values) {
- return map.putAll(key, values);
- }
-
- public boolean putAll(Multimap<? extends String,? extends FieldValue> multimap) {
- return map.putAll(multimap);
- }
-
- public void clear() {
- map.clear();
- }
-
- public Set<String> keySet() {
- return map.keySet();
- }
-
- public Multiset<String> keys() {
- return map.keys();
- }
-
- public Collection<FieldValue> values() {
- return map.values();
- }
-
- public Set<FieldValue> get(String key) {
- return (Set<FieldValue>) map.get(key);
- }
-
- public Set<FieldValue> removeAll(Object key) {
- return (Set<FieldValue>) map.removeAll(key);
- }
-
- public Set<FieldValue> replaceValues(String key, Iterable<? extends FieldValue> values) {
- return (Set<FieldValue>) map.replaceValues(key, values);
- }
-
- public Set<Entry<String,FieldValue>> entries() {
- return (Set<Entry<String,FieldValue>>) map.entries();
- }
-
- public Map<String,Collection<FieldValue>> asMap() {
- return map.asMap();
- }
-
- public int getByteSize() {
- int count = 0;
- for (Entry<String,FieldValue> e : map.entries()) {
- count += e.getKey().getBytes().length + e.getValue().size();
- }
- return count;
- }
-
- @Override
- public String toString() {
- StringBuilder buf = new StringBuilder();
- for (Entry<String,FieldValue> entry : map.entries()) {
- buf.append("\tkey: ").append(entry.getKey()).append(" -> ").append(entry.getValue().toString()).append("\n");
- }
- return buf.toString();
- }
-
- public static synchronized void initializeKryo(Kryo kryo) {
- if (kryoInitialized)
- return;
- valueSerializer = new ArraySerializer(kryo);
- valueSerializer.setDimensionCount(1);
- valueSerializer.setElementsAreSameType(true);
- valueSerializer.setCanBeNull(false);
- valueSerializer.setElementsCanBeNull(false);
- kryo.register(byte[].class, valueSerializer);
- kryoInitialized = true;
- }
-
- public void readObjectData(Kryo kryo, ByteBuffer buf) {
- if (!kryoInitialized)
- EventFields.initializeKryo(kryo);
- // Read in the number of map entries
- int entries = IntSerializer.get(buf, true);
- for (int i = 0; i < entries; i++) {
- // Read in the key
- String key = StringSerializer.get(buf);
- // Read in the fields in the value
- ColumnVisibility vis = new ColumnVisibility(valueSerializer.readObjectData(buf, byte[].class));
- byte[] value = valueSerializer.readObjectData(buf, byte[].class);
- map.put(key, new FieldValue(vis, value));
- }
-
- }
-
- public void writeObjectData(Kryo kryo, ByteBuffer buf) {
- if (!kryoInitialized)
- EventFields.initializeKryo(kryo);
- // Write out the number of entries;
- IntSerializer.put(buf, map.size(), true);
- for (Entry<String,FieldValue> entry : map.entries()) {
- // Write the key
- StringSerializer.put(buf, entry.getKey());
- // Write the fields in the value
- valueSerializer.writeObjectData(buf, entry.getValue().getVisibility().flatten());
- valueSerializer.writeObjectData(buf, entry.getValue().getValue());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/FieldIndexQueryReWriter.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/FieldIndexQueryReWriter.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/FieldIndexQueryReWriter.java
deleted file mode 100644
index acfb4f4..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/FieldIndexQueryReWriter.java
+++ /dev/null
@@ -1,1139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.parser;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Enumeration;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import javax.swing.tree.DefaultMutableTreeNode;
-
-import org.apache.accumulo.examples.wikisearch.parser.QueryParser.QueryTerm;
-import org.apache.accumulo.examples.wikisearch.parser.RangeCalculator.RangeBounds;
-import org.apache.commons.jexl2.parser.ASTAndNode;
-import org.apache.commons.jexl2.parser.ASTEQNode;
-import org.apache.commons.jexl2.parser.ASTERNode;
-import org.apache.commons.jexl2.parser.ASTGENode;
-import org.apache.commons.jexl2.parser.ASTGTNode;
-import org.apache.commons.jexl2.parser.ASTJexlScript;
-import org.apache.commons.jexl2.parser.ASTLENode;
-import org.apache.commons.jexl2.parser.ASTLTNode;
-import org.apache.commons.jexl2.parser.ASTNENode;
-import org.apache.commons.jexl2.parser.ASTNRNode;
-import org.apache.commons.jexl2.parser.ASTNotNode;
-import org.apache.commons.jexl2.parser.ASTOrNode;
-import org.apache.commons.jexl2.parser.ParseException;
-import org.apache.commons.jexl2.parser.ParserTreeConstants;
-import org.apache.hadoop.io.Text;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-
-
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Multimap;
-
-/**
- * The server-side field index queries can only support operations on indexed fields. Additionally, queries that have differing ranges (i.e. one range at the
- * fieldname level and another at the fieldValue level) are not currently supported. This class removes these conflicts from the query as well as sets proper
- * capitalization configurations etc.
- *
- * Once the query has been modified, you can pass it to the BooleanLogicIterator on the server-side via the options map.
- *
- */
-public class FieldIndexQueryReWriter {
-
- protected static final Logger log = Logger.getLogger(FieldIndexQueryReWriter.class);
- public static final String INDEXED_TERMS_LIST = "INDEXED_TERMS_LIST"; // comma separated list of indexed terms.
- public static Set<Integer> rangeNodeSet;
-
- static {
- rangeNodeSet = new HashSet<Integer>();
- rangeNodeSet.add(ParserTreeConstants.JJTLENODE);
- rangeNodeSet.add(ParserTreeConstants.JJTLTNODE);
- rangeNodeSet.add(ParserTreeConstants.JJTGENODE);
- rangeNodeSet.add(ParserTreeConstants.JJTGTNODE);
- rangeNodeSet = Collections.unmodifiableSet(rangeNodeSet);
- }
-
- /*
- * Given a JEXL Query, rewrite it and return it.
- *
- * 1. ParseQuery 2. Transform query 3. Refactor query 4. remove non-indexed terms a. remove any tree conflicts b. collapse any branches 7. add normalized
- * values 8. adjust for case sensitivity 9. add prefix.. but jexl chokes on null byte
- */
- public static void setLogLevel(Level lev) {
- log.setLevel(lev);
- }
-
- /**
- *
- * @param query
- * @param options
- * @return String representation of a given query.
- * @throws ParseException
- * @throws Exception
- */
- public String removeNonIndexedTermsAndInvalidRanges(String query, Map<String,String> options) throws ParseException, Exception {
- Multimap<String,String> indexedTerms = parseIndexedTerms(options);
- RewriterTreeNode node = parseJexlQuery(query);
- if (log.isDebugEnabled()) {
- log.debug("Tree: " + node.getContents());
- }
- node = removeNonIndexedTerms(node, indexedTerms);
- node = removeTreeConflicts(node, indexedTerms);
- node = collapseBranches(node);
- node = removeNegationViolations(node);
-
- if (log.isDebugEnabled()) {
- log.debug("Tree -NonIndexed: " + node.getContents());
- }
- return rebuildQueryFromTree(node);
- }
-
- /**
- *
- * @param query
- * @param options
- * @return String representation of a given query.
- * @throws ParseException
- * @throws Exception
- */
- public String applyNormalizedTerms(String query, Map<String,String> options) throws ParseException, Exception {
- if (log.isDebugEnabled()) {
- log.debug("applyNormalizedTerms, query: " + query);
- }
- Multimap<String,String> normalizedTerms = parseIndexedTerms(options);
- RewriterTreeNode node = parseJexlQuery(query);
- if (log.isDebugEnabled()) {
- log.debug("applyNormalizedTerms, Tree: " + node.getContents());
- }
- node = orNormalizedTerms(node, normalizedTerms);
- if (log.isDebugEnabled()) {
- log.debug("applyNormalizedTerms,Normalized: " + node.getContents());
- }
- return rebuildQueryFromTree(node);
- }
-
- /**
- *
- * @param query
- * @param fNameUpper
- * @param fValueUpper
- * @return String representation of a given query.
- * @throws ParseException
- */
- public String applyCaseSensitivity(String query, boolean fNameUpper, boolean fValueUpper) throws ParseException {
- RewriterTreeNode node = parseJexlQuery(query);
- if (log.isDebugEnabled()) {
- log.debug("Tree: " + node.getContents());
- }
- node = applyCaseSensitivity(node, fNameUpper, fValueUpper);
- if (log.isDebugEnabled()) {
- log.debug("Case: " + node.getContents());
- }
- return rebuildQueryFromTree(node);
- }
-
- private String rebuildQueryFromTree(RewriterTreeNode node) {
- if (node.isLeaf()) {
- String fName = node.getFieldName();
- String fValue = node.getFieldValue();
- String operator = node.getOperator();
- if (node.isNegated()) {
- if (node.getType() == JexlOperatorConstants.JJTEQNODE) {
- operator = JexlOperatorConstants.getOperator(JexlOperatorConstants.JJTNENODE);
- } else if (node.getType() == JexlOperatorConstants.JJTERNODE) {
- operator = JexlOperatorConstants.getOperator(JexlOperatorConstants.JJTNRNODE);
- } else if (node.getType() == JexlOperatorConstants.JJTLTNODE) {
- operator = JexlOperatorConstants.getOperator(JexlOperatorConstants.JJTGENODE);
- } else if (node.getType() == JexlOperatorConstants.JJTLENODE) {
- operator = JexlOperatorConstants.getOperator(JexlOperatorConstants.JJTGTNODE);
- } else if (node.getType() == JexlOperatorConstants.JJTGTNODE) {
- operator = JexlOperatorConstants.getOperator(JexlOperatorConstants.JJTLENODE);
- } else if (node.getType() == JexlOperatorConstants.JJTGENODE) {
- operator = JexlOperatorConstants.getOperator(JexlOperatorConstants.JJTLTNODE);
- }
- }
- return fName + operator + "'" + fValue + "'";
- } else {
- List<String> parts = new ArrayList<String>();
- Enumeration<?> children = node.children();
- while (children.hasMoreElements()) {
- RewriterTreeNode child = (RewriterTreeNode) children.nextElement();
- parts.add(rebuildQueryFromTree(child));
- }
- if (node.getType() == ParserTreeConstants.JJTJEXLSCRIPT) {
- return org.apache.commons.lang.StringUtils.join(parts, "");
- }
- String op = " " + JexlOperatorConstants.getOperator(node.getType()) + " ";
- if (log.isDebugEnabled()) {
- log.debug("Operator: " + op);
- }
- String query = org.apache.commons.lang.StringUtils.join(parts, op);
- query = "(" + query + ")";
- return query;
- }
- }
-
- /*
- * Don't use this, Jexl currently chokes on null bytes in the query
- */
- // public String applyFieldNamePrefix(String query, String prefix) throws ParseException {
- // NuwaveTreeNode node = parseJexlQuery(query);
- // if (log.isDebugEnabled()) {
- // log.debug("Tree: " + node.getContents());
- // }
- // node = applyFieldNamePrefix(node, prefix);
- // if (log.isDebugEnabled()) {
- // log.debug("Prefix: " + node.getContents());
- // }
- // return null;
- // }
- private RewriterTreeNode parseJexlQuery(String query) throws ParseException {
- if (log.isDebugEnabled()) {
- log.debug("parseJexlQuery, query: " + query);
- }
- QueryParser parser = new QueryParser();
- parser.execute(query);
- TreeNode tree = parser.getIteratorTree();
- RewriterTreeNode root = transformTreeNode(tree);
- if (log.isDebugEnabled()) {
- log.debug("parseJexlQuery, transformedTree: " + root.getContents());
- }
- root = refactorTree(root);
- if (log.isDebugEnabled()) {
- log.debug("parseJexlQuery, refactorTree: " + root.getContents());
- }
- return root;
- }
-
- /*
- *
- */
- private RewriterTreeNode transformTreeNode(TreeNode node) throws ParseException {
- if (node.getType().equals(ASTEQNode.class) || node.getType().equals(ASTNENode.class)) {
- if (log.isDebugEnabled()) {
- log.debug("transformTreeNode, Equals Node");
- }
-
- Multimap<String,QueryTerm> terms = node.getTerms();
- for (String fName : terms.keySet()) {
- Collection<QueryTerm> values = terms.get(fName);
-
- for (QueryTerm t : values) {
- if (null == t || null == t.getValue()) {
- continue;
- }
- String fValue = t.getValue().toString();
- fValue = fValue.replaceAll("'", "");
- boolean negated = t.getOperator().equals("!=");
- RewriterTreeNode child = new RewriterTreeNode(ParserTreeConstants.JJTEQNODE, fName, fValue, negated);
- return child;
- }
- }
- }
-
- if (node.getType().equals(ASTERNode.class) || node.getType().equals(ASTNRNode.class)) {
- if (log.isDebugEnabled()) {
- log.debug("transformTreeNode, Regex Node");
- }
-
- Multimap<String,QueryTerm> terms = node.getTerms();
- for (String fName : terms.keySet()) {
- Collection<QueryTerm> values = terms.get(fName);
- for (QueryTerm t : values) {
- if (null == t || null == t.getValue()) {
- continue;
- }
- String fValue = t.getValue().toString();
- fValue = fValue.replaceAll("'", "");
- boolean negated = node.getType().equals(ASTNRNode.class);
- RewriterTreeNode child = new RewriterTreeNode(ParserTreeConstants.JJTERNODE, fName, fValue, negated);
- return child;
- }
- }
- }
-
- if (node.getType().equals(ASTLTNode.class) || node.getType().equals(ASTLENode.class) || node.getType().equals(ASTGTNode.class)
- || node.getType().equals(ASTGENode.class)) {
- if (log.isDebugEnabled()) {
- log.debug("transformTreeNode, LT/LE/GT/GE node");
- }
- Multimap<String,QueryTerm> terms = node.getTerms();
- for (String fName : terms.keySet()) {
- Collection<QueryTerm> values = terms.get(fName);
- for (QueryTerm t : values) {
- if (null == t || null == t.getValue()) {
- continue;
- }
- String fValue = t.getValue().toString();
- fValue = fValue.replaceAll("'", "").toLowerCase();
- boolean negated = false; // to be negated, must be child of Not, which is handled elsewhere.
- int mytype = JexlOperatorConstants.getJJTNodeType(t.getOperator());
- RewriterTreeNode child = new RewriterTreeNode(mytype, fName, fValue, negated);
- return child;
- }
- }
- }
-
- RewriterTreeNode returnNode = null;
-
- if (node.getType().equals(ASTAndNode.class) || node.getType().equals(ASTOrNode.class)) {
- int parentType = node.getType().equals(ASTAndNode.class) ? ParserTreeConstants.JJTANDNODE : ParserTreeConstants.JJTORNODE;
- if (log.isDebugEnabled()) {
- log.debug("transformTreeNode, AND/OR node: " + parentType);
- }
- if (node.isLeaf() || !node.getTerms().isEmpty()) {
- returnNode = new RewriterTreeNode(parentType);
- Multimap<String,QueryTerm> terms = node.getTerms();
- for (String fName : terms.keySet()) {
- Collection<QueryTerm> values = terms.get(fName);
- for (QueryTerm t : values) {
- if (null == t || null == t.getValue()) {
- continue;
- }
- String fValue = t.getValue().toString();
- fValue = fValue.replaceAll("'", "");
- boolean negated = t.getOperator().equals("!=");
- int childType = JexlOperatorConstants.getJJTNodeType(t.getOperator());
- RewriterTreeNode child = new RewriterTreeNode(childType, fName, fValue, negated);
- if (log.isDebugEnabled()) {
- log.debug("adding child node: " + child.getContents());
- }
- returnNode.add(child);
- }
- }
- } else {
- returnNode = new RewriterTreeNode(parentType);
- }
- } else if (node.getType().equals(ASTNotNode.class)) {
- if (log.isDebugEnabled()) {
- log.debug("transformTreeNode, NOT node");
- }
- if (node.isLeaf()) {
- // NOTE: this should be cleaned up a bit.
- Multimap<String,QueryTerm> terms = node.getTerms();
- for (String fName : terms.keySet()) {
- Collection<QueryTerm> values = terms.get(fName);
- for (QueryTerm t : values) {
- if (null == t || null == t.getValue()) {
- continue;
- }
- String fValue = t.getValue().toString();
- fValue = fValue.replaceAll("'", "").toLowerCase();
- boolean negated = !t.getOperator().equals("!=");
- int mytype = JexlOperatorConstants.getJJTNodeType(t.getOperator());
- return new RewriterTreeNode(mytype, fName, fValue, negated);
- }
- }
- } else {
- returnNode = new RewriterTreeNode(ParserTreeConstants.JJTNOTNODE);
- }
- } else if (node.getType().equals(ASTJexlScript.class) || node.getType().getSimpleName().equals("RootNode")) {
-
- if (log.isDebugEnabled()) {
- log.debug("transformTreeNode, ROOT/JexlScript node");
- }
- if (node.isLeaf()) {
- returnNode = new RewriterTreeNode(ParserTreeConstants.JJTJEXLSCRIPT);
- // NOTE: this should be cleaned up a bit.
- Multimap<String,QueryTerm> terms = node.getTerms();
- for (String fName : terms.keySet()) {
- Collection<QueryTerm> values = terms.get(fName);
- for (QueryTerm t : values) {
- if (null == t || null == t.getValue()) {
- continue;
- }
- String fValue = t.getValue().toString();
- fValue = fValue.replaceAll("'", "");
- boolean negated = t.getOperator().equals("!=");
- int mytype = JexlOperatorConstants.getJJTNodeType(t.getOperator());
- RewriterTreeNode child = new RewriterTreeNode(mytype, fName, fValue, negated);
- returnNode.add(child);
- return returnNode;
- }
- }
- } else {
- returnNode = new RewriterTreeNode(ParserTreeConstants.JJTJEXLSCRIPT);
- }
- } else {
- log.error("transformTreeNode, Currently Unsupported Node type: " + node.getClass().getName() + " \t" + node.getType());
- }
- for (TreeNode child : node.getChildren()) {
- returnNode.add(transformTreeNode(child));
- }
-
- return returnNode;
- }
-
- private RewriterTreeNode removeNonIndexedTerms(RewriterTreeNode root, Multimap<String,String> indexedTerms) throws Exception {
- // public void removeNonIndexedTerms(BooleanLogicTreeNodeJexl myroot, String indexedTerms) throws Exception {
- if (indexedTerms.isEmpty()) {
- throw new Exception("removeNonIndexedTerms, indexed Terms empty");
- }
-
- // NOTE: doing a depth first enumeration didn't work when I started
- // removing nodes halfway through. The following method does work,
- // it's essentially a reverse breadth first traversal.
- List<RewriterTreeNode> nodes = new ArrayList<RewriterTreeNode>();
- Enumeration<?> bfe = root.breadthFirstEnumeration();
-
- while (bfe.hasMoreElements()) {
- RewriterTreeNode node = (RewriterTreeNode) bfe.nextElement();
- nodes.add(node);
- }
-
- // walk backwards
- for (int i = nodes.size() - 1; i >= 0; i--) {
- RewriterTreeNode node = nodes.get(i);
- if (log.isDebugEnabled()) {
- log.debug("removeNonIndexedTerms, analyzing node: " + node.toString() + " " + node.printNode());
- }
- if (node.getType() == ParserTreeConstants.JJTANDNODE || node.getType() == ParserTreeConstants.JJTORNODE) {
- // If all of your children are gone, AND/OR has no purpose, remove
- if (node.getChildCount() == 0) {
- node.removeFromParent();
-
- // If AND/OR has only 1 child, attach it to the parent directly.
- } else if (node.getChildCount() == 1) {
- RewriterTreeNode p = (RewriterTreeNode) node.getParent();
- RewriterTreeNode c = (RewriterTreeNode) node.getFirstChild();
- node.removeFromParent();
- p.add(c);
- }
- } else if (node.getType() == ParserTreeConstants.JJTJEXLSCRIPT) { // Head node
- // If head node has no children, we have nothing to search on.
- if (node.getChildCount() == 0) {
- throw new Exception();
- }
- } else if (rangeNodeSet.contains(node.getType())) { // leave it alone
- // leave ranges untouched, they'll be handled elsewhere.
- continue;
- } else {
- if (log.isDebugEnabled()) {
- log.debug("removeNonIndexedTerms, Testing: " + node.getFieldName() + ":" + node.getFieldValue());
- }
-
- if (!indexedTerms.containsKey(node.getFieldName().toString() + ":" + node.getFieldValue().toString())) {
- if (log.isDebugEnabled()) {
- log.debug(node.getFieldName() + ":" + node.getFieldValue() + " is NOT indexed");
- }
- node.removeFromParent();
- } else {
- if (log.isDebugEnabled()) {
- log.debug(node.getFieldName() + ":" + node.getFieldValue() + " is indexed");
- }
- }
- }
- }
-
- return root;
- }
-
- private RewriterTreeNode orNormalizedTerms(RewriterTreeNode myroot, Multimap<String,String> indexedTerms) throws Exception {
- // we have multimap of FieldName to multiple FieldValues
- if (indexedTerms.isEmpty()) {
- throw new Exception("indexed Terms empty");
- }
- try {
- // NOTE: doing a depth first enumeration didn't work when I started
- // removing nodes halfway through. The following method does work,
- // it's essentially a reverse breadth first traversal.
- List<RewriterTreeNode> nodes = new ArrayList<RewriterTreeNode>();
- Enumeration<?> bfe = myroot.breadthFirstEnumeration();
-
- while (bfe.hasMoreElements()) {
- RewriterTreeNode node = (RewriterTreeNode) bfe.nextElement();
- nodes.add(node);
- }
-
- // walk backwards
- for (int i = nodes.size() - 1; i >= 0; i--) {
- RewriterTreeNode node = nodes.get(i);
- if (log.isDebugEnabled()) {
- log.debug("orNormalizedTerms, analyzing node: " + node.toString() + " " + node.printNode());
- }
- if (node.getType() == ParserTreeConstants.JJTANDNODE || node.getType() == ParserTreeConstants.JJTORNODE) {
- continue;
- } else if (node.getType() == ParserTreeConstants.JJTJEXLSCRIPT) {
- if (node.getChildCount() == 0) {
- if (log.isDebugEnabled()) {
- log.debug("orNormalizedTerms: Head node has no children!");
- }
- throw new Exception(); // Head node has no children.
- }
- } else {
- if (log.isDebugEnabled()) {
- log.debug("Testing data location: " + node.getFieldName());
- }
- String fName = node.getFieldName().toString();
- String fValue = node.getFieldValue().toString();
- if (indexedTerms.containsKey(fName + ":" + fValue)) {
-
- if (indexedTerms.get(fName + ":" + fValue).size() > 1) {
- // Replace node with an OR, and make children from the multimap collection
- node.setType(ParserTreeConstants.JJTORNODE);
- boolean neg = node.isNegated();
- node.setNegated(false);
- node.setFieldName(null);
- node.setFieldValue(null);
- Collection<String> values = indexedTerms.get(fName + ":" + fValue);
- for (String value : values) {
- RewriterTreeNode n = new RewriterTreeNode(ParserTreeConstants.JJTEQNODE, fName, value, neg);
- node.add(n);
- }
- } else if (indexedTerms.get(fName + ":" + fValue).size() == 1) {
- // Straight replace
- Collection<String> values = indexedTerms.get(fName + ":" + fValue);
- for (String val : values) {
- // should only be 1
- node.setFieldValue(val);
- }
- }
-
- } else {
- // throw new Exception("orNormalizedTerms, encountered a non-indexed term: " + node.getFieldName().toString());
- }
- }
- }
- } catch (Exception e) {
- log.debug("Caught exception in orNormalizedTerms(): " + e);
- throw new Exception("exception in: orNormalizedTerms");
- }
-
- return myroot;
- }
-
- /***
- * We only want to pass ranges on if they meet a very narrow set of conditions. All ranges must be bounded i.e. x between(1,5) so their parent is an AND. We
- * will only pass a range if 1. The AND is the direct child of HEAD node 2. The AND is a child of an OR which is a direct child of HEAD node.
- *
- * If there is an HEAD-AND[x,OR[b,AND[range]]], and you remove the range, this turns the tree into HEAD-AND[X,OR[B]] which becomes HEAD-AND[X,B] which will
- * miss entries, so you need to cut out the entire OR at this point and let the positive side of the AND pick it up.
- */
- private RewriterTreeNode removeTreeConflicts(RewriterTreeNode root, Multimap<String,String> indexedTerms) {
- if (log.isDebugEnabled()) {
- log.debug("removeTreeConflicts");
- }
-
- /*
- * You can't modify the enumeration, so save it into a list. We want to walk backwards in a breadthFirstEnumeration. So we don't throw null pointers when we
- * erase nodes and shorten our list.
- */
- List<RewriterTreeNode> nodeList = new ArrayList<RewriterTreeNode>();
- Enumeration<?> nodes = root.breadthFirstEnumeration();
- while (nodes.hasMoreElements()) {
- RewriterTreeNode child = (RewriterTreeNode) nodes.nextElement();
- nodeList.add(child);
- }
-
- // walk backwards
- for (int i = nodeList.size() - 1; i >= 0; i--) {
- RewriterTreeNode node = nodeList.get(i);
-
- if (node.isRemoval()) {
- node.removeFromParent();
- continue;
- }
-
- RewriterTreeNode parent = (RewriterTreeNode) node.getParent();
- /*
- * All ranges must be bounded! This means the range must be part of an AND, and the parent of AND must be a HEAD node or an OR whose parent is a HEAD
- * node.
- */
- if (node.getType() == ParserTreeConstants.JJTANDNODE
- && (node.getLevel() == 1 || (parent.getType() == ParserTreeConstants.JJTORNODE && parent.getLevel() == 1))) {
-
- if (log.isDebugEnabled()) {
- log.debug("AND at level 1 or with OR parent at level 1");
- }
- Map<Text,RangeBounds> rangeMap = getBoundedRangeMap(node);
-
- // can't modify the enumeration... save children to a list.
- List<RewriterTreeNode> childList = new ArrayList<RewriterTreeNode>();
- Enumeration<?> children = node.children();
- while (children.hasMoreElements()) {
- RewriterTreeNode child = (RewriterTreeNode) children.nextElement();
- childList.add(child);
- }
-
- for (int j = childList.size() - 1; j >= 0; j--) {
- RewriterTreeNode child = childList.get(j);
- // currently we are not allowing unbounded ranges, so they must sit under an AND node.
- if (rangeNodeSet.contains(child.getType())) {
- if (log.isDebugEnabled()) {
- log.debug("child type: " + JexlOperatorConstants.getOperator(child.getType()));
- }
- if (rangeMap == null) {
- // remove
- child.removeFromParent();
- } else {
- if (!rangeMap.containsKey(new Text(child.getFieldName()))) {
- child.removeFromParent();
- } else {
- // check if it has a single non-range sibling
- boolean singleSib = false;
- if (log.isDebugEnabled()) {
- log.debug("checking for singleSib.");
- }
- Enumeration<?> sibs = child.getParent().children();
- while (sibs.hasMoreElements()) {
- RewriterTreeNode sib = (RewriterTreeNode) sibs.nextElement();
- if (!rangeNodeSet.contains(sib.getType())) {
- singleSib = true;
- break;
- }
- }
- if (singleSib) {
- child.removeFromParent();
- } else {
- if (indexedTerms.containsKey(child.getFieldName() + ":" + child.getFieldValue())) {
- if (log.isDebugEnabled()) {
- log.debug("removeTreeConflicts, node: " + node.getContents());
- }
- // swap parent AND with an OR
- node.removeAllChildren();
- node.setType(ParserTreeConstants.JJTORNODE);
-
- Collection<String> values = indexedTerms.get(child.getFieldName() + ":" + child.getFieldValue());
- for (String value : values) {
- RewriterTreeNode n = new RewriterTreeNode(ParserTreeConstants.JJTEQNODE, child.getFieldName(), value, child.isNegated());
- node.add(n);
- }
- if (log.isDebugEnabled()) {
- log.debug("removeTreeConflicts, node: " + node.getContents());
- }
-
- break;
- } else {
- child.removeFromParent();
- }
-
- }
- }
- }
- }
- }// end inner for
-
- } else { // remove all ranges!
- if (node.isLeaf()) {
- continue;
- }
- // can't modify the enumeration...
- List<RewriterTreeNode> childList = new ArrayList<RewriterTreeNode>();
- Enumeration<?> children = node.children();
- while (children.hasMoreElements()) {
- RewriterTreeNode child = (RewriterTreeNode) children.nextElement();
- childList.add(child);
- }
-
- // walk backwards
- for (int j = childList.size() - 1; j >= 0; j--) {
-
- RewriterTreeNode child = childList.get(j);
- if (log.isDebugEnabled()) {
- log.debug("removeTreeConflicts, looking at node: " + node);
- }
- if (rangeNodeSet.contains(child.getType())) {
- // if grand parent is an OR and not top level, mark whole thing for removal.
- RewriterTreeNode grandParent = (RewriterTreeNode) child.getParent().getParent();
- if (grandParent.getType() == ParserTreeConstants.JJTORNODE && grandParent.getLevel() != 1) {
- grandParent.setRemoval(true);
- }
- child.removeFromParent();
- }
- }
- }
-
- }// end outer for
-
- return root;
- }
-
- private RewriterTreeNode removeNegationViolations(RewriterTreeNode node) throws Exception {
- // Double check the top level node for negation violations
- // if AND, one child must be positive, if OR, no negatives allowed.
- RewriterTreeNode one = (RewriterTreeNode) node.getFirstChild(); // Head node has only 1 child.
- ArrayList<RewriterTreeNode> childrenList = new ArrayList<RewriterTreeNode>();
- Enumeration<?> children = one.children();
- while (children.hasMoreElements()) {
- RewriterTreeNode child = (RewriterTreeNode) children.nextElement();
- childrenList.add(child);
- }
- if (one.getType() == JexlOperatorConstants.JJTORNODE) {
- for (RewriterTreeNode child : childrenList) {
- if (child.isNegated()) {
- child.removeFromParent();
- }
- }
- if (one.getChildCount() == 0) {
- throw new Exception("FieldIndexQueryReWriter: Top level query node cannot be processed.");
- }
- } else if (one.getType() == JexlOperatorConstants.JJTANDNODE) {
- boolean ok = false;
- for (RewriterTreeNode child : childrenList) {
- if (!child.isNegated()) {
- ok = true;
- break;
- }
- }
- if (!ok) {
- throw new Exception("FieldIndexQueryReWriter: Top level query node cannot be processed.");
- }
- }
-
- return node;
- }
-
- // After tree conflicts have been resolve, we can collapse branches where
- // leaves have been pruned.
- private RewriterTreeNode collapseBranches(RewriterTreeNode myroot) throws Exception {
-
- // NOTE: doing a depth first enumeration didn't wory when I started
- // removing nodes halfway through. The following method does work,
- // it's essentially a reverse breadth first traversal.
- List<RewriterTreeNode> nodes = new ArrayList<RewriterTreeNode>();
- Enumeration<?> bfe = myroot.breadthFirstEnumeration();
-
- while (bfe.hasMoreElements()) {
- RewriterTreeNode node = (RewriterTreeNode) bfe.nextElement();
- nodes.add(node);
- }
-
- // walk backwards
- for (int i = nodes.size() - 1; i >= 0; i--) {
- RewriterTreeNode node = nodes.get(i);
- if (log.isDebugEnabled()) {
- log.debug("collapseBranches, inspecting node: " + node.toString() + " " + node.printNode());
- }
-
- if (node.getType() == ParserTreeConstants.JJTANDNODE || node.getType() == ParserTreeConstants.JJTORNODE) {
- if (node.getChildCount() == 0) {
- node.removeFromParent();
- } else if (node.getChildCount() == 1) {
- RewriterTreeNode p = (RewriterTreeNode) node.getParent();
- RewriterTreeNode c = (RewriterTreeNode) node.getFirstChild();
- node.removeFromParent();
- p.add(c);
-
- }
- } else if (node.getType() == ParserTreeConstants.JJTJEXLSCRIPT) {
- if (node.getChildCount() == 0) {
- throw new Exception();
- }
- }
- }
- return myroot;
- }
-
- /**
- * @param options
- */
- public Multimap<String,String> parseIndexedTerms(Map<String,String> options) {
- if (options.get(INDEXED_TERMS_LIST) != null) {
- Multimap<String,String> mmap = HashMultimap.create();
- String[] items = options.get(INDEXED_TERMS_LIST).split(";");
- for (String item : items) {
- item = item.trim();
- if (log.isDebugEnabled()) {}
- String[] parts = item.split(":");
- if (log.isDebugEnabled()) {
- log.debug("adding: " + parts[0]);
- }
- for (int i = 2; i < parts.length; i++) {
- // key is original query token, i.e. color:red
- mmap.put(parts[0] + ":" + parts[1], parts[i]);
-
- }
-
- }
- if (log.isDebugEnabled()) {
- log.debug("multimap: " + mmap);
- }
- return mmap;
- }
- if (log.isDebugEnabled()) {
- log.debug("parseIndexedTerms: returning null");
- }
- return null;
- }
-
- /**
- * @param root
- */
- public RewriterTreeNode refactorTree(RewriterTreeNode root) {
- Enumeration<?> dfe = root.breadthFirstEnumeration();
-
- while (dfe.hasMoreElements()) {
- RewriterTreeNode n = (RewriterTreeNode) dfe.nextElement();
-
- if (n.getType() == ParserTreeConstants.JJTNOTNODE) {// BooleanLogicTreeNode.NodeType.NOT) {
- RewriterTreeNode child = (RewriterTreeNode) n.getChildAt(0);
- child.setNegated(true);
- RewriterTreeNode parent = (RewriterTreeNode) n.getParent();
- parent.remove(n);
- parent.add(child);
- }
- }
-
- // cycle through again and distribute nots
- Enumeration<?> bfe = root.breadthFirstEnumeration();
- RewriterTreeNode child;
-
- while (bfe.hasMoreElements()) {
- child = (RewriterTreeNode) bfe.nextElement();
-
- if (child.isNegated()) {
- if (child.getChildCount() > 0) {
- demorganSubTree(child);
-
- }
- }
- }
- return root;
-
- }
-
- private void demorganSubTree(RewriterTreeNode root) {
-
- root.setNegated(false);
- // root.setChildrenAllNegated(true);
-
- if (root.getType() == ParserTreeConstants.JJTANDNODE) {// BooleanLogicTreeNode.NodeType.AND) {
- // root.setType(BooleanLogicTreeNode.NodeType.OR);
- root.setType(ParserTreeConstants.JJTORNODE);
- } else if (root.getType() == ParserTreeConstants.JJTORNODE) {// BooleanLogicTreeNode.NodeType.OR) {
- // root.setType(BooleanLogicTreeNode.NodeType.AND);
- root.setType(ParserTreeConstants.JJTANDNODE);
- } else if (root.getType() == ParserTreeConstants.JJTEQNODE || root.getType() == ParserTreeConstants.JJTERNODE) {
- // do nothing
- } else {
- log.error("refactorSubTree, node type not supported");
- }
-
- Enumeration<?> children = root.children();
- RewriterTreeNode child = null;
- // now distribute the negative
-
- while (children.hasMoreElements()) {
- child = (RewriterTreeNode) children.nextElement();
- if (child.isNegated()) {
- child.setNegated(false);
- } else {
- child.setNegated(true);
- }
- }
- }
-
- private RewriterTreeNode applyCaseSensitivity(RewriterTreeNode root, boolean fnUpper, boolean fvUpper) {
- // for each leaf, apply case sensitivity
- Enumeration<?> bfe = root.breadthFirstEnumeration();
- while (bfe.hasMoreElements()) {
- RewriterTreeNode node = (RewriterTreeNode) bfe.nextElement();
- if (node.isLeaf()) {
- String fName = fnUpper ? node.getFieldName().toUpperCase() : node.getFieldName().toLowerCase();
- node.setFieldName(fName);
-
- String fValue = fvUpper ? node.getFieldValue().toUpperCase() : node.getFieldValue().toLowerCase();
- node.setFieldValue(fValue);
-
- }
- }
- return root;
- }
-
- private Map<Text,RangeBounds> getBoundedRangeMap(RewriterTreeNode node) {
-
- if (node.getType() == ParserTreeConstants.JJTANDNODE || node.getType() == ParserTreeConstants.JJTORNODE) {
- Enumeration<?> children = node.children();
- Map<Text,RangeBounds> rangeMap = new HashMap<Text,RangeBounds>();
- while (children.hasMoreElements()) {
- RewriterTreeNode child = (RewriterTreeNode) children.nextElement();
- if (child.getType() == ParserTreeConstants.JJTLENODE || child.getType() == ParserTreeConstants.JJTLTNODE) {
- Text fName = new Text(child.getFieldName());
- if (rangeMap.containsKey(fName)) {
- RangeBounds rb = rangeMap.get(fName);
- if (rb.getLower() != null) {
- log.error("testBoundedRangeExistence, two lower bounds exist for bounded range.");
- }
- rb.setLower(new Text(child.getFieldValue()));
- } else {
- RangeBounds rb = new RangeBounds();
- rb.setLower(new Text(child.getFieldValue()));
- rangeMap.put(new Text(child.getFieldName()), rb);
- }
-
- } else if (child.getType() == ParserTreeConstants.JJTGENODE || child.getType() == ParserTreeConstants.JJTGTNODE) {
- Text fName = new Text(child.getFieldName());
- if (rangeMap.containsKey(fName)) {
- RangeBounds rb = rangeMap.get(fName);
- if (rb.getUpper() != null) {
- log.error("testBoundedRangeExistence, two Upper bounds exist for bounded range.");
- }
- rb.setUpper(new Text(child.getFieldValue()));
- } else {
- RangeBounds rb = new RangeBounds();
- rb.setUpper(new Text(child.getFieldValue()));
- rangeMap.put(new Text(child.getFieldName()), rb);
- }
- }
- }
-
- for (Entry<Text,RangeBounds> entry : rangeMap.entrySet()) {
- RangeBounds rb = entry.getValue();
- if (rb.getLower() == null || rb.getUpper() == null) {
- // unbounded range, remove
- if (log.isDebugEnabled()) {
- log.debug("testBoundedRangeExistence: Unbounded Range detected, removing entry from rangeMap");
- }
- rangeMap.remove(entry.getKey());
- }
- }
- if (!rangeMap.isEmpty()) {
- return rangeMap;
- }
- }
-
- return null;
- }
-
- /**
- * INNER CLASSES
- */
- public class RewriterTreeNode extends DefaultMutableTreeNode {
-
- private static final long serialVersionUID = 1L;
- private boolean negated = false;
- private String fieldName;
- private String fieldValue;
- private String operator;
- private int type;
- private boolean removal = false;
-
- /**
- *
- * @param type
- */
- public RewriterTreeNode(int type) {
- super();
- this.type = type;
- }
-
- /**
- *
- * @param type
- * @param fName
- * @param fValue
- */
- public RewriterTreeNode(int type, String fName, String fValue) {
- super();
- init(type, fName, fValue);
- }
-
- /**
- *
- * @param type
- * @param fName
- * @param fValue
- * @param negate
- */
- public RewriterTreeNode(int type, String fName, String fValue, boolean negate) {
- super();
- init(type, fName, fValue, negate);
- }
-
- private void init(int type, String fName, String fValue) {
- init(type, fName, fValue, false);
- }
-
- private void init(int type, String fName, String fValue, boolean negate) {
- this.type = type;
- this.fieldName = fName;
- this.fieldValue = fValue;
- this.negated = negate;
- this.operator = JexlOperatorConstants.getOperator(type);
- if (log.isDebugEnabled()) {
- log.debug("FN: " + this.fieldName + " FV: " + this.fieldValue + " Op: " + this.operator);
- }
- }
-
- /**
- * @return The field name.
- */
- public String getFieldName() {
- return fieldName;
- }
-
- /**
- *
- * @param fieldName
- */
- public void setFieldName(String fieldName) {
- this.fieldName = fieldName;
- }
-
- /**
- *
- * @return The field value.
- */
- public String getFieldValue() {
- return fieldValue;
- }
-
- /**
- *
- * @param fieldValue
- */
- public void setFieldValue(String fieldValue) {
- this.fieldValue = fieldValue;
- }
-
- /**
- *
- * @return true if negated, otherwise false.
- */
- public boolean isNegated() {
- return negated;
- }
-
- /**
- *
- * @param negated
- */
- public void setNegated(boolean negated) {
- this.negated = negated;
- }
-
- /**
- *
- * @return The operator.
- */
- public String getOperator() {
- return operator;
- }
-
- /**
- *
- * @param operator
- */
- public void setOperator(String operator) {
- this.operator = operator;
- }
-
- /**
- *
- * @return The type.
- */
- public int getType() {
- return type;
- }
-
- /**
- *
- * @param type
- */
- public void setType(int type) {
- this.type = type;
- }
-
- public boolean isRemoval() {
- return removal;
- }
-
- public void setRemoval(boolean removal) {
- this.removal = removal;
- }
-
- public String getContents() {
- StringBuilder s = new StringBuilder("[");
- s.append(toString());
-
- if (children != null) {
- Enumeration<?> e = this.children();
- while (e.hasMoreElements()) {
- RewriterTreeNode n = (RewriterTreeNode) e.nextElement();
- s.append(",");
- s.append(n.getContents());
- }
- }
- s.append("]");
- return s.toString();
- }
-
- /**
- *
- * @return A string represenation of the field name and value.
- */
- public String printNode() {
- StringBuilder s = new StringBuilder("[");
- s.append("Full Location & Term = ");
- if (this.fieldName != null) {
- s.append(this.fieldName.toString());
- } else {
- s.append("BlankDataLocation");
- }
- s.append(" ");
- if (this.fieldValue != null) {
- s.append(this.fieldValue.toString());
- } else {
- s.append("BlankTerm");
- }
- s.append("]");
- return s.toString();
- }
-
- @Override
- public String toString() {
- switch (type) {
- case ParserTreeConstants.JJTEQNODE:
- return fieldName + ":" + fieldValue + ":negated=" + isNegated();
- case ParserTreeConstants.JJTNENODE:
- return fieldName + ":" + fieldValue + ":negated=" + isNegated();
- case ParserTreeConstants.JJTERNODE:
- return fieldName + ":" + fieldValue + ":negated=" + isNegated();
- case ParserTreeConstants.JJTNRNODE:
- return fieldName + ":" + fieldValue + ":negated=" + isNegated();
- case ParserTreeConstants.JJTLENODE:
- return fieldName + ":" + fieldValue + ":negated=" + isNegated();
- case ParserTreeConstants.JJTLTNODE:
- return fieldName + ":" + fieldValue + ":negated=" + isNegated();
- case ParserTreeConstants.JJTGENODE:
- return fieldName + ":" + fieldValue + ":negated=" + isNegated();
- case ParserTreeConstants.JJTGTNODE:
- return fieldName + ":" + fieldValue + ":negated=" + isNegated();
- case ParserTreeConstants.JJTJEXLSCRIPT:
- return "HEAD";
- case ParserTreeConstants.JJTANDNODE:
- return "AND";
- case ParserTreeConstants.JJTNOTNODE:
- return "NOT";
- case ParserTreeConstants.JJTORNODE:
- return "OR";
- default:
- System.out.println("Problem in NuwaveTreeNode.toString()");
- return null;
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/JexlOperatorConstants.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/JexlOperatorConstants.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/JexlOperatorConstants.java
deleted file mode 100644
index 29898a3..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/JexlOperatorConstants.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.parser;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import org.apache.commons.jexl2.parser.ASTAndNode;
-
-import org.apache.commons.jexl2.parser.ASTEQNode;
-import org.apache.commons.jexl2.parser.ASTERNode;
-import org.apache.commons.jexl2.parser.ASTFunctionNode;
-import org.apache.commons.jexl2.parser.ASTGENode;
-import org.apache.commons.jexl2.parser.ASTGTNode;
-import org.apache.commons.jexl2.parser.ASTLENode;
-import org.apache.commons.jexl2.parser.ASTLTNode;
-import org.apache.commons.jexl2.parser.ASTNENode;
-import org.apache.commons.jexl2.parser.ASTNRNode;
-import org.apache.commons.jexl2.parser.ASTOrNode;
-import org.apache.commons.jexl2.parser.JexlNode;
-import org.apache.commons.jexl2.parser.ParserTreeConstants;
-
-public class JexlOperatorConstants implements ParserTreeConstants {
-
- private static Map<Class<? extends JexlNode>,String> operatorMap = new ConcurrentHashMap<Class<? extends JexlNode>,String>();
- private static Map<String,Class<? extends JexlNode>> classMap = new ConcurrentHashMap<String,Class<? extends JexlNode>>();
- private static Map<Integer,String> jjtOperatorMap = new ConcurrentHashMap<Integer,String>();
- private static Map<String,Integer> jjtTypeMap = new ConcurrentHashMap<String,Integer>();
-
- static {
- operatorMap.put(ASTEQNode.class, "==");
- operatorMap.put(ASTNENode.class, "!=");
- operatorMap.put(ASTLTNode.class, "<");
- operatorMap.put(ASTLENode.class, "<=");
- operatorMap.put(ASTGTNode.class, ">");
- operatorMap.put(ASTGENode.class, ">=");
- operatorMap.put(ASTERNode.class, "=~");
- operatorMap.put(ASTNRNode.class, "!~");
- operatorMap.put(ASTFunctionNode.class, "f");
- operatorMap.put(ASTAndNode.class, "and");
- operatorMap.put(ASTOrNode.class, "or");
-
- classMap.put("==", ASTEQNode.class);
- classMap.put("!=", ASTNENode.class);
- classMap.put("<", ASTLTNode.class);
- classMap.put("<=", ASTLENode.class);
- classMap.put(">", ASTGTNode.class);
- classMap.put(">=", ASTGENode.class);
- classMap.put("=~", ASTERNode.class);
- classMap.put("!~", ASTNRNode.class);
- classMap.put("f", ASTFunctionNode.class);
-
- jjtOperatorMap.put(JJTEQNODE, "==");
- jjtOperatorMap.put(JJTNENODE, "!=");
- jjtOperatorMap.put(JJTLTNODE, "<");
- jjtOperatorMap.put(JJTLENODE, "<=");
- jjtOperatorMap.put(JJTGTNODE, ">");
- jjtOperatorMap.put(JJTGENODE, ">=");
- jjtOperatorMap.put(JJTERNODE, "=~");
- jjtOperatorMap.put(JJTNRNODE, "!~");
- jjtOperatorMap.put(JJTFUNCTIONNODE, "f");
- jjtOperatorMap.put(JJTANDNODE, "and");
- jjtOperatorMap.put(JJTORNODE, "or");
-
- jjtTypeMap.put("==", JJTEQNODE);
- jjtTypeMap.put("!=", JJTNENODE);
- jjtTypeMap.put("<", JJTLTNODE);
- jjtTypeMap.put("<=", JJTLENODE);
- jjtTypeMap.put(">", JJTGTNODE);
- jjtTypeMap.put(">=", JJTGENODE);
- jjtTypeMap.put("=~", JJTERNODE);
- jjtTypeMap.put("!~", JJTNRNODE);
- jjtTypeMap.put("f", JJTFUNCTIONNODE);
-
- }
-
- public static String getOperator(Class<? extends JexlNode> nodeType) {
- return operatorMap.get(nodeType);
- }
-
- public static String getOperator(Integer jjtNode) {
- return jjtOperatorMap.get(jjtNode);
- }
-
- public static Class<? extends JexlNode> getClass(String operator) {
- return classMap.get(operator);
- }
-
- public static int getJJTNodeType(String operator) {
- return jjtTypeMap.get(operator);
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/QueryEvaluator.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/QueryEvaluator.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/QueryEvaluator.java
deleted file mode 100644
index aaac3d8..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/QueryEvaluator.java
+++ /dev/null
@@ -1,291 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.parser;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-
-import org.apache.accumulo.examples.wikisearch.function.QueryFunctions;
-import org.apache.accumulo.examples.wikisearch.jexl.Arithmetic;
-import org.apache.accumulo.examples.wikisearch.parser.EventFields.FieldValue;
-import org.apache.accumulo.examples.wikisearch.parser.QueryParser.QueryTerm;
-import org.apache.commons.jexl2.Expression;
-import org.apache.commons.jexl2.JexlContext;
-import org.apache.commons.jexl2.JexlEngine;
-import org.apache.commons.jexl2.MapContext;
-import org.apache.commons.jexl2.Script;
-import org.apache.commons.jexl2.parser.ParseException;
-import org.apache.commons.jexl2.parser.ParserTreeConstants;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-
-
-import com.google.common.collect.Multimap;
-
-
-/**
- * This class evaluates events against a query. The query is passed to the constructor and then parsed. It is evaluated against an event in the evaluate method.
- */
-public class QueryEvaluator {
-
- private static Logger log = Logger.getLogger(QueryEvaluator.class);
- // According to the JEXL 2.0 docs, the engine is thread-safe. Let's create 1 engine per VM and
- // cache 128 expressions
- private static JexlEngine engine = new JexlEngine(null, new Arithmetic(false), null, null);
-
- static {
- engine.setSilent(false);
- engine.setCache(128);
- Map<String,Object> functions = new HashMap<String,Object>();
- functions.put("f", QueryFunctions.class);
- engine.setFunctions(functions);
- }
- private String query = null;
- private Set<String> literals = null;
- private Multimap<String,QueryTerm> terms = null;
- private String modifiedQuery = null;
- private JexlContext ctx = new MapContext();
- private boolean caseInsensitive = true;
-
- public QueryEvaluator(String query) throws ParseException {
- this.caseInsensitive = true; // default case insensitive matching.
- if (caseInsensitive) {
- query = query.toLowerCase();
- }
- this.query = query;
- QueryParser parser = new QueryParser();
- parser.execute(query);
- this.terms = parser.getQueryTerms();
- if (caseInsensitive) {
- literals = new HashSet<String>();
- for (String lit : parser.getQueryIdentifiers()) {
- literals.add(lit.toLowerCase());
- }
- } else {
- this.literals = parser.getQueryIdentifiers();
- }
- }
-
- public QueryEvaluator(String query, boolean insensitive) throws ParseException {
- this.caseInsensitive = insensitive;
- if (this.caseInsensitive) {
- query = query.toLowerCase();
- }
- this.query = query;
- QueryParser parser = new QueryParser();
- parser.execute(query);
- this.terms = parser.getQueryTerms();
-
- if (caseInsensitive) {
- literals = new HashSet<String>();
- for (String lit : parser.getQueryIdentifiers()) {
- literals.add(lit.toLowerCase());
- }
- } else {
- this.literals = parser.getQueryIdentifiers();
- }
- }
-
- public String getQuery() {
- return this.query;
- }
-
- public void printLiterals() {
- for (String s : literals) {
- System.out.println("literal: " + s);
- }
- }
-
- public void setLevel(Level lev) {
- log.setLevel(lev);
- }
-
- public StringBuilder rewriteQuery(StringBuilder query, String fieldName, Collection<FieldValue> fieldValues) {
- if (log.isDebugEnabled()) {
- log.debug("rewriteQuery");
- }
- // Here we have a field that has multiple values. In this case we need to put
- // all values into the jexl context as an array and rewrite the query to account for all
- // of the fields.
- if (caseInsensitive) {
- fieldName = fieldName.toLowerCase();
- }
- if (log.isDebugEnabled()) {
- log.debug("Modifying original query: " + query);
- }
- // Pull the values out of the FieldValue object
- String[] values = new String[fieldValues.size()];
- int idx = 0;
- for (FieldValue fv : fieldValues) {
- if (caseInsensitive) {
- values[idx] = (new String(fv.getValue())).toLowerCase();
- } else {
- values[idx] = new String(fv.getValue());
- }
- idx++;
- }
- // Add the array to the context
- ctx.set(fieldName, values);
-
- Collection<QueryTerm> qt = terms.get(fieldName);
-
- // Add a script to the beginning of the query for this multi-valued field
- StringBuilder script = new StringBuilder();
- script.append("_").append(fieldName).append(" = false;\n");
- script.append("for (field : ").append(fieldName).append(") {\n");
-
- for (QueryTerm t : qt) {
- if (!t.getOperator().equals(JexlOperatorConstants.getOperator(ParserTreeConstants.JJTFUNCTIONNODE))) {
- script.append("\tif (_").append(fieldName).append(" == false && field ").append(t.getOperator()).append(" ").append(t.getValue()).append(") { \n");
- } else {
- script.append("\tif (_").append(fieldName).append(" == false && ").append(t.getValue().toString().replace(fieldName, "field")).append(") { \n");
- }
- script.append("\t\t_").append(fieldName).append(" = true;\n");
- script.append("\t}\n");
- }
- script.append("}\n");
-
- // Add the script to the beginning of the query
- query.insert(0, script.toString());
-
- StringBuilder newPredicate = new StringBuilder();
- newPredicate.append("_").append(fieldName).append(" == true");
-
- for (QueryTerm t : qt) {
- // Find the location of this term in the query
- StringBuilder predicate = new StringBuilder();
- int start = 0;
- if (!t.getOperator().equals(JexlOperatorConstants.getOperator(ParserTreeConstants.JJTFUNCTIONNODE))) {
- predicate.append(fieldName).append(" ").append(t.getOperator()).append(" ").append(t.getValue());
- start = query.indexOf(predicate.toString());
- } else {
- predicate.append(t.getValue().toString());
- // need to find the second occurence of the string.
- start = query.indexOf(predicate.toString());
- }
- if (-1 == start) {
- log.warn("Unable to find predicate: " + predicate.toString() + " in rewritten query: " + query.toString());
- }
- int length = predicate.length();
-
- // Now modify the query to check the value of my.fieldName
- query.replace(start, start + length, newPredicate.toString());
- }
-
- if (log.isDebugEnabled()) {
- log.debug("leaving rewriteQuery with: " + query.toString());
- }
- return query;
- }
-
- /**
- * Evaluates the query against an event.
- *
- * @param eventFields
- */
- public boolean evaluate(EventFields eventFields) {
-
- this.modifiedQuery = null;
- boolean rewritten = false;
-
- // Copy the query
- StringBuilder q = new StringBuilder(query);
- // Copy the literals, we are going to remove elements from this set
- // when they are added to the JEXL context. This will allow us to
- // determine which items in the query where *NOT* in the data.
- HashSet<String> literalsCopy = new HashSet<String>(literals);
-
- // Loop through the event fields and add them to the JexlContext.
- for (Entry<String,Collection<FieldValue>> field : eventFields.asMap().entrySet()) {
- String fName = field.getKey();
- if (caseInsensitive) {
- fName = fName.toLowerCase();
- }
- // If this field is not part of the expression, then skip it.
- if (!literals.contains(fName)) {
- continue;
- } else {
- literalsCopy.remove(fName);
- }
-
- // This field may have multiple values.
- if (field.getValue().size() == 0) {
- continue;
- } else if (field.getValue().size() == 1) {
- // We are explicitly converting these bytes to a String.
- if (caseInsensitive) {
- ctx.set(field.getKey().toLowerCase(), (new String(field.getValue().iterator().next().getValue())).toLowerCase());
- } else {
- ctx.set(field.getKey(), new String(field.getValue().iterator().next().getValue()));
- }
-
- } else {
- // q = queryRewrite(q, field.getKey(), field.getValue());
- q = rewriteQuery(q, field.getKey(), field.getValue());
- rewritten = true;
- }// End of if
-
- }// End of loop
-
- // For any literals in the query that were not found in the data, add them to the context
- // with a null value.
- for (String lit : literalsCopy) {
- ctx.set(lit, null);
- }
-
- if (log.isDebugEnabled()) {
- log.debug("Evaluating query: " + q.toString());
- }
-
- this.modifiedQuery = q.toString();
-
- Boolean result = null;
- if (rewritten) {
- Script script = engine.createScript(this.modifiedQuery);
- try {
- result = (Boolean) script.execute(ctx);
- } catch (Exception e) {
- log.error("Error evaluating script: " + this.modifiedQuery + " against event" + eventFields.toString(), e);
- }
- } else {
- Expression expr = engine.createExpression(this.modifiedQuery);
- try {
- result = (Boolean) expr.evaluate(ctx);
- } catch (Exception e) {
- log.error("Error evaluating expression: " + this.modifiedQuery + " against event" + eventFields.toString(), e);
- }
- }
- if (null != result && result) {
- return true;
- } else {
- return false;
- }
- } // End of method
-
- /**
- *
- * @return rewritten query that was evaluated against the most recent event
- */
- public String getModifiedQuery() {
- return this.modifiedQuery;
- }
-}
[16/39] git commit: ACCUMULO-286 added context factory stuff
Posted by ec...@apache.org.
ACCUMULO-286 added context factory stuff
git-svn-id: https://svn.apache.org/repos/asf/accumulo/trunk@1328104 13f79535-47bb-0310-9956-ffa450edef68
(cherry picked from commit 0680b04bf03e2d6ad19ae3c368f6cb23f4e30056)
Reason: Testing
Author: Billie Rinaldi <bi...@apache.org>
Ref: ACCUMULO-1792
Signed-off-by: Eric Newton <er...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7fa0085a
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7fa0085a
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7fa0085a
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 7fa0085a48248e0fcb06ee38064f712026ed9609
Parents: 6bba940
Author: Billie Rinaldi <bi...@apache.org>
Authored: Thu Apr 19 20:22:16 2012 +0000
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Nov 25 16:06:42 2013 -0500
----------------------------------------------------------------------
.../accumulo/core/util/ContextFactory.java | 12 +++++++---
.../helloworld/InsertWithOutputFormat.java | 4 ++--
.../simple/filedata/ChunkInputFormatTest.java | 6 ++---
.../wikisearch/ingest/WikipediaMapperTest.java | 9 +++-----
.../reader/AggregatingRecordReaderTest.java | 24 +++++++++++---------
.../wikisearch/logic/TestQueryLogic.java | 9 ++++----
6 files changed, 34 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7fa0085a/src/core/src/main/java/org/apache/accumulo/core/util/ContextFactory.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/util/ContextFactory.java b/src/core/src/main/java/org/apache/accumulo/core/util/ContextFactory.java
index 61e853d..5a1c2ef 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/util/ContextFactory.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/util/ContextFactory.java
@@ -147,15 +147,21 @@ public class ContextFactory {
}
}
- @SuppressWarnings({"unchecked", "rawtypes"})
public static <K1,V1,K2,V2> Mapper<K1,V1,K2,V2>.Context createMapContext(Mapper<K1,V1,K2,V2> m, TaskAttemptContext tac, RecordReader<K1,V1> reader,
RecordWriter<K2,V2> writer, InputSplit split) {
+ return createMapContext(m, tac, reader, writer, null, null, split);
+ }
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public static <K1,V1,K2,V2> Mapper<K1,V1,K2,V2>.Context createMapContext(Mapper<K1,V1,K2,V2> m, TaskAttemptContext tac, RecordReader<K1,V1> reader,
+ RecordWriter<K2,V2> writer, OutputCommitter committer, StatusReporter reporter, InputSplit split) {
try {
if (useV21) {
- Object basis = MAP_CONTEXT_IMPL_CONSTRUCTOR.newInstance(tac.getConfiguration(), tac.getTaskAttemptID(), reader, writer, null, null, split);
+ Object basis = MAP_CONTEXT_IMPL_CONSTRUCTOR.newInstance(tac.getConfiguration(), tac.getTaskAttemptID(), reader, writer, committer, reporter, split);
return (Mapper.Context) MAP_CONTEXT_CONSTRUCTOR.newInstance((Mapper<K1,V1,K2,V2>) MAP_CONSTRUCTOR.newInstance(), basis);
} else {
- return (Mapper.Context) MAP_CONTEXT_CONSTRUCTOR.newInstance(m, tac.getConfiguration(), tac.getTaskAttemptID(), reader, writer, null, null, split);
+ return (Mapper.Context) MAP_CONTEXT_CONSTRUCTOR.newInstance(m, tac.getConfiguration(), tac.getTaskAttemptID(), reader, writer, committer, reporter,
+ split);
}
} catch (InstantiationException e) {
throw new IllegalArgumentException("Can't create object", e);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7fa0085a/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/InsertWithOutputFormat.java
----------------------------------------------------------------------
diff --git a/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/InsertWithOutputFormat.java b/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/InsertWithOutputFormat.java
index af03470..5c37d6c 100644
--- a/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/InsertWithOutputFormat.java
+++ b/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/InsertWithOutputFormat.java
@@ -45,8 +45,8 @@ public class InsertWithOutputFormat extends Configured implements Tool {
}
Text tableName = new Text(args[4]);
Job job = new Job(getConf());
- AccumuloOutputFormat.setZooKeeperInstance(job, args[0], args[1]);
- AccumuloOutputFormat.setOutputInfo(job, args[3], args[4].getBytes(), true, null);
+ AccumuloOutputFormat.setZooKeeperInstance(job.getConfiguration(), args[0], args[1]);
+ AccumuloOutputFormat.setOutputInfo(job.getConfiguration(), args[2], args[3].getBytes(), true, null);
job.setOutputFormatClass(AccumuloOutputFormat.class);
// when running a mapreduce, you won't need to instantiate the output
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7fa0085a/src/examples/simple/src/test/java/org/apache/accumulo/examples/simple/filedata/ChunkInputFormatTest.java
----------------------------------------------------------------------
diff --git a/src/examples/simple/src/test/java/org/apache/accumulo/examples/simple/filedata/ChunkInputFormatTest.java b/src/examples/simple/src/test/java/org/apache/accumulo/examples/simple/filedata/ChunkInputFormatTest.java
index 8937048..53cadac 100644
--- a/src/examples/simple/src/test/java/org/apache/accumulo/examples/simple/filedata/ChunkInputFormatTest.java
+++ b/src/examples/simple/src/test/java/org/apache/accumulo/examples/simple/filedata/ChunkInputFormatTest.java
@@ -90,7 +90,7 @@ public class ChunkInputFormatTest extends TestCase {
ChunkInputFormat.setMockInstance(job.getConfiguration(), "instance1");
ChunkInputFormat cif = new ChunkInputFormat();
RangeInputSplit ris = new RangeInputSplit();
- TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
+ TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job.getConfiguration());
RecordReader<List<Entry<Key,Value>>,InputStream> rr = cif.createRecordReader(ris, tac);
rr.initialize(ris, tac);
@@ -141,7 +141,7 @@ public class ChunkInputFormatTest extends TestCase {
ChunkInputFormat.setMockInstance(job.getConfiguration(), "instance2");
ChunkInputFormat cif = new ChunkInputFormat();
RangeInputSplit ris = new RangeInputSplit();
- TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
+ TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job.getConfiguration());
RecordReader<List<Entry<Key,Value>>,InputStream> crr = cif.createRecordReader(ris, tac);
crr.initialize(ris, tac);
@@ -180,7 +180,7 @@ public class ChunkInputFormatTest extends TestCase {
ChunkInputFormat.setMockInstance(job.getConfiguration(), "instance3");
ChunkInputFormat cif = new ChunkInputFormat();
RangeInputSplit ris = new RangeInputSplit();
- TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
+ TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job.getConfiguration());
RecordReader<List<Entry<Key,Value>>,InputStream> crr = cif.createRecordReader(ris, tac);
crr.initialize(ris, tac);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7fa0085a/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaMapperTest.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaMapperTest.java b/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaMapperTest.java
index a924aee..c659ec4 100644
--- a/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaMapperTest.java
+++ b/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaMapperTest.java
@@ -34,8 +34,7 @@ import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.examples.wikisearch.ingest.WikipediaConfiguration;
-import org.apache.accumulo.examples.wikisearch.ingest.WikipediaMapper;
+import org.apache.accumulo.core.util.ContextFactory;
import org.apache.accumulo.examples.wikisearch.reader.AggregatingRecordReader;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -47,7 +46,6 @@ import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.OutputCommitter;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
import org.junit.Before;
@@ -118,8 +116,7 @@ public class WikipediaMapperTest {
writerMap.put(new Text(INDEX_TABLE_NAME), c.createBatchWriter(INDEX_TABLE_NAME, 1000L, 1000L, 1));
writerMap.put(new Text(RINDEX_TABLE_NAME), c.createBatchWriter(RINDEX_TABLE_NAME, 1000L, 1000L, 1));
- TaskAttemptID id = new TaskAttemptID();
- TaskAttemptContext context = new TaskAttemptContext(conf, id);
+ TaskAttemptContext context = ContextFactory.createTaskAttemptContext(conf);
RawLocalFileSystem fs = new RawLocalFileSystem();
fs.setConf(conf);
@@ -141,7 +138,7 @@ public class WikipediaMapperTest {
WikipediaMapper mapper = new WikipediaMapper();
// Load data into Mock Accumulo
- Mapper<LongWritable,Text,Text,Mutation>.Context con = mapper.new Context(conf, id, rr, rw, oc, sr, split);
+ Mapper<LongWritable,Text,Text,Mutation>.Context con = ContextFactory.createMapContext(mapper, context, rr, rw, oc, sr, split);
mapper.run(con);
// Flush and close record writers.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7fa0085a/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/reader/AggregatingRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/reader/AggregatingRecordReaderTest.java b/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/reader/AggregatingRecordReaderTest.java
index c1cb263..c842da7 100644
--- a/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/reader/AggregatingRecordReaderTest.java
+++ b/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/reader/AggregatingRecordReaderTest.java
@@ -16,7 +16,10 @@
*/
package org.apache.accumulo.examples.wikisearch.reader;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.File;
import java.io.FileWriter;
@@ -28,13 +31,12 @@ import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathFactory;
+import org.apache.accumulo.core.util.ContextFactory;
import org.apache.accumulo.examples.wikisearch.ingest.WikipediaInputFormat.WikipediaInputSplit;
-import org.apache.accumulo.examples.wikisearch.reader.AggregatingRecordReader;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.junit.Before;
import org.junit.Test;
@@ -95,7 +97,7 @@ public class AggregatingRecordReaderTest {
conf.set(AggregatingRecordReader.START_TOKEN, "<doc");
conf.set(AggregatingRecordReader.END_TOKEN, "</doc>");
conf.set(AggregatingRecordReader.RETURN_PARTIAL_MATCHES, Boolean.toString(true));
- ctx = new TaskAttemptContext(conf, new TaskAttemptID());
+ ctx = ContextFactory.createTaskAttemptContext(conf);
XPath xp = xpFactory.newXPath();
EXPR_A = xp.compile("/doc/a");
EXPR_B = xp.compile("/doc/b");
@@ -141,7 +143,7 @@ public class AggregatingRecordReaderTest {
// Create FileSplit
Path p = new Path(f.toURI().toString());
- WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null),0);
+ WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null), 0);
AggregatingRecordReader reader = new AggregatingRecordReader();
try {
// Clear the values for BEGIN and STOP TOKEN
@@ -163,7 +165,7 @@ public class AggregatingRecordReaderTest {
// Create FileSplit
Path p = new Path(f.toURI().toString());
- WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null),0);
+ WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null), 0);
// Initialize the RecordReader
AggregatingRecordReader reader = new AggregatingRecordReader();
@@ -184,7 +186,7 @@ public class AggregatingRecordReaderTest {
// Create FileSplit
Path p = new Path(f.toURI().toString());
- WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null),0);
+ WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null), 0);
// Initialize the RecordReader
AggregatingRecordReader reader = new AggregatingRecordReader();
@@ -202,7 +204,7 @@ public class AggregatingRecordReaderTest {
// Create FileSplit
Path p = new Path(f.toURI().toString());
- WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null),0);
+ WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null), 0);
// Initialize the RecordReader
AggregatingRecordReader reader = new AggregatingRecordReader();
@@ -220,7 +222,7 @@ public class AggregatingRecordReaderTest {
// Create FileSplit
Path p = new Path(f.toURI().toString());
- WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null),0);
+ WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null), 0);
// Initialize the RecordReader
AggregatingRecordReader reader = new AggregatingRecordReader();
@@ -245,7 +247,7 @@ public class AggregatingRecordReaderTest {
// Create FileSplit
Path p = new Path(f.toURI().toString());
- WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null),0);
+ WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null), 0);
// Initialize the RecordReader
AggregatingRecordReader reader = new AggregatingRecordReader();
@@ -264,7 +266,7 @@ public class AggregatingRecordReaderTest {
File f = createFile(xml5);
// Create FileSplit
Path p = new Path(f.toURI().toString());
- WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null),0);
+ WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(p, 0, f.length(), null), 0);
// Initialize the RecordReader
AggregatingRecordReader reader = new AggregatingRecordReader();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7fa0085a/src/examples/wikisearch/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/TestQueryLogic.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/TestQueryLogic.java b/src/examples/wikisearch/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/TestQueryLogic.java
index 4b7aaee..938f01b 100644
--- a/src/examples/wikisearch/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/TestQueryLogic.java
+++ b/src/examples/wikisearch/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/TestQueryLogic.java
@@ -36,6 +36,7 @@ import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.ContextFactory;
import org.apache.accumulo.examples.wikisearch.ingest.WikipediaConfiguration;
import org.apache.accumulo.examples.wikisearch.ingest.WikipediaInputFormat.WikipediaInputSplit;
import org.apache.accumulo.examples.wikisearch.ingest.WikipediaMapper;
@@ -53,7 +54,6 @@ import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.OutputCommitter;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
import org.apache.log4j.Level;
@@ -125,8 +125,7 @@ public class TestQueryLogic {
writerMap.put(new Text(table), c.createBatchWriter(table, 1000L, 1000L, 1));
}
- TaskAttemptID id = new TaskAttemptID();
- TaskAttemptContext context = new TaskAttemptContext(conf, id);
+ TaskAttemptContext context = ContextFactory.createTaskAttemptContext(conf);
RawLocalFileSystem fs = new RawLocalFileSystem();
fs.setConf(conf);
@@ -137,7 +136,7 @@ public class TestQueryLogic {
Path tmpFile = new Path(data.getAbsolutePath());
// Setup the Mapper
- WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(tmpFile, 0, fs.pathToFile(tmpFile).length(), null),0);
+ WikipediaInputSplit split = new WikipediaInputSplit(new FileSplit(tmpFile, 0, fs.pathToFile(tmpFile).length(), null), 0);
AggregatingRecordReader rr = new AggregatingRecordReader();
Path ocPath = new Path(tmpFile, "oc");
OutputCommitter oc = new FileOutputCommitter(ocPath, context);
@@ -148,7 +147,7 @@ public class TestQueryLogic {
WikipediaMapper mapper = new WikipediaMapper();
// Load data into Mock Accumulo
- Mapper<LongWritable,Text,Text,Mutation>.Context con = mapper.new Context(conf, id, rr, rw, oc, sr, split);
+ Mapper<LongWritable,Text,Text,Mutation>.Context con = ContextFactory.createMapContext(mapper, context, rr, rw, oc, sr, split);
mapper.run(con);
// Flush and close record writers.
[10/39] ACCUMULO-600 removed wikisearch from trunk
Posted by ec...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/AbstractEvaluatingIterator.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/AbstractEvaluatingIterator.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/AbstractEvaluatingIterator.java
deleted file mode 100644
index 87b4da2..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/AbstractEvaluatingIterator.java
+++ /dev/null
@@ -1,323 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.iterator;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.accumulo.core.data.ByteSequence;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.PartialKey;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.IteratorEnvironment;
-import org.apache.accumulo.core.iterators.OptionDescriber;
-import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
-import org.apache.accumulo.examples.wikisearch.parser.EventFields;
-import org.apache.accumulo.examples.wikisearch.parser.QueryEvaluator;
-import org.apache.commons.jexl2.parser.ParseException;
-import org.apache.log4j.Logger;
-
-
-import com.esotericsoftware.kryo.Kryo;
-
-/**
- *
- * This iterator aggregates rows together using the specified key comparator. Subclasses will provide their own implementation of fillMap which will fill the
- * supplied EventFields object with field names (key) and field values (value). After all fields have been put into the aggregated object (by aggregating all
- * columns with the same key), the EventFields object will be compared against the supplied expression. If the expression returns true, then the return key and
- * return value can be retrieved via getTopKey() and getTopValue().
- *
- * Optionally, the caller can set an expression (field operator value) that should not be evaluated against the event. For example, if the query is
- * "A == 'foo' and B == 'bar'", but for some reason B may not be in the data, then setting the UNEVALUATED_EXPRESSIONS option to "B == 'bar'" will allow the
- * events to be evaluated against the remainder of the expression and still return as true.
- *
- * By default this iterator will return all Events in the shard. If the START_DATE and END_DATE are specified, then this iterator will evaluate the timestamp of
- * the key against the start and end dates. If the event date is not within the range of start to end, then it is skipped.
- *
- * This iterator will return up the stack an EventFields object serialized using Kryo in the cell Value.
- *
- */
-public abstract class AbstractEvaluatingIterator implements SortedKeyValueIterator<Key,Value>, OptionDescriber {
-
- private static Logger log = Logger.getLogger(AbstractEvaluatingIterator.class);
- protected static final byte[] NULL_BYTE = new byte[0];
- public static final String QUERY_OPTION = "expr";
- public static final String UNEVALUTED_EXPRESSIONS = "unevaluated.expressions";
-
- private PartialKey comparator = null;
- protected SortedKeyValueIterator<Key,Value> iterator;
- private Key currentKey = new Key();
- private Key returnKey;
- private Value returnValue;
- private String expression;
- private QueryEvaluator evaluator;
- private EventFields event = null;
- private static Kryo kryo = new Kryo();
- private Range seekRange = null;
- private Set<String> skipExpressions = null;
-
- protected AbstractEvaluatingIterator(AbstractEvaluatingIterator other, IteratorEnvironment env) {
- iterator = other.iterator.deepCopy(env);
- event = other.event;
- }
-
- public AbstractEvaluatingIterator() {}
-
- /**
- * Implementations will return the PartialKey value to use for comparing keys for aggregating events
- *
- * @return the type of comparator to use
- */
- public abstract PartialKey getKeyComparator();
-
- /**
- * When the query expression evaluates to true against the event, the event fields will be serialized into the Value and returned up the iterator stack.
- * Implemenations will need to provide a key to be used with the event.
- *
- * @param k
- * @return the key that should be returned with the map of values.
- */
- public abstract Key getReturnKey(Key k) throws Exception;
-
- /**
- * Implementations will need to fill the map with field visibilities, names, and values. When all fields have been aggregated the event will be evaluated
- * against the query expression.
- *
- * @param event
- * Multimap of event names and fields.
- * @param key
- * current Key
- * @param value
- * current Value
- */
- public abstract void fillMap(EventFields event, Key key, Value value) throws Exception;
-
- /**
- * Provides the ability to skip this key and all of the following ones that match using the comparator.
- *
- * @param key
- * @return true if the key should be acted upon, otherwise false.
- * @throws IOException
- */
- public abstract boolean isKeyAccepted(Key key) throws IOException;
-
- /**
- * Reset state.
- */
- public void reset() {
- event.clear();
- }
-
- private void aggregateRowColumn(EventFields event) throws IOException {
-
- currentKey.set(iterator.getTopKey());
-
- try {
- fillMap(event, iterator.getTopKey(), iterator.getTopValue());
- iterator.next();
-
- while (iterator.hasTop() && iterator.getTopKey().equals(currentKey, this.comparator)) {
- fillMap(event, iterator.getTopKey(), iterator.getTopValue());
- iterator.next();
- }
-
- // Get the return key
- returnKey = getReturnKey(currentKey);
- } catch (Exception e) {
- throw new IOException("Error aggregating event", e);
- }
-
- }
-
- private void findTop() throws IOException {
- do {
- reset();
- // check if aggregation is needed
- if (iterator.hasTop()) {
- // Check to see if the current key is accepted. For example in the wiki
- // table there are field index rows. We don't want to process those in
- // some cases so return right away. Consume all of the non-accepted keys
- while (iterator.hasTop() && !isKeyAccepted(iterator.getTopKey())) {
- iterator.next();
- }
-
- if (iterator.hasTop()) {
- aggregateRowColumn(event);
-
- // Evaluate the event against the expression
- if (event.size() > 0 && this.evaluator.evaluate(event)) {
- if (log.isDebugEnabled()) {
- log.debug("Event evaluated to true, key = " + returnKey);
- }
- // Create a byte array
- byte[] serializedMap = new byte[event.getByteSize() + (event.size() * 20)];
- // Wrap in ByteBuffer to work with Kryo
- ByteBuffer buf = ByteBuffer.wrap(serializedMap);
- // Serialize the EventFields object
- event.writeObjectData(kryo, buf);
- // Truncate array to the used size.
- returnValue = new Value(Arrays.copyOfRange(serializedMap, 0, buf.position()));
- } else {
- returnKey = null;
- returnValue = null;
- }
- } else {
- if (log.isDebugEnabled()) {
- log.debug("Iterator no longer has top.");
- }
- }
- } else {
- log.debug("Iterator.hasTop() == false");
- }
- } while (returnValue == null && iterator.hasTop());
-
- // Sanity check. Make sure both returnValue and returnKey are null or both are not null
- if (!((returnKey == null && returnValue == null) || (returnKey != null && returnValue != null))) {
- log.warn("Key: " + ((returnKey == null) ? "null" : returnKey.toString()));
- log.warn("Value: " + ((returnValue == null) ? "null" : returnValue.toString()));
- throw new IOException("Return values are inconsistent");
- }
- }
-
- public Key getTopKey() {
- if (returnKey != null) {
- return returnKey;
- }
- return iterator.getTopKey();
- }
-
- public Value getTopValue() {
- if (returnValue != null) {
- return returnValue;
- }
- return iterator.getTopValue();
- }
-
- public boolean hasTop() {
- return returnKey != null || iterator.hasTop();
- }
-
- public void next() throws IOException {
- if (returnKey != null) {
- returnKey = null;
- returnValue = null;
- } else if (iterator.hasTop()) {
- iterator.next();
- }
-
- findTop();
- }
-
- /**
- * Copy of IteratorUtil.maximizeStartKeyTimeStamp due to IllegalAccessError
- *
- * @param range
- * @return
- */
- static Range maximizeStartKeyTimeStamp(Range range) {
- Range seekRange = range;
-
- if (range.getStartKey() != null && range.getStartKey().getTimestamp() != Long.MAX_VALUE) {
- Key seekKey = new Key(seekRange.getStartKey());
- seekKey.setTimestamp(Long.MAX_VALUE);
- seekRange = new Range(seekKey, true, range.getEndKey(), range.isEndKeyInclusive());
- }
-
- return seekRange;
- }
-
- public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
- // do not want to seek to the middle of a value that should be
- // aggregated...
-
- seekRange = maximizeStartKeyTimeStamp(range);
-
- iterator.seek(seekRange, columnFamilies, inclusive);
- findTop();
-
- if (range.getStartKey() != null) {
- while (hasTop() && getTopKey().equals(range.getStartKey(), this.comparator) && getTopKey().getTimestamp() > range.getStartKey().getTimestamp()) {
- // the value has a more recent time stamp, so
- // pass it up
- // log.debug("skipping "+getTopKey());
- next();
- }
-
- while (hasTop() && range.beforeStartKey(getTopKey())) {
- next();
- }
- }
-
- }
-
- public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
- validateOptions(options);
- event = new EventFields();
- this.comparator = getKeyComparator();
- this.iterator = source;
- try {
- // Replace any expressions that we should not evaluate.
- if (null != this.skipExpressions && this.skipExpressions.size() != 0) {
- for (String skip : this.skipExpressions) {
- // Expression should have form: field<sp>operator<sp>literal.
- // We are going to replace the expression with field == null.
- String field = skip.substring(0, skip.indexOf(" ") - 1);
- this.expression = this.expression.replaceAll(skip, field + " == null");
- }
- }
- this.evaluator = new QueryEvaluator(this.expression);
- } catch (ParseException e) {
- throw new IllegalArgumentException("Failed to parse query", e);
- }
- EventFields.initializeKryo(kryo);
- }
-
- public IteratorOptions describeOptions() {
- Map<String,String> options = new HashMap<String,String>();
- options.put(QUERY_OPTION, "query expression");
- options.put(UNEVALUTED_EXPRESSIONS, "comma separated list of expressions to skip");
- return new IteratorOptions(getClass().getSimpleName(), "evaluates event objects against an expression", options, null);
- }
-
- public boolean validateOptions(Map<String,String> options) {
- if (!options.containsKey(QUERY_OPTION))
- return false;
- else
- this.expression = options.get(QUERY_OPTION);
-
- if (options.containsKey(UNEVALUTED_EXPRESSIONS)) {
- String expressionList = options.get(UNEVALUTED_EXPRESSIONS);
- if (expressionList != null && !expressionList.trim().equals("")) {
- this.skipExpressions = new HashSet<String>();
- for (String e : expressionList.split(","))
- this.skipExpressions.add(e);
- }
- }
- return true;
- }
-
- public String getQueryExpression() {
- return this.expression;
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/AndIterator.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/AndIterator.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/AndIterator.java
deleted file mode 100644
index 734d423..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/AndIterator.java
+++ /dev/null
@@ -1,921 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.iterator;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-
-import org.apache.accumulo.core.data.ArrayByteSequence;
-import org.apache.accumulo.core.data.ByteSequence;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.PartialKey;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.IteratorEnvironment;
-import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
-import org.apache.accumulo.core.util.TextUtil;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.hadoop.io.Text;
-import org.apache.log4j.Logger;
-
-public class AndIterator implements SortedKeyValueIterator<Key,Value> {
-
- protected static final Logger log = Logger.getLogger(AndIterator.class);
- private TermSource[] sources;
- private int sourcesCount = 0;
- protected Text nullText = new Text();
- protected final byte[] emptyByteArray = new byte[0];
- private Key topKey = null;
- protected Value value = new Value(emptyByteArray);
- private Range overallRange;
- private Text currentRow = null;
- private Text currentTerm = new Text(emptyByteArray);
- private Text currentDocID = new Text(emptyByteArray);
- private Text parentEndRow;
- private static boolean SEEK_INCLUSIVE = true;
-
- /**
- * Used in representing a Term that is intersected on.
- */
- protected static class TermSource {
-
- public SortedKeyValueIterator<Key,Value> iter;
- public Text dataLocation;
- public Text term;
- public boolean notFlag;
- private Collection<ByteSequence> seekColumnFamilies;
-
- private TermSource(TermSource other) {
- this(other.iter, other.dataLocation, other.term, other.notFlag);
- }
-
- public TermSource(SortedKeyValueIterator<Key,Value> iter, Text dataLocation, Text term) {
- this(iter, dataLocation, term, false);
- }
-
- public TermSource(SortedKeyValueIterator<Key,Value> iter, Text dataLocation, Text term, boolean notFlag) {
- this.iter = iter;
- this.dataLocation = dataLocation;
- ByteSequence bs = new ArrayByteSequence(dataLocation.getBytes(), 0, dataLocation.getLength());
- this.seekColumnFamilies = Collections.singletonList(bs);
- this.term = term;
- this.notFlag = notFlag;
- }
-
- public String getTermString() {
- return (this.term == null) ? new String("Iterator") : this.term.toString();
- }
- }
-
- /*
- * | Row | Column Family | Column Qualifier | Value
- * | {RowID} | {dataLocation} | {term}\0{dataType}\0{UID} | Empty
- */
- protected Text getPartition(Key key) {
- return key.getRow();
- }
-
- /**
- * Returns the given key's dataLocation
- *
- * @param key
- * @return The given key's dataLocation
- */
- protected Text getDataLocation(Key key) {
- return key.getColumnFamily();
- }
-
- /**
- * Returns the given key's term
- *
- * @param key
- * @return The given key's term
- */
- protected Text getTerm(Key key) {
- int idx = 0;
- String sKey = key.getColumnQualifier().toString();
-
- idx = sKey.indexOf("\0");
- return new Text(sKey.substring(0, idx));
- }
-
- /**
- * Returns the given key's DocID
- *
- * @param key
- * @return The given key's DocID
- */
- protected Text getDocID(Key key) {
- int idx = 0;
- String sKey = key.getColumnQualifier().toString();
-
- idx = sKey.indexOf("\0");
- return new Text(sKey.substring(idx + 1));
- }
-
- /**
- * Returns the given key's UID
- *
- * @param key
- * @return The given key's UID
- */
- protected String getUID(Key key) {
- int idx = 0;
- String sKey = key.getColumnQualifier().toString();
-
- idx = sKey.indexOf("\0");
- return sKey.substring(idx + 1);
- }
-
- /**
- * Build a key from the given row and dataLocation
- *
- * @param row
- * The desired row
- * @param dataLocation
- * The desired dataLocation
- * @return A Key object built from the given row and dataLocation.
- */
- protected Key buildKey(Text row, Text dataLocation) {
- return new Key(row, (dataLocation == null) ? nullText : dataLocation);
- }
-
- /**
- * Build a key from the given row, dataLocation, and term
- *
- * @param row
- * The desired row
- * @param dataLocation
- * The desired dataLocation
- * @param term
- * The desired term
- * @return A Key object built from the given row, dataLocation, and term.
- */
- protected Key buildKey(Text row, Text dataLocation, Text term) {
- return new Key(row, (dataLocation == null) ? nullText : dataLocation, (term == null) ? nullText : term);
- }
-
- /**
- * Return the key that directly follows the given key
- *
- * @param key
- * The key who will be directly before the returned key
- * @return The key directly following the given key.
- */
- protected Key buildFollowingPartitionKey(Key key) {
- return key.followingKey(PartialKey.ROW);
- }
-
- /**
- * Empty default constructor
- */
- public AndIterator() {}
-
- public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
- return new AndIterator(this, env);
- }
-
- public AndIterator(AndIterator other, IteratorEnvironment env) {
- if (other.sources != null) {
- sourcesCount = other.sourcesCount;
- sources = new TermSource[sourcesCount];
- for (int i = 0; i < sourcesCount; i++) {
- sources[i] = new TermSource(other.sources[i].iter.deepCopy(env), other.sources[i].dataLocation, other.sources[i].term);
- }
- }
- }
-
- public Key getTopKey() {
- return topKey;
- }
-
- public Value getTopValue() {
- return value;
- }
-
- public boolean hasTop() {
- return currentRow != null;
- }
-
- /**
- * Find the next key in the current TermSource that is at or beyond the cursor (currentRow, currentTerm, currentDocID).
- *
- * @param sourceID
- * The index of the current source in <code>sources</code>
- * @return True if the source advanced beyond the cursor
- * @throws IOException
- */
- private boolean seekOneSource(TermSource ts) throws IOException {
- /*
- * Within this loop progress must be made in one of the following forms: - currentRow, currentTerm, or curretDocID must be increased - the given source must
- * advance its iterator This loop will end when any of the following criteria are met - the iterator for the given source is pointing to the key
- * (currentRow, columnFamilies[sourceID], currentTerm, currentDocID) - the given source is out of data and currentRow is set to null - the given source has
- * advanced beyond the endRow and currentRow is set to null
- */
-
- // precondition: currentRow is not null
- boolean advancedCursor = false;
-
- while (true) {
- if (ts.iter.hasTop() == false) {
- if (log.isDebugEnabled()) {
- log.debug("The current iterator no longer has a top");
- }
-
- // If we got to the end of an iterator, found a Match if it's a NOT
- if (ts.notFlag) {
- break;
- }
-
- currentRow = null;
- // setting currentRow to null counts as advancing the cursor
- return true;
- }
-
- // check if we're past the end key
- int endCompare = -1;
-
- if (log.isDebugEnabled()) {
- log.debug("Current topKey = " + ts.iter.getTopKey());
- }
-
- // we should compare the row to the end of the range
- if (overallRange.getEndKey() != null) {
- if (log.isDebugEnabled()) {
- log.debug("II.seekOneSource overallRange.getEndKey() != null");
- }
-
- endCompare = overallRange.getEndKey().getRow().compareTo(ts.iter.getTopKey().getRow());
-
- if ((!overallRange.isEndKeyInclusive() && endCompare <= 0) || endCompare < 0) {
- if (log.isDebugEnabled()) {
- log.debug("II.seekOneSource at the end of the tablet server");
- }
-
- currentRow = null;
-
- // setting currentRow to null counts as advancing the cursor
- return true;
- }
- } else {
- if (log.isDebugEnabled()) {
- log.debug("II.seekOneSource overallRange.getEndKey() == null");
- }
- }
-
- // Compare the Row IDs
- int partitionCompare = currentRow.compareTo(getPartition(ts.iter.getTopKey()));
- if (log.isDebugEnabled()) {
- log.debug("Current partition: " + currentRow);
- }
-
- // check if this source is already at or beyond currentRow
- // if not, then seek to at least the current row
- if (partitionCompare > 0) {
- if (log.isDebugEnabled()) {
- log.debug("Need to seek to the current row");
-
- // seek to at least the currentRow
- log.debug("ts.dataLocation = " + ts.dataLocation.getBytes());
- log.debug("Term = " + new Text(ts.term + "\0" + currentDocID).getBytes());
- }
-
- Key seekKey = buildKey(currentRow, ts.dataLocation, nullText);// new Text(ts.term + "\0" + currentDocID));
-
- if (log.isDebugEnabled()) {
- log.debug("Seeking to: " + seekKey);
- }
- ts.iter.seek(new Range(seekKey, true, null, false), ts.seekColumnFamilies, SEEK_INCLUSIVE);
- continue;
- }
-
- // check if this source has gone beyond currentRow
- // if so, advance currentRow
- if (partitionCompare < 0) {
- if (log.isDebugEnabled()) {
- log.debug("Went too far beyond the currentRow");
- }
-
- if (ts.notFlag) {
- break;
- }
-
- currentRow.set(getPartition(ts.iter.getTopKey()));
- currentDocID.set(emptyByteArray);
-
- advancedCursor = true;
- continue;
- }
-
- // we have verified that the current source is positioned in currentRow
- // now we must make sure we're in the right columnFamily in the current row
- if (ts.dataLocation != null) {
- int dataLocationCompare = ts.dataLocation.compareTo(getDataLocation(ts.iter.getTopKey()));
-
- if (log.isDebugEnabled()) {
- log.debug("Comparing dataLocations");
- log.debug("dataLocation = " + ts.dataLocation);
- log.debug("newDataLocation = " + getDataLocation(ts.iter.getTopKey()));
- }
-
- // check if this source is already on the right columnFamily
- // if not, then seek forwards to the right columnFamily
- if (dataLocationCompare > 0) {
- if (log.isDebugEnabled()) {
- log.debug("Need to seek to the right dataLocation");
- }
-
- Key seekKey = buildKey(currentRow, ts.dataLocation, nullText);// , new Text(ts.term + "\0" + currentDocID));
-
- if (log.isDebugEnabled()) {
- log.debug("Seeking to: " + seekKey);
- }
-
- ts.iter.seek(new Range(seekKey, true, null, false), ts.seekColumnFamilies, SEEK_INCLUSIVE);
- if (!ts.iter.hasTop()) {
- currentRow = null;
- return true;
- }
-
- continue;
- }
- // check if this source is beyond the right columnFamily
- // if so, then seek to the next row
- if (dataLocationCompare < 0) {
- if (log.isDebugEnabled()) {
- log.debug("Went too far beyond the dataLocation");
- }
-
- if (endCompare == 0) {
- // we're done
- currentRow = null;
-
- // setting currentRow to null counts as advancing the cursor
- return true;
- }
-
- // Seeking beyond the current dataLocation gives a valid negated result
- if (ts.notFlag) {
- break;
- }
-
- Key seekKey = buildFollowingPartitionKey(ts.iter.getTopKey());
-
- if (log.isDebugEnabled()) {
- log.debug("Seeking to: " + seekKey);
- }
- ts.iter.seek(new Range(seekKey, true, null, false), ts.seekColumnFamilies, SEEK_INCLUSIVE);
- if (!ts.iter.hasTop()) {
- currentRow = null;
- return true;
- }
- continue;
- }
- }
-
- // Compare the Terms
- int termCompare = ts.term.compareTo(getTerm(ts.iter.getTopKey()));
- if (log.isDebugEnabled()) {
- log.debug("term = " + ts.term);
- log.debug("newTerm = " + getTerm(ts.iter.getTopKey()));
- }
-
- // We need to seek down farther into the data
- if (termCompare > 0) {
- if (log.isDebugEnabled()) {
- log.debug("Need to seek to the right term");
- }
- Key seekKey = buildKey(currentRow, ts.dataLocation, new Text(ts.term + "\0"));// new Text(ts.term + "\0" + currentDocID));
-
- if (log.isDebugEnabled()) {
- log.debug("Seeking to: " + seekKey);
- }
-
- ts.iter.seek(new Range(seekKey, true, null, false), ts.seekColumnFamilies, SEEK_INCLUSIVE);
- if (!ts.iter.hasTop()) {
- currentRow = null;
- return true;
- }
-
- // currentTerm = getTerm(ts.iter.getTopKey());
-
- if (log.isDebugEnabled()) {
- log.debug("topKey after seeking to correct term: " + ts.iter.getTopKey());
- }
-
- continue;
- }
-
- // We've jumped out of the current term, set the new term as currentTerm and start looking again
- if (termCompare < 0) {
- if (log.isDebugEnabled()) {
- log.debug("TERM: Need to jump to the next row");
- }
-
- if (endCompare == 0) {
- currentRow = null;
-
- return true;
- }
-
- if (ts.notFlag) {
- break;
- }
-
- Key seekKey = buildFollowingPartitionKey(ts.iter.getTopKey());
- if (log.isDebugEnabled()) {
- log.debug("Using this key to find the next key: " + ts.iter.getTopKey());
- log.debug("Seeking to: " + seekKey);
- }
-
- ts.iter.seek(new Range(seekKey, true, null, false), ts.seekColumnFamilies, SEEK_INCLUSIVE);
-
- if (!ts.iter.hasTop()) {
- currentRow = null;
- return true;
- }
-
- currentTerm = getTerm(ts.iter.getTopKey());
-
- continue;
- }
-
- // Compare the DocIDs
- Text docid = getDocID(ts.iter.getTopKey());
- int docidCompare = currentDocID.compareTo(docid);
-
- if (log.isDebugEnabled()) {
- log.debug("Comparing DocIDs");
- log.debug("currentDocID = " + currentDocID);
- log.debug("docid = " + docid);
- }
-
- // The source isn't at the right DOC
- if (docidCompare > 0) {
- if (log.isDebugEnabled()) {
- log.debug("Need to seek to the correct docid");
- }
-
- // seek forwards
- Key seekKey = buildKey(currentRow, ts.dataLocation, new Text(ts.term + "\0" + currentDocID));
-
- if (log.isDebugEnabled()) {
- log.debug("Seeking to: " + seekKey);
- }
-
- ts.iter.seek(new Range(seekKey, true, null, false), ts.seekColumnFamilies, SEEK_INCLUSIVE);
-
- continue;
- }
-
- // if this source has advanced beyond the current column qualifier then advance currentCQ and return true
- if (docidCompare < 0) {
- if (ts.notFlag) {
- break;
- }
-
- if (log.isDebugEnabled()) {
- log.debug("We went too far, update the currentDocID to be the location of where were seek'ed to");
- }
-
- currentDocID.set(docid);
- advancedCursor = true;
- break;
- }
-
- // Set the term as currentTerm (in case we found this record on the first try)
- currentTerm = getTerm(ts.iter.getTopKey());
-
- if (log.isDebugEnabled()) {
- log.debug("currentTerm = " + currentTerm);
- }
-
- // If we're negated, next() the first TermSource since we guaranteed it was not a NOT term
- if (ts.notFlag) {
- sources[0].iter.next();
- advancedCursor = true;
- }
-
- // If we got here, we have a match
- break;
- }
-
- return advancedCursor;
- }
-
- public void next() throws IOException {
- if (log.isDebugEnabled()) {
- log.debug("In ModifiedIntersectingIterator.next()");
- }
-
- if (currentRow == null) {
- return;
- }
-
- // precondition: the current row is set up and the sources all have the same column qualifier
- // while we don't have a match, seek in the source with the smallest column qualifier
- sources[0].iter.next();
-
- advanceToIntersection();
-
- if (hasTop()) {
- if (overallRange != null && !overallRange.contains(topKey)) {
- topKey = null;
- }
- }
- }
-
- protected void advanceToIntersection() throws IOException {
- if (log.isDebugEnabled()) {
- log.debug("In AndIterator.advanceToIntersection()");
- }
-
- boolean cursorChanged = true;
- while (cursorChanged) {
- // seek all of the sources to at least the highest seen column qualifier in the current row
- cursorChanged = false;
- for (TermSource ts : sources) {
- if (currentRow == null) {
- topKey = null;
- return;
- }
- if (seekOneSource(ts)) {
- cursorChanged = true;
- break;
- }
- }
- }
-
- topKey = buildKey(currentRow, currentTerm, currentDocID);
-
- if (log.isDebugEnabled()) {
- log.debug("ModifiedIntersectingIterator: Got a match: " + topKey);
- }
- }
-
- public static String stringTopKey(SortedKeyValueIterator<Key,Value> iter) {
- if (iter.hasTop()) {
- return iter.getTopKey().toString();
- }
- return "";
- }
-
- public static final String columnFamiliesOptionName = "columnFamilies";
- public static final String termValuesOptionName = "termValues";
- public static final String notFlagsOptionName = "notFlags";
-
- /**
- * Encode a <code>Text</code> array of all the columns to intersect on
- *
- * @param columns
- * The columns to be encoded
- * @return A Base64 encoded string (using a \n delimiter) of all columns to intersect on.
- */
- public static String encodeColumns(Text[] columns) {
- StringBuilder sb = new StringBuilder();
- for (int i = 0; i < columns.length; i++) {
- sb.append(new String(Base64.encodeBase64(TextUtil.getBytes(columns[i]))));
- sb.append('\n');
- }
- return sb.toString();
- }
-
- /**
- * Encode a <code>Text</code> array of all of the terms to intersect on. The terms should match the columns in a one-to-one manner
- *
- * @param terms
- * The terms to be encoded
- * @return A Base64 encoded string (using a \n delimiter) of all terms to intersect on.
- */
- public static String encodeTermValues(Text[] terms) {
- StringBuilder sb = new StringBuilder();
- for (int i = 0; i < terms.length; i++) {
- sb.append(new String(Base64.encodeBase64(TextUtil.getBytes(terms[i]))));
- sb.append('\n');
- }
-
- return sb.toString();
- }
-
- /**
- * Encode an array of <code>booleans</code> denoted which columns are NOT'ed
- *
- * @param flags
- * The array of NOTs
- * @return A base64 encoded string of which columns are NOT'ed
- */
- public static String encodeBooleans(boolean[] flags) {
- byte[] bytes = new byte[flags.length];
- for (int i = 0; i < flags.length; i++) {
- if (flags[i]) {
- bytes[i] = 1;
- } else {
- bytes[i] = 0;
- }
- }
- return new String(Base64.encodeBase64(bytes));
- }
-
- /**
- * Decode the encoded columns into a <code>Text</code> array
- *
- * @param columns
- * The Base64 encoded String of the columns
- * @return A Text array of the decoded columns
- */
- public static Text[] decodeColumns(String columns) {
- String[] columnStrings = columns.split("\n");
- Text[] columnTexts = new Text[columnStrings.length];
- for (int i = 0; i < columnStrings.length; i++) {
- columnTexts[i] = new Text(Base64.decodeBase64(columnStrings[i].getBytes()));
- }
-
- return columnTexts;
- }
-
- /**
- * Decode the encoded terms into a <code>Text</code> array
- *
- * @param terms
- * The Base64 encoded String of the terms
- * @return A Text array of decoded terms.
- */
- public static Text[] decodeTermValues(String terms) {
- String[] termStrings = terms.split("\n");
- Text[] termTexts = new Text[termStrings.length];
- for (int i = 0; i < termStrings.length; i++) {
- termTexts[i] = new Text(Base64.decodeBase64(termStrings[i].getBytes()));
- }
-
- return termTexts;
- }
-
- /**
- * Decode the encoded NOT flags into a <code>boolean</code> array
- *
- * @param flags
- * @return A boolean array of decoded NOT flags
- */
- public static boolean[] decodeBooleans(String flags) {
- // return null of there were no flags
- if (flags == null) {
- return null;
- }
-
- byte[] bytes = Base64.decodeBase64(flags.getBytes());
- boolean[] bFlags = new boolean[bytes.length];
- for (int i = 0; i < bytes.length; i++) {
- if (bytes[i] == 1) {
- bFlags[i] = true;
- } else {
- bFlags[i] = false;
- }
- }
-
- return bFlags;
- }
-
- public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
- if (log.isDebugEnabled()) {
- log.debug("In AndIterator.init()");
- }
-
- Text[] dataLocations = decodeColumns(options.get(columnFamiliesOptionName));
- Text[] terms = decodeTermValues(options.get(termValuesOptionName));
- boolean[] notFlags = decodeBooleans(options.get(notFlagsOptionName));
-
- if (terms.length < 2) {
- throw new IllegalArgumentException("AndIterator requires two or more columns families");
- }
-
- // Scan the not flags.
- // There must be at least one term that isn't negated
- // And we are going to re-order such that the first term is not a ! term
- if (notFlags == null) {
- notFlags = new boolean[terms.length];
- for (int i = 0; i < terms.length; i++) {
- notFlags[i] = false;
- }
- }
-
- // Make sure that the first dataLocation/Term is not a NOT by swapping it with a later dataLocation/Term
- if (notFlags[0]) {
- for (int i = 1; i < notFlags.length; i++) {
- if (notFlags[i] == false) {
- // Swap the terms
- Text swap = new Text(terms[0]);
- terms[0].set(terms[i]);
- terms[i].set(swap);
-
- // Swap the dataLocations
- swap.set(dataLocations[0]);
- dataLocations[0].set(dataLocations[i]);
- dataLocations[i].set(swap);
-
- // Flip the notFlags
- notFlags[0] = false;
- notFlags[i] = true;
- break;
- }
- }
-
- if (notFlags[0]) {
- throw new IllegalArgumentException("AndIterator requires at least one column family without not");
- }
- }
-
- // Build up the array of sources that are to be intersected
- sources = new TermSource[dataLocations.length];
- for (int i = 0; i < dataLocations.length; i++) {
- sources[i] = new TermSource(source.deepCopy(env), dataLocations[i], terms[i], notFlags[i]);
- }
-
- sourcesCount = dataLocations.length;
- }
-
- public void seek(Range range, Collection<ByteSequence> seekColumnFamilies, boolean inclusive) throws IOException {
- if (log.isDebugEnabled()) {
- log.debug("In AndIterator.seek()");
- log.debug("AndIterator.seek Given range => " + range);
- }
- currentRow = new Text();
- currentDocID.set(emptyByteArray);
- doSeek(range);
- }
-
- private void doSeek(Range range) throws IOException {
-
- overallRange = new Range(range);
-
- if (range.getEndKey() != null && range.getEndKey().getRow() != null) {
- this.parentEndRow = range.getEndKey().getRow();
- }
-
- // seek each of the sources to the right column family within the row given by key
- for (int i = 0; i < sourcesCount; i++) {
- Key sourceKey;
- Text dataLocation = (sources[i].dataLocation == null) ? nullText : sources[i].dataLocation;
- if (range.getStartKey() != null) {
- // Build a key with the DocID if one is given
- if (range.getStartKey().getColumnFamily() != null) {
- sourceKey = buildKey(getPartition(range.getStartKey()), dataLocation,
- (sources[i].term == null) ? nullText : new Text(sources[i].term + "\0" + range.getStartKey().getColumnFamily()));
- } // Build a key with just the term.
- else {
- sourceKey = buildKey(getPartition(range.getStartKey()), dataLocation,
- (sources[i].term == null) ? nullText : sources[i].term);
- }
- if (!range.isStartKeyInclusive())
- sourceKey = sourceKey.followingKey(PartialKey.ROW_COLFAM_COLQUAL);
- sources[i].iter.seek(new Range(sourceKey, true, null, false), sources[i].seekColumnFamilies, SEEK_INCLUSIVE);
- } else {
- sources[i].iter.seek(range, sources[i].seekColumnFamilies, SEEK_INCLUSIVE);
- }
- }
-
- advanceToIntersection();
-
- if (hasTop()) {
- if (overallRange != null && !overallRange.contains(topKey)) {
- topKey = null;
- if (log.isDebugEnabled()) {
- log.debug("doSeek, topKey is outside of overall range: " + overallRange);
- }
- }
- }
- }
-
- public void addSource(SortedKeyValueIterator<Key,Value> source, IteratorEnvironment env, Text term, boolean notFlag) {
- addSource(source, env, null, term, notFlag);
- }
-
- public void addSource(SortedKeyValueIterator<Key,Value> source, IteratorEnvironment env, Text dataLocation, Text term, boolean notFlag) {
- // Check if we have space for the added Source
- if (sources == null) {
- sources = new TermSource[1];
- } else {
- // allocate space for node, and copy current tree.
- // TODO: Should we change this to an ArrayList so that we can just add() ?
- TermSource[] localSources = new TermSource[sources.length + 1];
- int currSource = 0;
- for (TermSource myTerm : sources) {
- // TODO: Do I need to call new here? or can I just re-use the term?
- localSources[currSource] = new TermSource(myTerm);
- currSource++;
- }
- sources = localSources;
- }
-
- sources[sourcesCount] = new TermSource(source.deepCopy(env), dataLocation, term, notFlag);
- sourcesCount++;
- }
-
- public boolean jump(Key jumpKey) throws IOException {
- if (log.isDebugEnabled()) {
- log.debug("jump: " + jumpKey);
- }
-
- // is the jumpKey outside my overall range?
- if (parentEndRow != null && parentEndRow.compareTo(jumpKey.getRow()) < 0) {
- // can't go there.
- if (log.isDebugEnabled()) {
- log.debug("jumpRow: " + jumpKey.getRow() + " is greater than my parentEndRow: " + parentEndRow);
- }
- return false;
- }
-
- if (!hasTop()) {
- // TODO: will need to add current/last row if you want to measure if
- // we don't have topkey because we hit end of tablet.
-
- if (log.isDebugEnabled()) {
- log.debug("jump called, but topKey is null, must need to move to next row");
- }
- return false;
- } else {
-
- int comp = this.topKey.getRow().compareTo(jumpKey.getRow());
- // compare rows
- if (comp > 0) {
- if (log.isDebugEnabled()) {
- log.debug("jump, our row is ahead of jumpKey.");
- log.debug("jumpRow: " + jumpKey.getRow() + " myRow: " + topKey.getRow() + " parentEndRow" + parentEndRow);
- }
- return hasTop(); // do nothing, we're ahead of jumpKey row
- } else if (comp < 0) { // a row behind jump key, need to move forward
-
- if (log.isDebugEnabled()) {
- log.debug("II jump, row jump");
- }
- Key endKey = null;
- if (parentEndRow != null) {
- endKey = new Key(parentEndRow);
- }
- Key sKey = new Key(jumpKey.getRow());
- Range fake = new Range(sKey, true, endKey, false);
- this.seek(fake, null, false);
- return hasTop();
- } else {
- // need to check uid
- String myUid = this.topKey.getColumnQualifier().toString();
- String jumpUid = getUID(jumpKey);
- if (log.isDebugEnabled()) {
- if (myUid == null) {
- log.debug("myUid is null");
- } else {
- log.debug("myUid: " + myUid);
- }
-
- if (jumpUid == null) {
- log.debug("jumpUid is null");
- } else {
- log.debug("jumpUid: " + jumpUid);
- }
- }
-
- int ucomp = myUid.compareTo(jumpUid);
- if (ucomp < 0) { // need to move all sources forward
- if (log.isDebugEnabled()) {
- log.debug("jump, uid jump");
- }
- Text row = jumpKey.getRow();
- Range range = new Range(row);
- this.currentRow = row;
- this.currentDocID = new Text(this.getUID(jumpKey));
-
- doSeek(range);
-
- // make sure it is in the range if we have one.
- if (hasTop() && parentEndRow != null && topKey.getRow().compareTo(parentEndRow) > 0) {
- topKey = null;
- }
- if (log.isDebugEnabled() && hasTop()) {
- log.debug("jump, topKey is now: " + topKey);
- }
- return hasTop();
-
- }// else do nothing
- if (hasTop() && parentEndRow != null && topKey.getRow().compareTo(parentEndRow) > 0) {
- topKey = null;
- }
- return hasTop();
- }
- }
- }
-}
[22/39] git commit: ACCUMULO-564 changes for 0.23 compile compatiblity
Posted by ec...@apache.org.
ACCUMULO-564 changes for 0.23 compile compatiblity
git-svn-id: https://svn.apache.org/repos/asf/accumulo/trunk@1332674 13f79535-47bb-0310-9956-ffa450edef68
(cherry picked from commit a5765d111e67b5f1cd645909285f729b902c9b6a)
Reason: Hadoop2 Compat
Author: Billie Rinaldi <bi...@apache.org>
Ref: ACCUMULO-1792
This is a partial backport from 1.5 that excludes the wiki-search example portions but includes
changes to Accumulo.java and DefaultServlet.java
A few minor 1.4.3 specific hacks to get compliation to work against hadoop 2.0.2-alpha
- CoordinateRecoveryTask.java and LogSort.java
Author: Jonathan M Hsieh <jo...@cloudera.com>
Signed-off-by: Eric Newton <er...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/31e4dd15
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/31e4dd15
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/31e4dd15
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 31e4dd15f5cb8bc85372dd66849e3452bae3ee7f
Parents: 2c83ca3
Author: Jonathan M Hsieh <jo...@cloudera.com>
Authored: Thu May 30 17:41:58 2013 -0700
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Nov 25 16:06:42 2013 -0500
----------------------------------------------------------------------
.../src/main/java/org/apache/accumulo/server/Accumulo.java | 1 -
.../apache/accumulo/server/master/CoordinateRecoveryTask.java | 3 +++
.../main/java/org/apache/accumulo/server/master/LogSort.java | 2 +-
.../accumulo/server/monitor/servlets/DefaultServlet.java | 6 +++---
4 files changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/31e4dd15/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java b/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
index 32462b7..253962b 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
@@ -43,7 +43,6 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction;
import org.apache.log4j.Logger;
import org.apache.log4j.helpers.LogLog;
import org.apache.log4j.xml.DOMConfigurator;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/31e4dd15/src/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java b/src/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java
index 79065a2..64ed42e 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java
@@ -257,6 +257,9 @@ public class CoordinateRecoveryTask implements Runnable {
return new RecoveryStatus(logFile.server, logFile.file, (sortJob == null ? 0. : sortJob.mapProgress()), (sortJob == null ? 0.
: sortJob.reduceProgress()), (int) (System.currentTimeMillis() - copyStartTime), (sortJob != null) ? 1. : (copySize == 0 ? 0 : copiedSoFar()
/ (double) copySize));
+ } catch (InterruptedException ie) {
+ // Hadoop 2.0.2-alpha's Job.mapProgress throws Interrupted Exception. 1.x and 2.0.4 do not.
+ return new RecoveryStatus(logFile.server, logFile.file, 1.0, 1.0, (int) (System.currentTimeMillis() - copyStartTime), 1.0);
} catch (NullPointerException npe) {
return new RecoveryStatus(logFile.server, logFile.file, 1.0, 1.0, (int) (System.currentTimeMillis() - copyStartTime), 1.0);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/31e4dd15/src/server/src/main/java/org/apache/accumulo/server/master/LogSort.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/master/LogSort.java b/src/server/src/main/java/org/apache/accumulo/server/master/LogSort.java
index 1e7f29b..006d06e 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/master/LogSort.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/master/LogSort.java
@@ -81,8 +81,8 @@ public class LogSort extends Configured implements Tool {
@Override
public void abortTask(TaskAttemptContext context) {
- super.abortTask(context);
try {
+ super.abortTask(context);
outputFileSystem.delete(outputPath, true);
} catch (IOException ex) {
throw new RuntimeException(ex);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/31e4dd15/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java b/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java
index 0eb2e45..465b253 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java
@@ -53,7 +53,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-import org.apache.hadoop.hdfs.protocol.FSConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.mapred.ClusterStatus;
import org.apache.hadoop.mapred.JobClient;
@@ -307,8 +307,8 @@ public class DefaultServlet extends BasicServlet {
tableRow(sb, (highlight = !highlight), "Unreplicated Capacity", bytes(fs.getRawCapacity()));
tableRow(sb, (highlight = !highlight), "% Used", NumberType.commas(fs.getRawUsed() * 100. / fs.getRawCapacity(), 0, 90, 0, 100) + "%");
tableRow(sb, (highlight = !highlight), "Corrupt Blocks", NumberType.commas(fs.getCorruptBlocksCount(), 0, 0));
- DatanodeInfo[] liveNodes = fs.getClient().datanodeReport(DatanodeReportType.LIVE);
- DatanodeInfo[] deadNodes = fs.getClient().datanodeReport(DatanodeReportType.DEAD);
+ DatanodeInfo[] liveNodes = fs.getClient().datanodeReport(FSConstants.DatanodeReportType.LIVE);
+ DatanodeInfo[] deadNodes = fs.getClient().datanodeReport(FSConstants.DatanodeReportType.DEAD);
tableRow(sb, (highlight = !highlight), "<a href='" + liveUrl + "'>Live Data Nodes</a>", NumberType.commas(liveNodes.length));
tableRow(sb, (highlight = !highlight), "<a href='" + deadUrl + "'>Dead Data Nodes</a>", NumberType.commas(deadNodes.length));
long count = 0;
[29/39] git commit: ACCUMULO-804 pick up the new location of the
hadoop jars
Posted by ec...@apache.org.
ACCUMULO-804 pick up the new location of the hadoop jars
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.5@1456640 13f79535-47bb-0310-9956-ffa450edef68
(cherry picked from commit 21780bd2950df7e72a8b1bd3ddef5017b07b63ad)
Reason: Hadoop2 Compat
Author: Eric C. Newton <ec...@apache.org>
Ref: ACCUMULO-1792
Skips changes to MiniAccumuloCluster that were already present in backport.
Signed-off-by: Eric Newton <er...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4f7fb04b
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4f7fb04b
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4f7fb04b
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 4f7fb04b0b485377daa7d0bd0985994d1ce86173
Parents: 3a35096
Author: Sean Busbey <bu...@cloudera.com>
Authored: Sun Oct 27 23:24:49 2013 -0400
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Nov 25 16:06:44 2013 -0500
----------------------------------------------------------------------
bin/accumulo | 17 +++++++++++++++--
.../1GB/native-standalone/accumulo-env.sh | 2 ++
.../1GB/native-standalone/accumulo-site.xml | 7 +++++++
conf/examples/1GB/standalone/accumulo-env.sh | 2 ++
conf/examples/1GB/standalone/accumulo-site.xml | 7 +++++++
.../2GB/native-standalone/accumulo-env.sh | 2 ++
.../2GB/native-standalone/accumulo-site.xml | 7 +++++++
conf/examples/2GB/standalone/accumulo-env.sh | 2 ++
.../3GB/native-standalone/accumulo-env.sh | 2 ++
.../3GB/native-standalone/accumulo-site.xml | 7 +++++++
conf/examples/3GB/standalone/accumulo-env.sh | 2 ++
conf/examples/3GB/standalone/accumulo-site.xml | 7 +++++++
.../512MB/native-standalone/accumulo-env.sh | 2 ++
.../512MB/native-standalone/accumulo-site.xml | 7 +++++++
conf/examples/512MB/standalone/accumulo-env.sh | 2 ++
conf/examples/512MB/standalone/accumulo-site.xml | 7 +++++++
.../start/classloader/AccumuloClassLoader.java | 18 ++++++++++++------
17 files changed, 92 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f7fb04b/bin/accumulo
----------------------------------------------------------------------
diff --git a/bin/accumulo b/bin/accumulo
index bbca3f0..e2a89e9 100755
--- a/bin/accumulo
+++ b/bin/accumulo
@@ -131,6 +131,19 @@ CLASSPATH=${XML_FILES}:${START_JAR}:${COMMONS_JCI_JARS}
JAVA=$JAVA_HOME/bin/java
PLATFORM="`$JAVA -cp $CLASSPATH org.apache.accumulo.start.Platform`"
-
+# hadoop 1.0, 1.1
+LIB_PATH="$HADOOP_PREFIX/lib/native/$PLATFORM"
+if [ ! -d "$LIB_PATH" ]; then
+ # hadoop-2.0
+ LIB_PATH=$HADOOP_PREFIX/lib/native
+fi
+#
# app isn't used anywhere, but it makes the process easier to spot when ps/top/snmp truncate the command line
-exec $JAVA "-Dapp=$1" -classpath $CLASSPATH $ACCUMULO_OPTS -XX:OnOutOfMemoryError="${ACCUMULO_KILL_CMD:-kill -9 %p}" -Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl -Djava.library.path="$HADOOP_PREFIX/lib/native/$PLATFORM" -Dorg.apache.accumulo.core.home.dir="$ACCUMULO_HOME" -Dhadoop.home.dir="$HADOOP_PREFIX" -Dhadoop.conf.dir="$HADOOP_CONF_DIR" -Dzookeeper.home.dir="$ZOOKEEPER_HOME" org.apache.accumulo.start.Main "$@"
+exec $JAVA "-Dapp=$1" $ACCUMULO_OPTS -classpath $CLASSPATH -XX:OnOutOfMemoryError="${ACCUMULO_KILL_CMD:-kill -9 %p}" \
+ -Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl \
+ -Djava.library.path="$LIB_PATH" \
+ -Dorg.apache.accumulo.core.home.dir="$ACCUMULO_HOME" \
+ -Dhadoop.home.dir="$HADOOP_PREFIX" \
+ -Dzookeeper.home.dir="$ZOOKEEPER_HOME" \
+ org.apache.accumulo.start.Main \
+ "$@"
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f7fb04b/conf/examples/1GB/native-standalone/accumulo-env.sh
----------------------------------------------------------------------
diff --git a/conf/examples/1GB/native-standalone/accumulo-env.sh b/conf/examples/1GB/native-standalone/accumulo-env.sh
index 15caef6..e301eb8 100755
--- a/conf/examples/1GB/native-standalone/accumulo-env.sh
+++ b/conf/examples/1GB/native-standalone/accumulo-env.sh
@@ -35,6 +35,8 @@ else
unset HADOOP_HOME
fi
test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/conf"
+# hadoop-2.0:
+# test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/etc/hadoop"
test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f7fb04b/conf/examples/1GB/native-standalone/accumulo-site.xml
----------------------------------------------------------------------
diff --git a/conf/examples/1GB/native-standalone/accumulo-site.xml b/conf/examples/1GB/native-standalone/accumulo-site.xml
index 97f70d5..e995bfe 100644
--- a/conf/examples/1GB/native-standalone/accumulo-site.xml
+++ b/conf/examples/1GB/native-standalone/accumulo-site.xml
@@ -94,6 +94,13 @@
<property>
<name>general.classpaths</name>
+ <!--
+ Add the following for hadoop-2.0
+ $HADOOP_PREFIX/share/hadoop/common/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,
+ -->
<value>
$ACCUMULO_HOME/src/server/target/classes/,
$ACCUMULO_HOME/src/core/target/classes/,
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f7fb04b/conf/examples/1GB/standalone/accumulo-env.sh
----------------------------------------------------------------------
diff --git a/conf/examples/1GB/standalone/accumulo-env.sh b/conf/examples/1GB/standalone/accumulo-env.sh
index 5c54a39..9abb597 100755
--- a/conf/examples/1GB/standalone/accumulo-env.sh
+++ b/conf/examples/1GB/standalone/accumulo-env.sh
@@ -35,6 +35,8 @@ else
unset HADOOP_HOME
fi
test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/conf"
+# hadoop-2.0:
+# test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/etc/hadoop"
test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f7fb04b/conf/examples/1GB/standalone/accumulo-site.xml
----------------------------------------------------------------------
diff --git a/conf/examples/1GB/standalone/accumulo-site.xml b/conf/examples/1GB/standalone/accumulo-site.xml
index f857fd0..cf081a7 100644
--- a/conf/examples/1GB/standalone/accumulo-site.xml
+++ b/conf/examples/1GB/standalone/accumulo-site.xml
@@ -89,6 +89,13 @@
<property>
<name>general.classpaths</name>
+ <!--
+ Add the following for hadoop-2.0
+ $HADOOP_PREFIX/share/hadoop/common/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,
+ -->
<value>
$ACCUMULO_HOME/src/server/target/classes/,
$ACCUMULO_HOME/src/core/target/classes/,
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f7fb04b/conf/examples/2GB/native-standalone/accumulo-env.sh
----------------------------------------------------------------------
diff --git a/conf/examples/2GB/native-standalone/accumulo-env.sh b/conf/examples/2GB/native-standalone/accumulo-env.sh
index 4f93bb4..701d6e5 100755
--- a/conf/examples/2GB/native-standalone/accumulo-env.sh
+++ b/conf/examples/2GB/native-standalone/accumulo-env.sh
@@ -35,6 +35,8 @@ else
unset HADOOP_HOME
fi
test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/conf"
+# hadoop-2.0:
+# test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/etc/hadoop"
test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java
test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/path/to/zookeeper
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f7fb04b/conf/examples/2GB/native-standalone/accumulo-site.xml
----------------------------------------------------------------------
diff --git a/conf/examples/2GB/native-standalone/accumulo-site.xml b/conf/examples/2GB/native-standalone/accumulo-site.xml
index 3988e66..835e0fb 100644
--- a/conf/examples/2GB/native-standalone/accumulo-site.xml
+++ b/conf/examples/2GB/native-standalone/accumulo-site.xml
@@ -84,6 +84,13 @@
<property>
<name>general.classpaths</name>
+ <!--
+ Add the following for hadoop-2.0
+ $HADOOP_PREFIX/share/hadoop/common/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,
+ -->
<value>
$ACCUMULO_HOME/src/server/target/classes/,
$ACCUMULO_HOME/src/core/target/classes/,
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f7fb04b/conf/examples/2GB/standalone/accumulo-env.sh
----------------------------------------------------------------------
diff --git a/conf/examples/2GB/standalone/accumulo-env.sh b/conf/examples/2GB/standalone/accumulo-env.sh
index 4595783..adb95b1 100755
--- a/conf/examples/2GB/standalone/accumulo-env.sh
+++ b/conf/examples/2GB/standalone/accumulo-env.sh
@@ -35,6 +35,8 @@ else
unset HADOOP_HOME
fi
test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/conf"
+# hadoop-2.0:
+# test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/etc/hadoop"
test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f7fb04b/conf/examples/3GB/native-standalone/accumulo-env.sh
----------------------------------------------------------------------
diff --git a/conf/examples/3GB/native-standalone/accumulo-env.sh b/conf/examples/3GB/native-standalone/accumulo-env.sh
index d5484be..51f96a1 100755
--- a/conf/examples/3GB/native-standalone/accumulo-env.sh
+++ b/conf/examples/3GB/native-standalone/accumulo-env.sh
@@ -35,6 +35,8 @@ else
unset HADOOP_HOME
fi
test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/conf"
+# hadoop-2.0:
+# test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/etc/hadoop"
test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java
test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/path/to/zookeeper
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f7fb04b/conf/examples/3GB/native-standalone/accumulo-site.xml
----------------------------------------------------------------------
diff --git a/conf/examples/3GB/native-standalone/accumulo-site.xml b/conf/examples/3GB/native-standalone/accumulo-site.xml
index 4509c96..5709556 100644
--- a/conf/examples/3GB/native-standalone/accumulo-site.xml
+++ b/conf/examples/3GB/native-standalone/accumulo-site.xml
@@ -79,6 +79,13 @@
<property>
<name>general.classpaths</name>
+ <!--
+ Add the following for hadoop-2.0
+ $HADOOP_PREFIX/share/hadoop/common/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,
+ -->
<value>
$ACCUMULO_HOME/src/server/target/classes/,
$ACCUMULO_HOME/src/core/target/classes/,
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f7fb04b/conf/examples/3GB/standalone/accumulo-env.sh
----------------------------------------------------------------------
diff --git a/conf/examples/3GB/standalone/accumulo-env.sh b/conf/examples/3GB/standalone/accumulo-env.sh
index 10b1803..ad89755 100755
--- a/conf/examples/3GB/standalone/accumulo-env.sh
+++ b/conf/examples/3GB/standalone/accumulo-env.sh
@@ -35,6 +35,8 @@ else
unset HADOOP_HOME
fi
test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/conf"
+# hadoop-2.0:
+# test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/etc/hadoop"
test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java
test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/path/to/zookeeper
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f7fb04b/conf/examples/3GB/standalone/accumulo-site.xml
----------------------------------------------------------------------
diff --git a/conf/examples/3GB/standalone/accumulo-site.xml b/conf/examples/3GB/standalone/accumulo-site.xml
index 4509c96..5709556 100644
--- a/conf/examples/3GB/standalone/accumulo-site.xml
+++ b/conf/examples/3GB/standalone/accumulo-site.xml
@@ -79,6 +79,13 @@
<property>
<name>general.classpaths</name>
+ <!--
+ Add the following for hadoop-2.0
+ $HADOOP_PREFIX/share/hadoop/common/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,
+ -->
<value>
$ACCUMULO_HOME/src/server/target/classes/,
$ACCUMULO_HOME/src/core/target/classes/,
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f7fb04b/conf/examples/512MB/native-standalone/accumulo-env.sh
----------------------------------------------------------------------
diff --git a/conf/examples/512MB/native-standalone/accumulo-env.sh b/conf/examples/512MB/native-standalone/accumulo-env.sh
index 750da89..b3b733d 100755
--- a/conf/examples/512MB/native-standalone/accumulo-env.sh
+++ b/conf/examples/512MB/native-standalone/accumulo-env.sh
@@ -35,6 +35,8 @@ else
unset HADOOP_HOME
fi
test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/conf"
+# hadoop-2.0:
+# test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/etc/hadoop"
test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java
test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/path/to/zookeeper
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f7fb04b/conf/examples/512MB/native-standalone/accumulo-site.xml
----------------------------------------------------------------------
diff --git a/conf/examples/512MB/native-standalone/accumulo-site.xml b/conf/examples/512MB/native-standalone/accumulo-site.xml
index 7d22ab2..7f7d88d 100644
--- a/conf/examples/512MB/native-standalone/accumulo-site.xml
+++ b/conf/examples/512MB/native-standalone/accumulo-site.xml
@@ -89,6 +89,13 @@
<property>
<name>general.classpaths</name>
+ <!--
+ Add the following for hadoop-2.0
+ $HADOOP_PREFIX/share/hadoop/common/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,
+ -->
<value>
$ACCUMULO_HOME/src/server/target/classes/,
$ACCUMULO_HOME/src/core/target/classes/,
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f7fb04b/conf/examples/512MB/standalone/accumulo-env.sh
----------------------------------------------------------------------
diff --git a/conf/examples/512MB/standalone/accumulo-env.sh b/conf/examples/512MB/standalone/accumulo-env.sh
index bf21762..975e20c 100755
--- a/conf/examples/512MB/standalone/accumulo-env.sh
+++ b/conf/examples/512MB/standalone/accumulo-env.sh
@@ -35,6 +35,8 @@ else
unset HADOOP_HOME
fi
test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/conf"
+# hadoop-2.0:
+# test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/etc/hadoop"
test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java
test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/path/to/zookeeper
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f7fb04b/conf/examples/512MB/standalone/accumulo-site.xml
----------------------------------------------------------------------
diff --git a/conf/examples/512MB/standalone/accumulo-site.xml b/conf/examples/512MB/standalone/accumulo-site.xml
index 34868b5..99a5153 100644
--- a/conf/examples/512MB/standalone/accumulo-site.xml
+++ b/conf/examples/512MB/standalone/accumulo-site.xml
@@ -89,6 +89,13 @@
<property>
<name>general.classpaths</name>
+ <!--
+ Add the following for hadoop-2.0
+ $HADOOP_PREFIX/share/hadoop/common/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,
+ $HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,
+ -->
<value>
$ACCUMULO_HOME/src/server/target/classes/,
$ACCUMULO_HOME/src/core/target/classes/,
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4f7fb04b/src/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
----------------------------------------------------------------------
diff --git a/src/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java b/src/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
index ff02d13..5a30b78 100644
--- a/src/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
+++ b/src/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
@@ -106,12 +106,18 @@ public class AccumuloClassLoader {
public static final String DYNAMIC_CLASSPATH_PROPERTY_NAME = "general.dynamic.classpaths";
- public static final String ACCUMULO_CLASSPATH_VALUE =
- "$ACCUMULO_CONF_DIR,\n"
- + "$ACCUMULO_HOME/lib/[^.].*.jar,\n"
- + "$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n"
- + "$HADOOP_PREFIX/[^.].*.jar,\n" + "$HADOOP_CONF_DIR,\n"
- + "$HADOOP_PREFIX/lib/[^.].*.jar,\n";
+ public static final String ACCUMULO_CLASSPATH_VALUE =
+ "$ACCUMULO_CONF_DIR,\n" +
+ "$ACCUMULO_HOME/lib/[^.].*.jar,\n" +
+ "$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n" +
+ "$HADOOP_CONF_DIR,\n" +
+ "$HADOOP_PREFIX/[^.].*.jar,\n" +
+ "$HADOOP_PREFIX/lib/[^.].*.jar,\n" +
+ "$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n" +
+ "$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n" +
+ "$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n" +
+ "$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n"
+ ;
/**
* Dynamic classpath. These locations will be monitored for changes.
[25/39] git commit: ACCUMULO-287 added mock instance to
AccumuloOutputFormat - merged to trunk
Posted by ec...@apache.org.
ACCUMULO-287 added mock instance to AccumuloOutputFormat - merged to trunk
git-svn-id: https://svn.apache.org/repos/asf/incubator/accumulo/trunk@1229605 13f79535-47bb-0310-9956-ffa450edef68
(cherry picked from commit 63545d307599b3f1c1db4884a61fe880911cef7a)
Reason: Testing
Author: Billie Rinaldi <bi...@apache.org>
merged uncleanly, left in deprecated things removed in 1.5. squashed in test changes because AccumuloOutputFormatTest got created by ACCUMULO-287 independently in the 1.5 and 1.4 branches
Author: Sean Busbey <bu...@cloudera.com>
Signed-off-by: Eric Newton <er...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/792af9a1
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/792af9a1
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/792af9a1
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 792af9a19133d81edb1e49e9e51407724a53cfc3
Parents: a3264e4
Author: Billie Rinaldi <bi...@apache.org>
Authored: Tue Jan 10 15:47:22 2012 +0000
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Nov 25 16:06:42 2013 -0500
----------------------------------------------------------------------
.../client/mapreduce/AccumuloOutputFormatTest.java | 17 +++++++----------
1 file changed, 7 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/792af9a1/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java b/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java
index 8bc2b45..94e8bf0 100644
--- a/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java
+++ b/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java
@@ -33,7 +33,7 @@ import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.accumulo.core.util.ContextFactory;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Job;
@@ -41,7 +41,6 @@ import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.junit.Test;
/**
@@ -94,11 +93,10 @@ public class AccumuloOutputFormatTest {
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Mutation.class);
job.setNumReduceTasks(0);
- Configuration conf = job.getConfiguration();
- AccumuloInputFormat.setInputInfo(conf, "root", "".getBytes(), "testtable1", new Authorizations());
- AccumuloInputFormat.setMockInstance(conf, "testmrinstance");
- AccumuloOutputFormat.setOutputInfo(conf, "root", "".getBytes(), false, "testtable2");
- AccumuloOutputFormat.setMockInstance(conf, "testmrinstance");
+ AccumuloInputFormat.setInputInfo(job.getConfiguration(), "root", "".getBytes(), "testtable1", new Authorizations());
+ AccumuloInputFormat.setMockInstance(job.getConfiguration(), "testmrinstance");
+ AccumuloOutputFormat.setOutputInfo(job, "root", "".getBytes(), false, "testtable2");
+ AccumuloOutputFormat.setMockInstance(job, "testmrinstance");
AccumuloInputFormat input = new AccumuloInputFormat();
List<InputSplit> splits = input.getSplits(job);
@@ -108,11 +106,10 @@ public class AccumuloOutputFormatTest {
TestMapper mapper = (TestMapper) job.getMapperClass().newInstance();
for (InputSplit split : splits) {
- TaskAttemptID id = new TaskAttemptID();
- TaskAttemptContext tac = new TaskAttemptContext(job.getConfiguration(), id);
+ TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
RecordReader<Key,Value> reader = input.createRecordReader(split, tac);
RecordWriter<Text,Mutation> writer = output.getRecordWriter(tac);
- Mapper<Key,Value,Text,Mutation>.Context context = mapper.new Context(job.getConfiguration(), id, reader, writer, null, null, split);
+ Mapper<Key,Value,Text,Mutation>.Context context = ContextFactory.createMapContext(mapper, tac, reader, writer, split);
reader.initialize(split, context);
mapper.run(context);
writer.close(context);
[05/39] ACCUMULO-600 removed wikisearch from trunk
Posted by ec...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/QueryParser.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/QueryParser.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/QueryParser.java
deleted file mode 100644
index 73fa2cf..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/QueryParser.java
+++ /dev/null
@@ -1,845 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.parser;
-
-import java.io.StringReader;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.collections.map.LRUMap;
-import org.apache.commons.jexl2.parser.ASTAdditiveNode;
-import org.apache.commons.jexl2.parser.ASTAdditiveOperator;
-import org.apache.commons.jexl2.parser.ASTAmbiguous;
-import org.apache.commons.jexl2.parser.ASTAndNode;
-import org.apache.commons.jexl2.parser.ASTArrayAccess;
-import org.apache.commons.jexl2.parser.ASTArrayLiteral;
-import org.apache.commons.jexl2.parser.ASTAssignment;
-import org.apache.commons.jexl2.parser.ASTBitwiseAndNode;
-import org.apache.commons.jexl2.parser.ASTBitwiseComplNode;
-import org.apache.commons.jexl2.parser.ASTBitwiseOrNode;
-import org.apache.commons.jexl2.parser.ASTBitwiseXorNode;
-import org.apache.commons.jexl2.parser.ASTBlock;
-import org.apache.commons.jexl2.parser.ASTConstructorNode;
-import org.apache.commons.jexl2.parser.ASTDivNode;
-import org.apache.commons.jexl2.parser.ASTEQNode;
-import org.apache.commons.jexl2.parser.ASTERNode;
-import org.apache.commons.jexl2.parser.ASTEmptyFunction;
-import org.apache.commons.jexl2.parser.ASTFalseNode;
-import org.apache.commons.jexl2.parser.ASTFloatLiteral;
-import org.apache.commons.jexl2.parser.ASTForeachStatement;
-import org.apache.commons.jexl2.parser.ASTFunctionNode;
-import org.apache.commons.jexl2.parser.ASTGENode;
-import org.apache.commons.jexl2.parser.ASTGTNode;
-import org.apache.commons.jexl2.parser.ASTIdentifier;
-import org.apache.commons.jexl2.parser.ASTIfStatement;
-import org.apache.commons.jexl2.parser.ASTIntegerLiteral;
-import org.apache.commons.jexl2.parser.ASTJexlScript;
-import org.apache.commons.jexl2.parser.ASTLENode;
-import org.apache.commons.jexl2.parser.ASTLTNode;
-import org.apache.commons.jexl2.parser.ASTMapEntry;
-import org.apache.commons.jexl2.parser.ASTMapLiteral;
-import org.apache.commons.jexl2.parser.ASTMethodNode;
-import org.apache.commons.jexl2.parser.ASTModNode;
-import org.apache.commons.jexl2.parser.ASTMulNode;
-import org.apache.commons.jexl2.parser.ASTNENode;
-import org.apache.commons.jexl2.parser.ASTNRNode;
-import org.apache.commons.jexl2.parser.ASTNotNode;
-import org.apache.commons.jexl2.parser.ASTNullLiteral;
-import org.apache.commons.jexl2.parser.ASTOrNode;
-import org.apache.commons.jexl2.parser.ASTReference;
-import org.apache.commons.jexl2.parser.ASTSizeFunction;
-import org.apache.commons.jexl2.parser.ASTSizeMethod;
-import org.apache.commons.jexl2.parser.ASTStringLiteral;
-import org.apache.commons.jexl2.parser.ASTTernaryNode;
-import org.apache.commons.jexl2.parser.ASTTrueNode;
-import org.apache.commons.jexl2.parser.ASTUnaryMinusNode;
-import org.apache.commons.jexl2.parser.ASTWhileStatement;
-import org.apache.commons.jexl2.parser.ParseException;
-import org.apache.commons.jexl2.parser.Parser;
-import org.apache.commons.jexl2.parser.ParserVisitor;
-import org.apache.commons.jexl2.parser.SimpleNode;
-import org.apache.hadoop.util.hash.Hash;
-import org.apache.hadoop.util.hash.MurmurHash;
-
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Multimap;
-
-/**
- * Parses the query for the purposes of extracting terms, operators, and literals for query optimization. This class does not necessarily understand how to
- * parse all of the possible combinations of the JEXL syntax, but that does not mean that the query will not evaluate against the event objects. It means that
- * the unsupported operators will not be parsed and included in the optimization step.
- *
- */
-public class QueryParser implements ParserVisitor {
-
- public static class QueryTerm {
- private boolean negated = false;
- private String operator = null;
- private Object value = null;
-
- public QueryTerm(boolean negated, String operator, Object value) {
- super();
- this.negated = negated;
- this.operator = operator;
- this.value = value;
- }
-
- public boolean isNegated() {
- return negated;
- }
-
- public String getOperator() {
- return operator;
- }
-
- public Object getValue() {
- return value;
- }
-
- public void setNegated(boolean negated) {
- this.negated = negated;
- }
-
- public void setOperator(String operator) {
- this.operator = operator;
- }
-
- public void setValue(Object value) {
- this.value = value;
- }
-
- public String toString() {
- StringBuilder buf = new StringBuilder();
- buf.append("negated: ").append(negated).append(", operator: ").append(operator).append(", value: ").append(value);
- return buf.toString();
- }
- }
-
- /**
- * Holder object
- */
- static class ObjectHolder {
- Object object;
-
- public Object getObject() {
- return object;
- }
-
- public void setObject(Object object) {
- this.object = object;
- }
- }
-
- static class FunctionResult {
- private List<TermResult> terms = new ArrayList<TermResult>();
-
- public List<TermResult> getTerms() {
- return terms;
- }
- }
-
- /**
- * Holder object for a term (i.e. field name)
- */
- static class TermResult {
- Object value;
-
- public TermResult(Object value) {
- this.value = value;
- }
- }
-
- /**
- * Holder object for a literal (integer, float, string, or null literal) value
- */
- static class LiteralResult {
- Object value;
-
- public LiteralResult(Object value) {
- this.value = value;
- }
- }
-
- /**
- * Object used to store context information as the AST is being iterated over.
- */
- static class EvaluationContext {
- boolean inOrContext = false;
- boolean inNotContext = false;
- boolean inAndContext = false;
- }
-
- /**
- * Object to store information from previously parsed queries.
- */
- private static class CacheEntry {
- private Set<String> negatedTerms = null;
- private Set<String> andTerms = null;
- private Set<String> orTerms = null;
- private Set<Object> literals = null;
- private Multimap<String,QueryTerm> terms = null;
- private ASTJexlScript rootNode = null;
- private TreeNode tree = null;
-
- public CacheEntry(Set<String> negatedTerms, Set<String> andTerms, Set<String> orTerms, Set<Object> literals, Multimap<String,QueryTerm> terms,
- ASTJexlScript rootNode, TreeNode tree) {
- super();
- this.negatedTerms = negatedTerms;
- this.andTerms = andTerms;
- this.orTerms = orTerms;
- this.literals = literals;
- this.terms = terms;
- this.rootNode = rootNode;
- this.tree = tree;
- }
-
- public Set<String> getNegatedTerms() {
- return negatedTerms;
- }
-
- public Set<String> getAndTerms() {
- return andTerms;
- }
-
- public Set<String> getOrTerms() {
- return orTerms;
- }
-
- public Set<Object> getLiterals() {
- return literals;
- }
-
- public Multimap<String,QueryTerm> getTerms() {
- return terms;
- }
-
- public ASTJexlScript getRootNode() {
- return rootNode;
- }
-
- public TreeNode getTree() {
- return tree;
- }
- }
-
- private static final int SEED = 650567;
-
- private static LRUMap cache = new LRUMap();
-
- protected Set<String> negatedTerms = new HashSet<String>();
-
- private Set<String> andTerms = new HashSet<String>();
-
- private Set<String> orTerms = new HashSet<String>();
-
- /**
- * List of String, Integer, Float, etc literals that were passed in the query
- */
- private Set<Object> literals = new HashSet<Object>();
-
- /**
- * Map of terms (field names) to QueryTerm objects.
- */
- private Multimap<String,QueryTerm> terms = HashMultimap.create();
-
- private ASTJexlScript rootNode = null;
-
- private TreeNode tree = null;
-
- private int hashVal = 0;
-
- public QueryParser() {}
-
- private void reset() {
- this.negatedTerms.clear();
- this.andTerms.clear();
- this.orTerms.clear();
- this.literals.clear();
- this.terms = HashMultimap.create();
- }
-
- public void execute(String query) throws ParseException {
- reset();
- query = query.replaceAll("\\s+AND\\s+", " and ");
- query = query.replaceAll("\\s+OR\\s+", " or ");
- query = query.replaceAll("\\s+NOT\\s+", " not ");
-
- // Check to see if its in the cache
- Hash hash = MurmurHash.getInstance();
- this.hashVal = hash.hash(query.getBytes(), SEED);
- CacheEntry entry = null;
- synchronized (cache) {
- entry = (CacheEntry) cache.get(hashVal);
- }
- if (entry != null) {
- this.negatedTerms = entry.getNegatedTerms();
- this.andTerms = entry.getAndTerms();
- this.orTerms = entry.getOrTerms();
- this.literals = entry.getLiterals();
- this.terms = entry.getTerms();
- this.rootNode = entry.getRootNode();
- this.tree = entry.getTree();
- } else {
- Parser p = new Parser(new StringReader(";"));
- rootNode = p.parse(new StringReader(query), null);
- rootNode.childrenAccept(this, null);
- TreeBuilder builder = new TreeBuilder(rootNode);
- tree = builder.getRootNode();
- entry = new CacheEntry(this.negatedTerms, this.andTerms, this.orTerms, this.literals, this.terms, rootNode, tree);
- synchronized (cache) {
- cache.put(hashVal, entry);
- }
- }
-
- }
-
- /**
- *
- * @return this queries hash value
- */
- public int getHashValue() {
- return this.hashVal;
- }
-
- public TreeNode getIteratorTree() {
- return this.tree;
- }
-
- /**
- *
- * @return JEXL abstract syntax tree
- */
- public ASTJexlScript getAST() {
- return this.rootNode;
- }
-
- /**
- *
- * @return Set of field names to use in the optimizer for nots. As a general rule none of these terms should be used to find an event and should they should
- * be evaluated on each event after being found.
- */
- public Set<String> getNegatedTermsForOptimizer() {
- return negatedTerms;
- }
-
- /**
- *
- * @return Set of field names to use in the optimizer for ands. As a general rule any one term of an and clause can be used to find associated events.
- */
- public Set<String> getAndTermsForOptimizer() {
- return andTerms;
- }
-
- /**
- *
- * @return Set of field names to use in the optimizer for ors. As a general rule any terms that are part of an or clause need to be searched to find the
- * associated events.
- */
- public Set<String> getOrTermsForOptimizer() {
- return orTerms;
- }
-
- /**
- *
- * @return String, Integer, and Float literals used in the query.
- */
- public Set<Object> getQueryLiterals() {
- return literals;
- }
-
- /**
- *
- * @return Set of all identifiers (field names) in the query.
- */
- public Set<String> getQueryIdentifiers() {
- return terms.keySet();
- }
-
- /**
- *
- * @return map of term (field name) to QueryTerm object
- */
- public Multimap<String,QueryTerm> getQueryTerms() {
- return terms;
- }
-
- public Object visit(SimpleNode node, Object data) {
- return null;
- }
-
- public Object visit(ASTJexlScript node, Object data) {
- return null;
- }
-
- public Object visit(ASTBlock node, Object data) {
- return null;
- }
-
- public Object visit(ASTAmbiguous node, Object data) {
- return null;
- }
-
- public Object visit(ASTIfStatement node, Object data) {
- return null;
- }
-
- public Object visit(ASTWhileStatement node, Object data) {
- return null;
- }
-
- public Object visit(ASTForeachStatement node, Object data) {
- return null;
- }
-
- public Object visit(ASTAssignment node, Object data) {
- return null;
- }
-
- public Object visit(ASTTernaryNode node, Object data) {
- return null;
- }
-
- public Object visit(ASTOrNode node, Object data) {
- boolean previouslyInOrContext = false;
- EvaluationContext ctx = null;
- if (null != data && data instanceof EvaluationContext) {
- ctx = (EvaluationContext) data;
- previouslyInOrContext = ctx.inOrContext;
- } else {
- ctx = new EvaluationContext();
- }
- ctx.inOrContext = true;
- // Process both sides of this node.
- node.jjtGetChild(0).jjtAccept(this, ctx);
- node.jjtGetChild(1).jjtAccept(this, ctx);
- // reset the state
- if (null != data && !previouslyInOrContext)
- ctx.inOrContext = false;
- return null;
- }
-
- public Object visit(ASTAndNode node, Object data) {
- boolean previouslyInAndContext = false;
- EvaluationContext ctx = null;
- if (null != data && data instanceof EvaluationContext) {
- ctx = (EvaluationContext) data;
- previouslyInAndContext = ctx.inAndContext;
- } else {
- ctx = new EvaluationContext();
- }
- ctx.inAndContext = true;
- // Process both sides of this node.
- node.jjtGetChild(0).jjtAccept(this, ctx);
- node.jjtGetChild(1).jjtAccept(this, ctx);
- // reset the state
- if (null != data && !previouslyInAndContext)
- ctx.inAndContext = false;
- return null;
- }
-
- public Object visit(ASTBitwiseOrNode node, Object data) {
- return null;
- }
-
- public Object visit(ASTBitwiseXorNode node, Object data) {
- return null;
- }
-
- public Object visit(ASTBitwiseAndNode node, Object data) {
- return null;
- }
-
- public Object visit(ASTEQNode node, Object data) {
- StringBuilder fieldName = new StringBuilder();
- ObjectHolder value = new ObjectHolder();
- // Process both sides of this node.
- Object left = node.jjtGetChild(0).jjtAccept(this, data);
- Object right = node.jjtGetChild(1).jjtAccept(this, data);
- // Ignore functions in the query
- if (left instanceof FunctionResult || right instanceof FunctionResult)
- return null;
- decodeResults(left, right, fieldName, value);
- // We need to check to see if we are in a NOT context. If so,
- // then we need to reverse the negation.
- boolean negated = false;
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- if (ctx.inNotContext)
- negated = !negated;
- }
- QueryTerm term = new QueryTerm(negated, JexlOperatorConstants.getOperator(node.getClass()), value.getObject());
- terms.put(fieldName.toString(), term);
- return null;
- }
-
- public Object visit(ASTNENode node, Object data) {
- StringBuilder fieldName = new StringBuilder();
- ObjectHolder value = new ObjectHolder();
- // Process both sides of this node.
- Object left = node.jjtGetChild(0).jjtAccept(this, data);
- Object right = node.jjtGetChild(1).jjtAccept(this, data);
- // Ignore functions in the query
- if (left instanceof FunctionResult || right instanceof FunctionResult)
- return null;
- decodeResults(left, right, fieldName, value);
- // We need to check to see if we are in a NOT context. If so,
- // then we need to reverse the negation.
- boolean negated = true;
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- if (ctx.inNotContext)
- negated = !negated;
- }
- if (negated)
- negatedTerms.add(fieldName.toString());
- QueryTerm term = new QueryTerm(negated, JexlOperatorConstants.getOperator(node.getClass()), value.getObject());
- terms.put(fieldName.toString(), term);
- return null;
- }
-
- public Object visit(ASTLTNode node, Object data) {
- StringBuilder fieldName = new StringBuilder();
- ObjectHolder value = new ObjectHolder();
- // Process both sides of this node.
- Object left = node.jjtGetChild(0).jjtAccept(this, data);
- Object right = node.jjtGetChild(1).jjtAccept(this, data);
- // Ignore functions in the query
- if (left instanceof FunctionResult || right instanceof FunctionResult)
- return null;
- decodeResults(left, right, fieldName, value);
- // We need to check to see if we are in a NOT context. If so,
- // then we need to reverse the negation.
- boolean negated = false;
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- if (ctx.inNotContext)
- negated = !negated;
- }
- QueryTerm term = new QueryTerm(negated, JexlOperatorConstants.getOperator(node.getClass()), value.getObject());
- terms.put(fieldName.toString(), term);
- return null;
- }
-
- public Object visit(ASTGTNode node, Object data) {
- StringBuilder fieldName = new StringBuilder();
- ObjectHolder value = new ObjectHolder();
- // Process both sides of this node.
- Object left = node.jjtGetChild(0).jjtAccept(this, data);
- Object right = node.jjtGetChild(1).jjtAccept(this, data);
- // Ignore functions in the query
- if (left instanceof FunctionResult || right instanceof FunctionResult)
- return null;
- decodeResults(left, right, fieldName, value);
- // We need to check to see if we are in a NOT context. If so,
- // then we need to reverse the negation.
- boolean negated = false;
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- if (ctx.inNotContext)
- negated = !negated;
- }
- QueryTerm term = new QueryTerm(negated, JexlOperatorConstants.getOperator(node.getClass()), value.getObject());
- terms.put(fieldName.toString(), term);
- return null;
- }
-
- public Object visit(ASTLENode node, Object data) {
- StringBuilder fieldName = new StringBuilder();
- ObjectHolder value = new ObjectHolder();
- // Process both sides of this node.
- Object left = node.jjtGetChild(0).jjtAccept(this, data);
- Object right = node.jjtGetChild(1).jjtAccept(this, data);
- // Ignore functions in the query
- if (left instanceof FunctionResult || right instanceof FunctionResult)
- return null;
- decodeResults(left, right, fieldName, value);
- // We need to check to see if we are in a NOT context. If so,
- // then we need to reverse the negation.
- boolean negated = false;
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- if (ctx.inNotContext)
- negated = !negated;
- }
- QueryTerm term = new QueryTerm(negated, JexlOperatorConstants.getOperator(node.getClass()), value.getObject());
- terms.put(fieldName.toString(), term);
- return null;
- }
-
- public Object visit(ASTGENode node, Object data) {
- StringBuilder fieldName = new StringBuilder();
- ObjectHolder value = new ObjectHolder();
- // Process both sides of this node.
- Object left = node.jjtGetChild(0).jjtAccept(this, data);
- Object right = node.jjtGetChild(1).jjtAccept(this, data);
- // Ignore functions in the query
- if (left instanceof FunctionResult || right instanceof FunctionResult)
- return null;
- decodeResults(left, right, fieldName, value);
- // We need to check to see if we are in a NOT context. If so,
- // then we need to reverse the negation.
- boolean negated = false;
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- if (ctx.inNotContext)
- negated = !negated;
- }
- QueryTerm term = new QueryTerm(negated, JexlOperatorConstants.getOperator(node.getClass()), value.getObject());
- terms.put(fieldName.toString(), term);
- return null;
- }
-
- public Object visit(ASTERNode node, Object data) {
- StringBuilder fieldName = new StringBuilder();
- ObjectHolder value = new ObjectHolder();
- // Process both sides of this node.
- Object left = node.jjtGetChild(0).jjtAccept(this, data);
- Object right = node.jjtGetChild(1).jjtAccept(this, data);
- // Ignore functions in the query
- if (left instanceof FunctionResult || right instanceof FunctionResult)
- return null;
- decodeResults(left, right, fieldName, value);
- // We need to check to see if we are in a NOT context. If so,
- // then we need to reverse the negation.
- boolean negated = false;
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- if (ctx.inNotContext)
- negated = !negated;
- }
- QueryTerm term = new QueryTerm(negated, JexlOperatorConstants.getOperator(node.getClass()), value.getObject());
- terms.put(fieldName.toString(), term);
- return null;
- }
-
- public Object visit(ASTNRNode node, Object data) {
- StringBuilder fieldName = new StringBuilder();
- ObjectHolder value = new ObjectHolder();
- // Process both sides of this node.
- Object left = node.jjtGetChild(0).jjtAccept(this, data);
- Object right = node.jjtGetChild(1).jjtAccept(this, data);
- // Ignore functions in the query
- if (left instanceof FunctionResult || right instanceof FunctionResult)
- return null;
- decodeResults(left, right, fieldName, value);
- // We need to check to see if we are in a NOT context. If so,
- // then we need to reverse the negation.
- boolean negated = true;
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- if (ctx.inNotContext)
- negated = !negated;
- }
- if (negated)
- negatedTerms.add(fieldName.toString());
- QueryTerm term = new QueryTerm(negated, JexlOperatorConstants.getOperator(node.getClass()), value.getObject());
- terms.put(fieldName.toString(), term);
- return null;
- }
-
- public Object visit(ASTAdditiveNode node, Object data) {
- return null;
- }
-
- public Object visit(ASTAdditiveOperator node, Object data) {
- return null;
- }
-
- public Object visit(ASTMulNode node, Object data) {
- return null;
- }
-
- public Object visit(ASTDivNode node, Object data) {
- return null;
- }
-
- public Object visit(ASTModNode node, Object data) {
- return null;
- }
-
- public Object visit(ASTUnaryMinusNode node, Object data) {
- return null;
- }
-
- public Object visit(ASTBitwiseComplNode node, Object data) {
- return null;
- }
-
- public Object visit(ASTNotNode node, Object data) {
- boolean previouslyInNotContext = false;
- EvaluationContext ctx = null;
- if (null != data && data instanceof EvaluationContext) {
- ctx = (EvaluationContext) data;
- previouslyInNotContext = ctx.inNotContext;
- } else {
- ctx = new EvaluationContext();
- }
- ctx.inNotContext = true;
- // Process both sides of this node.
- node.jjtGetChild(0).jjtAccept(this, ctx);
- // reset the state
- if (null != data && !previouslyInNotContext)
- ctx.inNotContext = false;
- return null;
- }
-
- public Object visit(ASTIdentifier node, Object data) {
- if (data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- if (ctx.inAndContext)
- andTerms.add(node.image);
- if (ctx.inNotContext)
- negatedTerms.add(node.image);
- if (ctx.inOrContext)
- orTerms.add(node.image);
- }
- return new TermResult(node.image);
- }
-
- public Object visit(ASTNullLiteral node, Object data) {
- literals.add(node.image);
- return new LiteralResult(node.image);
- }
-
- public Object visit(ASTTrueNode node, Object data) {
- return new LiteralResult(node.image);
- }
-
- public Object visit(ASTFalseNode node, Object data) {
- return new LiteralResult(node.image);
- }
-
- public Object visit(ASTIntegerLiteral node, Object data) {
- literals.add(node.image);
- return new LiteralResult(node.image);
- }
-
- public Object visit(ASTFloatLiteral node, Object data) {
- literals.add(node.image);
- return new LiteralResult(node.image);
- }
-
- public Object visit(ASTStringLiteral node, Object data) {
- literals.add("'" + node.image + "'");
- return new LiteralResult("'" + node.image + "'");
- }
-
- public Object visit(ASTArrayLiteral node, Object data) {
- return null;
- }
-
- public Object visit(ASTMapLiteral node, Object data) {
- return null;
- }
-
- public Object visit(ASTMapEntry node, Object data) {
- return null;
- }
-
- public Object visit(ASTEmptyFunction node, Object data) {
- return null;
- }
-
- public Object visit(ASTSizeFunction node, Object data) {
- return null;
- }
-
- public Object visit(ASTFunctionNode node, Object data) {
- // We need to check to see if we are in a NOT context. If so,
- // then we need to reverse the negation.
- boolean negated = true;
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- if (ctx.inNotContext)
- negated = !negated;
- }
- // used to rebuild function call from the AST
- StringBuilder buf = new StringBuilder();
- String sep = "";
- // objectNode 0 is the prefix
- buf.append(node.jjtGetChild(0).image).append(":");
- // objectNode 1 is the identifier , the others are parameters.
- buf.append(node.jjtGetChild(1).image).append("(");
- // process the remaining arguments
- FunctionResult fr = new FunctionResult();
- int argc = node.jjtGetNumChildren() - 2;
- for (int i = 0; i < argc; i++) {
- // Process both sides of this node.
- Object result = node.jjtGetChild(i + 2).jjtAccept(this, data);
- if (result instanceof TermResult) {
- TermResult tr = (TermResult) result;
- fr.getTerms().add(tr);
- buf.append(sep).append(tr.value);
- sep = ", ";
- } else {
- buf.append(sep).append(node.jjtGetChild(i + 2).image);
- sep = ", ";
- }
- }
- buf.append(")");
- // Capture the entire function call for each function parameter
- for (TermResult tr : fr.terms)
- terms.put((String) tr.value, new QueryTerm(negated, JexlOperatorConstants.getOperator(node.getClass()), buf.toString()));
- return fr;
- }
-
- public Object visit(ASTMethodNode node, Object data) {
- return null;
- }
-
- public Object visit(ASTSizeMethod node, Object data) {
- return null;
- }
-
- public Object visit(ASTConstructorNode node, Object data) {
- return null;
- }
-
- public Object visit(ASTArrayAccess node, Object data) {
- return null;
- }
-
- public Object visit(ASTReference node, Object data) {
- return node.jjtGetChild(0).jjtAccept(this, data);
- }
-
- protected void decodeResults(Object left, Object right, StringBuilder fieldName, ObjectHolder holder) {
- if (left instanceof TermResult) {
- TermResult tr = (TermResult) left;
- fieldName.append((String) tr.value);
- // Then the right has to be the value
- if (right instanceof LiteralResult) {
- holder.setObject(((LiteralResult) right).value);
- } else {
- throw new IllegalArgumentException("Object mismatch");
- }
- } else if (right instanceof TermResult) {
- TermResult tr = (TermResult) right;
- fieldName.append((String) tr.value);
- if (left instanceof LiteralResult) {
- holder.setObject(((LiteralResult) left).value);
- } else {
- throw new IllegalArgumentException("Object mismatch");
- }
-
- } else {
- throw new IllegalArgumentException("No Term specified in query");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/RangeCalculator.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/RangeCalculator.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/RangeCalculator.java
deleted file mode 100644
index 8a5474b..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/parser/RangeCalculator.java
+++ /dev/null
@@ -1,1199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.parser;
-
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.TreeSet;
-
-import org.apache.accumulo.core.client.BatchScanner;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.PartialKey;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.examples.wikisearch.iterator.EvaluatingIterator;
-import org.apache.accumulo.examples.wikisearch.logic.AbstractQueryLogic;
-import org.apache.accumulo.examples.wikisearch.normalizer.Normalizer;
-import org.apache.accumulo.examples.wikisearch.protobuf.Uid;
-import org.apache.accumulo.examples.wikisearch.util.TextUtil;
-import org.apache.commons.jexl2.parser.ASTAndNode;
-import org.apache.commons.jexl2.parser.ASTEQNode;
-import org.apache.commons.jexl2.parser.ASTERNode;
-import org.apache.commons.jexl2.parser.ASTFalseNode;
-import org.apache.commons.jexl2.parser.ASTFunctionNode;
-import org.apache.commons.jexl2.parser.ASTGENode;
-import org.apache.commons.jexl2.parser.ASTGTNode;
-import org.apache.commons.jexl2.parser.ASTLENode;
-import org.apache.commons.jexl2.parser.ASTLTNode;
-import org.apache.commons.jexl2.parser.ASTNENode;
-import org.apache.commons.jexl2.parser.ASTNRNode;
-import org.apache.commons.jexl2.parser.ASTNullLiteral;
-import org.apache.commons.jexl2.parser.ASTOrNode;
-import org.apache.commons.jexl2.parser.ASTTrueNode;
-import org.apache.commons.jexl2.parser.ParseException;
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.commons.lang.builder.ToStringBuilder;
-import org.apache.hadoop.io.Text;
-import org.apache.log4j.Logger;
-
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Multimap;
-import com.google.protobuf.InvalidProtocolBufferException;
-
-/**
- * This class is used to query the global indices to determine that set of ranges to use when querying the shard table. The RangeCalculator looks at each term
- * in the query to determine if it is a equivalence, range, or wildcard comparison, and queries the appropriate index to find the ranges for the terms which are
- * then cached. The final set of ranges is computed as the AST is traversed.
- */
-public class RangeCalculator extends QueryParser {
-
- /**
- * Container used as map keys in this class
- *
- */
- public static class MapKey implements Comparable<MapKey> {
- private String fieldName = null;
- private String fieldValue = null;
- private String originalQueryValue = null;
-
- public MapKey(String fieldName, String fieldValue) {
- super();
- this.fieldName = fieldName;
- this.fieldValue = fieldValue;
- }
-
- public String getFieldName() {
- return fieldName;
- }
-
- public String getFieldValue() {
- return fieldValue;
- }
-
- public void setFieldName(String fieldName) {
- this.fieldName = fieldName;
- }
-
- public void setFieldValue(String fieldValue) {
- this.fieldValue = fieldValue;
- }
-
- public String getOriginalQueryValue() {
- return originalQueryValue;
- }
-
- public void setOriginalQueryValue(String originalQueryValue) {
- this.originalQueryValue = originalQueryValue;
- }
-
- @Override
- public int hashCode() {
- return new HashCodeBuilder(17, 37).append(fieldName).append(fieldValue).toHashCode();
- }
-
- @Override
- public String toString() {
- return this.fieldName + " " + this.fieldValue;
- }
-
- @Override
- public boolean equals(Object other) {
- if (other == null)
- return false;
- if (other instanceof MapKey) {
- MapKey o = (MapKey) other;
- return (this.fieldName.equals(o.fieldName) && this.fieldValue.equals(o.fieldValue));
- } else
- return false;
- }
-
- public int compareTo(MapKey o) {
- int result = this.fieldName.compareTo(o.fieldName);
- if (result != 0) {
- return this.fieldValue.compareTo(o.fieldValue);
- } else {
- return result;
- }
- }
-
- }
-
- /**
- * Container used to hold the lower and upper bound of a range
- *
- */
- public static class RangeBounds {
- private String originalLower = null;
- private Text lower = null;
- private String originalUpper = null;
- private Text upper = null;
-
- public Text getLower() {
- return lower;
- }
-
- public Text getUpper() {
- return upper;
- }
-
- public void setLower(Text lower) {
- this.lower = lower;
- }
-
- public void setUpper(Text upper) {
- this.upper = upper;
- }
-
- public String getOriginalLower() {
- return originalLower;
- }
-
- public String getOriginalUpper() {
- return originalUpper;
- }
-
- public void setOriginalLower(String originalLower) {
- this.originalLower = originalLower;
- }
-
- public void setOriginalUpper(String originalUpper) {
- this.originalUpper = originalUpper;
- }
- }
-
- /**
- *
- * Object that is used to hold ranges found in the index. Subclasses may compute the final range set in various ways.
- */
- protected static class TermRange implements Comparable<TermRange> {
-
- private String fieldName = null;
- private Object fieldValue = null;
- private Set<Range> ranges = new TreeSet<Range>();
-
- public TermRange(String name, Object fieldValue) {
- this.fieldName = name;
- this.fieldValue = fieldValue;
- }
-
- public String getFieldName() {
- return this.fieldName;
- }
-
- public Object getFieldValue() {
- return this.fieldValue;
- }
-
- public void addAll(Set<Range> r) {
- ranges.addAll(r);
- }
-
- public void add(Range r) {
- ranges.add(r);
- }
-
- public Set<Range> getRanges() {
- return ranges;
- }
-
- @Override
- public String toString() {
- ToStringBuilder tsb = new ToStringBuilder(this);
- tsb.append("fieldName", fieldName);
- tsb.append("fieldValue", fieldValue);
- tsb.append("ranges", ranges);
- return tsb.toString();
- }
-
- public int compareTo(TermRange o) {
- int result = this.fieldName.compareTo(o.fieldName);
- if (result == 0) {
- return ((Integer) ranges.size()).compareTo(o.ranges.size());
- } else {
- return result;
- }
- }
- }
-
- /**
- * Object used to store context information as the AST is being traversed.
- */
- static class EvaluationContext {
- boolean inOrContext = false;
- boolean inNotContext = false;
- boolean inAndContext = false;
- TermRange lastRange = null;
- String lastProcessedTerm = null;
- }
-
- protected static Logger log = Logger.getLogger(RangeCalculator.class);
- private static String WILDCARD = ".*";
- private static String SINGLE_WILDCARD = "\\.";
-
- protected Connector c;
- protected Authorizations auths;
- protected Multimap<String,Normalizer> indexedTerms;
- protected Multimap<String,QueryTerm> termsCopy = HashMultimap.create();
- protected String indexTableName;
- protected String reverseIndexTableName;
- protected int queryThreads = 8;
-
- /* final results of index lookups, ranges for the shard table */
- protected Set<Range> result = null;
- /* map of field names to values found in the index */
- protected Multimap<String,String> indexEntries = HashMultimap.create();
- /* map of value in the index to the original query values */
- protected Map<String,String> indexValues = new HashMap<String,String>();
- /* map of values in the query to map keys used */
- protected Multimap<String,MapKey> originalQueryValues = HashMultimap.create();
- /* map of field name to cardinality */
- protected Map<String,Long> termCardinalities = new HashMap<String,Long>();
- /* cached results of all ranges found global index lookups */
- protected Map<MapKey,TermRange> globalIndexResults = new HashMap<MapKey,TermRange>();
-
- /**
- *
- * @param c
- * @param auths
- * @param indexedTerms
- * @param terms
- * @param query
- * @param logic
- * @param typeFilter
- * @throws ParseException
- */
- public void execute(Connector c, Authorizations auths, Multimap<String,Normalizer> indexedTerms, Multimap<String,QueryTerm> terms, String query,
- AbstractQueryLogic logic, Set<String> typeFilter) throws ParseException {
- super.execute(query);
- this.c = c;
- this.auths = auths;
- this.indexedTerms = indexedTerms;
- this.termsCopy.putAll(terms);
- this.indexTableName = logic.getIndexTableName();
- this.reverseIndexTableName = logic.getReverseIndexTableName();
- this.queryThreads = logic.getQueryThreads();
-
- Map<MapKey,Set<Range>> indexRanges = new HashMap<MapKey,Set<Range>>();
- Map<MapKey,Set<Range>> trailingWildcardRanges = new HashMap<MapKey,Set<Range>>();
- Map<MapKey,Set<Range>> leadingWildcardRanges = new HashMap<MapKey,Set<Range>>();
- Map<Text,RangeBounds> rangeMap = new HashMap<Text,RangeBounds>();
-
- // Here we iterate over all of the terms in the query to determine if they are an equivalence,
- // wildcard, or range type operator
- for (Entry<String,QueryTerm> entry : terms.entries()) {
- if (entry.getValue().getOperator().equals(JexlOperatorConstants.getOperator(ASTEQNode.class))
- || entry.getValue().getOperator().equals(JexlOperatorConstants.getOperator(ASTERNode.class))
- || entry.getValue().getOperator().equals(JexlOperatorConstants.getOperator(ASTLTNode.class))
- || entry.getValue().getOperator().equals(JexlOperatorConstants.getOperator(ASTLENode.class))
- || entry.getValue().getOperator().equals(JexlOperatorConstants.getOperator(ASTGTNode.class))
- || entry.getValue().getOperator().equals(JexlOperatorConstants.getOperator(ASTGENode.class))) {
- // If this term is not in the set of indexed terms, then bail
- if (!indexedTerms.containsKey(entry.getKey())) {
- termCardinalities.put(entry.getKey().toUpperCase(), 0L);
- continue;
- }
- // In the case of function calls, the query term could be null. Dont query the index for it.
- if (null == entry.getValue()) {
- termCardinalities.put(entry.getKey().toUpperCase(), 0L);
- continue;
- }
- // In the case where we are looking for 'null', then skip.
- if (null == entry.getValue().getValue() || ((String) entry.getValue().getValue()).equals("null")) {
- termCardinalities.put(entry.getKey().toUpperCase(), 0L);
- continue;
- }
-
- // Remove the begin and end ' marks
- String value = null;
- if (((String) entry.getValue().getValue()).startsWith("'") && ((String) entry.getValue().getValue()).endsWith("'"))
- value = ((String) entry.getValue().getValue()).substring(1, ((String) entry.getValue().getValue()).length() - 1);
- else
- value = (String) entry.getValue().getValue();
- // The entries in the index are normalized
- for (Normalizer normalizer : indexedTerms.get(entry.getKey())) {
- String normalizedFieldValue = normalizer.normalizeFieldValue(null, value);
- Text fieldValue = new Text(normalizedFieldValue);
- Text fieldName = new Text(entry.getKey().toUpperCase());
-
- // EQUALS
- if (entry.getValue().getOperator().equals(JexlOperatorConstants.getOperator(ASTEQNode.class))) {
- Key startRange = new Key(fieldValue, fieldName);
- Range r = new Range(startRange, true, startRange.followingKey(PartialKey.ROW), true);
-
- MapKey key = new MapKey(fieldName.toString(), fieldValue.toString());
- key.setOriginalQueryValue(value);
- this.originalQueryValues.put(value, key);
- if (!indexRanges.containsKey(key))
- indexRanges.put(key, new HashSet<Range>());
- indexRanges.get(key).add(r);
- // WILDCARD
- } else if (entry.getValue().getOperator().equals(JexlOperatorConstants.getOperator(ASTERNode.class))) {
- // This is a wildcard query using regex. We can only support leading and trailing wildcards at this time. Leading
- // wildcards will need be reversed and sent to the global reverse index. Trailing wildcard queries will be sent to the
- // global index. In all cases, the range for the wilcard will be the range of possible UNICODE codepoints, hex 0 to 10FFFF.
- int loc = normalizedFieldValue.indexOf(WILDCARD);
- if (-1 == loc)
- loc = normalizedFieldValue.indexOf(SINGLE_WILDCARD);
- if (-1 == loc) {
- // Then no wildcard in the query? Treat like the equals case above.
- Key startRange = new Key(fieldValue, fieldName);
- Range r = new Range(startRange, true, startRange.followingKey(PartialKey.ROW), true);
-
- MapKey key = new MapKey(fieldName.toString(), fieldValue.toString());
- key.setOriginalQueryValue(value);
- this.originalQueryValues.put(value, key);
- if (!indexRanges.containsKey(key))
- indexRanges.put(key, new HashSet<Range>());
- indexRanges.get(key).add(r);
- } else {
- if (loc == 0) {
- // Then we have a leading wildcard, reverse the term and use the global reverse index.
- StringBuilder buf = new StringBuilder(normalizedFieldValue.substring(2));
- normalizedFieldValue = buf.reverse().toString();
- Key startRange = new Key(new Text(normalizedFieldValue + "\u0000"), fieldName);
- Key endRange = new Key(new Text(normalizedFieldValue + "\u10FFFF"), fieldName);
- Range r = new Range(startRange, true, endRange, true);
-
- MapKey key = new MapKey(fieldName.toString(), normalizedFieldValue);
- key.setOriginalQueryValue(value);
- this.originalQueryValues.put(value, key);
- if (!leadingWildcardRanges.containsKey(key))
- leadingWildcardRanges.put(key, new HashSet<Range>());
- leadingWildcardRanges.get(key).add(r);
- } else if (loc == (normalizedFieldValue.length() - 2)) {
- normalizedFieldValue = normalizedFieldValue.substring(0, loc);
- // Then we have a trailing wildcard character.
- Key startRange = new Key(new Text(normalizedFieldValue + "\u0000"), fieldName);
- Key endRange = new Key(new Text(normalizedFieldValue + "\u10FFFF"), fieldName);
- Range r = new Range(startRange, true, endRange, true);
-
- MapKey key = new MapKey(fieldName.toString(), normalizedFieldValue);
- key.setOriginalQueryValue(value);
- this.originalQueryValues.put(value, key);
- if (!trailingWildcardRanges.containsKey(key))
- trailingWildcardRanges.put(key, new HashSet<Range>());
- trailingWildcardRanges.get(key).add(r);
- } else {
- // throw new RuntimeException("Unsupported wildcard location. Only trailing or leading wildcards are supported: " + normalizedFieldValue);
- // Don't throw an exception, there must be a wildcard in the query, we'll treat it as a filter on the results since it is not
- // leading or trailing.
- }
- }
- // RANGES
- } else if (entry.getValue().getOperator().equals(JexlOperatorConstants.getOperator(ASTGTNode.class))
- || entry.getValue().getOperator().equals(JexlOperatorConstants.getOperator(ASTGENode.class))) {
- // Then we have a lower bound to a range query
- if (!rangeMap.containsKey(fieldName))
- rangeMap.put(fieldName, new RangeBounds());
- rangeMap.get(fieldName).setLower(fieldValue);
- rangeMap.get(fieldName).setOriginalLower(value);
- } else if (entry.getValue().getOperator().equals(JexlOperatorConstants.getOperator(ASTLTNode.class))
- || entry.getValue().getOperator().equals(JexlOperatorConstants.getOperator(ASTLENode.class))) {
- // Then we have an upper bound to a range query
- if (!rangeMap.containsKey(fieldName))
- rangeMap.put(fieldName, new RangeBounds());
- rangeMap.get(fieldName).setUpper(fieldValue);
- rangeMap.get(fieldName).setOriginalUpper(value);
- }
- }
- }
- }
-
- // INDEX RANGE QUERY
- // Now that we have figured out the range bounds, create the index ranges.
- for (Entry<Text,RangeBounds> entry : rangeMap.entrySet()) {
- if (entry.getValue().getLower() != null && entry.getValue().getUpper() != null) {
- // Figure out the key order
- Key lk = new Key(entry.getValue().getLower());
- Key up = new Key(entry.getValue().getUpper());
- Text lower = lk.getRow();
- Text upper = up.getRow();
- // Swith the order if needed.
- if (lk.compareTo(up) > 0) {
- lower = up.getRow();
- upper = lk.getRow();
- }
- Key startRange = new Key(lower, entry.getKey());
- Key endRange = new Key(upper, entry.getKey());
- Range r = new Range(startRange, true, endRange, true);
- // For the range queries we need to query the global index and then handle the results a little differently.
- Map<MapKey,Set<Range>> ranges = new HashMap<MapKey,Set<Range>>();
- MapKey key = new MapKey(entry.getKey().toString(), entry.getValue().getLower().toString());
- key.setOriginalQueryValue(entry.getValue().getOriginalLower().toString());
- this.originalQueryValues.put(entry.getValue().getOriginalLower().toString(), key);
- ranges.put(key, new HashSet<Range>());
- ranges.get(key).add(r);
-
- // Now query the global index and override the field value used in the results map
- try {
- Map<MapKey,TermRange> lowerResults = queryGlobalIndex(ranges, entry.getKey().toString(), this.indexTableName, false, key, typeFilter);
- // Add the results to the global index results for both the upper and lower field values.
- Map<MapKey,TermRange> upperResults = new HashMap<MapKey,TermRange>();
- for (Entry<MapKey,TermRange> e : lowerResults.entrySet()) {
- MapKey key2 = new MapKey(e.getKey().getFieldName(), entry.getValue().getUpper().toString());
- key2.setOriginalQueryValue(entry.getValue().getOriginalUpper().toString());
- upperResults.put(key2, e.getValue());
- this.originalQueryValues.put(entry.getValue().getOriginalUpper(), key2);
-
- }
-
- this.globalIndexResults.putAll(lowerResults);
- this.globalIndexResults.putAll(upperResults);
-
- } catch (TableNotFoundException e) {
- log.error("index table not found", e);
- throw new RuntimeException(" index table not found", e);
- }
- } else {
- log.warn("Unbounded range detected, not querying index for it. Field " + entry.getKey().toString() + " in query: " + query);
- }
- }
- // Now that we have calculated all of the ranges, query the global index.
- try {
-
- // Query for the trailing wildcards if we have any
- for (Entry<MapKey,Set<Range>> trailing : trailingWildcardRanges.entrySet()) {
- Map<MapKey,Set<Range>> m = new HashMap<MapKey,Set<Range>>();
- m.put(trailing.getKey(), trailing.getValue());
- if (log.isDebugEnabled())
- log.debug("Ranges for Wildcard Global Index query: " + m.toString());
- this.globalIndexResults.putAll(queryGlobalIndex(m, trailing.getKey().getFieldName(), this.indexTableName, false, trailing.getKey(), typeFilter));
- }
-
- // Query for the leading wildcards if we have any
- for (Entry<MapKey,Set<Range>> leading : leadingWildcardRanges.entrySet()) {
- Map<MapKey,Set<Range>> m = new HashMap<MapKey,Set<Range>>();
- m.put(leading.getKey(), leading.getValue());
- if (log.isDebugEnabled())
- log.debug("Ranges for Wildcard Global Reverse Index query: " + m.toString());
- this.globalIndexResults.putAll(queryGlobalIndex(m, leading.getKey().getFieldName(), this.reverseIndexTableName, true, leading.getKey(), typeFilter));
- }
-
- // Query for the equals case
- for (Entry<MapKey,Set<Range>> equals : indexRanges.entrySet()) {
- Map<MapKey,Set<Range>> m = new HashMap<MapKey,Set<Range>>();
- m.put(equals.getKey(), equals.getValue());
- if (log.isDebugEnabled())
- log.debug("Ranges for Global Index query: " + m.toString());
- this.globalIndexResults.putAll(queryGlobalIndex(m, equals.getKey().getFieldName(), this.indexTableName, false, equals.getKey(), typeFilter));
- }
- } catch (TableNotFoundException e) {
- log.error("index table not found", e);
- throw new RuntimeException(" index table not found", e);
- }
-
- if (log.isDebugEnabled())
- log.debug("Ranges from Global Index query: " + globalIndexResults.toString());
-
- // Now traverse the AST
- EvaluationContext ctx = new EvaluationContext();
- this.getAST().childrenAccept(this, ctx);
-
- if (ctx.lastRange.getRanges().size() == 0) {
- log.debug("No resulting range set");
- } else {
- if (log.isDebugEnabled())
- log.debug("Setting range results to: " + ctx.lastRange.getRanges().toString());
- this.result = ctx.lastRange.getRanges();
- }
- }
-
- /**
- *
- * @return set of ranges to use for the shard table
- */
- public Set<Range> getResult() {
- return result;
- }
-
- /**
- *
- * @return map of field names to index field values
- */
- public Multimap<String,String> getIndexEntries() {
- return indexEntries;
- }
-
- public Map<String,String> getIndexValues() {
- return indexValues;
- }
-
- /**
- *
- * @return Cardinality for each field name.
- */
- public Map<String,Long> getTermCardinalities() {
- return termCardinalities;
- }
-
- /**
- *
- * @param indexRanges
- * @param tableName
- * @param isReverse
- * switch that determines whether or not to reverse the results
- * @param override
- * mapKey for wildcard and range queries that specify which mapkey to use in the results
- * @param typeFilter
- * - optional list of datatypes
- * @throws TableNotFoundException
- */
- protected Map<MapKey,TermRange> queryGlobalIndex(Map<MapKey,Set<Range>> indexRanges, String specificFieldName, String tableName, boolean isReverse,
- MapKey override, Set<String> typeFilter) throws TableNotFoundException {
-
- // The results map where the key is the field name and field value and the
- // value is a set of ranges. The mapkey will always be the field name
- // and field value that was passed in the original query. The TermRange
- // will contain the field name and field value found in the index.
- Map<MapKey,TermRange> results = new HashMap<MapKey,TermRange>();
-
- // Seed the results map and create the range set for the batch scanner
- Set<Range> rangeSuperSet = new HashSet<Range>();
- for (Entry<MapKey,Set<Range>> entry : indexRanges.entrySet()) {
- rangeSuperSet.addAll(entry.getValue());
- TermRange tr = new TermRange(entry.getKey().getFieldName(), entry.getKey().getFieldValue());
- if (null == override)
- results.put(entry.getKey(), tr);
- else
- results.put(override, tr);
- }
-
- if (log.isDebugEnabled())
- log.debug("Querying global index table: " + tableName + ", range: " + rangeSuperSet.toString() + " colf: " + specificFieldName);
- BatchScanner bs = this.c.createBatchScanner(tableName, this.auths, this.queryThreads);
- bs.setRanges(rangeSuperSet);
- if (null != specificFieldName) {
- bs.fetchColumnFamily(new Text(specificFieldName));
- }
-
- for (Entry<Key,Value> entry : bs) {
- if (log.isDebugEnabled()) {
- log.debug("Index entry: " + entry.getKey().toString());
- }
- String fieldValue = null;
- if (!isReverse) {
- fieldValue = entry.getKey().getRow().toString();
- } else {
- StringBuilder buf = new StringBuilder(entry.getKey().getRow().toString());
- fieldValue = buf.reverse().toString();
- }
-
- String fieldName = entry.getKey().getColumnFamily().toString();
- // Get the shard id and datatype from the colq
- String colq = entry.getKey().getColumnQualifier().toString();
- int separator = colq.indexOf(EvaluatingIterator.NULL_BYTE_STRING);
- String shardId = null;
- String datatype = null;
- if (separator != -1) {
- shardId = colq.substring(0, separator);
- datatype = colq.substring(separator + 1);
- } else {
- shardId = colq;
- }
- // Skip this entry if the type is not correct
- if (null != datatype && null != typeFilter && !typeFilter.contains(datatype))
- continue;
- // Parse the UID.List object from the value
- Uid.List uidList = null;
- try {
- uidList = Uid.List.parseFrom(entry.getValue().get());
- } catch (InvalidProtocolBufferException e) {
- // Don't add UID information, at least we know what shards
- // it is located in.
- }
-
- // Add the count for this shard to the total count for the term.
- long count = 0;
- Long storedCount = termCardinalities.get(fieldName);
- if (null == storedCount || 0 == storedCount) {
- count = uidList.getCOUNT();
- } else {
- count = uidList.getCOUNT() + storedCount;
- }
- termCardinalities.put(fieldName, count);
- this.indexEntries.put(fieldName, fieldValue);
-
- if (null == override)
- this.indexValues.put(fieldValue, fieldValue);
- else
- this.indexValues.put(fieldValue, override.getOriginalQueryValue());
-
- // Create the keys
- Text shard = new Text(shardId);
- if (uidList.getIGNORE()) {
- // Then we create a scan range that is the entire shard
- if (null == override)
- results.get(new MapKey(fieldName, fieldValue)).add(new Range(shard));
- else
- results.get(override).add(new Range(shard));
- } else {
- // We should have UUIDs, create event ranges
- for (String uuid : uidList.getUIDList()) {
- Text cf = new Text(datatype);
- TextUtil.textAppend(cf, uuid);
- Key startKey = new Key(shard, cf);
- Key endKey = new Key(shard, new Text(cf.toString() + EvaluatingIterator.NULL_BYTE_STRING));
- Range eventRange = new Range(startKey, true, endKey, false);
- if (null == override)
- results.get(new MapKey(fieldName, fieldValue)).add(eventRange);
- else
- results.get(override).add(eventRange);
- }
- }
- }
- bs.close();
- return results;
- }
-
- @Override
- public Object visit(ASTOrNode node, Object data) {
- boolean previouslyInOrContext = false;
- EvaluationContext ctx = null;
- if (null != data && data instanceof EvaluationContext) {
- ctx = (EvaluationContext) data;
- previouslyInOrContext = ctx.inOrContext;
- } else {
- ctx = new EvaluationContext();
- }
- ctx.inOrContext = true;
- // Process both sides of this node. Left branch first
- node.jjtGetChild(0).jjtAccept(this, ctx);
- Long leftCardinality = this.termCardinalities.get(ctx.lastProcessedTerm);
- if (null == leftCardinality)
- leftCardinality = 0L;
- TermRange leftRange = ctx.lastRange;
- if (log.isDebugEnabled())
- log.debug("[OR-left] term: " + ctx.lastProcessedTerm + ", cardinality: " + leftCardinality + ", ranges: " + leftRange.getRanges().size());
-
- // Process the right branch
- node.jjtGetChild(1).jjtAccept(this, ctx);
- Long rightCardinality = this.termCardinalities.get(ctx.lastProcessedTerm);
- if (null == rightCardinality)
- rightCardinality = 0L;
- TermRange rightRange = ctx.lastRange;
- if (log.isDebugEnabled())
- log.debug("[OR-right] term: " + ctx.lastProcessedTerm + ", cardinality: " + rightCardinality + ", ranges: " + rightRange.getRanges().size());
-
- // reset the state
- if (null != data && !previouslyInOrContext)
- ctx.inOrContext = false;
- // Add the ranges for the left and right branches to a TreeSet to sort them
- Set<Range> ranges = new TreeSet<Range>();
- ranges.addAll(leftRange.getRanges());
- ranges.addAll(rightRange.getRanges());
- // Now create the union set
- Set<Text> shardsAdded = new HashSet<Text>();
- Set<Range> returnSet = new HashSet<Range>();
- for (Range r : ranges) {
- if (!shardsAdded.contains(r.getStartKey().getRow())) {
- // Only add ranges with a start key for the entire shard.
- if (r.getStartKey().getColumnFamily() == null) {
- shardsAdded.add(r.getStartKey().getRow());
- }
- returnSet.add(r);
- } else {
- // if (log.isTraceEnabled())
- log.info("Skipping event specific range: " + r.toString() + " because shard range has already been added: "
- + shardsAdded.contains(r.getStartKey().getRow()));
- }
- }
- // Clear the ranges from the context and add the result in its place
- TermRange orRange = new TermRange("OR_RESULT", "foo");
- orRange.addAll(returnSet);
- if (log.isDebugEnabled())
- log.debug("[OR] results: " + orRange.getRanges().toString());
- ctx.lastRange = orRange;
- ctx.lastProcessedTerm = "OR_RESULT";
- this.termCardinalities.put("OR_RESULT", (leftCardinality + rightCardinality));
- return null;
- }
-
- @Override
- public Object visit(ASTAndNode node, Object data) {
- boolean previouslyInAndContext = false;
- EvaluationContext ctx = null;
- if (null != data && data instanceof EvaluationContext) {
- ctx = (EvaluationContext) data;
- previouslyInAndContext = ctx.inAndContext;
- } else {
- ctx = new EvaluationContext();
- }
- ctx.inAndContext = true;
- // Process both sides of this node.
- node.jjtGetChild(0).jjtAccept(this, ctx);
- String leftTerm = ctx.lastProcessedTerm;
- Long leftCardinality = this.termCardinalities.get(leftTerm);
- if (null == leftCardinality)
- leftCardinality = 0L;
- TermRange leftRange = ctx.lastRange;
- if (log.isDebugEnabled())
- log.debug("[AND-left] term: " + ctx.lastProcessedTerm + ", cardinality: " + leftCardinality + ", ranges: " + leftRange.getRanges().size());
-
- // Process the right branch
- node.jjtGetChild(1).jjtAccept(this, ctx);
- String rightTerm = ctx.lastProcessedTerm;
- Long rightCardinality = this.termCardinalities.get(rightTerm);
- if (null == rightCardinality)
- rightCardinality = 0L;
- TermRange rightRange = ctx.lastRange;
- if (log.isDebugEnabled())
- log.debug("[AND-right] term: " + ctx.lastProcessedTerm + ", cardinality: " + rightCardinality + ", ranges: " + rightRange.getRanges().size());
-
- // reset the state
- if (null != data && !previouslyInAndContext)
- ctx.inAndContext = false;
-
- long card = 0L;
- TermRange andRange = new TermRange("AND_RESULT", "foo");
- if ((leftCardinality > 0 && leftCardinality <= rightCardinality) || rightCardinality == 0) {
- card = leftCardinality;
- andRange.addAll(leftRange.getRanges());
- } else if ((rightCardinality > 0 && rightCardinality <= leftCardinality) || leftCardinality == 0) {
- card = rightCardinality;
- andRange.addAll(rightRange.getRanges());
- }
- if (log.isDebugEnabled())
- log.debug("[AND] results: " + andRange.getRanges().toString());
- ctx.lastRange = andRange;
- ctx.lastProcessedTerm = "AND_RESULT";
- this.termCardinalities.put("AND_RESULT", card);
-
- return null;
- }
-
- @Override
- public Object visit(ASTEQNode node, Object data) {
- StringBuilder fieldName = new StringBuilder();
- ObjectHolder value = new ObjectHolder();
- // Process both sides of this node.
- Object left = node.jjtGetChild(0).jjtAccept(this, data);
- Object right = node.jjtGetChild(1).jjtAccept(this, data);
- // Ignore functions in the query
- if (left instanceof FunctionResult || right instanceof FunctionResult)
- return null;
- decodeResults(left, right, fieldName, value);
- // We need to check to see if we are in a NOT context. If so,
- // then we need to reverse the negation.
- boolean negated = false;
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- if (ctx.inNotContext)
- negated = !negated;
- }
- QueryTerm term = new QueryTerm(negated, JexlOperatorConstants.getOperator(node.getClass()), value.getObject());
- termsCopy.put(fieldName.toString(), term);
- // Get the terms from the global index
- // Remove the begin and end ' marks
- String termValue = null;
- if (((String) term.getValue()).startsWith("'") && ((String) term.getValue()).endsWith("'"))
- termValue = ((String) term.getValue()).substring(1, ((String) term.getValue()).length() - 1);
- else
- termValue = (String) term.getValue();
- // Get the values found in the index for this query term
- TermRange ranges = null;
- for (MapKey key : this.originalQueryValues.get(termValue)) {
- if (key.getFieldName().equalsIgnoreCase(fieldName.toString())) {
- ranges = this.globalIndexResults.get(key);
- if (log.isDebugEnabled())
- log.debug("Results for cached index ranges for key: " + key + " are " + ranges);
- }
- }
- // If no result for this field name and value, then add empty range
- if (null == ranges)
- ranges = new TermRange(fieldName.toString(), (String) term.getValue());
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- ctx.lastRange = ranges;
- ctx.lastProcessedTerm = fieldName.toString();
- }
- return null;
- }
-
- @Override
- public Object visit(ASTNENode node, Object data) {
- StringBuilder fieldName = new StringBuilder();
- ObjectHolder value = new ObjectHolder();
- // Process both sides of this node.
- Object left = node.jjtGetChild(0).jjtAccept(this, data);
- Object right = node.jjtGetChild(1).jjtAccept(this, data);
- // Ignore functions in the query
- if (left instanceof FunctionResult || right instanceof FunctionResult)
- return null;
- decodeResults(left, right, fieldName, value);
- // We need to check to see if we are in a NOT context. If so,
- // then we need to reverse the negation.
- boolean negated = true;
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- if (ctx.inNotContext)
- negated = !negated;
- }
- if (negated)
- negatedTerms.add(fieldName.toString());
- QueryTerm term = new QueryTerm(negated, JexlOperatorConstants.getOperator(node.getClass()), value.getObject());
- termsCopy.put(fieldName.toString(), term);
- // We can only use the global index for equality, put in fake results
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- ctx.lastRange = new TermRange(fieldName.toString(), term.getValue());
- ctx.lastProcessedTerm = fieldName.toString();
- termCardinalities.put(fieldName.toString(), 0L);
- }
- return null;
- }
-
- @Override
- public Object visit(ASTLTNode node, Object data) {
- StringBuilder fieldName = new StringBuilder();
- ObjectHolder value = new ObjectHolder();
- // Process both sides of this node.
- Object left = node.jjtGetChild(0).jjtAccept(this, data);
- Object right = node.jjtGetChild(1).jjtAccept(this, data);
- // Ignore functions in the query
- if (left instanceof FunctionResult || right instanceof FunctionResult)
- return null;
- decodeResults(left, right, fieldName, value);
- // We need to check to see if we are in a NOT context. If so,
- // then we need to reverse the negation.
- boolean negated = false;
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- if (ctx.inNotContext)
- negated = !negated;
- }
- QueryTerm term = new QueryTerm(negated, JexlOperatorConstants.getOperator(node.getClass()), value.getObject());
- termsCopy.put(fieldName.toString(), term);
- // Get the terms from the global index
- // Remove the begin and end ' marks
- String termValue = null;
- if (((String) term.getValue()).startsWith("'") && ((String) term.getValue()).endsWith("'"))
- termValue = ((String) term.getValue()).substring(1, ((String) term.getValue()).length() - 1);
- else
- termValue = (String) term.getValue();
- // Get the values found in the index for this query term
- TermRange ranges = null;
- for (MapKey key : this.originalQueryValues.get(termValue)) {
- if (key.getFieldName().equalsIgnoreCase(fieldName.toString())) {
- ranges = this.globalIndexResults.get(key);
- if (log.isDebugEnabled())
- log.debug("Results for cached index ranges for key: " + key + " are " + ranges);
- }
- }
- // If no result for this field name and value, then add empty range
- if (null == ranges)
- ranges = new TermRange(fieldName.toString(), (String) term.getValue());
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- ctx.lastRange = ranges;
- ctx.lastProcessedTerm = fieldName.toString();
- }
- return null;
- }
-
- @Override
- public Object visit(ASTGTNode node, Object data) {
- StringBuilder fieldName = new StringBuilder();
- ObjectHolder value = new ObjectHolder();
- // Process both sides of this node.
- Object left = node.jjtGetChild(0).jjtAccept(this, data);
- Object right = node.jjtGetChild(1).jjtAccept(this, data);
- // Ignore functions in the query
- if (left instanceof FunctionResult || right instanceof FunctionResult)
- return null;
- decodeResults(left, right, fieldName, value);
- // We need to check to see if we are in a NOT context. If so,
- // then we need to reverse the negation.
- boolean negated = false;
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- if (ctx.inNotContext)
- negated = !negated;
- }
- QueryTerm term = new QueryTerm(negated, JexlOperatorConstants.getOperator(node.getClass()), value.getObject());
- termsCopy.put(fieldName.toString(), term);
- // Get the terms from the global index
- // Remove the begin and end ' marks
- String termValue = null;
- if (((String) term.getValue()).startsWith("'") && ((String) term.getValue()).endsWith("'"))
- termValue = ((String) term.getValue()).substring(1, ((String) term.getValue()).length() - 1);
- else
- termValue = (String) term.getValue();
- // Get the values found in the index for this query term
- TermRange ranges = null;
- for (MapKey key : this.originalQueryValues.get(termValue)) {
- if (key.getFieldName().equalsIgnoreCase(fieldName.toString())) {
- ranges = this.globalIndexResults.get(key);
- if (log.isDebugEnabled())
- log.debug("Results for cached index ranges for key: " + key + " are " + ranges);
- }
- }
- // If no result for this field name and value, then add empty range
- if (null == ranges)
- ranges = new TermRange(fieldName.toString(), (String) term.getValue());
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- ctx.lastRange = ranges;
- ctx.lastProcessedTerm = fieldName.toString();
- }
- return null;
- }
-
- @Override
- public Object visit(ASTLENode node, Object data) {
- StringBuilder fieldName = new StringBuilder();
- ObjectHolder value = new ObjectHolder();
- // Process both sides of this node.
- Object left = node.jjtGetChild(0).jjtAccept(this, data);
- Object right = node.jjtGetChild(1).jjtAccept(this, data);
- // Ignore functions in the query
- if (left instanceof FunctionResult || right instanceof FunctionResult)
- return null;
- decodeResults(left, right, fieldName, value);
- // We need to check to see if we are in a NOT context. If so,
- // then we need to reverse the negation.
- boolean negated = false;
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- if (ctx.inNotContext)
- negated = !negated;
- }
- QueryTerm term = new QueryTerm(negated, JexlOperatorConstants.getOperator(node.getClass()), value.getObject());
- termsCopy.put(fieldName.toString(), term);
- // Get the terms from the global index
- // Remove the begin and end ' marks
- String termValue = null;
- if (((String) term.getValue()).startsWith("'") && ((String) term.getValue()).endsWith("'"))
- termValue = ((String) term.getValue()).substring(1, ((String) term.getValue()).length() - 1);
- else
- termValue = (String) term.getValue();
- // Get the values found in the index for this query term
- TermRange ranges = null;
- for (MapKey key : this.originalQueryValues.get(termValue)) {
- if (key.getFieldName().equalsIgnoreCase(fieldName.toString())) {
- ranges = this.globalIndexResults.get(key);
- if (log.isDebugEnabled())
- log.debug("Results for cached index ranges for key: " + key + " are " + ranges);
- }
- }
- // If no result for this field name and value, then add empty range
- if (null == ranges)
- ranges = new TermRange(fieldName.toString(), (String) term.getValue());
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- ctx.lastRange = ranges;
- ctx.lastProcessedTerm = fieldName.toString();
- }
- return null;
- }
-
- @Override
- public Object visit(ASTGENode node, Object data) {
- StringBuilder fieldName = new StringBuilder();
- ObjectHolder value = new ObjectHolder();
- // Process both sides of this node.
- Object left = node.jjtGetChild(0).jjtAccept(this, data);
- Object right = node.jjtGetChild(1).jjtAccept(this, data);
- // Ignore functions in the query
- if (left instanceof FunctionResult || right instanceof FunctionResult)
- return null;
- decodeResults(left, right, fieldName, value);
- // We need to check to see if we are in a NOT context. If so,
- // then we need to reverse the negation.
- boolean negated = false;
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- if (ctx.inNotContext)
- negated = !negated;
- }
- QueryTerm term = new QueryTerm(negated, JexlOperatorConstants.getOperator(node.getClass()), value.getObject());
- termsCopy.put(fieldName.toString(), term);
- // Get the terms from the global index
- // Remove the begin and end ' marks
- String termValue = null;
- if (((String) term.getValue()).startsWith("'") && ((String) term.getValue()).endsWith("'"))
- termValue = ((String) term.getValue()).substring(1, ((String) term.getValue()).length() - 1);
- else
- termValue = (String) term.getValue();
- // Get the values found in the index for this query term
- TermRange ranges = null;
- for (MapKey key : this.originalQueryValues.get(termValue)) {
- if (key.getFieldName().equalsIgnoreCase(fieldName.toString())) {
- ranges = this.globalIndexResults.get(key);
- if (log.isDebugEnabled())
- log.debug("Results for cached index ranges for key: " + key + " are " + ranges);
- }
- }
- // If no result for this field name and value, then add empty range
- if (null == ranges)
- ranges = new TermRange(fieldName.toString(), (String) term.getValue());
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- ctx.lastRange = ranges;
- ctx.lastProcessedTerm = fieldName.toString();
- }
- return null;
- }
-
- @Override
- public Object visit(ASTERNode node, Object data) {
- StringBuilder fieldName = new StringBuilder();
- ObjectHolder value = new ObjectHolder();
- // Process both sides of this node.
- Object left = node.jjtGetChild(0).jjtAccept(this, data);
- Object right = node.jjtGetChild(1).jjtAccept(this, data);
- // Ignore functions in the query
- if (left instanceof FunctionResult || right instanceof FunctionResult)
- return null;
- decodeResults(left, right, fieldName, value);
- // We need to check to see if we are in a NOT context. If so,
- // then we need to reverse the negation.
- boolean negated = false;
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- if (ctx.inNotContext)
- negated = !negated;
- }
- QueryTerm term = new QueryTerm(negated, JexlOperatorConstants.getOperator(node.getClass()), value.getObject());
- termsCopy.put(fieldName.toString(), term);
- // Get the terms from the global index
- // Remove the begin and end ' marks
- String termValue = null;
- if (((String) term.getValue()).startsWith("'") && ((String) term.getValue()).endsWith("'"))
- termValue = ((String) term.getValue()).substring(1, ((String) term.getValue()).length() - 1);
- else
- termValue = (String) term.getValue();
- // Get the values found in the index for this query term
- TermRange ranges = null;
- for (MapKey key : this.originalQueryValues.get(termValue)) {
- if (key.getFieldName().equalsIgnoreCase(fieldName.toString())) {
- ranges = this.globalIndexResults.get(key);
- if (log.isDebugEnabled())
- log.debug("Results for cached index ranges for key: " + key + " are " + ranges);
- }
- }
- // If no result for this field name and value, then add empty range
- if (null == ranges)
- ranges = new TermRange(fieldName.toString(), (String) term.getValue());
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- ctx.lastRange = ranges;
- ctx.lastProcessedTerm = fieldName.toString();
- }
-
- return null;
- }
-
- @Override
- public Object visit(ASTNRNode node, Object data) {
- StringBuilder fieldName = new StringBuilder();
- ObjectHolder value = new ObjectHolder();
- // Process both sides of this node.
- Object left = node.jjtGetChild(0).jjtAccept(this, data);
- Object right = node.jjtGetChild(1).jjtAccept(this, data);
- // Ignore functions in the query
- if (left instanceof FunctionResult || right instanceof FunctionResult)
- return null;
- decodeResults(left, right, fieldName, value);
- // We need to check to see if we are in a NOT context. If so,
- // then we need to reverse the negation.
- boolean negated = true;
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- if (ctx.inNotContext)
- negated = !negated;
- }
- if (negated)
- negatedTerms.add(fieldName.toString());
- QueryTerm term = new QueryTerm(negated, JexlOperatorConstants.getOperator(node.getClass()), value.getObject());
- termsCopy.put(fieldName.toString(), term);
- // We can only use the global index for equality, put in fake results
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- ctx.lastRange = new TermRange(fieldName.toString(), term.getValue());
- ctx.lastProcessedTerm = fieldName.toString();
- termCardinalities.put(fieldName.toString(), 0L);
- }
-
- return null;
- }
-
- @Override
- public Object visit(ASTNullLiteral node, Object data) {
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- ctx.lastRange = new TermRange("null", "null");
- ctx.lastProcessedTerm = "null";
- termCardinalities.put("null", 0L);
- }
- return new LiteralResult(node.image);
- }
-
- @Override
- public Object visit(ASTTrueNode node, Object data) {
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- ctx.lastRange = new TermRange("true", "true");
- ctx.lastProcessedTerm = "true";
- termCardinalities.put("true", 0L);
- }
- return new LiteralResult(node.image);
- }
-
- @Override
- public Object visit(ASTFalseNode node, Object data) {
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- ctx.lastRange = new TermRange("false", "false");
- ctx.lastProcessedTerm = "false";
- termCardinalities.put("false", 0L);
- }
- return new LiteralResult(node.image);
- }
-
- @Override
- public Object visit(ASTFunctionNode node, Object data) {
- // objectNode 0 is the prefix
- // objectNode 1 is the identifier , the others are parameters.
- // process the remaining arguments
- FunctionResult fr = new FunctionResult();
- int argc = node.jjtGetNumChildren() - 2;
- for (int i = 0; i < argc; i++) {
- // Process both sides of this node.
- Object result = node.jjtGetChild(i + 2).jjtAccept(this, data);
- if (result instanceof TermResult) {
- TermResult tr = (TermResult) result;
- fr.getTerms().add(tr);
- termsCopy.put((String) tr.value, null);
- }
- }
- if (null != data && data instanceof EvaluationContext) {
- EvaluationContext ctx = (EvaluationContext) data;
- ctx.lastRange = new TermRange(node.jjtGetChild(0).image, node.jjtGetChild(1).image);
- ctx.lastProcessedTerm = node.jjtGetChild(0).image;
- termCardinalities.put(node.jjtGetChild(0).image, 0L);
- }
- return fr;
- }
-
-}
[33/39] git commit: ACCUMULO-645 found and removed many other
references to HADOOP_HOME
Posted by ec...@apache.org.
ACCUMULO-645 found and removed many other references to HADOOP_HOME
git-svn-id: https://svn.apache.org/repos/asf/accumulo/trunk@1433049 13f79535-47bb-0310-9956-ffa450edef68
(cherry picked from commit 15f71f96c6850cbc1712e1693307f8f8f978d069)
Reason: Hadoop2 Compat
Author: Eric C. Newton <ec...@apache.org>
Ref: ACCUMULO-1792
Signed-off-by: Eric Newton <er...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d16fd77f
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d16fd77f
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d16fd77f
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: d16fd77fa10afedab3bffe17235a3d700fd8de8e
Parents: d70e5a1
Author: Sean Busbey <bu...@cloudera.com>
Authored: Sun Oct 27 15:47:25 2013 -0500
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Nov 25 16:06:44 2013 -0500
----------------------------------------------------------------------
bin/accumulo | 15 +++++++++------
bin/config.sh | 6 +++---
.../start/classloader/AccumuloClassLoader.java | 7 ++++---
test/system/auto/TestUtils.py | 16 ++++++++++++++--
4 files changed, 30 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d16fd77f/bin/accumulo
----------------------------------------------------------------------
diff --git a/bin/accumulo b/bin/accumulo
index 6642818..bbca3f0 100755
--- a/bin/accumulo
+++ b/bin/accumulo
@@ -48,9 +48,9 @@ if [ -z "$JAVA_HOME" -o ! -d "$JAVA_HOME" ]; then
echo "JAVA_HOME is not set. Please make sure it's set globally or in conf/accumulo-env.sh"
exit 1
fi
-if [ -z "$HADOOP_HOME" -o ! -d "$HADOOP_HOME" ]; then
- echo "HADOOP_HOME is not set. Please make sure it's set globally or in conf/accumulo-env.sh"
- exit 1
+if [ -z $HADOOP_PREFIX -o ! -d $HADOOP_PREFIX ]; then
+ echo "HADOOP_PREFIX is not set. Please make sure it's set globally or in conf/accumulo-env.sh"
+ exit 1
fi
if [ -z "$ZOOKEEPER_HOME" -o ! -d "$ZOOKEEPER_HOME" ]; then
echo "ZOOKEEPER_HOME is not set. Please make sure it's set globally or in conf/accumulo-env.sh"
@@ -60,8 +60,11 @@ fi
#Export the variables just in case they are not exported
#This makes them available to java
export JAVA_HOME=$JAVA_HOME
-export HADOOP_HOME=$HADOOP_HOME
+export HADOOP_PREFIX=$HADOOP_PREFIX
export ZOOKEEPER_HOME=$ZOOKEEPER_HOME
+# backwards compatible with old configurations
+export HADOOP_HOME=$HADOOP_PREFIX
+export HADOOP_HOME_WARN_SUPPRESS=1
START_JAR=$ACCUMULO_HOME/lib/accumulo-start-$ACCUMULO_VERSION.jar
COMMONS_JCI_JARS=$ACCUMULO_HOME/lib/commons-jci-core-1.0.jar:$ACCUMULO_HOME/lib/commons-jci-fam-1.0.jar:$ACCUMULO_HOME/lib/log4j-1.2.16.jar:$ACCUMULO_HOME/lib/commons-logging-1.0.4.jar:$ACCUMULO_HOME/lib/commons-logging-api-1.0.4.jar
@@ -79,7 +82,7 @@ locationByProgram()
}
test -n "$JAVA_HOME" || export JAVA_HOME=`locationByProgram java`
-test -n "$HADOOP_HOME" || export HADOOP_HOME=`locationByProgram hadoop`
+test -n "$HADOOP_PREFIX" || export HADOOP_PREFIX=`locationByProgram hadoop`
test -n "$ZOOKEEPER_HOME" || export ZOOKEEPER_HOME=`locationByProgram zkCli.sh`
@@ -130,4 +133,4 @@ JAVA=$JAVA_HOME/bin/java
PLATFORM="`$JAVA -cp $CLASSPATH org.apache.accumulo.start.Platform`"
# app isn't used anywhere, but it makes the process easier to spot when ps/top/snmp truncate the command line
-exec $JAVA "-Dapp=$1" -classpath $CLASSPATH $ACCUMULO_OPTS -XX:OnOutOfMemoryError="${ACCUMULO_KILL_CMD:-kill -9 %p}" -Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl -Djava.library.path="$HADOOP_HOME/lib/native/$PLATFORM" -Dorg.apache.accumulo.core.home.dir="$ACCUMULO_HOME" -Dhadoop.home.dir="$HADOOP_HOME" -Dzookeeper.home.dir="$ZOOKEEPER_HOME" org.apache.accumulo.start.Main "$@"
+exec $JAVA "-Dapp=$1" -classpath $CLASSPATH $ACCUMULO_OPTS -XX:OnOutOfMemoryError="${ACCUMULO_KILL_CMD:-kill -9 %p}" -Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl -Djava.library.path="$HADOOP_PREFIX/lib/native/$PLATFORM" -Dorg.apache.accumulo.core.home.dir="$ACCUMULO_HOME" -Dhadoop.home.dir="$HADOOP_PREFIX" -Dhadoop.conf.dir="$HADOOP_CONF_DIR" -Dzookeeper.home.dir="$ZOOKEEPER_HOME" org.apache.accumulo.start.Main "$@"
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d16fd77f/bin/config.sh
----------------------------------------------------------------------
diff --git a/bin/config.sh b/bin/config.sh
index d9300d5..08a8bbd 100755
--- a/bin/config.sh
+++ b/bin/config.sh
@@ -74,14 +74,14 @@ fi
if [ -z "$HADOOP_PREFIX" ]
then
- HADOOP_HOME="`which hadoop`"
+ HADOOP_PREFIX="`which hadoop`"
if [ -z "$HADOOP_PREFIX" ]
then
echo "You must set HADOOP_PREFIX"
exit 1
fi
- HADOOP_HOME=`dirname $HADOOP_PREFIX`
- HADOOP_HOME=`dirname $HADOOP_PREFIX`
+ HADOOP_PREFIX=`dirname $HADOOP_PREFIX`
+ HADOOP_PREFIX=`dirname $HADOOP_PREFIX`
fi
if [ ! -d "$HADOOP_PREFIX" ]
then
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d16fd77f/src/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
----------------------------------------------------------------------
diff --git a/src/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java b/src/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
index 8d217d7..fd41938 100644
--- a/src/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
+++ b/src/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java
@@ -106,10 +106,11 @@ public class AccumuloClassLoader {
public static final String DYNAMIC_CLASSPATH_PROPERTY_NAME = "general.dynamic.classpaths";
- public static final String ACCUMULO_CLASSPATH_VALUE =
+ public static final String ACCUMULO_CLASSPATH_VALUE =
"$ACCUMULO_CONF_DIR,\n"
- + "$ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar,\n"
- + "$ACCUMULO_HOME/lib/[^.].*.jar,\n" + "$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n" + "$HADOOP_HOME/[^.].*.jar,\n" + "$HADOOP_HOME/conf,\n"
+ + "$ACCUMULO_HOME/lib/[^.].*.jar,\n"
+ + "$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n"
+ + "$HADOOP_PREFIX/[^.].*.jar,\n" + "$HADOOP_CONF_DIR,\n"
+ "$HADOOP_HOME/lib/[^.].*.jar,\n";
/**
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d16fd77f/test/system/auto/TestUtils.py
----------------------------------------------------------------------
diff --git a/test/system/auto/TestUtils.py b/test/system/auto/TestUtils.py
index b7659a2..0e1bf1a 100755
--- a/test/system/auto/TestUtils.py
+++ b/test/system/auto/TestUtils.py
@@ -54,8 +54,20 @@ LOG_PROPERTIES= os.path.join(ACCUMULO_CONF_DIR, 'log4j.properties')
LOG_GENERIC = os.path.join(ACCUMULO_CONF_DIR, 'generic_logger.xml')
LOG_MONITOR = os.path.join(ACCUMULO_CONF_DIR, 'monitor_logger.xml')
-General_CLASSPATH = ("$ACCUMULO_CONF_DIR,$ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar, $ACCUMULO_HOME/lib/[^.].*.jar, $ZOOKEEPER_HOME/zookeeper[^.].*.jar,"
-"$HADOOP_HOME/conf,$HADOOP_HOME/[^.].*.jar, $HADOOP_HOME/lib/[^.].*.jar")
+General_CLASSPATH = """
+$ACCUMULO_CONF_DIR,
+$ACCUMULO_HOME/server/target/classes/,
+ $ACCUMULO_HOME/core/target/classes/,
+ $ACCUMULO_HOME/start/target/classes/,
+ $ACCUMULO_HOME/fate/target/classes/,
+ $ACCUMULO_HOME/examples/instamo/target/classes,
+ $ACCUMULO_HOME/examples/simple/target/classes,
+ $ACCUMULO_HOME/lib/[^.].*.jar,
+ $ZOOKEEPER_HOME/zookeeper[^.].*.jar,
+ $HADOOP_CONF_DIR,
+ $HADOOP_PREFIX/[^.].*.jar,
+ $HADOOP_PREFIX/lib/[^.].*.jar,
+"""
log = logging.getLogger('test.auto')
[24/39] git commit: ACCUMULO-873 removed hdfs info from accumulo
monitor page
Posted by ec...@apache.org.
ACCUMULO-873 removed hdfs info from accumulo monitor page
git-svn-id: https://svn.apache.org/repos/asf/accumulo/trunk@1427887 13f79535-47bb-0310-9956-ffa450edef68
(cherry picked from commit e05ca37e5f89b620c990723b2a41396415fdb8b9)
Reason: Polish
Author: Billie Rinaldi <bi...@apache.org>
Ref: ACCUMULO-1792
Signed-off-by: Eric Newton <er...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5d22af49
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5d22af49
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5d22af49
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 5d22af4903f49682db98f39d46509f089aeebe86
Parents: 4c16d8b
Author: Billie Rinaldi <bi...@apache.org>
Authored: Wed Jan 2 18:07:53 2013 +0000
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Nov 25 16:06:42 2013 -0500
----------------------------------------------------------------------
.../server/monitor/servlets/DefaultServlet.java | 40 --------------------
1 file changed, 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5d22af49/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java b/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java
index 9591d4a..5603f69 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java
@@ -50,10 +50,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-import org.apache.hadoop.hdfs.protocol.FSConstants;
-import org.apache.hadoop.ipc.RemoteException;
public class DefaultServlet extends BasicServlet {
@@ -213,10 +209,6 @@ public class DefaultServlet extends BasicServlet {
sb.append("</td>\n");
sb.append("<td class='noborder'>\n");
- doHdfsTable(sb);
- sb.append("</td>\n");
-
- sb.append("<td class='noborder'>\n");
doZooKeeperTable(sb);
sb.append("</td>\n");
@@ -296,38 +288,6 @@ public class DefaultServlet extends BasicServlet {
sb.append("</table>\n");
}
- private void doHdfsTable(StringBuilder sb) throws IOException {
- // HDFS
- Configuration conf = CachedConfiguration.getInstance();
- DistributedFileSystem fs = (DistributedFileSystem) FileSystem.get(conf);
- String httpAddress = conf.get("dfs.http.address");
- String port = httpAddress.split(":")[1];
- String href = "http://" + fs.getUri().getHost() + ":" + port;
- String liveUrl = href + "/dfsnodelist.jsp?whatNodes=LIVE";
- String deadUrl = href + "/dfsnodelist.jsp?whatNodes=DEAD";
- sb.append("<table>\n");
- sb.append("<tr><th colspan='2'><a href='" + href + "'>NameNode</a></th></tr>\n");
- try {
- boolean highlight = false;
- tableRow(sb, (highlight = !highlight), "Unreplicated Capacity", bytes(fs.getRawCapacity()));
- tableRow(sb, (highlight = !highlight), "% Used", NumberType.commas(fs.getRawUsed() * 100. / fs.getRawCapacity(), 0, 90, 0, 100) + "%");
- tableRow(sb, (highlight = !highlight), "Corrupt Blocks", NumberType.commas(fs.getCorruptBlocksCount(), 0, 0));
- DatanodeInfo[] liveNodes = fs.getClient().datanodeReport(FSConstants.DatanodeReportType.LIVE);
- DatanodeInfo[] deadNodes = fs.getClient().datanodeReport(FSConstants.DatanodeReportType.DEAD);
- tableRow(sb, (highlight = !highlight), "<a href='" + liveUrl + "'>Live Data Nodes</a>", NumberType.commas(liveNodes.length));
- tableRow(sb, (highlight = !highlight), "<a href='" + deadUrl + "'>Dead Data Nodes</a>", NumberType.commas(deadNodes.length));
- long count = 0;
- for (DatanodeInfo stat : liveNodes)
- count += stat.getXceiverCount();
- tableRow(sb, (highlight = !highlight), "Xceivers", NumberType.commas(count));
- } catch (RemoteException ex) {
- sb.append("<tr><td colspan='2'>Permission Denied</td></tr>\n");
- } catch (Exception ex) {
- sb.append("<tr><td colspan='2'><span class='error'>Down</span></td></tr>\n");
- }
- sb.append("</table>\n");
- }
-
private void doZooKeeperTable(StringBuilder sb) throws IOException {
// Zookeepers
sb.append("<table>\n");
[26/39] git commit: ACCUMULO-286 introduced ContextFactory to assist
with testing InputFormats and OutputFormats under different versions of
hadoop
Posted by ec...@apache.org.
ACCUMULO-286 introduced ContextFactory to assist with testing InputFormats and OutputFormats under different versions of hadoop
git-svn-id: https://svn.apache.org/repos/asf/incubator/accumulo/trunk@1229596 13f79535-47bb-0310-9956-ffa450edef68
(cherry picked from commit c0a0afdcceedba2947949761ca13ecc42ff8f9c1)
Reason: Testing
Author: Billie Rinaldi <bi...@apache.org>
Ref: ACCUMULO-1792
Expands change to CoordinateRecoveryTask to remove previous addition of InterruptedException catching in CoordinateRecoveryTask to maintain compile compatibility with non-2.0.2-alpha versions.
Differs from upstream by expanding test modifications to include depecrated ones removed in 1.5.x and tests only found in the 1.4.x branch.
Author: Sean Busbey <bu...@cloudera.com>
Signed-off-by: Eric Newton <er...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a3264e4f
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a3264e4f
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a3264e4f
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: a3264e4f1b188e0f4808120fa39d5c1695b6b01c
Parents: 5d22af4
Author: Billie Rinaldi <bi...@apache.org>
Authored: Tue Jan 10 15:27:57 2012 +0000
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Nov 25 16:06:42 2013 -0500
----------------------------------------------------------------------
.../accumulo/core/util/ContextFactory.java | 169 +++++++++++++++++++
.../mapreduce/AccumuloFileOutputFormatTest.java | 8 +-
.../mapreduce/AccumuloInputFormatTest.java | 71 ++++----
.../mapreduce/AccumuloRowInputFormatTest.java | 8 +-
.../lib/partition/RangePartitionerTest.java | 5 +-
.../helloworld/InsertWithOutputFormat.java | 10 +-
.../simple/filedata/ChunkInputFormatTest.java | 16 +-
.../server/master/CoordinateRecoveryTask.java | 5 +-
.../apache/accumulo/server/master/LogSort.java | 2 +-
9 files changed, 224 insertions(+), 70 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3264e4f/src/core/src/main/java/org/apache/accumulo/core/util/ContextFactory.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/util/ContextFactory.java b/src/core/src/main/java/org/apache/accumulo/core/util/ContextFactory.java
new file mode 100644
index 0000000..67819da
--- /dev/null
+++ b/src/core/src/main/java/org/apache/accumulo/core/util/ContextFactory.java
@@ -0,0 +1,169 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.core.util;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.MapContext;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.StatusReporter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+
+/**
+ * A factory to allow applications to deal with inconsistencies between MapReduce Context Objects API between hadoop-0.20 and later versions. This code is based
+ * on org.apache.hadoop.mapreduce.ContextFactory in hadoop-mapred-0.22.0.
+ */
+public class ContextFactory {
+
+ private static final Constructor<?> JOB_CONTEXT_CONSTRUCTOR;
+ private static final Constructor<?> TASK_CONTEXT_CONSTRUCTOR;
+ private static final Constructor<?> TASK_ID_CONSTRUCTOR;
+ private static final Constructor<?> MAP_CONSTRUCTOR;
+ private static final Constructor<?> MAP_CONTEXT_CONSTRUCTOR;
+ private static final Constructor<?> MAP_CONTEXT_IMPL_CONSTRUCTOR;
+ private static final Class<?> TASK_TYPE_CLASS;
+ private static final boolean useV21;
+
+ static {
+ boolean v21 = true;
+ final String PACKAGE = "org.apache.hadoop.mapreduce";
+ try {
+ Class.forName(PACKAGE + ".task.JobContextImpl");
+ } catch (ClassNotFoundException cnfe) {
+ v21 = false;
+ }
+ useV21 = v21;
+ Class<?> jobContextCls;
+ Class<?> taskContextCls;
+ Class<?> mapCls;
+ Class<?> mapContextCls;
+ Class<?> innerMapContextCls;
+ try {
+ if (v21) {
+ jobContextCls = Class.forName(PACKAGE + ".task.JobContextImpl");
+ taskContextCls = Class.forName(PACKAGE + ".task.TaskAttemptContextImpl");
+ TASK_TYPE_CLASS = Class.forName(PACKAGE + ".TaskType");
+ mapContextCls = Class.forName(PACKAGE + ".task.MapContextImpl");
+ mapCls = Class.forName(PACKAGE + ".lib.map.WrappedMapper");
+ innerMapContextCls = Class.forName(PACKAGE + ".lib.map.WrappedMapper$Context");
+ } else {
+ jobContextCls = Class.forName(PACKAGE + ".JobContext");
+ taskContextCls = Class.forName(PACKAGE + ".TaskAttemptContext");
+ TASK_TYPE_CLASS = null;
+ mapContextCls = Class.forName(PACKAGE + ".MapContext");
+ mapCls = Class.forName(PACKAGE + ".Mapper");
+ innerMapContextCls = Class.forName(PACKAGE + ".Mapper$Context");
+ }
+ } catch (ClassNotFoundException e) {
+ throw new IllegalArgumentException("Can't find class", e);
+ }
+ try {
+ JOB_CONTEXT_CONSTRUCTOR = jobContextCls.getConstructor(Configuration.class, JobID.class);
+ JOB_CONTEXT_CONSTRUCTOR.setAccessible(true);
+ TASK_CONTEXT_CONSTRUCTOR = taskContextCls.getConstructor(Configuration.class, TaskAttemptID.class);
+ TASK_CONTEXT_CONSTRUCTOR.setAccessible(true);
+ if (useV21) {
+ TASK_ID_CONSTRUCTOR = TaskAttemptID.class.getConstructor(String.class, int.class, TASK_TYPE_CLASS, int.class, int.class);
+ TASK_ID_CONSTRUCTOR.setAccessible(true);
+ MAP_CONSTRUCTOR = mapCls.getConstructor();
+ MAP_CONTEXT_CONSTRUCTOR = innerMapContextCls.getConstructor(mapCls, MapContext.class);
+ MAP_CONTEXT_IMPL_CONSTRUCTOR = mapContextCls.getDeclaredConstructor(Configuration.class, TaskAttemptID.class, RecordReader.class, RecordWriter.class,
+ OutputCommitter.class, StatusReporter.class, InputSplit.class);
+ MAP_CONTEXT_IMPL_CONSTRUCTOR.setAccessible(true);
+ } else {
+ TASK_ID_CONSTRUCTOR = TaskAttemptID.class.getConstructor(String.class, int.class, boolean.class, int.class, int.class);
+ TASK_ID_CONSTRUCTOR.setAccessible(true);
+ MAP_CONSTRUCTOR = null;
+ MAP_CONTEXT_CONSTRUCTOR = innerMapContextCls.getConstructor(mapCls, Configuration.class, TaskAttemptID.class, RecordReader.class, RecordWriter.class,
+ OutputCommitter.class, StatusReporter.class, InputSplit.class);
+ MAP_CONTEXT_IMPL_CONSTRUCTOR = null;
+ }
+ MAP_CONTEXT_CONSTRUCTOR.setAccessible(true);
+ } catch (SecurityException e) {
+ throw new IllegalArgumentException("Can't run constructor ", e);
+ } catch (NoSuchMethodException e) {
+ throw new IllegalArgumentException("Can't find constructor ", e);
+ }
+ }
+
+ public static JobContext createJobContext() {
+ return createJobContext(new Configuration());
+ }
+
+ public static JobContext createJobContext(Configuration conf) {
+ try {
+ return (JobContext) JOB_CONTEXT_CONSTRUCTOR.newInstance(conf, new JobID("local", 0));
+ } catch (InstantiationException e) {
+ throw new IllegalArgumentException("Can't create object", e);
+ } catch (IllegalAccessException e) {
+ throw new IllegalArgumentException("Can't create object", e);
+ } catch (InvocationTargetException e) {
+ throw new IllegalArgumentException("Can't create object", e);
+ }
+ }
+
+ public static TaskAttemptContext createTaskAttemptContext(JobContext job) {
+ return createTaskAttemptContext(job.getConfiguration());
+ }
+
+ public static TaskAttemptContext createTaskAttemptContext(Configuration conf) {
+ try {
+ if (useV21)
+ return (TaskAttemptContext) TASK_CONTEXT_CONSTRUCTOR.newInstance(conf,
+ TASK_ID_CONSTRUCTOR.newInstance("local", 0, TASK_TYPE_CLASS.getEnumConstants()[0], 0, 0));
+ else
+ return (TaskAttemptContext) TASK_CONTEXT_CONSTRUCTOR.newInstance(conf, TASK_ID_CONSTRUCTOR.newInstance("local", 0, true, 0, 0));
+ } catch (InstantiationException e) {
+ throw new IllegalArgumentException("Can't create object", e);
+ } catch (IllegalAccessException e) {
+ throw new IllegalArgumentException("Can't create object", e);
+ } catch (InvocationTargetException e) {
+ throw new IllegalArgumentException("Can't create object", e);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public static <K1,V1,K2,V2> Mapper<K1,V1,K2,V2>.Context createMapContext(Mapper<K1,V1,K2,V2> m, TaskAttemptContext tac, RecordReader<K1,V1> reader,
+ RecordWriter<K2,V2> writer, InputSplit split) {
+ try {
+ if (useV21) {
+ Object basis = MAP_CONTEXT_IMPL_CONSTRUCTOR.newInstance(tac.getConfiguration(), tac.getTaskAttemptID(), reader, writer, null, null, split);
+ return (Mapper<K1,V1,K2,V2>.Context) MAP_CONTEXT_CONSTRUCTOR.newInstance((Mapper<K1,V1,K2,V2>) MAP_CONSTRUCTOR.newInstance(), basis);
+ } else {
+ return (Mapper<K1,V1,K2,V2>.Context) MAP_CONTEXT_CONSTRUCTOR.newInstance(m, tac.getConfiguration(), tac.getTaskAttemptID(), reader, writer, null, null,
+ split);
+ }
+ } catch (InstantiationException e) {
+ throw new IllegalArgumentException("Can't create object", e);
+ } catch (IllegalAccessException e) {
+ throw new IllegalArgumentException("Can't create object", e);
+ } catch (InvocationTargetException e) {
+ throw new IllegalArgumentException("Can't create object", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3264e4f/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormatTest.java b/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormatTest.java
index 342455f..84dce27 100644
--- a/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormatTest.java
+++ b/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormatTest.java
@@ -26,13 +26,11 @@ import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.accumulo.core.util.ContextFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -44,13 +42,13 @@ public class AccumuloFileOutputFormatTest {
@Before
public void setup() {
- job = new JobContext(new Configuration(), new JobID());
+ job = ContextFactory.createJobContext();
Path file = new Path(System.getenv("ACCUMULO_HOME") + "/target/");
f = new Path(file, "_temporary");
job.getConfiguration().set("mapred.output.dir", file.toString());
- tac = new TaskAttemptContext(job.getConfiguration(), new TaskAttemptID());
+ tac = ContextFactory.createTaskAttemptContext(job);
}
@After
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3264e4f/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
----------------------------------------------------------------------
diff --git a/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java b/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
index 7239b01..f2e2f2c 100644
--- a/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
+++ b/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
@@ -39,16 +39,15 @@ import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.user.WholeRowIterator;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.ContextFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.log4j.Level;
import org.junit.After;
import org.junit.Assert;
@@ -67,7 +66,7 @@ public class AccumuloInputFormatTest {
*/
@Test
public void testMaxVersions() throws IOException {
- JobContext job = new JobContext(new Configuration(), new JobID());
+ JobContext job = ContextFactory.createJobContext();
AccumuloInputFormat.setMaxVersions(job.getConfiguration(), 1);
int version = AccumuloInputFormat.getMaxVersions(job.getConfiguration());
assertEquals(1, version);
@@ -81,7 +80,7 @@ public class AccumuloInputFormatTest {
*/
@Test(expected = IOException.class)
public void testMaxVersionsLessThan1() throws IOException {
- JobContext job = new JobContext(new Configuration(), new JobID());
+ JobContext job = ContextFactory.createJobContext();
AccumuloInputFormat.setMaxVersions(job.getConfiguration(), 0);
}
@@ -90,7 +89,7 @@ public class AccumuloInputFormatTest {
*/
@Test
public void testNoMaxVersion() {
- JobContext job = new JobContext(new Configuration(), new JobID());
+ JobContext job = ContextFactory.createJobContext();
assertEquals(-1, AccumuloInputFormat.getMaxVersions(job.getConfiguration()));
}
@@ -100,8 +99,8 @@ public class AccumuloInputFormatTest {
@SuppressWarnings("deprecation")
@Test
public void testSetIterator() {
- JobContext job = new JobContext(new Configuration(), new JobID());
-
+ JobContext job = ContextFactory.createJobContext();
+
AccumuloInputFormat.setIterator(job, 1, "org.apache.accumulo.core.iterators.WholeRowIterator", "WholeRow");
Configuration conf = job.getConfiguration();
String iterators = conf.get("AccumuloInputFormat.iterators");
@@ -110,8 +109,8 @@ public class AccumuloInputFormatTest {
@Test
public void testAddIterator() {
- JobContext job = new JobContext(new Configuration(), new JobID());
-
+ JobContext job = ContextFactory.createJobContext();
+
AccumuloInputFormat.addIterator(job.getConfiguration(), new IteratorSetting(1, "WholeRow", WholeRowIterator.class));
AccumuloInputFormat.addIterator(job.getConfiguration(), new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
IteratorSetting iter = new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator");
@@ -192,8 +191,8 @@ public class AccumuloInputFormatTest {
@SuppressWarnings("deprecation")
@Test
public void testGetIteratorSettings() {
- JobContext job = new JobContext(new Configuration(), new JobID());
-
+ JobContext job = ContextFactory.createJobContext();
+
AccumuloInputFormat.setIterator(job, 1, "org.apache.accumulo.core.iterators.WholeRowIterator", "WholeRow");
AccumuloInputFormat.setIterator(job, 2, "org.apache.accumulo.core.iterators.VersioningIterator", "Versions");
AccumuloInputFormat.setIterator(job, 3, "org.apache.accumulo.core.iterators.CountingIterator", "Count");
@@ -227,7 +226,7 @@ public class AccumuloInputFormatTest {
@SuppressWarnings("deprecation")
@Test
public void testSetIteratorOption() {
- JobContext job = new JobContext(new Configuration(), new JobID());
+ JobContext job = ContextFactory.createJobContext();
AccumuloInputFormat.setIteratorOption(job, "someIterator", "aKey", "aValue");
Configuration conf = job.getConfiguration();
@@ -241,8 +240,8 @@ public class AccumuloInputFormatTest {
@SuppressWarnings("deprecation")
@Test
public void testGetIteratorOption() {
- JobContext job = new JobContext(new Configuration(), new JobID());
-
+ JobContext job = ContextFactory.createJobContext();
+
AccumuloInputFormat.setIteratorOption(job, "iterator1", "key1", "value1");
AccumuloInputFormat.setIteratorOption(job, "iterator2", "key2", "value2");
AccumuloInputFormat.setIteratorOption(job, "iterator3", "key3", "value3");
@@ -272,8 +271,8 @@ public class AccumuloInputFormatTest {
@SuppressWarnings("deprecation")
@Test
public void testSetRegex() {
- JobContext job = new JobContext(new Configuration(), new JobID());
-
+ JobContext job = ContextFactory.createJobContext();
+
String regex = ">\"*%<>\'\\";
AccumuloInputFormat.setRegex(job, RegexType.ROW, regex);
@@ -329,10 +328,9 @@ public class AccumuloInputFormatTest {
Assert.assertEquals(new Authorizations(), risplit.getAuths());
Assert.assertEquals("testmapinstance", risplit.getInstanceName());
- TaskAttemptID id = new TaskAttemptID();
- TaskAttemptContext attempt = new TaskAttemptContext(job.getConfiguration(), id);
- RecordReader<Key,Value> reader = input.createRecordReader(split, attempt);
- Mapper<Key,Value,Key,Value>.Context context = mapper.new Context(job.getConfiguration(), id, reader, null, null, null, split);
+ TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
+ RecordReader<Key,Value> reader = input.createRecordReader(split, tac);
+ Mapper<Key,Value,Key,Value>.Context context = ContextFactory.createMapContext(mapper, tac, reader, null, split);
reader.initialize(split, context);
mapper.run(context);
}
@@ -350,20 +348,21 @@ public class AccumuloInputFormatTest {
bw.addMutation(m);
}
bw.close();
-
- JobContext job = new JobContext(new Configuration(), new JobID());
+
+ JobContext job = ContextFactory.createJobContext();
AccumuloInputFormat.setInputInfo(job.getConfiguration(), "root", "".getBytes(), "testtable2", new Authorizations());
AccumuloInputFormat.setMockInstance(job.getConfiguration(), "testmapinstance");
AccumuloInputFormat input = new AccumuloInputFormat();
RangeInputSplit ris = new RangeInputSplit();
- TaskAttemptContext tac = new TaskAttemptContext(job.getConfiguration(), new TaskAttemptID());
+ TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
RecordReader<Key,Value> rr = input.createRecordReader(ris, tac);
rr.initialize(ris, tac);
TestMapper mapper = new TestMapper();
- Mapper<Key,Value,Key,Value>.Context context = mapper.new Context(job.getConfiguration(), tac.getTaskAttemptID(), rr, null, null, null, ris);
+ Mapper<Key,Value,Key,Value>.Context context = ContextFactory.createMapContext(mapper, tac, rr, null, ris);
+ rr.initialize(ris, tac);
while (rr.nextKeyValue()) {
- mapper.map(rr.getCurrentKey(), rr.getCurrentValue(), context);
+ mapper.map(rr.getCurrentKey(), rr.getCurrentValue(), (TestMapper.Context) context);
}
}
@@ -380,15 +379,15 @@ public class AccumuloInputFormatTest {
bw.addMutation(m);
}
bw.close();
-
- JobContext job = new JobContext(new Configuration(), new JobID());
+
+ JobContext job = ContextFactory.createJobContext();
AccumuloInputFormat.setInputInfo(job.getConfiguration(), "root", "".getBytes(), "testtable3", new Authorizations());
AccumuloInputFormat.setMockInstance(job.getConfiguration(), "testmapinstance");
final String regex = ".*1.*";
AccumuloInputFormat.setRegex(job, RegexType.ROW, regex);
AccumuloInputFormat input = new AccumuloInputFormat();
RangeInputSplit ris = new RangeInputSplit();
- TaskAttemptContext tac = new TaskAttemptContext(job.getConfiguration(), new TaskAttemptID());
+ TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
RecordReader<Key,Value> rr = input.createRecordReader(ris, tac);
rr.initialize(ris, tac);
@@ -401,7 +400,7 @@ public class AccumuloInputFormatTest {
@SuppressWarnings("deprecation")
@Test
public void testCorrectRangeInputSplits() throws Exception {
- JobContext job = new JobContext(new Configuration(), new JobID());
+ JobContext job = ContextFactory.createJobContext();
String username = "user", table = "table", rowRegex = "row.*", colfRegex = "colf.*", colqRegex = "colq.*";
String valRegex = "val.*", instance = "instance";
@@ -485,10 +484,9 @@ public class AccumuloInputFormatTest {
RangeInputSplit emptySplit = new RangeInputSplit();
// Using an empty split should fall back to the information in the Job's Configuration
- TaskAttemptID id = new TaskAttemptID();
- TaskAttemptContext attempt = new TaskAttemptContext(job.getConfiguration(), id);
- RecordReader<Key,Value> reader = input.createRecordReader(emptySplit, attempt);
- Mapper<Key,Value,Key,Value>.Context context = mapper.new Context(job.getConfiguration(), id, reader, null, null, null, emptySplit);
+ TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
+ RecordReader<Key,Value> reader = input.createRecordReader(emptySplit, tac);
+ Mapper<Key,Value,Key,Value>.Context context = ContextFactory.createMapContext(mapper, tac, reader, null, emptySplit);
reader.initialize(emptySplit, context);
mapper.run(context);
}
@@ -524,10 +522,9 @@ public class AccumuloInputFormatTest {
emptySplit.setPassword("anythingelse".getBytes());
// Using an empty split should fall back to the information in the Job's Configuration
- TaskAttemptID id = new TaskAttemptID();
- TaskAttemptContext attempt = new TaskAttemptContext(job.getConfiguration(), id);
- RecordReader<Key,Value> reader = input.createRecordReader(emptySplit, attempt);
- Mapper<Key,Value,Key,Value>.Context context = mapper.new Context(job.getConfiguration(), id, reader, null, null, null, emptySplit);
+ TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
+ RecordReader<Key,Value> reader = input.createRecordReader(emptySplit, tac);
+ Mapper<Key,Value,Key,Value>.Context context = ContextFactory.createMapContext(mapper, tac, reader, null, emptySplit);
reader.initialize(emptySplit, context);
mapper.run(context);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3264e4f/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloRowInputFormatTest.java
----------------------------------------------------------------------
diff --git a/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloRowInputFormatTest.java b/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloRowInputFormatTest.java
index d9f9da0..5199352 100644
--- a/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloRowInputFormatTest.java
+++ b/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloRowInputFormatTest.java
@@ -34,14 +34,12 @@ import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.util.ContextFactory;
import org.apache.accumulo.core.util.PeekingIterator;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.junit.Test;
public class AccumuloRowInputFormatTest {
@@ -99,12 +97,12 @@ public class AccumuloRowInputFormatTest {
insertList(bw, row3);
bw.close();
- JobContext job = new JobContext(new Configuration(), new JobID());
+ JobContext job = ContextFactory.createJobContext();
AccumuloRowInputFormat.setInputInfo(job.getConfiguration(), "root", "".getBytes(), "test", new Authorizations());
AccumuloRowInputFormat.setMockInstance(job.getConfiguration(), "instance1");
AccumuloRowInputFormat crif = new AccumuloRowInputFormat();
RangeInputSplit ris = new RangeInputSplit();
- TaskAttemptContext tac = new TaskAttemptContext(job.getConfiguration(), new TaskAttemptID());
+ TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
RecordReader<Text,PeekingIterator<Entry<Key,Value>>> rr = crif.createRecordReader(ris, tac);
rr.initialize(ris, tac);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3264e4f/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitionerTest.java
----------------------------------------------------------------------
diff --git a/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitionerTest.java b/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitionerTest.java
index 8b95c57..87059c1 100644
--- a/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitionerTest.java
+++ b/src/core/src/test/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitionerTest.java
@@ -18,10 +18,9 @@ package org.apache.accumulo.core.client.mapreduce.lib.partition;
import static org.junit.Assert.assertTrue;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.accumulo.core.util.ContextFactory;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.JobID;
import org.junit.Test;
public class RangePartitionerTest {
@@ -53,7 +52,7 @@ public class RangePartitionerTest {
}
private RangePartitioner prepPartitioner(int numSubBins) {
- JobContext job = new JobContext(new Configuration(), new JobID());
+ JobContext job = ContextFactory.createJobContext();
RangePartitioner.setNumSubBins(job, numSubBins);
RangePartitioner rp = new RangePartitioner();
rp.setConf(job.getConfiguration());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3264e4f/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/InsertWithOutputFormat.java
----------------------------------------------------------------------
diff --git a/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/InsertWithOutputFormat.java b/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/InsertWithOutputFormat.java
index 7583494..af03470 100644
--- a/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/InsertWithOutputFormat.java
+++ b/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/helloworld/InsertWithOutputFormat.java
@@ -20,13 +20,12 @@ import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.util.CachedConfiguration;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.accumulo.core.util.ContextFactory;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
@@ -46,16 +45,15 @@ public class InsertWithOutputFormat extends Configured implements Tool {
}
Text tableName = new Text(args[4]);
Job job = new Job(getConf());
- Configuration conf = job.getConfiguration();
- AccumuloOutputFormat.setZooKeeperInstance(conf, args[0], args[1]);
- AccumuloOutputFormat.setOutputInfo(conf, args[2], args[3].getBytes(), true, null);
+ AccumuloOutputFormat.setZooKeeperInstance(job, args[0], args[1]);
+ AccumuloOutputFormat.setOutputInfo(job, args[3], args[4].getBytes(), true, null);
job.setOutputFormatClass(AccumuloOutputFormat.class);
// when running a mapreduce, you won't need to instantiate the output
// format and record writer
// mapreduce will do that for you, and you will just use
// output.collect(tableName, mutation)
- TaskAttemptContext context = new TaskAttemptContext(conf, new TaskAttemptID());
+ TaskAttemptContext context = ContextFactory.createTaskAttemptContext(job);
RecordWriter<Text,Mutation> rw = new AccumuloOutputFormat().getRecordWriter(context);
Text colf = new Text("colfam");
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3264e4f/src/examples/simple/src/test/java/org/apache/accumulo/examples/simple/filedata/ChunkInputFormatTest.java
----------------------------------------------------------------------
diff --git a/src/examples/simple/src/test/java/org/apache/accumulo/examples/simple/filedata/ChunkInputFormatTest.java b/src/examples/simple/src/test/java/org/apache/accumulo/examples/simple/filedata/ChunkInputFormatTest.java
index af12302..8937048 100644
--- a/src/examples/simple/src/test/java/org/apache/accumulo/examples/simple/filedata/ChunkInputFormatTest.java
+++ b/src/examples/simple/src/test/java/org/apache/accumulo/examples/simple/filedata/ChunkInputFormatTest.java
@@ -37,12 +37,10 @@ import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.accumulo.core.util.ContextFactory;
import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.log4j.Logger;
public class ChunkInputFormatTest extends TestCase {
@@ -87,12 +85,12 @@ public class ChunkInputFormatTest extends TestCase {
}
bw.close();
- JobContext job = new JobContext(new Configuration(), new JobID());
+ JobContext job = ContextFactory.createJobContext();
ChunkInputFormat.setInputInfo(job.getConfiguration(), "root", "".getBytes(), "test", new Authorizations("A", "B", "C", "D"));
ChunkInputFormat.setMockInstance(job.getConfiguration(), "instance1");
ChunkInputFormat cif = new ChunkInputFormat();
RangeInputSplit ris = new RangeInputSplit();
- TaskAttemptContext tac = new TaskAttemptContext(job.getConfiguration(), new TaskAttemptID());
+ TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
RecordReader<List<Entry<Key,Value>>,InputStream> rr = cif.createRecordReader(ris, tac);
rr.initialize(ris, tac);
@@ -138,12 +136,12 @@ public class ChunkInputFormatTest extends TestCase {
}
bw.close();
- JobContext job = new JobContext(new Configuration(), new JobID());
+ JobContext job = ContextFactory.createJobContext();
ChunkInputFormat.setInputInfo(job.getConfiguration(), "root", "".getBytes(), "test", new Authorizations("A", "B", "C", "D"));
ChunkInputFormat.setMockInstance(job.getConfiguration(), "instance2");
ChunkInputFormat cif = new ChunkInputFormat();
RangeInputSplit ris = new RangeInputSplit();
- TaskAttemptContext tac = new TaskAttemptContext(job.getConfiguration(), new TaskAttemptID());
+ TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
RecordReader<List<Entry<Key,Value>>,InputStream> crr = cif.createRecordReader(ris, tac);
crr.initialize(ris, tac);
@@ -177,12 +175,12 @@ public class ChunkInputFormatTest extends TestCase {
}
bw.close();
- JobContext job = new JobContext(new Configuration(), new JobID());
+ JobContext job = ContextFactory.createJobContext();
ChunkInputFormat.setInputInfo(job.getConfiguration(), "root", "".getBytes(), "test", new Authorizations("A", "B", "C", "D"));
ChunkInputFormat.setMockInstance(job.getConfiguration(), "instance3");
ChunkInputFormat cif = new ChunkInputFormat();
RangeInputSplit ris = new RangeInputSplit();
- TaskAttemptContext tac = new TaskAttemptContext(job.getConfiguration(), new TaskAttemptID());
+ TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
RecordReader<List<Entry<Key,Value>>,InputStream> crr = cif.createRecordReader(ris, tac);
crr.initialize(ris, tac);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3264e4f/src/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java b/src/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java
index 64ed42e..b98f29d 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/master/CoordinateRecoveryTask.java
@@ -257,10 +257,7 @@ public class CoordinateRecoveryTask implements Runnable {
return new RecoveryStatus(logFile.server, logFile.file, (sortJob == null ? 0. : sortJob.mapProgress()), (sortJob == null ? 0.
: sortJob.reduceProgress()), (int) (System.currentTimeMillis() - copyStartTime), (sortJob != null) ? 1. : (copySize == 0 ? 0 : copiedSoFar()
/ (double) copySize));
- } catch (InterruptedException ie) {
- // Hadoop 2.0.2-alpha's Job.mapProgress throws Interrupted Exception. 1.x and 2.0.4 do not.
- return new RecoveryStatus(logFile.server, logFile.file, 1.0, 1.0, (int) (System.currentTimeMillis() - copyStartTime), 1.0);
- } catch (NullPointerException npe) {
+ } catch (Exception e) {
return new RecoveryStatus(logFile.server, logFile.file, 1.0, 1.0, (int) (System.currentTimeMillis() - copyStartTime), 1.0);
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a3264e4f/src/server/src/main/java/org/apache/accumulo/server/master/LogSort.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/master/LogSort.java b/src/server/src/main/java/org/apache/accumulo/server/master/LogSort.java
index 006d06e..4666331 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/master/LogSort.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/master/LogSort.java
@@ -31,9 +31,9 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.MapFile;
+import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.DefaultCodec;
import org.apache.hadoop.mapreduce.Job;
[32/39] git commit: ACCUMULO-645: found more references to HADOOP_HOME
Posted by ec...@apache.org.
ACCUMULO-645: found more references to HADOOP_HOME
git-svn-id: https://svn.apache.org/repos/asf/accumulo/trunk@1431596 13f79535-47bb-0310-9956-ffa450edef68
(cherry picked from commit 02829fffc3ad9261648da5dd12a02dfcafa5a0fa)
Reason: Hadoop2 compat
Author: Eric C. Newton <ec...@apache.org>
Ref: ACCUMULO-1792
Signed-off-by: Eric Newton <er...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d70e5a12
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d70e5a12
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d70e5a12
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: d70e5a12739a7775995340be3dca5d460cdf7be2
Parents: 344e725
Author: Eric C. Newton <ec...@apache.org>
Authored: Thu Jan 10 20:10:42 2013 +0000
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Nov 25 16:06:44 2013 -0500
----------------------------------------------------------------------
bin/tool.sh | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d70e5a12/bin/tool.sh
----------------------------------------------------------------------
diff --git a/bin/tool.sh b/bin/tool.sh
index 3c114e8..cefd6b9 100755
--- a/bin/tool.sh
+++ b/bin/tool.sh
@@ -20,8 +20,8 @@ bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
. "$bin"/config.sh
-if [ -z $HADOOP_HOME ] ; then
- echo "HADOOP_HOME is not set. Please make sure it's set globally or in conf/accumulo-env.sh"
+if [ -z $HADOOP_PREFIX ] ; then
+ echo "HADOOP_PREFIX is not set. Please make sure it's set globally or in conf/accumulo-env.sh"
exit 1
fi
if [ -z $ZOOKEEPER_HOME ] ; then
@@ -98,5 +98,5 @@ fi
#echo USERJARS=$USERJARS
#echo CLASSNAME=$CLASSNAME
#echo HADOOP_CLASSPATH=$HADOOP_CLASSPATH
-#echo exec "$HADOOP_HOME/bin/hadoop" jar "$TOOLJAR" $CLASSNAME -libjars \"$LIB_JARS\" $ARGS
-exec "$HADOOP_HOME/bin/hadoop" jar "$TOOLJAR" $CLASSNAME -libjars \"$LIB_JARS\" "$@"
+#echo exec "$HADOOP_PREFIX/bin/hadoop" jar "$TOOLJAR" $CLASSNAME -libjars \"$LIB_JARS\" $ARGS
+exec "$HADOOP_PREFIX/bin/hadoop" jar "$TOOLJAR" $CLASSNAME -libjars \"$LIB_JARS\" "$@"
[11/39] ACCUMULO-600 removed wikisearch from trunk
Posted by ec...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query-war/src/test/resources/test.xml
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query-war/src/test/resources/test.xml b/src/examples/wikisearch/query-war/src/test/resources/test.xml
deleted file mode 100644
index 895bb5c..0000000
--- a/src/examples/wikisearch/query-war/src/test/resources/test.xml
+++ /dev/null
@@ -1,1651 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<results>
- <document>
- <id>54744</id>
- <field name="COMMENTS">r2.6.3) (روبوت إضافة: [[simple:Astro Boy]]</field>
- <field name="ID">54744</field>
- <field name="TIMESTAMP">1303786076000</field>
- <field name="TITLE">الفتى أسترو</field>
- <field name="DOCUMENT">DOCUMENT:44/arwiki/54744</field>
- </document>
- <document>
- <id>615944</id>
- <field name="COMMENTS">r2.5.2) (روبوت إضافة: [[hu:Game Boy]] إزالة: [[sv:Game Boy]] تعديل: [[nl:Game Boy]]</field>
- <field name="ID">615944</field>
- <field name="TIMESTAMP">1300883423000</field>
- <field name="TITLE">جيم بوي</field>
- <field name="DOCUMENT">DOCUMENT:44/arwiki/615944</field>
- </document>
- <document>
- <id>843644</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[uk:Останній з могікан]]</field>
- <field name="ID">843644</field>
- <field name="TIMESTAMP">1295303327000</field>
- <field name="TITLE">آخر الموهيكان</field>
- <field name="DOCUMENT">DOCUMENT:44/arwiki/843644</field>
- </document>
- <document>
- <id>465963</id>
- <field name="COMMENTS">روبوت إضافة: [[bar:Werner Herzog]]; تغييرات تجميلية</field>
- <field name="ID">465963</field>
- <field name="TIMESTAMP">1305983438000</field>
- <field name="TITLE">فرنر هرتزوغ</field>
- <field name="DOCUMENT">DOCUMENT:63/arwiki/465963</field>
- </document>
- <document>
- <id>975163</id>
- <field name="COMMENTS">أنشأ الصفحة ب'{{Navbox |name = محمد عبده يماني |title = أعمال [[محمد عبده يماني|الدكتور محمد عبده يماني]] |style = text-align:right; |...'</field>
- <field name="ID">975163</field>
- <field name="TIMESTAMP">1292442667000</field>
- <field name="TITLE">قالب:محمد عبده يماني</field>
- <field name="DOCUMENT">DOCUMENT:63/arwiki/975163</field>
- </document>
- <document>
- <id>731769</id>
- <field name="COMMENTS">/* إدراك اليابانيين بحدوث الانفجار */</field>
- <field name="ID">731769</field>
- <field name="TIMESTAMP">1306136710000</field>
- <field name="TITLE">الهجوم النووي على هيروشيما وناجازاكي</field>
- <field name="DOCUMENT">DOCUMENT:69/arwiki/731769</field>
- </document>
- <document>
- <id>734469</id>
- <field name="COMMENTS">الرجوع عن التعديل 6870445 بواسطة [[Special:Contributions/Zxs|Zxs]] ([[User talk:Zxs|نقاش]])</field>
- <field name="ID">734469</field>
- <field name="TIMESTAMP">1306382343000</field>
- <field name="TITLE">تروبيك ثاندر</field>
- <field name="DOCUMENT">DOCUMENT:69/arwiki/734469</field>
- </document>
- <document>
- <id>496886</id>
- <field name="COMMENTS">/* " " " الأغاني التي حطمت المبيعات في العالم " " " */</field>
- <field name="ID">496886</field>
- <field name="TIMESTAMP">1306053297000</field>
- <field name="TITLE">كوين</field>
- <field name="DOCUMENT">DOCUMENT:86/arwiki/496886</field>
- </document>
- <document>
- <id>65386</id>
- <field name="COMMENTS">2011 image</field>
- <field name="ID">65386</field>
- <field name="TIMESTAMP">1307421099000</field>
- <field name="TITLE">روبرت دي نيرو</field>
- <field name="DOCUMENT">DOCUMENT:86/arwiki/65386</field>
- </document>
- <document>
- <id>1008068</id>
- <field name="COMMENTS">تدقيق إملائي بالاعتماد على [[تعابير نمطية|التعابير النمطية]]، يرجى الإبلاغ عن [[نقاش المستخدم:Bassem JARKAS|الأخطاء والاقتراحات]]</field>
- <field name="ID">1008068</field>
- <field name="TIMESTAMP">1301730227000</field>
- <field name="TITLE">تصفية حيوية</field>
- <field name="DOCUMENT">DOCUMENT:68/arwiki/1008068</field>
- </document>
- <document>
- <id>267641</id>
- <field name="COMMENTS">تدقيق إملائي وتنسيق,</field>
- <field name="ID">267641</field>
- <field name="TIMESTAMP">1264886458000</field>
- <field name="TITLE">واتش ذيم رول</field>
- <field name="DOCUMENT">DOCUMENT:41/arwiki/267641</field>
- </document>
- <document>
- <id>486341</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[ca:Can't Get You Out of My Head]]</field>
- <field name="ID">486341</field>
- <field name="TIMESTAMP">1306355262000</field>
- <field name="TITLE">كانت غيت يو آوت أوف ماي هيد</field>
- <field name="DOCUMENT">DOCUMENT:41/arwiki/486341</field>
- </document>
- <document>
- <id>1005366</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[ro:Eric Saade]]</field>
- <field name="ID">1005366</field>
- <field name="TIMESTAMP">1306350232000</field>
- <field name="TITLE">إريك سعادة</field>
- <field name="DOCUMENT">DOCUMENT:66/arwiki/1005366</field>
- </document>
- <document>
- <id>123766</id>
- <field name="COMMENTS">روبوت إضافة: [[fr:Imagi Animation Studios]]</field>
- <field name="ID">123766</field>
- <field name="TIMESTAMP">1276531414000</field>
- <field name="TITLE">إيماجي للرسوم المتحركة</field>
- <field name="DOCUMENT">DOCUMENT:66/arwiki/123766</field>
- </document>
- <document>
- <id>266990</id>
- <field name="COMMENTS">روبوت: تغييرات تجميلية</field>
- <field name="ID">266990</field>
- <field name="TIMESTAMP">1260205590000</field>
- <field name="TITLE">جيت بيزي (أغنية)</field>
- <field name="DOCUMENT">DOCUMENT:90/arwiki/266990</field>
- </document>
- <document>
- <id>615990</id>
- <field name="COMMENTS"/>
- <field name="ID">615990</field>
- <field name="TIMESTAMP">1304572496000</field>
- <field name="TITLE">جيمي نيوترون: الفتى العبقري</field>
- <field name="DOCUMENT">DOCUMENT:90/arwiki/615990</field>
- </document>
- <document>
- <id>717290</id>
- <field name="COMMENTS">r2.7.1) (روبوت تعديل: [[es:Ashanti (cantante)]]</field>
- <field name="ID">717290</field>
- <field name="TIMESTAMP">1301812271000</field>
- <field name="TITLE">آشانتي (مغنية)</field>
- <field name="DOCUMENT">DOCUMENT:90/arwiki/717290</field>
- </document>
- <document>
- <id>922690</id>
- <field name="COMMENTS">إضافة [[تصنيف:قوائم متعلقة بالسياسة]] (باستخدام [[ويكيبيديا:المصناف الفوري|المصناف الفوري]])</field>
- <field name="ID">922690</field>
- <field name="TIMESTAMP">1306008169000</field>
- <field name="TITLE">ملحق:قائمة الكتب الممنوعة من الحكومات</field>
- <field name="DOCUMENT">DOCUMENT:90/arwiki/922690</field>
- </document>
- <document>
- <id>67465</id>
- <field name="COMMENTS">تدقيق إملائي بالاعتماد على [[تعابير نمطية|التعابير النمطية]]، يرجى الإبلاغ عن [[نقاش المستخدم:Bassem JARKAS|الأخطاء والاقتراحات]]</field>
- <field name="ID">67465</field>
- <field name="TIMESTAMP">1301755042000</field>
- <field name="TITLE">ذا كلاش</field>
- <field name="DOCUMENT">DOCUMENT:65/arwiki/67465</field>
- </document>
- <document>
- <id>339764</id>
- <field name="COMMENTS">نقل إلى ← [[ويكيبيديا:تغيير اسم المستخدم/انتزاعات|الصفحة الرئيسية للطلبات]]</field>
- <field name="ID">339764</field>
- <field name="TIMESTAMP">1270564389000</field>
- <field name="TITLE">ويكيبيديا:تغيير اسم المستخدم/انتزاعات/أرشيف 1</field>
- <field name="DOCUMENT">DOCUMENT:64/arwiki/339764</field>
- </document>
- <document>
- <id>393064</id>
- <field name="COMMENTS">روبوت تعديل: [[fr:Taito (entreprise)]]</field>
- <field name="ID">393064</field>
- <field name="TIMESTAMP">1246537262000</field>
- <field name="TITLE">تايتو (شركة)</field>
- <field name="DOCUMENT">DOCUMENT:64/arwiki/393064</field>
- </document>
- <document>
- <id>922295</id>
- <field name="COMMENTS">تنسيق وتصنيف</field>
- <field name="ID">922295</field>
- <field name="TIMESTAMP">1289906570000</field>
- <field name="TITLE">قالب:تصفح مجموعات مطوية/شرح</field>
- <field name="DOCUMENT">DOCUMENT:95/arwiki/922295</field>
- </document>
- <document>
- <id>891122</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[lv:Džons Denvers]]</field>
- <field name="ID">891122</field>
- <field name="TIMESTAMP">1295227154000</field>
- <field name="TITLE">جون دنفر</field>
- <field name="DOCUMENT">DOCUMENT:22/arwiki/891122</field>
- </document>
- <document>
- <id>948322</id>
- <field name="COMMENTS">[[دفع به الى أقصى حد]] تم نقلها إلى [[دفع به إلى أقصى حد]]: الالف</field>
- <field name="ID">948322</field>
- <field name="TIMESTAMP">1301433389000</field>
- <field name="TITLE">دفع به إلى أقصى حد</field>
- <field name="DOCUMENT">DOCUMENT:22/arwiki/948322</field>
- </document>
- <document>
- <id>196549</id>
- <field name="COMMENTS">إزالة al-moharer.net بناء على [[ويكيبيديا:الميدان/أرشيف/سياسات/04/2011#القائمة السوداء للمواقع الخارجية|طلب]] من [[User:Aa2-2004|Aa2-2004]]</field>
- <field name="ID">196549</field>
- <field name="TIMESTAMP">1302853926000</field>
- <field name="TITLE">ميدياويكي:Spam-blacklist</field>
- <field name="DOCUMENT">DOCUMENT:49/arwiki/196549</field>
- </document>
- <document>
- <id>318349</id>
- <field name="COMMENTS">بوت: [[es:Cowboy]] هي مقالة جيدة</field>
- <field name="ID">318349</field>
- <field name="TIMESTAMP">1293745409000</field>
- <field name="TITLE">راعي البقر</field>
- <field name="DOCUMENT">DOCUMENT:49/arwiki/318349</field>
- </document>
- <document>
- <id>846284</id>
- <field name="COMMENTS">تدقيق إملائي بالاعتماد على [[تعابير نمطية|التعابير النمطية]]، يرجى الإبلاغ عن [[نقاش المستخدم:Bassem JARKAS|الأخطاء والاقتراحات]]</field>
- <field name="ID">846284</field>
- <field name="TIMESTAMP">1301731198000</field>
- <field name="TITLE">تفاعل أوكسو</field>
- <field name="DOCUMENT">DOCUMENT:84/arwiki/846284</field>
- </document>
- <document>
- <id>59623</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[ko:핵 군비 경쟁]]</field>
- <field name="ID">59623</field>
- <field name="TIMESTAMP">1301386533000</field>
- <field name="TITLE">سباق التسلح النووي</field>
- <field name="DOCUMENT">DOCUMENT:23/arwiki/59623</field>
- </document>
- <document>
- <id>824123</id>
- <field name="COMMENTS">روبوت: تغييرات تجميلية</field>
- <field name="ID">824123</field>
- <field name="TIMESTAMP">1293184454000</field>
- <field name="TITLE">Cooking? Cooking! (أغنية)</field>
- <field name="DOCUMENT">DOCUMENT:23/arwiki/824123</field>
- </document>
- <document>
- <id>50546</id>
- <field name="COMMENTS">/* اللاعبون */</field>
- <field name="ID">50546</field>
- <field name="TIMESTAMP">1307224650000</field>
- <field name="TITLE">إيه سي ميلان</field>
- <field name="DOCUMENT">DOCUMENT:46/arwiki/50546</field>
- </document>
- <document>
- <id>653583</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[no:Hudson Soft]]</field>
- <field name="ID">653583</field>
- <field name="TIMESTAMP">1296129654000</field>
- <field name="TITLE">هادسن سوفت</field>
- <field name="DOCUMENT">DOCUMENT:83/arwiki/653583</field>
- </document>
- <document>
- <id>137647</id>
- <field name="COMMENTS">+ [[تصنيف:صور أيام شهر مارس]]</field>
- <field name="ID">137647</field>
- <field name="TIMESTAMP">1307060653000</field>
- <field name="TITLE">قالب:صورة يوم 2 مارس</field>
- <field name="DOCUMENT">DOCUMENT:47/arwiki/137647</field>
- </document>
- <document>
- <id>328647</id>
- <field name="COMMENTS">r2.5.1) (روبوت إضافة: [[ro:Gary Oldman]]</field>
- <field name="ID">328647</field>
- <field name="TIMESTAMP">1302910778000</field>
- <field name="TITLE">غاري أولدمان</field>
- <field name="DOCUMENT">DOCUMENT:47/arwiki/328647</field>
- </document>
- <document>
- <id>343747</id>
- <field name="COMMENTS">added [[Category:مغنيات أمريكيات]] using [[Help:Gadget-HotCat|HotCat]]</field>
- <field name="ID">343747</field>
- <field name="TIMESTAMP">1307284026000</field>
- <field name="TITLE">أليسا ميلانو</field>
- <field name="DOCUMENT">DOCUMENT:47/arwiki/343747</field>
- </document>
- <document>
- <id>942682</id>
- <field name="COMMENTS">r2.5.1) (روبوت إضافة: [[sk:Progéria]]</field>
- <field name="ID">942682</field>
- <field name="TIMESTAMP">1298372772000</field>
- <field name="TITLE">شياخ</field>
- <field name="DOCUMENT">DOCUMENT:82/arwiki/942682</field>
- </document>
- <document>
- <id>130821</id>
- <field name="COMMENTS">روبوت تعديل: [[lv:Sāra Halke]]</field>
- <field name="ID">130821</field>
- <field name="TIMESTAMP">1303071486000</field>
- <field name="TITLE">سارة تشالك</field>
- <field name="DOCUMENT">DOCUMENT:21/arwiki/130821</field>
- </document>
- <document>
- <id>280421</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[an:My Chemical Romance]]</field>
- <field name="ID">280421</field>
- <field name="TIMESTAMP">1304666063000</field>
- <field name="TITLE">ماي كيميكال رومانس</field>
- <field name="DOCUMENT">DOCUMENT:21/arwiki/280421</field>
- </document>
- <document>
- <id>24960</id>
- <field name="COMMENTS"/>
- <field name="ID">24960</field>
- <field name="TIMESTAMP">1306777446000</field>
- <field name="TITLE">توباك شاكور</field>
- <field name="DOCUMENT">DOCUMENT:60/arwiki/24960</field>
- </document>
- <document>
- <id>613060</id>
- <field name="COMMENTS">ناني بوت: تغييرات تجمييلية</field>
- <field name="ID">613060</field>
- <field name="TIMESTAMP">1294935062000</field>
- <field name="TITLE">ويب بيرس</field>
- <field name="DOCUMENT">DOCUMENT:60/arwiki/613060</field>
- </document>
- <document>
- <id>988360</id>
- <field name="COMMENTS">/* مؤلفاتها */ إضافة وصلات داخلية</field>
- <field name="ID">988360</field>
- <field name="TIMESTAMP">1298504641000</field>
- <field name="TITLE">جيرمين غرير</field>
- <field name="DOCUMENT">DOCUMENT:60/arwiki/988360</field>
- </document>
- <document>
- <id>52489</id>
- <field name="COMMENTS">/* التسعينات */</field>
- <field name="ID">52489</field>
- <field name="TIMESTAMP">1305348070000</field>
- <field name="TITLE">توم هانكس</field>
- <field name="DOCUMENT">DOCUMENT:89/arwiki/52489</field>
- </document>
- <document>
- <id>220403</id>
- <field name="COMMENTS"/>
- <field name="ID">220403</field>
- <field name="TIMESTAMP">1305851828000</field>
- <field name="TITLE">دعاء خليل أسود</field>
- <field name="DOCUMENT">DOCUMENT:3/arwiki/220403</field>
- </document>
- <document>
- <id>56403</id>
- <field name="COMMENTS">روبوت إضافة: [[my:ကာလဝမ်းရောဂါ]]; تغييرات تجميلية</field>
- <field name="ID">56403</field>
- <field name="TIMESTAMP">1306106972000</field>
- <field name="TITLE">كوليرا</field>
- <field name="DOCUMENT">DOCUMENT:3/arwiki/56403</field>
- </document>
- <document>
- <id>707703</id>
- <field name="COMMENTS"/>
- <field name="ID">707703</field>
- <field name="TIMESTAMP">1303737856000</field>
- <field name="TITLE">أليشا ديكسون</field>
- <field name="DOCUMENT">DOCUMENT:3/arwiki/707703</field>
- </document>
- <document>
- <id>842703</id>
- <field name="COMMENTS">+</field>
- <field name="ID">842703</field>
- <field name="TIMESTAMP">1270730932000</field>
- <field name="TITLE">ويكيبيديا:تقارير قاعدة البيانات/Users by bytes uploaded</field>
- <field name="DOCUMENT">DOCUMENT:3/arwiki/842703</field>
- </document>
- <document>
- <id>135588</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[eu:Jim Carrey]]</field>
- <field name="ID">135588</field>
- <field name="TIMESTAMP">1299268677000</field>
- <field name="TITLE">جيم كاري</field>
- <field name="DOCUMENT">DOCUMENT:88/arwiki/135588</field>
- </document>
- <document>
- <id>5550</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[fo:Ættbólkur]]</field>
- <field name="ID">5550</field>
- <field name="TIMESTAMP">1305428030000</field>
- <field name="TITLE">قبيلة</field>
- <field name="DOCUMENT">DOCUMENT:50/arwiki/5550</field>
- </document>
- <document>
- <id>749250</id>
- <field name="COMMENTS">/* الاستقبال والتراث */</field>
- <field name="ID">749250</field>
- <field name="TIMESTAMP">1306384756000</field>
- <field name="TITLE">سوبر ماريو بروس</field>
- <field name="DOCUMENT">DOCUMENT:50/arwiki/749250</field>
- </document>
- <document>
- <id>20787</id>
- <field name="COMMENTS"/>
- <field name="ID">20787</field>
- <field name="TIMESTAMP">1299740757000</field>
- <field name="TITLE">بول سيزان</field>
- <field name="DOCUMENT">DOCUMENT:87/arwiki/20787</field>
- </document>
- <document>
- <id>254287</id>
- <field name="COMMENTS">روبوت إضافة: [[fr:Diner Dash]]</field>
- <field name="ID">254287</field>
- <field name="TIMESTAMP">1269964040000</field>
- <field name="TITLE">داينر داش</field>
- <field name="DOCUMENT">DOCUMENT:87/arwiki/254287</field>
- </document>
- <document>
- <id>342987</id>
- <field name="COMMENTS"/>
- <field name="ID">342987</field>
- <field name="TIMESTAMP">1293393656000</field>
- <field name="TITLE">ديفيد نيومان (ملحن)</field>
- <field name="DOCUMENT">DOCUMENT:87/arwiki/342987</field>
- </document>
- <document>
- <id>881487</id>
- <field name="COMMENTS">==الترخيص=={{Non-free promotional|image_has_rationale=yes}}== Fair use for [[نوتوريوس بي. آي. جي.|The Notorious B.I.G.]] =={{Non-free use rationale|Article=نوتوريوس بي. آي. جي.|Description=Promotional photograph of</field>
- <field name="ID">881487</field>
- <field name="TIMESTAMP">1276681681000</field>
- <field name="TITLE">ملف:The Notorious B.I.G.jpg</field>
- <field name="DOCUMENT">DOCUMENT:87/arwiki/881487</field>
- </document>
- <document>
- <id>222907</id>
- <field name="COMMENTS">/* جوائز */</field>
- <field name="ID">222907</field>
- <field name="TIMESTAMP">1297325280000</field>
- <field name="TITLE">شون بول</field>
- <field name="DOCUMENT">DOCUMENT:7/arwiki/222907</field>
- </document>
- <document>
- <id>6307</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[gl:Hayao Miyazaki]]</field>
- <field name="ID">6307</field>
- <field name="TIMESTAMP">1305798856000</field>
- <field name="TITLE">هاياو ميازاكي</field>
- <field name="DOCUMENT">DOCUMENT:7/arwiki/6307</field>
- </document>
- <document>
- <id>747907</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[no:Paranthropus]]</field>
- <field name="ID">747907</field>
- <field name="TIMESTAMP">1302080785000</field>
- <field name="TITLE">بارانثروبوس</field>
- <field name="DOCUMENT">DOCUMENT:7/arwiki/747907</field>
- </document>
- <document>
- <id>26720</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[kaa:Mark Twain]]</field>
- <field name="ID">26720</field>
- <field name="TIMESTAMP">1306827822000</field>
- <field name="TITLE">مارك توين</field>
- <field name="DOCUMENT">DOCUMENT:20/arwiki/26720</field>
- </document>
- <document>
- <id>768620</id>
- <field name="COMMENTS">r2.6.4) (روبوت تعديل: [[pt:Kaiketsu Zorro]]</field>
- <field name="ID">768620</field>
- <field name="TIMESTAMP">1307073099000</field>
- <field name="TITLE">أسطورة زورو (أنيمي)</field>
- <field name="DOCUMENT">DOCUMENT:20/arwiki/768620</field>
- </document>
- <document>
- <id>1054429</id>
- <field name="COMMENTS">r2.7.1) (روبوت تعديل: [[hy:Բլու]]</field>
- <field name="ID">1054429</field>
- <field name="TIMESTAMP">1305887208000</field>
- <field name="TITLE">بلو</field>
- <field name="DOCUMENT">DOCUMENT:29/arwiki/1054429</field>
- </document>
- <document>
- <id>173229</id>
- <field name="COMMENTS"/>
- <field name="ID">173229</field>
- <field name="TIMESTAMP">1266720325000</field>
- <field name="TITLE">قالب:بوب</field>
- <field name="DOCUMENT">DOCUMENT:29/arwiki/173229</field>
- </document>
- <document>
- <id>267629</id>
- <field name="COMMENTS"/>
- <field name="ID">267629</field>
- <field name="TIMESTAMP">1307279020000</field>
- <field name="TITLE">ريانا</field>
- <field name="DOCUMENT">DOCUMENT:29/arwiki/267629</field>
- </document>
- <document>
- <id>1066228</id>
- <field name="COMMENTS"/>
- <field name="ID">1066228</field>
- <field name="TIMESTAMP">1307191631000</field>
- <field name="TITLE">ملحق:قائمة ممثلين هاري بوتر</field>
- <field name="DOCUMENT">DOCUMENT:28/arwiki/1066228</field>
- </document>
- <document>
- <id>277228</id>
- <field name="COMMENTS">إزالة نقطتين من عنوان الفقرة</field>
- <field name="ID">277228</field>
- <field name="TIMESTAMP">1304433258000</field>
- <field name="TITLE">شيا لابوف</field>
- <field name="DOCUMENT">DOCUMENT:28/arwiki/277228</field>
- </document>
- <document>
- <id>300428</id>
- <field name="COMMENTS">تدقيق إملائي بالاعتماد على [[تعابير نمطية|التعابير النمطية]]، يرجى الإبلاغ عن [[نقاش المستخدم:Bassem JARKAS|الأخطاء والاقتراحات]]</field>
- <field name="ID">300428</field>
- <field name="TIMESTAMP">1301724610000</field>
- <field name="TITLE">بول تيبيتس</field>
- <field name="DOCUMENT">DOCUMENT:28/arwiki/300428</field>
- </document>
- <document>
- <id>353696</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[an:Rob Lowe]]</field>
- <field name="ID">353696</field>
- <field name="TIMESTAMP">1300364854000</field>
- <field name="TITLE">روب لوو</field>
- <field name="DOCUMENT">DOCUMENT:96/arwiki/353696</field>
- </document>
- <document>
- <id>629096</id>
- <field name="COMMENTS"/>
- <field name="ID">629096</field>
- <field name="TIMESTAMP">1296988327000</field>
- <field name="TITLE">قالب:قائمة أغاني/شرح</field>
- <field name="DOCUMENT">DOCUMENT:96/arwiki/629096</field>
- </document>
- <document>
- <id>171939</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[zh-yue:Morning娘。]]</field>
- <field name="ID">171939</field>
- <field name="TIMESTAMP">1304655457000</field>
- <field name="TITLE">مورنغ موسومي</field>
- <field name="DOCUMENT">DOCUMENT:39/arwiki/171939</field>
- </document>
- <document>
- <id>646739</id>
- <field name="COMMENTS">روبوت إضافة: [[pl:Hektor Malot]]</field>
- <field name="ID">646739</field>
- <field name="TIMESTAMP">1273835959000</field>
- <field name="TITLE">هيكتور مالو</field>
- <field name="DOCUMENT">DOCUMENT:39/arwiki/646739</field>
- </document>
- <document>
- <id>169093</id>
- <field name="COMMENTS">r2.6.4) (روبوت تعديل: [[ca:Codi d'aeroports IATA]]</field>
- <field name="ID">169093</field>
- <field name="TIMESTAMP">1306630044000</field>
- <field name="TITLE">رمز مطار اتحاد النقل الجوي الدولي</field>
- <field name="DOCUMENT">DOCUMENT:93/arwiki/169093</field>
- </document>
- <document>
- <id>853493</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[fr:Dan Byrd]]</field>
- <field name="ID">853493</field>
- <field name="TIMESTAMP">1304591914000</field>
- <field name="TITLE">دان بيرد</field>
- <field name="DOCUMENT">DOCUMENT:93/arwiki/853493</field>
- </document>
- <document>
- <id>341280</id>
- <field name="COMMENTS">تدقيق إملائي وتنسيق</field>
- <field name="ID">341280</field>
- <field name="TIMESTAMP">1292144772000</field>
- <field name="TITLE">فريدي هايمور</field>
- <field name="DOCUMENT">DOCUMENT:80/arwiki/341280</field>
- </document>
- <document>
- <id>51580</id>
- <field name="COMMENTS">Rejected the last 1 changes (by [[مستخدم:94.97.75.64|94.97.75.64]]) and restored revision 6751743 by Σύμμαχος</field>
- <field name="ID">51580</field>
- <field name="TIMESTAMP">1305864563000</field>
- <field name="TITLE">إم بي سي</field>
- <field name="DOCUMENT">DOCUMENT:80/arwiki/51580</field>
- </document>
- <document>
- <id>723280</id>
- <field name="COMMENTS"/>
- <field name="ID">723280</field>
- <field name="TIMESTAMP">1303913428000</field>
- <field name="TITLE">نيوز</field>
- <field name="DOCUMENT">DOCUMENT:80/arwiki/723280</field>
- </document>
- <document>
- <id>92980</id>
- <field name="COMMENTS">/* العودة والعداء مع تربل اتش (2002-2004) */</field>
- <field name="ID">92980</field>
- <field name="TIMESTAMP">1303055617000</field>
- <field name="TITLE">شون مايكلز</field>
- <field name="DOCUMENT">DOCUMENT:80/arwiki/92980</field>
- </document>
- <document>
- <id>168294</id>
- <field name="COMMENTS">r2.6.2) (روبوت إضافة: [[bo:ཕྲུ་གུ]]</field>
- <field name="ID">168294</field>
- <field name="TIMESTAMP">1307376628000</field>
- <field name="TITLE">طفل</field>
- <field name="DOCUMENT">DOCUMENT:94/arwiki/168294</field>
- </document>
- <document>
- <id>62094</id>
- <field name="COMMENTS">تدقيق إملائي وتنسيق</field>
- <field name="ID">62094</field>
- <field name="TIMESTAMP">1292155455000</field>
- <field name="TITLE">قنابل ذات انشطار مصوب</field>
- <field name="DOCUMENT">DOCUMENT:94/arwiki/62094</field>
- </document>
- <document>
- <id>653494</id>
- <field name="COMMENTS">r2.7.1) (روبوت تعديل: [[zh:Camelot]]</field>
- <field name="ID">653494</field>
- <field name="TIMESTAMP">1297652789000</field>
- <field name="TITLE">كامالوت سوفتوير بلانينغ</field>
- <field name="DOCUMENT">DOCUMENT:94/arwiki/653494</field>
- </document>
- <document>
- <id>57691</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[zh-yue:任天堂]]</field>
- <field name="ID">57691</field>
- <field name="TIMESTAMP">1304916871000</field>
- <field name="TITLE">نينتندو</field>
- <field name="DOCUMENT">DOCUMENT:91/arwiki/57691</field>
- </document>
- <document>
- <id>671491</id>
- <field name="COMMENTS">روبوت: تعريب</field>
- <field name="ID">671491</field>
- <field name="TIMESTAMP">1252578813000</field>
- <field name="TITLE">تمثال الفتى ذو الحذاء المرشح</field>
- <field name="DOCUMENT">DOCUMENT:91/arwiki/671491</field>
- </document>
- <document>
- <id>676291</id>
- <field name="COMMENTS">تدقيق إملائي بالاعتماد على [[تعابير نمطية|التعابير النمطية]]، يرجى الإبلاغ عن [[نقاش المستخدم:Bassem JARKAS|الأخطاء والاقتراحات]]</field>
- <field name="ID">676291</field>
- <field name="TIMESTAMP">1301666718000</field>
- <field name="TITLE">آرون كارتر</field>
- <field name="DOCUMENT">DOCUMENT:91/arwiki/676291</field>
- </document>
- <document>
- <id>267018</id>
- <field name="COMMENTS">روبوت تعديل: [[es:Baby Boy]]</field>
- <field name="ID">267018</field>
- <field name="TIMESTAMP">1307148303000</field>
- <field name="TITLE">بيبي بوي (أغنية)</field>
- <field name="DOCUMENT">DOCUMENT:18/arwiki/267018</field>
- </document>
- <document>
- <id>64292</id>
- <field name="COMMENTS">/* الأهداف التربوية للكشاف المتقدم */</field>
- <field name="ID">64292</field>
- <field name="TIMESTAMP">1307319382000</field>
- <field name="TITLE">كشافة</field>
- <field name="DOCUMENT">DOCUMENT:92/arwiki/64292</field>
- </document>
- <document>
- <id>429617</id>
- <field name="COMMENTS">روبوت إزالة: [[tr:Oğlan (İnsan)]]</field>
- <field name="ID">429617</field>
- <field name="TIMESTAMP">1302566586000</field>
- <field name="TITLE">ولد</field>
- <field name="DOCUMENT">DOCUMENT:17/arwiki/429617</field>
- </document>
- <document>
- <id>266509</id>
- <field name="COMMENTS">روبوت: تغييرات تجميلية</field>
- <field name="ID">266509</field>
- <field name="TIMESTAMP">1260250217000</field>
- <field name="TITLE">دوتي روك (ألبوم)</field>
- <field name="DOCUMENT">DOCUMENT:9/arwiki/266509</field>
- </document>
- <document>
- <id>652309</id>
- <field name="COMMENTS">ناني بوت: تغييرات تجمييلية</field>
- <field name="ID">652309</field>
- <field name="TIMESTAMP">1294922741000</field>
- <field name="TITLE">ورلد أوف جوو</field>
- <field name="DOCUMENT">DOCUMENT:9/arwiki/652309</field>
- </document>
- <document>
- <id>1053026</id>
- <field name="COMMENTS">أنشأ الصفحة ب'{{Navbox |name =الفائزون بجائزة الفتى الذهبي|title = [[توتوسبورت#جائزة الفتى الذهبي|جائزة الفتى الذهبي]] |...'</field>
- <field name="ID">1053026</field>
- <field name="TIMESTAMP">1304610914000</field>
- <field name="TITLE">قالب:الفائزون بجائزة الفتى الذهبي</field>
- <field name="DOCUMENT">DOCUMENT:26/arwiki/1053026</field>
- </document>
- <document>
- <id>283426</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[la:Marius (persona ficticia)]]</field>
- <field name="ID">283426</field>
- <field name="TIMESTAMP">1298884095000</field>
- <field name="TITLE">ماريو (شخصية خيالية)</field>
- <field name="DOCUMENT">DOCUMENT:26/arwiki/283426</field>
- </document>
- <document>
- <id>35626</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[lv:Diddy]]</field>
- <field name="ID">35626</field>
- <field name="TIMESTAMP">1306794172000</field>
- <field name="TITLE">شون كومز</field>
- <field name="DOCUMENT">DOCUMENT:26/arwiki/35626</field>
- </document>
- <document>
- <id>523626</id>
- <field name="COMMENTS">[r2.5.2] روبوت إضافة: [[ga:Robert Redford]]</field>
- <field name="ID">523626</field>
- <field name="TIMESTAMP">1290712154000</field>
- <field name="TITLE">روبرت ريدفورد</field>
- <field name="DOCUMENT">DOCUMENT:26/arwiki/523626</field>
- </document>
- <document>
- <id>319281</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[sr:Џони Кеш]]</field>
- <field name="ID">319281</field>
- <field name="TIMESTAMP">1306719153000</field>
- <field name="TITLE">جوني كاش</field>
- <field name="DOCUMENT">DOCUMENT:81/arwiki/319281</field>
- </document>
- <document>
- <id>319681</id>
- <field name="COMMENTS"/>
- <field name="ID">319681</field>
- <field name="TIMESTAMP">1305581575000</field>
- <field name="TITLE">بلدية العناصر</field>
- <field name="DOCUMENT">DOCUMENT:81/arwiki/319681</field>
- </document>
- <document>
- <id>412981</id>
- <field name="COMMENTS"/>
- <field name="ID">412981</field>
- <field name="TIMESTAMP">1304701104000</field>
- <field name="TITLE">جيم بوي أدفانس</field>
- <field name="DOCUMENT">DOCUMENT:81/arwiki/412981</field>
- </document>
- <document>
- <id>83381</id>
- <field name="COMMENTS">/* وصلات خارجية */ إزالة فراغات</field>
- <field name="ID">83381</field>
- <field name="TIMESTAMP">1305754301000</field>
- <field name="TITLE">آفريل لافين</field>
- <field name="DOCUMENT">DOCUMENT:81/arwiki/83381</field>
- </document>
- <document>
- <id>595327</id>
- <field name="COMMENTS"/>
- <field name="ID">595327</field>
- <field name="TIMESTAMP">1306342745000</field>
- <field name="TITLE">كيفين ناش</field>
- <field name="DOCUMENT">DOCUMENT:27/arwiki/595327</field>
- </document>
- <document>
- <id>724427</id>
- <field name="COMMENTS">إضافة سريعة للتصنيف "جراحة عظمية" (باستخدام [[وب:المصناف الفوري|المصناف الفوري]])</field>
- <field name="ID">724427</field>
- <field name="TIMESTAMP">1266806028000</field>
- <field name="TITLE">ارتفاق الأصابع</field>
- <field name="DOCUMENT">DOCUMENT:27/arwiki/724427</field>
- </document>
- <document>
- <id>838042</id>
- <field name="COMMENTS"/>
- <field name="ID">838042</field>
- <field name="TIMESTAMP">1302426763000</field>
- <field name="TITLE">قالب:Fixbunching/doc</field>
- <field name="DOCUMENT">DOCUMENT:42/arwiki/838042</field>
- </document>
- <document>
- <id>351761</id>
- <field name="COMMENTS">تدقيق إملائي بالاعتماد على [[تعابير نمطية|التعابير النمطية]]، يرجى الإبلاغ عن [[نقاش المستخدم:Bassem JARKAS|الأخطاء والاقتراحات]]</field>
- <field name="ID">351761</field>
- <field name="TIMESTAMP">1301837112000</field>
- <field name="TITLE">نيوبي</field>
- <field name="DOCUMENT">DOCUMENT:61/arwiki/351761</field>
- </document>
- <document>
- <id>209024</id>
- <field name="COMMENTS">/* تعريف عام بسيتو كايبا */</field>
- <field name="ID">209024</field>
- <field name="TIMESTAMP">1306661354000</field>
- <field name="TITLE">سيتو كايبا</field>
- <field name="DOCUMENT">DOCUMENT:24/arwiki/209024</field>
- </document>
- <document>
- <id>407424</id>
- <field name="COMMENTS">ناني بوت: تغييرات تجمييلية</field>
- <field name="ID">407424</field>
- <field name="TIMESTAMP">1294934880000</field>
- <field name="TITLE">وي سبورتز</field>
- <field name="DOCUMENT">DOCUMENT:24/arwiki/407424</field>
- </document>
- <document>
- <id>449324</id>
- <field name="COMMENTS">تدقيق إملائي. 528 كلمة مستهدفة حاليًا.</field>
- <field name="ID">449324</field>
- <field name="TIMESTAMP">1301333406000</field>
- <field name="TITLE">دخيل عربي في الفرنسية</field>
- <field name="DOCUMENT">DOCUMENT:24/arwiki/449324</field>
- </document>
- <document>
- <id>717924</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[sl:Chris Martin]]</field>
- <field name="ID">717924</field>
- <field name="TIMESTAMP">1307303285000</field>
- <field name="TITLE">كريس مارتن</field>
- <field name="DOCUMENT">DOCUMENT:24/arwiki/717924</field>
- </document>
- <document>
- <id>942424</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[fa:گیم بوی کالر]]</field>
- <field name="ID">942424</field>
- <field name="TIMESTAMP">1301079637000</field>
- <field name="TITLE">جيم بوي كولر</field>
- <field name="DOCUMENT">DOCUMENT:24/arwiki/942424</field>
- </document>
- <document>
- <id>297743</id>
- <field name="COMMENTS">/* ألبوم كلامي 2009 */</field>
- <field name="ID">297743</field>
- <field name="TIMESTAMP">1302460058000</field>
- <field name="TITLE">لطفي دوبل كانون</field>
- <field name="DOCUMENT">DOCUMENT:43/arwiki/297743</field>
- </document>
- <document>
- <id>340743</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[es:Soulja Boy]]</field>
- <field name="ID">340743</field>
- <field name="TIMESTAMP">1304204793000</field>
- <field name="TITLE">سولجا بوي تل إم</field>
- <field name="DOCUMENT">DOCUMENT:43/arwiki/340743</field>
- </document>
- <document>
- <id>209125</id>
- <field name="COMMENTS"/>
- <field name="ID">209125</field>
- <field name="TIMESTAMP">1306031791000</field>
- <field name="TITLE">شلل دماغي</field>
- <field name="DOCUMENT">DOCUMENT:25/arwiki/209125</field>
- </document>
- <document>
- <id>266625</id>
- <field name="COMMENTS">روبوت: تغييرات تجميلية</field>
- <field name="ID">266625</field>
- <field name="TIMESTAMP">1260257965000</field>
- <field name="TITLE">ذا ترينتي (ألبوم)</field>
- <field name="DOCUMENT">DOCUMENT:25/arwiki/266625</field>
- </document>
- <document>
- <id>749125</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[lv:Deivids Arčuleta]]</field>
- <field name="ID">749125</field>
- <field name="TIMESTAMP">1306768791000</field>
- <field name="TITLE">ديفيد أرتشولينا</field>
- <field name="DOCUMENT">DOCUMENT:25/arwiki/749125</field>
- </document>
- <document>
- <id>841048</id>
- <field name="COMMENTS"/>
- <field name="ID">841048</field>
- <field name="TIMESTAMP">1305117637000</field>
- <field name="TITLE">ريك فلير(مصارع)</field>
- <field name="DOCUMENT">DOCUMENT:48/arwiki/841048</field>
- </document>
- <document>
- <id>509185</id>
- <field name="COMMENTS">روبوت: نقل الصفحات من تصنيف محول</field>
- <field name="ID">509185</field>
- <field name="TIMESTAMP">1305378630000</field>
- <field name="TITLE">دوشيرمة</field>
- <field name="DOCUMENT">DOCUMENT:85/arwiki/509185</field>
- </document>
- <document>
- <id>632985</id>
- <field name="COMMENTS">تدقيق إملائي. 528 كلمة مستهدفة حاليًا.</field>
- <field name="ID">632985</field>
- <field name="TIMESTAMP">1301310572000</field>
- <field name="TITLE">تسجيل السبايس جيرلز</field>
- <field name="DOCUMENT">DOCUMENT:85/arwiki/632985</field>
- </document>
- <document>
- <id>50176</id>
- <field name="COMMENTS">r2.6.4) (روبوت إضافة: [[be-x-old:Рынга Стар]]</field>
- <field name="ID">50176</field>
- <field name="TIMESTAMP">1300395173000</field>
- <field name="TITLE">رينغو ستار</field>
- <field name="DOCUMENT">DOCUMENT:76/arwiki/50176</field>
- </document>
- <document>
- <id>777576</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[es:Nicki Aycox]]</field>
- <field name="ID">777576</field>
- <field name="TIMESTAMP">1306460652000</field>
- <field name="TITLE">نيكي أيكوكس</field>
- <field name="DOCUMENT">DOCUMENT:76/arwiki/777576</field>
- </document>
- <document>
- <id>960876</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[yo:James Cagney]]</field>
- <field name="ID">960876</field>
- <field name="TIMESTAMP">1306172520000</field>
- <field name="TITLE">جيمس كاجنى</field>
- <field name="DOCUMENT">DOCUMENT:76/arwiki/960876</field>
- </document>
- <document>
- <id>972376</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[mk:Кери Хилсон]]</field>
- <field name="ID">972376</field>
- <field name="TIMESTAMP">1307261498000</field>
- <field name="TITLE">كيري هيلسون</field>
- <field name="DOCUMENT">DOCUMENT:76/arwiki/972376</field>
- </document>
- <document>
- <id>682158</id>
- <field name="COMMENTS">[r2.6.1] روبوت إضافة: [[gl:Jeff Buckley]]</field>
- <field name="ID">682158</field>
- <field name="TIMESTAMP">1290888101000</field>
- <field name="TITLE">جيف بوكلي</field>
- <field name="DOCUMENT">DOCUMENT:58/arwiki/682158</field>
- </document>
- <document>
- <id>737775</id>
- <field name="COMMENTS">/* البدائل */إزالة نقطتين من العنوان, replaced: : == → ==</field>
- <field name="ID">737775</field>
- <field name="TIMESTAMP">1304425289000</field>
- <field name="TITLE">فلورة المياه</field>
- <field name="DOCUMENT">DOCUMENT:75/arwiki/737775</field>
- </document>
- <document>
- <id>773875</id>
- <field name="COMMENTS">تدقيق إملائي. 528 كلمة مستهدفة حاليًا.</field>
- <field name="ID">773875</field>
- <field name="TIMESTAMP">1301322822000</field>
- <field name="TITLE">جوش جروبان</field>
- <field name="DOCUMENT">DOCUMENT:75/arwiki/773875</field>
- </document>
- <document>
- <id>910575</id>
- <field name="COMMENTS">روبوت تعديل: [[fa:حدیثه (خواننده)]]; تغييرات تجميلية</field>
- <field name="ID">910575</field>
- <field name="TIMESTAMP">1305666341000</field>
- <field name="TITLE">حديثة (مغنية)</field>
- <field name="DOCUMENT">DOCUMENT:75/arwiki/910575</field>
- </document>
- <document>
- <id>671257</id>
- <field name="COMMENTS">تدقيق إملائي بالاعتماد على [[تعابير نمطية|التعابير النمطية]]، يرجى الإبلاغ عن [[نقاش المستخدم:Bassem JARKAS|الأخطاء والاقتراحات]]</field>
- <field name="ID">671257</field>
- <field name="TIMESTAMP">1301666073000</field>
- <field name="TITLE">90210 (مسلسل تلفزيوني)</field>
- <field name="DOCUMENT">DOCUMENT:57/arwiki/671257</field>
- </document>
- <document>
- <id>881457</id>
- <field name="COMMENTS"/>
- <field name="ID">881457</field>
- <field name="TIMESTAMP">1306719992000</field>
- <field name="TITLE">عداوة جي-يونيت ضد ذا غيم</field>
- <field name="DOCUMENT">DOCUMENT:57/arwiki/881457</field>
- </document>
- <document>
- <id>396905</id>
- <field name="COMMENTS">r2.7.1) (روبوت تعديل: [[uk:Губка Боб Квадратні Штани]]</field>
- <field name="ID">396905</field>
- <field name="TIMESTAMP">1305509933000</field>
- <field name="TITLE">سبونجبوب</field>
- <field name="DOCUMENT">DOCUMENT:5/arwiki/396905</field>
- </document>
- <document>
- <id>897170</id>
- <field name="COMMENTS">روبوت: تغييرات تجميلية</field>
- <field name="ID">897170</field>
- <field name="TIMESTAMP">1293187426000</field>
- <field name="TITLE">بوابة:ألعاب فيديو/أحداث جارية</field>
- <field name="DOCUMENT">DOCUMENT:70/arwiki/897170</field>
- </document>
- <document>
- <id>267059</id>
- <field name="COMMENTS">تدقيق إملائي وتنسيق,</field>
- <field name="ID">267059</field>
- <field name="TIMESTAMP">1264686390000</field>
- <field name="TITLE">بابي بوي</field>
- <field name="DOCUMENT">DOCUMENT:59/arwiki/267059</field>
- </document>
- <document>
- <id>535259</id>
- <field name="COMMENTS">863</field>
- <field name="ID">535259</field>
- <field name="TIMESTAMP">1249447619000</field>
- <field name="TITLE">ويكيبيديا:قوائم المراقبة الاختصاصية/الإمارات</field>
- <field name="DOCUMENT">DOCUMENT:59/arwiki/535259</field>
- </document>
- <document>
- <id>66259</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[mrj:Торий]]</field>
- <field name="ID">66259</field>
- <field name="TIMESTAMP">1306760041000</field>
- <field name="TITLE">ثوريوم</field>
- <field name="DOCUMENT">DOCUMENT:59/arwiki/66259</field>
- </document>
- <document>
- <id>546497</id>
- <field name="COMMENTS">r2.6.4) (روبوت إضافة: [[yo:Sophia Loren]]</field>
- <field name="ID">546497</field>
- <field name="TIMESTAMP">1307304523000</field>
- <field name="TITLE">صوفيا لورين</field>
- <field name="DOCUMENT">DOCUMENT:97/arwiki/546497</field>
- </document>
- <document>
- <id>727097</id>
- <field name="COMMENTS">removed [[Category:أفلام]]; added [[Category:أفلام ناطقة بالإنجليزية]] using [[Help:Gadget-HotCat|HotCat]]</field>
- <field name="ID">727097</field>
- <field name="TIMESTAMP">1305333545000</field>
- <field name="TITLE">عودة المومياء</field>
- <field name="DOCUMENT">DOCUMENT:97/arwiki/727097</field>
- </document>
- <document>
- <id>132440</id>
- <field name="COMMENTS">r2.6.4) (روبوت إضافة: [[lbe:Щаллуну иссаву]]</field>
- <field name="ID">132440</field>
- <field name="TIMESTAMP">1291892845000</field>
- <field name="TITLE">غسل</field>
- <field name="DOCUMENT">DOCUMENT:40/arwiki/132440</field>
- </document>
- <document>
- <id>334240</id>
- <field name="COMMENTS"/>
- <field name="ID">334240</field>
- <field name="TIMESTAMP">1303767903000</field>
- <field name="TITLE">قالب:أجهزة نينتندو</field>
- <field name="DOCUMENT">DOCUMENT:40/arwiki/334240</field>
- </document>
- <document>
- <id>330398</id>
- <field name="COMMENTS">removed [[Category:رسوم متحركة]]; added [[Category:مسلسلات رسوم متحركة]] using [[Help:Gadget-HotCat|HotCat]]</field>
- <field name="ID">330398</field>
- <field name="TIMESTAMP">1305334411000</field>
- <field name="TITLE">مدرسة الكونغ فو</field>
- <field name="DOCUMENT">DOCUMENT:98/arwiki/330398</field>
- </document>
- <document>
- <id>428098</id>
- <field name="COMMENTS"/>
- <field name="ID">428098</field>
- <field name="TIMESTAMP">1301860914000</field>
- <field name="TITLE">بدر هاري</field>
- <field name="DOCUMENT">DOCUMENT:98/arwiki/428098</field>
- </document>
- <document>
- <id>730098</id>
- <field name="COMMENTS">تدقيق إملائي بالاعتماد على [[تعابير نمطية|التعابير النمطية]]، يرجى الإبلاغ عن [[نقاش المستخدم:Bassem JARKAS|الأخطاء والاقتراحات]]</field>
- <field name="ID">730098</field>
- <field name="TIMESTAMP">1301671339000</field>
- <field name="TITLE">أراشي</field>
- <field name="DOCUMENT">DOCUMENT:98/arwiki/730098</field>
- </document>
- <document>
- <id>19601</id>
- <field name="COMMENTS">/* وصلات خارجية */ إزالة فراغات</field>
- <field name="ID">19601</field>
- <field name="TIMESTAMP">1305815831000</field>
- <field name="TITLE">ميكيلانجيلو</field>
- <field name="DOCUMENT">DOCUMENT:1/arwiki/19601</field>
- </document>
- <document>
- <id>48901</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[ckb:مارادۆنا]]</field>
- <field name="ID">48901</field>
- <field name="TIMESTAMP">1306919031000</field>
- <field name="TITLE">دييغو مارادونا</field>
- <field name="DOCUMENT">DOCUMENT:1/arwiki/48901</field>
- </document>
- <document>
- <id>964501</id>
- <field name="COMMENTS"/>
- <field name="ID">964501</field>
- <field name="TIMESTAMP">1307144766000</field>
- <field name="TITLE">قالب:كارتون نتورك</field>
- <field name="DOCUMENT">DOCUMENT:1/arwiki/964501</field>
- </document>
- <document>
- <id>173230</id>
- <field name="COMMENTS">روبوت: تغييرات تجميلية</field>
- <field name="ID">173230</field>
- <field name="TIMESTAMP">1262037002000</field>
- <field name="TITLE">قالب:موسيقى البوب</field>
- <field name="DOCUMENT">DOCUMENT:30/arwiki/173230</field>
- </document>
- <document>
- <id>223330</id>
- <field name="COMMENTS">r2.6.4) (روبوت إضافة: [[vi:Thung lũng các vị vua]]</field>
- <field name="ID">223330</field>
- <field name="TIMESTAMP">1301562410000</field>
- <field name="TITLE">وادي الملوك</field>
- <field name="DOCUMENT">DOCUMENT:30/arwiki/223330</field>
- </document>
- <document>
- <id>37030</id>
- <field name="COMMENTS">تدقيق إملائي بالاعتماد على [[تعابير نمطية|التعابير النمطية]]، يرجى الإبلاغ عن [[نقاش المستخدم:Bassem JARKAS|الأخطاء والاقتراحات]]</field>
- <field name="ID">37030</field>
- <field name="TIMESTAMP">1301722687000</field>
- <field name="TITLE">بلد+</field>
- <field name="DOCUMENT">DOCUMENT:30/arwiki/37030</field>
- </document>
- <document>
- <id>54030</id>
- <field name="COMMENTS"/>
- <field name="ID">54030</field>
- <field name="TIMESTAMP">1296598440000</field>
- <field name="TITLE">ريمي</field>
- <field name="DOCUMENT">DOCUMENT:30/arwiki/54030</field>
- </document>
- <document>
- <id>743630</id>
- <field name="COMMENTS">تدقيق إملائي. 528 كلمة مستهدفة حاليًا.</field>
- <field name="ID">743630</field>
- <field name="TIMESTAMP">1301311716000</field>
- <field name="TITLE">تطبيقات الإنترنت الغنية</field>
- <field name="DOCUMENT">DOCUMENT:30/arwiki/743630</field>
- </document>
- <document>
- <id>805030</id>
- <field name="COMMENTS">روبوت إضافة: [[tr:Selena albüm diskografisi]]</field>
- <field name="ID">805030</field>
- <field name="TIMESTAMP">1306619302000</field>
- <field name="TITLE">ديسكغرفي سيلينا</field>
- <field name="DOCUMENT">DOCUMENT:30/arwiki/805030</field>
- </document>
- <document>
- <id>8230</id>
- <field name="COMMENTS">روبوت إضافة: [[ky:Аниме]]; تغييرات تجميلية</field>
- <field name="ID">8230</field>
- <field name="TIMESTAMP">1303622065000</field>
- <field name="TITLE">أنمي</field>
- <field name="DOCUMENT">DOCUMENT:30/arwiki/8230</field>
- </document>
- <document>
- <id>1057974</id>
- <field name="COMMENTS"/>
- <field name="ID">1057974</field>
- <field name="TIMESTAMP">1305439164000</field>
- <field name="TITLE">روني مارا</field>
- <field name="DOCUMENT">DOCUMENT:74/arwiki/1057974</field>
- </document>
- <document>
- <id>486074</id>
- <field name="COMMENTS">تدقيق إملائي بالاعتماد على [[تعابير نمطية|التعابير النمطية]]، يرجى الإبلاغ عن [[نقاش المستخدم:Bassem JARKAS|الأخطاء والاقتراحات]]</field>
- <field name="ID">486074</field>
- <field name="TIMESTAMP">1301734365000</field>
- <field name="TITLE">تيم برتون</field>
- <field name="DOCUMENT">DOCUMENT:74/arwiki/486074</field>
- </document>
- <document>
- <id>2538</id>
- <field name="COMMENTS">/* لينا */</field>
- <field name="ID">2538</field>
- <field name="TIMESTAMP">1307426171000</field>
- <field name="TITLE">مغامرات عدنان</field>
- <field name="DOCUMENT">DOCUMENT:38/arwiki/2538</field>
- </document>
- <document>
- <id>471338</id>
- <field name="COMMENTS">r2.5.2) (روبوت إضافة: cs, en, eu إزالة: ro تعديل: es, ru, th, zh</field>
- <field name="ID">471338</field>
- <field name="TIMESTAMP">1299488636000</field>
- <field name="TITLE">تطبيق إنترنت غني</field>
- <field name="DOCUMENT">DOCUMENT:38/arwiki/471338</field>
- </document>
- <document>
- <id>171712</id>
- <field name="COMMENTS">تدقيق إملائي وتنسيق</field>
- <field name="ID">171712</field>
- <field name="TIMESTAMP">1292248012000</field>
- <field name="TITLE">لي هيوري</field>
- <field name="DOCUMENT">DOCUMENT:12/arwiki/171712</field>
- </document>
- <document>
- <id>391612</id>
- <field name="COMMENTS">تدقيق إملائي بالاعتماد على [[تعابير نمطية|التعابير النمطية]]، يرجى الإبلاغ عن [[نقاش المستخدم:Bassem JARKAS|الأخطاء والاقتراحات]]</field>
- <field name="ID">391612</field>
- <field name="TIMESTAMP">1301724600000</field>
- <field name="TITLE">بول تبيتس</field>
- <field name="DOCUMENT">DOCUMENT:12/arwiki/391612</field>
- </document>
- <document>
- <id>717312</id>
- <field name="COMMENTS">تدقيق إملائي وتنسيق</field>
- <field name="ID">717312</field>
- <field name="TIMESTAMP">1288377674000</field>
- <field name="TITLE">واما</field>
- <field name="DOCUMENT">DOCUMENT:12/arwiki/717312</field>
- </document>
- <document>
- <id>1067873</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: de, es, fr, hu, it, ko, nl, no, pl, pt, sv, wuu, zh, zh-yue</field>
- <field name="ID">1067873</field>
- <field name="TIMESTAMP">1307052301000</field>
- <field name="TITLE">متلازمة اللكنة الأجنبية</field>
- <field name="DOCUMENT">DOCUMENT:73/arwiki/1067873</field>
- </document>
- <document>
- <id>162473</id>
- <field name="COMMENTS">تدقيق إملائي بالاعتماد على [[تعابير نمطية|التعابير النمطية]]، يرجى الإبلاغ عن [[نقاش المستخدم:Bassem JARKAS|الأخطاء والاقتراحات]]</field>
- <field name="ID">162473</field>
- <field name="TIMESTAMP">1301829236000</field>
- <field name="TITLE">مي بلان</field>
- <field name="DOCUMENT">DOCUMENT:73/arwiki/162473</field>
- </document>
- <document>
- <id>182231</id>
- <field name="COMMENTS">[[كلمات أعجمية من أصل عربي]] تم نقلها إلى [[دخيل عربي في الإنجليزية]]: انظر "دخيل عربي"</field>
- <field name="ID">182231</field>
- <field name="TIMESTAMP">1298669805000</field>
- <field name="TITLE">دخيل عربي في الإنجليزية</field>
- <field name="DOCUMENT">DOCUMENT:31/arwiki/182231</field>
- </document>
- <document>
- <id>192831</id>
- <field name="COMMENTS"/>
- <field name="ID">192831</field>
- <field name="TIMESTAMP">1306135157000</field>
- <field name="TITLE">حلاق</field>
- <field name="DOCUMENT">DOCUMENT:31/arwiki/192831</field>
- </document>
- <document>
- <id>42631</id>
- <field name="COMMENTS">/* كلمات الأغنية */</field>
- <field name="ID">42631</field>
- <field name="TIMESTAMP">1307366905000</field>
- <field name="TITLE">نينجا المغامر</field>
- <field name="DOCUMENT">DOCUMENT:31/arwiki/42631</field>
- </document>
- <document>
- <id>709513</id>
- <field name="COMMENTS">/* الأفلام والمسلسلات التي اشترك بها */</field>
- <field name="ID">709513</field>
- <field name="TIMESTAMP">1300978971000</field>
- <field name="TITLE">جو جوناس</field>
- <field name="DOCUMENT">DOCUMENT:13/arwiki/709513</field>
- </document>
- <document>
- <id>277232</id>
- <field name="COMMENTS">تدقيق إملائي. 528 كلمة مستهدفة حاليًا.</field>
- <field name="ID">277232</field>
- <field name="TIMESTAMP">1301228507000</field>
- <field name="TITLE">ميغان فوكس</field>
- <field name="DOCUMENT">DOCUMENT:32/arwiki/277232</field>
- </document>
- <document>
- <id>574432</id>
- <field name="COMMENTS">استرجاع تعديلات [[خاص:مساهمات/196.221.134.123|196.221.134.123]] ([[نقاش المستخدم:196.221.134.123|نقاش]]) حتى آخر نسخة بواسطة [[مستخدم:Faris knight|Faris knight]]</field>
- <field name="ID">574432</field>
- <field name="TIMESTAMP">1307214309000</field>
- <field name="TITLE">إبادة هيروشيما</field>
- <field name="DOCUMENT">DOCUMENT:32/arwiki/574432</field>
- </document>
- <document>
- <id>81332</id>
- <field name="COMMENTS">r2.7.1) (روبوت تعديل: [[ru:Мартин Шин]]</field>
- <field name="ID">81332</field>
- <field name="TIMESTAMP">1299598748000</field>
- <field name="TITLE">مارتن شين</field>
- <field name="DOCUMENT">DOCUMENT:32/arwiki/81332</field>
- </document>
- <document>
- <id>324114</id>
- <field name="COMMENTS">/* نشأته */</field>
- <field name="ID">324114</field>
- <field name="TIMESTAMP">1307290775000</field>
- <field name="TITLE">كيانو ريفز</field>
- <field name="DOCUMENT">DOCUMENT:14/arwiki/324114</field>
- </document>
- <document>
- <id>676214</id>
- <field name="COMMENTS">/* روابط إضافية */</field>
- <field name="ID">676214</field>
- <field name="TIMESTAMP">1303817963000</field>
- <field name="TITLE">اندريه ارشافين</field>
- <field name="DOCUMENT">DOCUMENT:14/arwiki/676214</field>
- </document>
- <document>
- <id>855752</id>
- <field name="COMMENTS">الرجوع عن التعديل 6765271 بواسطة [[Special:Contributions/Bravo|Bravo]] ([[User talk:Bravo|نقاش]])</field>
- <field name="ID">855752</field>
- <field name="TIMESTAMP">1304375326000</field>
- <field name="TITLE">خمن من سيأتي للعشاء (فيلم)</field>
- <field name="DOCUMENT">DOCUMENT:52/arwiki/855752</field>
- </document>
- <document>
- <id>45379</id>
- <field name="COMMENTS">الرجوع عن التعديل 6730985 بواسطة [[Special:Contributions/217.164.24.196|217.164.24.196]] ([[User talk:217.164.24.196|نقاش]])</field>
- <field name="ID">45379</field>
- <field name="TIMESTAMP">1303589039000</field>
- <field name="TITLE">فيتامين ألف</field>
- <field name="DOCUMENT">DOCUMENT:79/arwiki/45379</field>
- </document>
- <document>
- <id>844479</id>
- <field name="COMMENTS">تنسيق الصور</field>
- <field name="ID">844479</field>
- <field name="TIMESTAMP">1304890879000</field>
- <field name="TITLE">نيكولاس هولت</field>
- <field name="DOCUMENT">DOCUMENT:79/arwiki/844479</field>
- </document>
- <document>
- <id>975279</id>
- <field name="COMMENTS"/>
- <field name="ID">975279</field>
- <field name="TIMESTAMP">1301307502000</field>
- <field name="TITLE">لقاح شلل الأطفال</field>
- <field name="DOCUMENT">DOCUMENT:79/arwiki/975279</field>
- </document>
- <document>
- <id>725353</id>
- <field name="COMMENTS">تدقيق إملائي. 528 كلمة مستهدفة حاليًا.</field>
- <field name="ID">725353</field>
- <field name="TIMESTAMP">1301322828000</field>
- <field name="TITLE">جوش غروبان</field>
- <field name="DOCUMENT">DOCUMENT:53/arwiki/725353</field>
- </document>
- <document>
- <id>854353</id>
- <field name="COMMENTS">روبوت تعديل: [[it:Template:Discografia]], [[pt:Predefinição:Info/Discografia de artista]], [[ru:Шаблон:Дискография артиста]]</field>
- <field name="ID">854353</field>
- <field name="TIMESTAMP">1286357410000</field>
- <field name="TITLE">قالب:دسكغرافيا/شرح</field>
- <field name="DOCUMENT">DOCUMENT:53/arwiki/854353</field>
- </document>
- <document>
- <id>236308</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[zh:天方夜譚 (電視動畫)]]</field>
- <field name="ID">236308</field>
- <field name="TIMESTAMP">1306290121000</field>
- <field name="TITLE">مغامرات سندباد</field>
- <field name="DOCUMENT">DOCUMENT:8/arwiki/236308</field>
- </document>
- <document>
- <id>519356</id>
- <field name="COMMENTS">اضافة رابط لصفحة فيلم mimic بالعربية</field>
- <field name="ID">519356</field>
- <field name="TIMESTAMP">1306050078000</field>
- <field name="TITLE">جيرمي نورثام</field>
- <field name="DOCUMENT">DOCUMENT:56/arwiki/519356</field>
- </document>
- <document>
- <id>65256</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[yo:John Wayne]]</field>
- <field name="ID">65256</field>
- <field name="TIMESTAMP">1306172155000</field>
- <field name="TITLE">جون وين</field>
- <field name="DOCUMENT">DOCUMENT:56/arwiki/65256</field>
- </document>
- <document>
- <id>557955</id>
- <field name="COMMENTS">تدقيق إملائي بالإعتماد على الصيغة القياسية</field>
- <field name="ID">557955</field>
- <field name="TIMESTAMP">1301124962000</field>
- <field name="TITLE">مسألة مونتي هول</field>
- <field name="DOCUMENT">DOCUMENT:55/arwiki/557955</field>
- </document>
- <document>
- <id>892855</id>
- <field name="COMMENTS">/* معنى Google */إصلاح عناوين الفقرات</field>
- <field name="ID">892855</field>
- <field name="TIMESTAMP">1304431057000</field>
- <field name="TITLE">خصائص البحث في google</field>
- <field name="DOCUMENT">DOCUMENT:55/arwiki/892855</field>
- </document>
- <document>
- <id>57833</id>
- <field name="COMMENTS">تدقيق إملائي بالاعتماد على [[تعابير نمطية|التعابير النمطية]]، يرجى الإبلاغ عن [[نقاش المستخدم:Bassem JARKAS|الأخطاء والاقتراحات]]</field>
- <field name="ID">57833</field>
- <field name="TIMESTAMP">1301837060000</field>
- <field name="TITLE">نينتندو دي إس</field>
- <field name="DOCUMENT">DOCUMENT:33/arwiki/57833</field>
- </document>
- <document>
- <id>50115</id>
- <field name="COMMENTS">r2.6.4) (روبوت تعديل: [[pl:Sega (firma)]]</field>
- <field name="ID">50115</field>
- <field name="TIMESTAMP">1303725850000</field>
- <field name="TITLE">سيجا</field>
- <field name="DOCUMENT">DOCUMENT:15/arwiki/50115</field>
- </document>
- <document>
- <id>274151</id>
- <field name="COMMENTS">/* حياته */</field>
- <field name="ID">274151</field>
- <field name="TIMESTAMP">1293119956000</field>
- <field name="TITLE">باتريك ديمبسي</field>
- <field name="DOCUMENT">DOCUMENT:51/arwiki/274151</field>
- </document>
- <document>
- <id>948351</id>
- <field name="COMMENTS">أضافة صورة</field>
- <field name="ID">948351</field>
- <field name="TIMESTAMP">1300792092000</field>
- <field name="TITLE">تشغيله مرة أخرى</field>
- <field name="DOCUMENT">DOCUMENT:51/arwiki/948351</field>
- </document>
- <document>
- <id>16234</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[sa:अकबरः]]</field>
- <field name="ID">16234</field>
- <field name="TIMESTAMP">1307205637000</field>
- <field name="TITLE">جلال الدين أكبر</field>
- <field name="DOCUMENT">DOCUMENT:34/arwiki/16234</field>
- </document>
- <document>
- <id>24234</id>
- <field name="COMMENTS">روبوت تعديل: [[it:Little Boy]]</field>
- <field name="ID">24234</field>
- <field name="TIMESTAMP">1305272763000</field>
- <field name="TITLE">الولد الصغير (قنبلة)</field>
- <field name="DOCUMENT">DOCUMENT:34/arwiki/24234</field>
- </document>
- <document>
- <id>242634</id>
- <field name="COMMENTS">r2.6.3) (روبوت إضافة: [[ro:Plânset]]</field>
- <field name="ID">242634</field>
- <field name="TIMESTAMP">1295379427000</field>
- <field name="TITLE">بكاء</field>
- <field name="DOCUMENT">DOCUMENT:34/arwiki/242634</field>
- </document>
- <document>
- <id>38534</id>
- <field name="COMMENTS">تدقيق إملائي وتنسيق</field>
- <field name="ID">38534</field>
- <field name="TIMESTAMP">1292155719000</field>
- <field name="TITLE">قنبلة ذات انشطار مصوب</field>
- <field name="DOCUMENT">DOCUMENT:34/arwiki/38534</field>
- </document>
- <document>
- <id>730534</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[es:Boy A]]</field>
- <field name="ID">730534</field>
- <field name="TIMESTAMP">1306808480000</field>
- <field name="TITLE">الصبي أ (فيلم)</field>
- <field name="DOCUMENT">DOCUMENT:34/arwiki/730534</field>
- </document>
- <document>
- <id>778034</id>
- <field name="COMMENTS">ناني بوت: تغييرات تجمييلية</field>
- <field name="ID">778034</field>
- <field name="TIMESTAMP">1295126691000</field>
- <field name="TITLE">قاعة الشهرة (دبليو دبليو إي)</field>
- <field name="DOCUMENT">DOCUMENT:34/arwiki/778034</field>
- </document>
- <document>
- <id>805334</id>
- <field name="COMMENTS">قالب غير موقع</field>
- <field name="ID">805334</field>
- <field name="TIMESTAMP">1267344122000</field>
- <field name="TITLE">ويكيبيديا:نقاش الحذف/وسام عبد الله</field>
- <field name="DOCUMENT">DOCUMENT:34/arwiki/805334</field>
- </document>
- <document>
- <id>923034</id>
- <field name="COMMENTS">/* المراجع */</field>
- <field name="ID">923034</field>
- <field name="TIMESTAMP">1303744763000</field>
- <field name="TITLE">الموت السريري</field>
- <field name="DOCUMENT">DOCUMENT:34/arwiki/923034</field>
- </document>
- <document>
- <id>131904</id>
- <field name="COMMENTS">/* أهداف المشروع */سايفربوت: إملائي</field>
- <field name="ID">131904</field>
- <field name="TIMESTAMP">1259919949000</field>
- <field name="TITLE">ويكيبيديا:مشاريع ويكي/أنظمة ألعاب فيديو</field>
- <field name="DOCUMENT">DOCUMENT:4/arwiki/131904</field>
- </document>
- <document>
- <id>32304</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[az:Alanis Morissette]]</field>
- <field name="ID">32304</field>
- <field name="TIMESTAMP">1306494689000</field>
- <field name="TITLE">ألانيس موريسيت</field>
- <field name="DOCUMENT">DOCUMENT:4/arwiki/32304</field>
- </document>
- <document>
- <id>52204</id>
- <field name="COMMENTS">r2.6.4) (روبوت إضافة: [[rue:Сатьяджит Рей]]</field>
- <field name="ID">52204</field>
- <field name="TIMESTAMP">1296319866000</field>
- <field name="TITLE">ساتياجيت راي</field>
- <field name="DOCUMENT">DOCUMENT:4/arwiki/52204</field>
- </document>
- <document>
- <id>601704</id>
- <field name="COMMENTS">/* وصلات خارجية */</field>
- <field name="ID">601704</field>
- <field name="TIMESTAMP">1304579461000</field>
- <field name="TITLE">الحركة العامة للكشافة والمرشدات</field>
- <field name="DOCUMENT">DOCUMENT:4/arwiki/601704</field>
- </document>
- <document>
- <id>161535</id>
- <field name="COMMENTS">/* الألبوم الأول */</field>
- <field name="ID">161535</field>
- <field name="TIMESTAMP">1297987689000</field>
- <field name="TITLE">دكلن جالبرايث</field>
- <field name="DOCUMENT">DOCUMENT:35/arwiki/161535</field>
- </document>
- <document>
- <id>200335</id>
- <field name="COMMENTS">r2.6.4) (روبوت تعديل: [[hu:Sean Astin]]</field>
- <field name="ID">200335</field>
- <field name="TIMESTAMP">1307246737000</field>
- <field name="TITLE">شون أستين</field>
- <field name="DOCUMENT">DOCUMENT:35/arwiki/200335</field>
- </document>
- <document>
- <id>907035</id>
- <field name="COMMENTS">أنشأ الصفحة ب'<span class="IPA">[[ويكيبيديا:أصد للإنكليزية|{{#switch:{{{1}}} | b ={{H:title| 'b' in 'buy' |b}} | d ...'</field>
- <field name="ID">907035</field>
- <field name="TIMESTAMP">1281011614000</field>
- <field name="TITLE">قالب:H:IPA</field>
- <field name="DOCUMENT">DOCUMENT:35/arwiki/907035</field>
- </document>
- <document>
- <id>182736</id>
- <field name="COMMENTS">/* مجموعة شركات inter public برياطانية */</field>
- <field name="ID">182736</field>
- <field name="TIMESTAMP">1295113520000</field>
- <field name="TITLE">أكبر مؤسسات إعلانية بالعالم</field>
- <field name="DOCUMENT">DOCUMENT:36/arwiki/182736</field>
- </document>
- <document>
- <id>322336</id>
- <field name="COMMENTS">إزالة قالب بذرة (أكثر من 5000 حرف و10 وصلات داخلية)</field>
- <field name="ID">322336</field>
- <field name="TIMESTAMP">1301573975000</field>
- <field name="TITLE">تيم روث</field>
- <field name="DOCUMENT">DOCUMENT:36/arwiki/322336</field>
- </document>
- <document>
- <id>493436</id>
- <field name="COMMENTS">5 - مشروع تطوير [[بوابة:رياضيات]]</field>
- <field name="ID">493436</field>
- <field name="TIMESTAMP">1232127458000</field>
- <field name="TITLE">بوابة:رياضيات/صورة مختارة/48</field>
- <field name="DOCUMENT">DOCUMENT:36/arwiki/493436</field>
- </document>
- <document>
- <id>508736</id>
- <field name="COMMENTS"/>
- <field name="ID">508736</field>
- <field name="TIMESTAMP">1289425402000</field>
- <field name="TITLE">ايثيل مانين</field>
- <field name="DOCUMENT">DOCUMENT:36/arwiki/508736</field>
- </document>
- <document>
- <id>979136</id>
- <field name="COMMENTS">تدقيق إملائي بالاعتماد على [[تعابير نمطية|التعابير النمطية]]، يرجى الإبلاغ عن [[نقاش المستخدم:Bassem JARKAS|الأخطاء والاقتراحات]]</field>
- <field name="ID">979136</field>
- <field name="TIMESTAMP">1301803397000</field>
- <field name="TITLE">كمير (وراثة)</field>
- <field name="DOCUMENT">DOCUMENT:36/arwiki/979136</field>
- </document>
- <document>
- <id>142671</id>
- <field name="COMMENTS">r2.6.5) (روبوت تعديل: [[ru:Кейдж, Николас]]</field>
- <field name="ID">142671</field>
- <field name="TIMESTAMP">1302820266000</field>
- <field name="TITLE">نيكولاس كيج</field>
- <field name="DOCUMENT">DOCUMENT:71/arwiki/142671</field>
- </document>
- <document>
- <id>232371</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[fa:اسکیتر بوی]]</field>
- <field name="ID">232371</field>
- <field name="TIMESTAMP">1304700855000</field>
- <field name="TITLE">سكيتر بوي</field>
- <field name="DOCUMENT">DOCUMENT:71/arwiki/232371</field>
- </document>
- <document>
- <id>5471</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[cy:Ffilm fer]]</field>
- <field name="ID">5471</field>
- <field name="TIMESTAMP">1306171126000</field>
- <field name="TITLE">فيلم قصير</field>
- <field name="DOCUMENT">DOCUMENT:71/arwiki/5471</field>
- </document>
- <document>
- <id>828471</id>
- <field name="COMMENTS">تدقيق إملائي. 528 كلمة مستهدفة حاليًا.</field>
- <field name="ID">828471</field>
- <field name="TIMESTAMP">1301232163000</field>
- <field name="TITLE">نخبة بروكلين (فيلم)</field>
- <field name="DOCUMENT">DOCUMENT:71/arwiki/828471</field>
- </document>
- <document>
- <id>170937</id>
- <field name="COMMENTS"/>
- <field name="ID">170937</field>
- <field name="TIMESTAMP">1287146222000</field>
- <field name="TITLE">نيبون أنيميشن</field>
- <field name="DOCUMENT">DOCUMENT:37/arwiki/170937</field>
- </document>
- <document>
- <id>31737</id>
- <field name="COMMENTS">/* بدايات حياتها */</field>
- <field name="ID">31737</field>
- <field name="TIMESTAMP">1306282843000</field>
- <field name="TITLE">اشلي سيمبسون</field>
- <field name="DOCUMENT">DOCUMENT:37/arwiki/31737</field>
- </document>
- <document>
- <id>348137</id>
- <field name="COMMENTS">r2.7.1) (روبوت إضافة: [[bg:Фаюмски портрети]]</field>
- <field name="ID">348137</field>
- <field name="TIMESTAMP">1299358018000</field>
- <field name="TITLE">لوحات مومياوات الفيوم</field>
- <field name="DOCUMENT">DOCUMENT:37/arwiki/348137</field>
- </document>
- <document>
- <id>189711</id>
- <field name="COMMENTS">روبوت إضافة: [[fa:مدرسه گرامر اوکلند]]</field>
- <field name="ID">189711</field>
- <field name="TIMESTAMP">1288472858000</field>
- <field name="TITLE">مدرسة أوكلاند الثانوية</field>
- <field name="DOCUMENT">DOCUMENT:11/arwiki/189711</field>
- </document>
- <document>
- <id>420311</id>
- <field name="COMMENTS"/>
- <field name="ID">420311</field>
- <field name="TIMESTAMP">1302826054000</field>
- <field name="TITLE">ويكيبيديا:مشروع ويكي سينما/مقالات أفلام مطلوبة</field>
- <field name="DOCUMENT">DOCUMENT:11/arwiki/420311</field>
- </document>
- <document>
- <id>659511</id>
- <field name="COMMENTS">روبوت إضافة: [[ka:ბიჭი ზოლიან პიჟამაში]]</field>
- <field name="ID">659511</field>
- <field name="TIMESTAMP">1304894077000</field>
- <field name="TITLE">الصبي في البيجامة المخططة (فيلم)</field>
- <field name="DOCUMENT">DOCUMENT:11/arwiki/659511</field>
- </document>
-</results>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/pom.xml
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/pom.xml b/src/examples/wikisearch/query/pom.xml
deleted file mode 100644
index 8c6e6d3..0000000
--- a/src/examples/wikisearch/query/pom.xml
+++ /dev/null
@@ -1,180 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <parent>
- <artifactId>accumulo-wikisearch</artifactId>
- <groupId>org.apache.accumulo</groupId>
- <version>1.4.5-SNAPSHOT</version>
- <relativePath>../</relativePath>
- </parent>
-
- <artifactId>wikisearch-query</artifactId>
- <packaging>ejb</packaging>
- <name>wikisearch-query</name>
- <dependencies>
- <dependency>
- <groupId>javaee</groupId>
- <artifactId>javaee-api</artifactId>
- <version>5</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- <scope>runtime</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>cloudtrace</artifactId>
- <scope>runtime</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libthrift</artifactId>
- <scope>runtime</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-jexl</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.collections</groupId>
- <artifactId>google-collections</artifactId>
- </dependency>
- <dependency>
- <groupId>com.googlecode</groupId>
- <artifactId>kryo</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>wikisearch-ingest</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.googlecode</groupId>
- <artifactId>minlog</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>copy-dependencies</id>
- <phase>process-resources</phase>
- <goals>
- <goal>copy-dependencies</goal>
- </goals>
- <configuration>
- <outputDirectory>lib</outputDirectory>
- <!-- just grab the non-provided runtime dependencies -->
- <includeArtifactIds>commons-lang,commons-codec,protobuf-java,libthrift,zookeeper,hadoop-core,commons-jexl,google-collections,kryo,asm,minlog,reflectasm,wikisearch-ingest,accumulo-core,cloudtrace</includeArtifactIds>
- <excludeTransitive>true</excludeTransitive>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <configuration>
- <descriptors>
- <descriptor>src/assembly/dist.xml</descriptor>
- </descriptors>
- <tarLongFileMode>gnu</tarLongFileMode>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <source>1.6</source>
- <target>1.6</target>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-ejb-plugin</artifactId>
- <version>2.1</version>
- <configuration>
- <ejbVersion>3.1</ejbVersion>
- </configuration>
- </plugin>
- </plugins>
- </build>
- <modelVersion>4.0.0</modelVersion>
-
- <profiles>
- <!-- profile for building against Hadoop 1.0.x
- Activate by not specifying hadoop.profile -->
- <profile>
- <id>hadoop-1.0</id>
- <activation>
- <property>
- <name>!hadoop.profile</name>
- </property>
- </activation>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- </dependency>
- </dependencies>
- </profile>
- <!-- profile for building against Hadoop 2.0.x
- Activate using: mvn -Dhadoop.profile=2.0 -->
- <profile>
- <id>hadoop-2.0</id>
- <activation>
- <property>
- <name>hadoop.profile</name>
- <value>2.0</value>
- </property>
- </activation>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- </dependency>
- </dependencies>
- </profile>
- </profiles>
-
-</project>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/assembly/dist.xml
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/assembly/dist.xml b/src/examples/wikisearch/query/src/assembly/dist.xml
deleted file mode 100644
index 66dd52e..0000000
--- a/src/examples/wikisearch/query/src/assembly/dist.xml
+++ /dev/null
@@ -1,40 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<assembly>
- <id>dist</id>
- <formats>
- <format>tar.gz</format>
- </formats>
- <baseDirectory></baseDirectory>
- <fileSets>
- <fileSet>
- <directory>lib</directory>
- <outputDirectory>lib</outputDirectory>
- <excludes>
- <exclude>${project.name}-${project.version}.jar</exclude>
- </excludes>
- <fileMode>0644</fileMode>
- </fileSet>
- </fileSets>
- <files>
- <file>
- <source>target/${project.name}-${project.version}.jar</source>
- <outputDirectory>deploy</outputDirectory>
- </file>
- </files>
-</assembly>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/function/QueryFunctions.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/function/QueryFunctions.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/function/QueryFunctions.java
deleted file mode 100644
index ff6662c..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/function/QueryFunctions.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.function;
-
-import org.apache.commons.lang.math.NumberUtils;
-import org.apache.log4j.Logger;
-
-public class QueryFunctions {
-
- protected static Logger log = Logger.getLogger(QueryFunctions.class);
-
- public static boolean between(String fieldValue, double left, double right) {
- try {
- Double value = Double.parseDouble(fieldValue);
- if (value >= left && value <= right)
- return true;
- return false;
- } catch (NumberFormatException nfe) {
- return false;
- }
- }
-
- public static boolean between(String fieldValue, long left, long right) {
- try {
- Long value = Long.parseLong(fieldValue);
- if (value >= left && value <= right)
- return true;
- return false;
- } catch (NumberFormatException nfe) {
- return false;
- }
- }
-
- public static Number abs(String fieldValue) {
- Number retval = null;
- try {
- Number value = NumberUtils.createNumber(fieldValue);
- if (null == value)
- retval = (Number) Integer.MIN_VALUE;
- else if (value instanceof Long)
- retval = Math.abs(value.longValue());
- else if (value instanceof Double)
- retval = Math.abs(value.doubleValue());
- else if (value instanceof Float)
- retval = Math.abs(value.floatValue());
- else if (value instanceof Integer)
- retval = Math.abs(value.intValue());
- } catch (NumberFormatException nfe) {
- return (Number) Integer.MIN_VALUE;
- }
- return retval;
- }
-
-}
[17/39] git commit: ACCUMULO-804 defend against
FileNotFoundExceptions for hadoop-2.0
Posted by ec...@apache.org.
ACCUMULO-804 defend against FileNotFoundExceptions for hadoop-2.0
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.5@1458344 13f79535-47bb-0310-9956-ffa450edef68
(cherry picked from commit aa0ca4a30fcbae39e33fd0ae7e9c6e5dd44c7cd8)
Reason: Hadoop2 Compat
Author: Eric C. Newton <ec...@apache.org>
Ref: ACCUMULO-1792
only one ref to fs.delete() in GarbageCollectWriteAheadLog
Signed-off-by: Eric Newton <er...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/1b1334f5
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/1b1334f5
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/1b1334f5
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 1b1334f5d26bda6a6be435dd277e3eca225a98c3
Parents: ec2aaa6
Author: Jonathan M Hsieh <jo...@cloudera.com>
Authored: Wed May 29 13:57:46 2013 -0700
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Nov 25 16:06:42 2013 -0500
----------------------------------------------------------------------
.../apache/accumulo/core/client/ZooKeeperInstance.java | 8 +++++++-
.../util/shell/commands/ImportDirectoryCommand.java | 13 ++++++++++---
.../server/gc/GarbageCollectWriteAheadLogs.java | 3 +++
.../accumulo/server/gc/SimpleGarbageCollector.java | 12 +++++++++---
.../accumulo/server/master/tableOps/BulkImport.java | 7 ++++++-
.../apache/accumulo/server/tabletserver/Tablet.java | 10 +++++++++-
6 files changed, 44 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1b1334f5/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java b/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
index 069d217..fcf8f55 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.core.client;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Collections;
@@ -272,7 +273,12 @@ public class ZooKeeperInstance implements Instance {
public static String getInstanceIDFromHdfs(Path instanceDirectory) {
try {
FileSystem fs = FileUtil.getFileSystem(CachedConfiguration.getInstance(), AccumuloConfiguration.getSiteConfiguration());
- FileStatus[] files = fs.listStatus(instanceDirectory);
+ FileStatus[] files = null;
+ try {
+ files = fs.listStatus(instanceDirectory);
+ } catch (FileNotFoundException ex) {
+ // ignored
+ }
log.debug("Trying to read instance id from " + instanceDirectory);
if (files == null || files.length == 0) {
log.error("unable obtain instance id at " + instanceDirectory);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1b1334f5/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ImportDirectoryCommand.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ImportDirectoryCommand.java b/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ImportDirectoryCommand.java
index 3deefa7..31479a1 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ImportDirectoryCommand.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ImportDirectoryCommand.java
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.core.util.shell.commands;
+import java.io.FileNotFoundException;
import java.io.IOException;
import org.apache.accumulo.core.client.AccumuloException;
@@ -45,10 +46,16 @@ public class ImportDirectoryCommand extends Command {
String failureDir = cl.getArgs()[1];
boolean setTime = Boolean.parseBoolean(cl.getArgs()[2]);
- FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
- FileStatus failStatus = fs.getFileStatus(new Path(failureDir));
- if (failStatus == null || !failStatus.isDir() || fs.listStatus(new Path(failureDir)).length != 0)
+ final FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
+ FileStatus failStatus = null;
+ try {
+ failStatus = fs.getFileStatus(new Path(failureDir));
+ } catch (FileNotFoundException ex) {
+ // ignored
+ }
+ if (failStatus == null || !failStatus.isDir() || fs.listStatus(new Path(failureDir)).length != 0) {
throw new AccumuloException(failureDir + " is not an empty directory");
+ }
shellState.getConnector().tableOperations().importDirectory(shellState.getTableName(), dir, failureDir, setTime);
return 0;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1b1334f5/src/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java b/src/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java
index 38f00be..06e5349 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.server.gc;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
@@ -138,6 +139,8 @@ public class GarbageCollectWriteAheadLogs {
for (FileStatus match : fs.globStatus(new Path(ServerConstants.getRecoveryDir(), file + "*"))) {
fs.delete(match.getPath(), true);
}
+ } catch (FileNotFoundException ex) {
+ // ignored
} catch (IOException ex) {
log.warn("Error deleting recovery data: ", ex);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1b1334f5/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java b/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
index 50bdfed..22c3c0e 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
@@ -355,9 +355,15 @@ public class SimpleGarbageCollector implements Iface {
// tableIdsWithDeletes should now contain the set of deleted tables that had dirs deleted
for (String delTableId : tableIdsWithDeletes) {
- // if dir exist and is empty, then empty list is returned... if dir does not exist
- // then null is returned
- FileStatus[] tabletDirs = fs.listStatus(new Path(ServerConstants.getTablesDir() + "/" + delTableId));
+ // if dir exist and is empty, then empty list is returned...
+ // hadoop 1.0 will return null if the file doesn't exist
+ // hadoop 2.0 will throw an exception if the file does not exist
+ FileStatus[] tabletDirs = null;
+ try {
+ tabletDirs = fs.listStatus(new Path(ServerConstants.getTablesDir() + "/" + delTableId));
+ } catch (FileNotFoundException ex) {
+ // ignored
+ }
if (tabletDirs == null)
continue;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1b1334f5/src/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java b/src/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java
index 0438563..e9d241c 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java
@@ -154,7 +154,12 @@ public class BulkImport extends MasterRepo {
ServerConfiguration.getSiteConfiguration()));
;
Path errorPath = new Path(errorDir);
- FileStatus errorStatus = fs.getFileStatus(errorPath);
+ FileStatus errorStatus = null;
+ try {
+ errorStatus = fs.getFileStatus(errorPath);
+ } catch (FileNotFoundException ex) {
+ // ignored
+ }
if (errorStatus == null)
throw new ThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_ERROR_DIRECTORY, errorDir
+ " does not exist");
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1b1334f5/src/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java b/src/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
index ef4f27a..09903bf 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
@@ -23,6 +23,9 @@ package org.apache.accumulo.server.tabletserver;
*
*/
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -471,7 +474,12 @@ public class Tablet {
private void checkTabletDir(Path tabletDir) throws IOException {
- FileStatus[] files = fs.listStatus(tabletDir);
+ FileStatus[] files = null;
+ try {
+ files = fs.listStatus(tabletDir);
+ } catch (FileNotFoundException ex) {
+ // ignored
+ }
if (files == null) {
if (tabletDir.getName().startsWith("c-"))
[38/39] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Posted by ec...@apache.org.
Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ad9ee5e0
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ad9ee5e0
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ad9ee5e0
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: ad9ee5e06aae970516f2fb9b2400c07c45a94a5a
Parents: f896c95 6446ad5
Author: Eric Newton <er...@gmail.com>
Authored: Tue Nov 26 10:46:39 2013 -0500
Committer: Eric Newton <er...@gmail.com>
Committed: Tue Nov 26 10:46:39 2013 -0500
----------------------------------------------------------------------
----------------------------------------------------------------------
[19/39] git commit: ACCUMULO-804 initial attempt to make accumulo
binary compatible with 2.0
Posted by ec...@apache.org.
ACCUMULO-804 initial attempt to make accumulo binary compatible with 2.0
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.5@1483399 13f79535-47bb-0310-9956-ffa450edef68
(cherry picked from commit d6c612d087e7f922c9935888cf443d4a9f1999c2)
Reason: Hadoop2 Compat
Author: Eric C. Newton <ec...@apache.org>
Ref: ACCUMULO-1792
Signed-off-by: Eric Newton <er...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/2c83ca33
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/2c83ca33
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/2c83ca33
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 2c83ca337ac54e3c71aff1596bcf6ea04aea6491
Parents: 1b1334f
Author: Jonathan M Hsieh <jo...@cloudera.com>
Authored: Wed May 29 14:23:01 2013 -0700
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Nov 25 16:06:42 2013 -0500
----------------------------------------------------------------------
.../accumulo/core/util/TTimeoutTransport.java | 12 ++++-
.../org/apache/accumulo/server/Accumulo.java | 51 ++++++++++++++++++--
2 files changed, 58 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2c83ca33/src/core/src/main/java/org/apache/accumulo/core/util/TTimeoutTransport.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/util/TTimeoutTransport.java b/src/core/src/main/java/org/apache/accumulo/core/util/TTimeoutTransport.java
index 3c1fa6a..0aebc39 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/util/TTimeoutTransport.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/util/TTimeoutTransport.java
@@ -21,6 +21,7 @@ import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.lang.reflect.Method;
import java.net.Socket;
import java.net.SocketAddress;
import java.nio.channels.spi.SelectorProvider;
@@ -31,12 +32,21 @@ import org.apache.thrift.transport.TTransport;
public class TTimeoutTransport {
+ private static InputStream getInputStream(Socket socket, long timeout) {
+ try {
+ Method m = NetUtils.class.getMethod("getInputStream", Socket.class, Long.TYPE);
+ return (InputStream)m.invoke(null, socket, timeout);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
public static TTransport create(SocketAddress addr, long timeoutMillis) throws IOException {
Socket socket = SelectorProvider.provider().openSocketChannel().socket();
socket.setSoLinger(false, 0);
socket.setTcpNoDelay(true);
socket.connect(addr);
- InputStream input = new BufferedInputStream(NetUtils.getInputStream(socket, timeoutMillis), 1024 * 10);
+ InputStream input = new BufferedInputStream(getInputStream(socket, timeoutMillis), 1024 * 10);
OutputStream output = new BufferedOutputStream(NetUtils.getOutputStream(socket, timeoutMillis), 1024 * 10);
return new TIOStreamTransport(input, output);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2c83ca33/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java b/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
index b2feb5c..32462b7 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
@@ -20,6 +20,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.lang.reflect.Method;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Map.Entry;
@@ -212,10 +213,7 @@ public class Accumulo {
while (true) {
try {
FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
- if (!(fs instanceof DistributedFileSystem))
- break;
- DistributedFileSystem dfs = (DistributedFileSystem) FileSystem.get(CachedConfiguration.getInstance());
- if (!dfs.setSafeMode(SafeModeAction.SAFEMODE_GET))
+ if (!isInSafeMode(fs))
break;
log.warn("Waiting for the NameNode to leave safemode");
} catch (IOException ex) {
@@ -227,4 +225,49 @@ public class Accumulo {
}
log.info("Connected to HDFS");
}
+
+ private static boolean isInSafeMode(FileSystem fs) throws IOException {
+ if (!(fs instanceof DistributedFileSystem))
+ return false;
+ DistributedFileSystem dfs = (DistributedFileSystem) FileSystem.get(CachedConfiguration.getInstance());
+ // So this: if (!dfs.setSafeMode(SafeModeAction.SAFEMODE_GET))
+ // Becomes this:
+ Class<?> constantClass;
+ try {
+ // hadoop 2.0
+ constantClass = Class.forName("org.apache.hadoop.hdfs.protocol.HdfsConstants");
+ } catch (ClassNotFoundException ex) {
+ // hadoop 1.0
+ try {
+ constantClass = Class.forName("org.apache.hadoop.hdfs.protocol.FSConstants");
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException("Cannot figure out the right class for Constants");
+ }
+ }
+ Class<?> safeModeAction = null;
+ for (Class<?> klass : constantClass.getDeclaredClasses()) {
+ if (klass.getSimpleName().equals("SafeModeAction")) {
+ safeModeAction = klass;
+ break;
+ }
+ }
+ if (safeModeAction == null) {
+ throw new RuntimeException("Cannot find SafeModeAction in constants class");
+ }
+
+ Object get = null;
+ for (Object obj : safeModeAction.getEnumConstants()) {
+ if (obj.toString().equals("SAFEMODE_GET"))
+ get = obj;
+ }
+ if (get == null) {
+ throw new RuntimeException("cannot find SAFEMODE_GET");
+ }
+ try {
+ Method setSafeMode = dfs.getClass().getMethod("setSafeMode", safeModeAction);
+ return (Boolean)setSafeMode.invoke(dfs, get);
+ } catch (Exception ex) {
+ throw new RuntimeException("cannot find method setSafeMode");
+ }
+ }
}
[21/39] git commit: ACCUMULO-769 Fix Hadoop 2.0 breakage by removing
methods marked as deprecated that didn't exist in prior version. ACCUMULO-975
Fix proxy pom.xml to support Hadoop 2.0
Posted by ec...@apache.org.
ACCUMULO-769 Fix Hadoop 2.0 breakage by removing methods marked as deprecated that didn't exist in prior version. ACCUMULO-975 Fix proxy pom.xml to support Hadoop 2.0
git-svn-id: https://svn.apache.org/repos/asf/accumulo/trunk@1435230 13f79535-47bb-0310-9956-ffa450edef68
(cherry picked from commit ec4055c698d7735093e964e30729ede9df71157f)
Reason: Hadoop2 compat
Author: Christopher Tubbs <ct...@apache.org>
Ref: ACCUMULO-1792
This patch differs from upstream by only including the changes relevant to ACCUMULO-975. It also does not reformat as much of proxy/pom.xml as the original.
Author: Sean Busbey <bu...@cloudera.com>
Signed-off-by: Eric Newton <er...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4c16d8b8
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4c16d8b8
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4c16d8b8
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 4c16d8b82713cb888e33a224a57d0d41b156b50b
Parents: 0f97284
Author: Sean Busbey <bu...@cloudera.com>
Authored: Mon Nov 4 15:39:14 2013 -0600
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Nov 25 16:06:42 2013 -0500
----------------------------------------------------------------------
src/proxy/pom.xml | 96 ++++++++++++++++++++++++++++++++++----------------
1 file changed, 65 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4c16d8b8/src/proxy/pom.xml
----------------------------------------------------------------------
diff --git a/src/proxy/pom.xml b/src/proxy/pom.xml
index 5d0414d..e1daa29 100644
--- a/src/proxy/pom.xml
+++ b/src/proxy/pom.xml
@@ -33,22 +33,22 @@
<build>
<pluginManagement>
<plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifestSections>
- <manifestSection>
- <name>accumulo/proxy/</name>
- <manifestEntries>
- <Sealed>true</Sealed>
- </manifestEntries>
- </manifestSection>
- </manifestSections>
- </archive>
- </configuration>
- </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <configuration>
+ <archive>
+ <manifestSections>
+ <manifestSection>
+ <name>accumulo/proxy/</name>
+ <manifestEntries>
+ <Sealed>true</Sealed>
+ </manifestEntries>
+ </manifestSection>
+ </manifestSections>
+ </archive>
+ </configuration>
+ </plugin>
</plugins>
</pluginManagement>
<plugins>
@@ -72,9 +72,9 @@
</build>
<dependencies>
<dependency>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-core</artifactId>
- <scope>compile</scope>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-core</artifactId>
+ <scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
@@ -82,11 +82,6 @@
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<scope>compile</scope>
@@ -113,16 +108,55 @@
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>examples-simple</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>examples-simple</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
</dependency>
<dependency>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-server</artifactId>
- <version>${project.version}</version>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-server</artifactId>
+ <version>${project.version}</version>
</dependency>
</dependencies>
+ <profiles>
+ <!-- profile for building against Hadoop 1.0.x
+ Activate by not specifying hadoop.profile -->
+ <profile>
+ <id>hadoop-1.0</id>
+ <activation>
+ <property>
+ <name>!hadoop.profile</name>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+ <!-- profile for building against Hadoop 2.0.x
+ Activate using: mvn -Dhadoop.profile=2.0 -->
+ <profile>
+ <id>hadoop-2.0</id>
+ <activation>
+ <property>
+ <name>hadoop.profile</name>
+ <value>2.0</value>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
</project>
[07/39] ACCUMULO-600 removed wikisearch from trunk
Posted by ec...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/ReadAheadIterator.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/ReadAheadIterator.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/ReadAheadIterator.java
deleted file mode 100644
index 880ae40..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/ReadAheadIterator.java
+++ /dev/null
@@ -1,297 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.iterator;
-
-import java.io.IOException;
-import java.lang.Thread.State;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.locks.ReentrantLock;
-
-import org.apache.log4j.Logger;
-
-import org.apache.accumulo.core.data.ByteSequence;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.IteratorEnvironment;
-import org.apache.accumulo.core.iterators.OptionDescriber;
-import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
-
-/**
- * This iterator takes the source iterator (the one below it in the iterator stack) and puts it in a background thread. The background thread continues
- * processing and fills a queue with the Keys and Values from the source iterator. When seek() is called on this iterator, it pauses the background thread,
- * clears the queue, calls seek() on the source iterator, then resumes the thread filling the queue.
- *
- * Users of this iterator can set the queue size, default is five elements. Users must be aware of the potential for OutOfMemory errors when using this iterator
- * with large queue sizes or large objects. This iterator copies the Key and Value from the source iterator and puts them into the queue.
- *
- * This iterator introduces some parallelism into the server side iterator stack. One use case for this would be when an iterator takes a relatively long time
- * to process each K,V pair and causes the iterators above it to wait. By putting the longer running iterator in a background thread we should be able to
- * achieve greater throughput.
- *
- * NOTE: Experimental!
- *
- */
-public class ReadAheadIterator implements SortedKeyValueIterator<Key,Value>, OptionDescriber {
-
- private static Logger log = Logger.getLogger(ReadAheadIterator.class);
-
- public static final String QUEUE_SIZE = "queue.size";
-
- public static final String TIMEOUT = "timeout";
-
- private static final QueueElement noMoreDataElement = new QueueElement();
-
- private int queueSize = 5;
-
- private int timeout = 60;
-
- /**
- *
- * Class to hold key and value from the producing thread.
- *
- */
- static class QueueElement {
- Key key = null;
- Value value = null;
-
- public QueueElement() {}
-
- public QueueElement(Key key, Value value) {
- super();
- this.key = new Key(key);
- this.value = new Value(value.get(), true);
- }
-
- public Key getKey() {
- return key;
- }
-
- public Value getValue() {
- return value;
- }
- }
-
- /**
- *
- * Thread that produces data from the source iterator and places the results in a queue.
- *
- */
- class ProducerThread extends ReentrantLock implements Runnable {
-
- private static final long serialVersionUID = 1L;
-
- private Exception e = null;
-
- private int waitTime = timeout;
-
- private SortedKeyValueIterator<Key,Value> sourceIter = null;
-
- public ProducerThread(SortedKeyValueIterator<Key,Value> source) {
- this.sourceIter = source;
- }
-
- public void run() {
- boolean hasMoreData = true;
- // Keep this thread running while there is more data to read
- // and items left in the queue to be read off.
- while (hasMoreData || queue.size() > 0) {
- try {
- // Acquire the lock, this will wait if the lock is being
- // held by the ReadAheadIterator.seek() method.
- this.lock();
- // Check to see if there is more data from the iterator below.
- hasMoreData = sourceIter.hasTop();
- // Break out of the loop if no more data.
- if (!hasMoreData)
- continue;
- // Put the next K,V onto the queue.
- try {
- QueueElement e = new QueueElement(sourceIter.getTopKey(), sourceIter.getTopValue());
- boolean inserted = false;
- try {
- inserted = queue.offer(e, this.waitTime, TimeUnit.SECONDS);
- } catch (InterruptedException ie) {
- this.e = ie;
- break;
- }
- if (!inserted) {
- // Then we either got a timeout, set the error and break out of the loop
- this.e = new TimeoutException("Background thread has exceeded wait time of " + this.waitTime + " seconds, aborting...");
- break;
- }
- // Move the iterator to the next K,V for the next iteration of this loop
- sourceIter.next();
- } catch (Exception e) {
- this.e = e;
- log.error("Error calling next on source iterator", e);
- break;
- }
- } finally {
- this.unlock();
- }
- }
- // If we broke out of the loop because of an error, then don't put the marker on the queue, just to do end.
- if (!hasError()) {
- // Put the special end of data marker into the queue
- try {
- queue.put(noMoreDataElement);
- } catch (InterruptedException e) {
- this.e = e;
- log.error("Error putting End of Data marker onto queue");
- }
- }
- }
-
- public boolean hasError() {
- return (this.e != null);
- }
-
- public Exception getError() {
- return this.e;
- }
- }
-
- private SortedKeyValueIterator<Key,Value> source;
- private ArrayBlockingQueue<QueueElement> queue = null;
- private QueueElement currentElement = new QueueElement();
- private ProducerThread thread = null;
- private Thread t = null;
-
- protected ReadAheadIterator(ReadAheadIterator other, IteratorEnvironment env) {
- source = other.source.deepCopy(env);
- }
-
- public ReadAheadIterator() {}
-
- public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
- return new ReadAheadIterator(this, env);
- }
-
- public Key getTopKey() {
- return currentElement.getKey();
- }
-
- public Value getTopValue() {
- return currentElement.getValue();
- }
-
- public boolean hasTop() {
- if (currentElement == noMoreDataElement)
- return false;
- return currentElement != null || queue.size() > 0 || source.hasTop();
- }
-
- public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
- validateOptions(options);
- this.source = source;
- queue = new ArrayBlockingQueue<QueueElement>(queueSize);
- thread = new ProducerThread(this.source);
- t = new Thread(thread, "ReadAheadIterator-SourceThread");
- t.start();
- }
-
- /**
- * Populate the key and value
- */
- public void next() throws IOException {
- // Thread startup race condition, need to make sure that the
- // thread has started before we call this the first time.
- while (t.getState().equals(State.NEW)) {
- try {
- Thread.sleep(1);
- } catch (InterruptedException e) {}
- }
-
- if (t.getState().equals(State.TERMINATED)) {
- // Thread encountered an error.
- if (thread.hasError()) {
- // and it should
- throw new IOException("Background thread has died", thread.getError());
- }
- }
-
- // Pull an element off the queue, this will wait if there is no data yet.
- try {
- if (thread.hasError())
- throw new IOException("background thread has error", thread.getError());
-
- QueueElement nextElement = null;
- while (null == nextElement) {
- try {
- nextElement = queue.poll(1, TimeUnit.SECONDS);
- } catch (InterruptedException e) {
- // TODO: Do we need to do anything here?
- }
- if (null == nextElement) {
- // Then we have no data and timed out, check for error condition in the read ahead thread
- if (thread.hasError()) {
- throw new IOException("background thread has error", thread.getError());
- }
- }
- }
- currentElement = nextElement;
- } catch (IOException e) {
- throw new IOException("Error getting element from source iterator", e);
- }
- }
-
- /**
- * Seek to the next matching cell and call next to populate the key and value.
- */
- public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
- if (t.isAlive()) {
- // Check for error
- if (thread.hasError())
- throw new IOException("background thread has error", thread.getError());
-
- try {
- // Acquire the lock, or wait until its unlocked by the producer thread.
- thread.lock();
- queue.clear();
- currentElement = null;
- source.seek(range, columnFamilies, inclusive);
- } finally {
- thread.unlock();
- }
- next();
- } else {
- throw new IOException("source iterator thread has died.");
- }
- }
-
- public IteratorOptions describeOptions() {
- Map<String,String> options = new HashMap<String,String>();
- options.put(QUEUE_SIZE, "read ahead queue size");
- options.put(TIMEOUT, "timeout in seconds before background thread thinks that the client has aborted");
- return new IteratorOptions(getClass().getSimpleName(), "Iterator that puts the source in another thread", options, null);
- }
-
- public boolean validateOptions(Map<String,String> options) {
- if (options.containsKey(QUEUE_SIZE))
- queueSize = Integer.parseInt(options.get(QUEUE_SIZE));
- if (options.containsKey(TIMEOUT))
- timeout = Integer.parseInt(options.get(TIMEOUT));
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/UniqFieldNameValueIterator.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/UniqFieldNameValueIterator.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/UniqFieldNameValueIterator.java
deleted file mode 100644
index 4c7201d..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/UniqFieldNameValueIterator.java
+++ /dev/null
@@ -1,342 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.iterator;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Map;
-
-import org.apache.accumulo.core.data.ByteSequence;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.PartialKey;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.IteratorEnvironment;
-import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
-import org.apache.accumulo.core.iterators.WrappingIterator;
-import org.apache.accumulo.examples.wikisearch.util.FieldIndexKeyParser;
-import org.apache.hadoop.io.Text;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-
-
-public class UniqFieldNameValueIterator extends WrappingIterator {
-
- protected static final Logger log = Logger.getLogger(UniqFieldNameValueIterator.class);
- // Wrapping iterator only accesses its private source in setSource and getSource
- // Since this class overrides these methods, it's safest to keep the source declaration here
- private SortedKeyValueIterator<Key,Value> source;
- private FieldIndexKeyParser keyParser;
- private Key topKey = null;
- private Value topValue = null;
- private Range overallRange = null;
- private Range currentSubRange;
- private Text fieldName = null;
- private Text fieldValueLowerBound = null;
- private Text fieldValueUpperBound = null;
- private static final Collection<ByteSequence> EMPTY_COL_FAMS = new ArrayList<ByteSequence>();
- private static final String ONE_BYTE = "\1";
- private boolean multiRow = false;
- private boolean seekInclusive = false;
-
- // -------------------------------------------------------------------------
- // ------------- Static Methods
- public static void setLogLevel(Level l) {
- log.setLevel(l);
- }
-
- // -------------------------------------------------------------------------
- // ------------- Constructors
- public UniqFieldNameValueIterator(Text fName, Text fValLower, Text fValUpper) {
- this.fieldName = fName;
- this.fieldValueLowerBound = fValLower;
- this.fieldValueUpperBound = fValUpper;
- keyParser = createDefaultKeyParser();
-
- }
-
- public UniqFieldNameValueIterator(UniqFieldNameValueIterator other, IteratorEnvironment env) {
- source = other.getSource().deepCopy(env);
- // Set a default KeyParser
- keyParser = createDefaultKeyParser();
- }
-
- // -------------------------------------------------------------------------
- // ------------- Overrides
- @Override
- public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
- super.init(source, options, env);
- source = super.getSource();
- }
-
- @Override
- protected void setSource(SortedKeyValueIterator<Key,Value> source) {
- this.source = source;
- }
-
- @Override
- protected SortedKeyValueIterator<Key,Value> getSource() {
- return source;
- }
-
- @Override
- public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
- return new UniqFieldNameValueIterator(this, env);
- }
-
- @Override
- public Key getTopKey() {
- return this.topKey;
- }
-
- @Override
- public Value getTopValue() {
- return this.topValue;
- }
-
- @Override
- public boolean hasTop() {
- return (topKey != null);
- }
-
- @Override
- public void next() throws IOException {
- if (log.isDebugEnabled()) {
- log.debug("next()");
- }
- if (!source.hasTop()) {
- topKey = null;
- topValue = null;
- return;
- }
-
- Key currentKey = topKey;
- keyParser.parse(topKey);
- String fValue = keyParser.getFieldValue();
-
- Text currentRow = currentKey.getRow();
- Text currentFam = currentKey.getColumnFamily();
-
- if (overallRange.getEndKey() != null && overallRange.getEndKey().getRow().compareTo(currentRow) < 0) {
- if (log.isDebugEnabled()) {
- log.debug("next, overall endRow: " + overallRange.getEndKey().getRow() + " currentRow: " + currentRow);
- }
- topKey = null;
- topValue = null;
- return;
- }
-
- if (fValue.compareTo(this.fieldValueUpperBound.toString()) > 0) {
- topKey = null;
- topValue = null;
- return;
- }
- Key followingKey = new Key(currentKey.getRow(), this.fieldName, new Text(fValue + ONE_BYTE));
- if (log.isDebugEnabled()) {
- log.debug("next, followingKey to seek on: " + followingKey);
- }
- Range r = new Range(followingKey, followingKey);
- source.seek(r, EMPTY_COL_FAMS, false);
- while (true) {
- if (!source.hasTop()) {
- topKey = null;
- topValue = null;
- return;
- }
-
- Key k = source.getTopKey();
- if (!overallRange.contains(k)) {
- topKey = null;
- topValue = null;
- return;
- }
- if (log.isDebugEnabled()) {
- log.debug("next(), key: " + k + " subrange: " + this.currentSubRange);
- }
- // if (this.currentSubRange.contains(k)) {
- keyParser.parse(k);
- Text currentVal = new Text(keyParser.getFieldValue());
- if (k.getRow().equals(currentRow) && k.getColumnFamily().equals(currentFam) && currentVal.compareTo(fieldValueUpperBound) <= 0) {
- topKey = k;
- topValue = source.getTopValue();
- return;
-
- } else { // need to move to next row.
- if (this.overallRange.contains(k) && this.multiRow) {
- // need to find the next sub range
- // STEPS
- // 1. check if you moved past your current row on last call to next
- // 2. figure out next row
- // 3. build new start key with lowerbound fvalue
- // 4. seek the source
- // 5. test the subrange.
- if (k.getRow().equals(currentRow)) {
- // get next row
- currentRow = getNextRow();
- if (currentRow == null) {
- topKey = null;
- topValue = null;
- return;
- }
- } else {
- // i'm already in the next row
- currentRow = source.getTopKey().getRow();
- }
-
- // build new startKey
- Key sKey = new Key(currentRow, fieldName, fieldValueLowerBound);
- Key eKey = new Key(currentRow, fieldName, fieldValueUpperBound);
- currentSubRange = new Range(sKey, eKey);
- source.seek(currentSubRange, EMPTY_COL_FAMS, seekInclusive);
-
- } else { // not multi-row or outside overall range, we're done
- topKey = null;
- topValue = null;
- return;
- }
- }
-
- }
-
- }
-
- @Override
- public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
- if (log.isDebugEnabled()) {
- log.debug("seek, range: " + range);
- }
- this.overallRange = range;
- this.seekInclusive = inclusive;
- source.seek(range, EMPTY_COL_FAMS, inclusive);
- topKey = null;
- topValue = null;
- Key sKey;
- Key eKey;
-
- if (range.isInfiniteStartKey()) {
- sKey = source.getTopKey();
- if (sKey == null) {
- return;
- }
- } else {
- sKey = range.getStartKey();
- }
-
- if (range.isInfiniteStopKey()) {
- eKey = null;
- this.multiRow = true; // assume we will go to the end of the tablet.
- } else {
- eKey = range.getEndKey();
- if (sKey.getRow().equals(eKey.getRow())) {
- this.multiRow = false;
- } else {
- this.multiRow = true;
- }
- }
-
- if (log.isDebugEnabled()) {
- log.debug("seek, multiRow:" + multiRow + " range:" + range);
- }
-
- /*
- * NOTE: If the seek range spans multiple rows, we are only interested in the fieldName:fieldValue subranges in each row. Keys will exist in the
- * overallRange that we will want to skip over so we need to create subranges per row so we don't have to examine every key in between.
- */
-
- Text sRow = sKey.getRow();
- Key ssKey = new Key(sRow, this.fieldName, this.fieldValueLowerBound);
- Key eeKey = new Key(sRow, this.fieldName, this.fieldValueUpperBound);
- this.currentSubRange = new Range(ssKey, eeKey);
-
- if (log.isDebugEnabled()) {
- log.debug("seek, currentSubRange: " + currentSubRange);
- }
- source.seek(this.currentSubRange, columnFamilies, inclusive);
- // cycle until we find a valid topKey, or we get ejected b/c we hit the
- // end of the tablet or exceeded the overallRange.
- while (topKey == null) {
- if (source.hasTop()) {
- Key k = source.getTopKey();
- if (log.isDebugEnabled()) {
- log.debug("seek, source.topKey: " + k);
- }
- if (currentSubRange.contains(k)) {
- topKey = k;
- topValue = source.getTopValue();
-
- if (log.isDebugEnabled()) {
- log.debug("seek, source has top in valid range");
- }
-
- } else { // outside of subRange.
- // if multiRow mode, get the next row and seek to it
- if (multiRow && overallRange.contains(k)) {
-
- Key fKey = sKey.followingKey(PartialKey.ROW);
- Range fRange = new Range(fKey, eKey);
- source.seek(fRange, columnFamilies, inclusive);
-
- if (source.hasTop()) {
- Text row = source.getTopKey().getRow();
- Key nKey = new Key(row, this.fieldName, this.fieldValueLowerBound);
- this.currentSubRange = new Range(nKey, eKey);
- sKey = this.currentSubRange.getStartKey();
- Range nextRange = new Range(sKey, eKey);
- source.seek(nextRange, columnFamilies, inclusive);
- } else {
- topKey = null;
- topValue = null;
- return;
- }
-
- } else { // not multi row & outside range, we're done.
- topKey = null;
- topValue = null;
- return;
- }
- }
- } else { // source does not have top, we're done
- topKey = null;
- topValue = null;
- return;
- }
- }
-
- }
-
- // -------------------------------------------------------------------------
- // ------------- Internal Methods
- private FieldIndexKeyParser createDefaultKeyParser() {
- FieldIndexKeyParser parser = new FieldIndexKeyParser();
- return parser;
- }
-
- private Text getNextRow() throws IOException {
- if (log.isDebugEnabled()) {
- log.debug("getNextRow()");
- }
- Key fakeKey = new Key(source.getTopKey().followingKey(PartialKey.ROW));
- Range fakeRange = new Range(fakeKey, fakeKey);
- source.seek(fakeRange, EMPTY_COL_FAMS, false);
- if (source.hasTop()) {
- return source.getTopKey().getRow();
- } else {
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/jexl/Arithmetic.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/jexl/Arithmetic.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/jexl/Arithmetic.java
deleted file mode 100644
index c59f573..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/jexl/Arithmetic.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.jexl;
-
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.jexl2.JexlArithmetic;
-import org.apache.commons.lang.math.NumberUtils;
-
-public class Arithmetic extends JexlArithmetic {
-
- public Arithmetic(boolean lenient) {
- super(lenient);
- }
-
- /**
- * This method differs from the parent in that we are not calling String.matches() because it does not match on a newline. Instead we are handling this case.
- *
- * @param left
- * first value
- * @param right
- * second value
- * @return test result.
- */
- @Override
- public boolean matches(Object left, Object right) {
- if (left == null && right == null) {
- // if both are null L == R
- return true;
- }
- if (left == null || right == null) {
- // we know both aren't null, therefore L != R
- return false;
- }
- final String arg = left.toString();
- if (right instanceof java.util.regex.Pattern) {
- return ((java.util.regex.Pattern) right).matcher(arg).matches();
- } else {
- // return arg.matches(right.toString());
- Pattern p = Pattern.compile(right.toString(), Pattern.DOTALL);
- Matcher m = p.matcher(arg);
- return m.matches();
-
- }
- }
-
- /**
- * This method differs from the parent class in that we are going to try and do a better job of coercing the types. As a last resort we will do a string
- * comparison and try not to throw a NumberFormatException. The JexlArithmetic class performs coercion to a particular type if either the left or the right
- * match a known type. We will look at the type of the right operator and try to make the left of the same type.
- */
- @Override
- public boolean equals(Object left, Object right) {
- Object fixedLeft = fixLeft(left, right);
- return super.equals(fixedLeft, right);
- }
-
- @Override
- public boolean lessThan(Object left, Object right) {
- Object fixedLeft = fixLeft(left, right);
- return super.lessThan(fixedLeft, right);
- }
-
- protected Object fixLeft(Object left, Object right) {
-
- if (null == left || null == right)
- return left;
-
- if (!(right instanceof Number) && left instanceof Number) {
- right = NumberUtils.createNumber(right.toString());
- }
-
- if (right instanceof Number && left instanceof Number) {
- if (right instanceof Double)
- return ((Double) right).doubleValue();
- else if (right instanceof Float)
- return ((Float) right).floatValue();
- else if (right instanceof Long)
- return ((Long) right).longValue();
- else if (right instanceof Integer)
- return ((Integer) right).intValue();
- else if (right instanceof Short)
- return ((Short) right).shortValue();
- else if (right instanceof Byte)
- return ((Byte) right).byteValue();
- else
- return right;
- }
- if (right instanceof Number && left instanceof String) {
- Number num = NumberUtils.createNumber(left.toString());
- // Let's try to cast left as right's type.
- if (this.isFloatingPointNumber(right) && this.isFloatingPointNumber(left))
- return num;
- else if (this.isFloatingPointNumber(right))
- return num.doubleValue();
- else if (right instanceof Number)
- return num.longValue();
- } else if (right instanceof Boolean && left instanceof String) {
- if (left.equals("true") || left.equals("false"))
- return Boolean.parseBoolean(left.toString());
-
- Number num = NumberUtils.createNumber(left.toString());
- if (num.intValue() == 1)
- return (Boolean) true;
- else if (num.intValue() == 0)
- return (Boolean) false;
- }
- return left;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/logic/AbstractQueryLogic.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/logic/AbstractQueryLogic.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/logic/AbstractQueryLogic.java
deleted file mode 100644
index 5c7c20c..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/logic/AbstractQueryLogic.java
+++ /dev/null
@@ -1,883 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.logic;
-
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.TreeSet;
-
-import org.apache.accumulo.core.client.BatchScanner;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.IteratorSetting;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.user.RegExFilter;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.examples.wikisearch.ingest.WikipediaMapper;
-import org.apache.accumulo.examples.wikisearch.iterator.BooleanLogicIterator;
-import org.apache.accumulo.examples.wikisearch.iterator.EvaluatingIterator;
-import org.apache.accumulo.examples.wikisearch.iterator.OptimizedQueryIterator;
-import org.apache.accumulo.examples.wikisearch.iterator.ReadAheadIterator;
-import org.apache.accumulo.examples.wikisearch.normalizer.LcNoDiacriticsNormalizer;
-import org.apache.accumulo.examples.wikisearch.normalizer.Normalizer;
-import org.apache.accumulo.examples.wikisearch.parser.EventFields;
-import org.apache.accumulo.examples.wikisearch.parser.EventFields.FieldValue;
-import org.apache.accumulo.examples.wikisearch.parser.FieldIndexQueryReWriter;
-import org.apache.accumulo.examples.wikisearch.parser.JexlOperatorConstants;
-import org.apache.accumulo.examples.wikisearch.parser.QueryParser;
-import org.apache.accumulo.examples.wikisearch.parser.QueryParser.QueryTerm;
-import org.apache.accumulo.examples.wikisearch.parser.RangeCalculator;
-import org.apache.accumulo.examples.wikisearch.sample.Document;
-import org.apache.accumulo.examples.wikisearch.sample.Field;
-import org.apache.accumulo.examples.wikisearch.sample.Results;
-import org.apache.commons.jexl2.parser.ParserTreeConstants;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.time.StopWatch;
-import org.apache.hadoop.io.Text;
-import org.apache.log4j.Logger;
-
-import com.esotericsoftware.kryo.Kryo;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Multimap;
-
-/**
- * <pre>
- * <h2>Overview</h2>
- * Query implementation that works with the JEXL grammar. This
- * uses the metadata, global index, and partitioned table to return
- * results based on the query. Example queries:
- *
- * <b>Single Term Query</b>
- * 'foo' - looks in global index for foo, and if any entries are found, then the query
- * is rewritten to be field1 == 'foo' or field2 == 'foo', etc. This is then passed
- * down the optimized query path which uses the intersecting iterators on the partitioned
- * table.
- *
- * <b>Boolean expression</b>
- * field == 'foo' - For fielded queries, those that contain a field, an operator, and a literal (string or number),
- * the query is parsed and the set of eventFields in the query that are indexed is determined by
- * querying the metadata table. Depending on the conjunctions in the query (or, and, not) and the
- * eventFields that are indexed, the query may be sent down the optimized path or the full scan path.
- *
- * We are not supporting all of the operators that JEXL supports at this time. We are supporting the following operators:
- *
- * ==, !=, >, ≥, <, ≤, =~, and !~
- *
- * Custom functions can be created and registered with the Jexl engine. The functions can be used in the queries in conjunction
- * with other supported operators. A sample function has been created, called between, and is bound to the 'f' namespace. An
- * example using this function is : "f:between(LATITUDE,60.0, 70.0)"
- *
- * <h2>Constraints on Query Structure</h2>
- * Queries that are sent to this class need to be formatted such that there is a space on either side of the operator. We are
- * rewriting the query in some cases and the current implementation is expecting a space on either side of the operator. If
- * an error occurs in the evaluation we are skipping the event.
- *
- * <h2>Notes on Optimization</h2>
- * Queries that meet any of the following criteria will perform a full scan of the events in the partitioned table:
- *
- * 1. An 'or' conjunction exists in the query but not all of the terms are indexed.
- * 2. No indexed terms exist in the query
- * 3. An unsupported operator exists in the query
- *
- * </pre>
- *
- */
-public abstract class AbstractQueryLogic {
-
- protected static Logger log = Logger.getLogger(AbstractQueryLogic.class);
-
- /**
- * Set of datatypes to limit the query to.
- */
- public static final String DATATYPE_FILTER_SET = "datatype.filter.set";
-
- private static class DoNotPerformOptimizedQueryException extends Exception {
- private static final long serialVersionUID = 1L;
- }
-
- /**
- * Object that is used to hold ranges found in the index. Subclasses may compute the final range set in various ways.
- */
- public static abstract class IndexRanges {
-
- private Map<String,String> indexValuesToOriginalValues = null;
- private Multimap<String,String> fieldNamesAndValues = HashMultimap.create();
- private Map<String,Long> termCardinality = new HashMap<String,Long>();
- protected Map<String,TreeSet<Range>> ranges = new HashMap<String,TreeSet<Range>>();
-
- public Multimap<String,String> getFieldNamesAndValues() {
- return fieldNamesAndValues;
- }
-
- public void setFieldNamesAndValues(Multimap<String,String> fieldNamesAndValues) {
- this.fieldNamesAndValues = fieldNamesAndValues;
- }
-
- public final Map<String,Long> getTermCardinality() {
- return termCardinality;
- }
-
- public Map<String,String> getIndexValuesToOriginalValues() {
- return indexValuesToOriginalValues;
- }
-
- public void setIndexValuesToOriginalValues(Map<String,String> indexValuesToOriginalValues) {
- this.indexValuesToOriginalValues = indexValuesToOriginalValues;
- }
-
- public abstract void add(String term, Range r);
-
- public abstract Set<Range> getRanges();
- }
-
- /**
- * Object that computes the ranges by unioning all of the ranges for all of the terms together. In the case where ranges overlap, the largest range is used.
- */
- public static class UnionIndexRanges extends IndexRanges {
-
- public static String DEFAULT_KEY = "default";
-
- public UnionIndexRanges() {
- this.ranges.put(DEFAULT_KEY, new TreeSet<Range>());
- }
-
- public Set<Range> getRanges() {
- // So the set of ranges is ordered. It *should* be the case that
- // ranges with partition ids will sort before ranges that point to
- // a specific event. Populate a new set of ranges but don't add a
- // range for an event where that range is contained in a range already
- // added.
- Set<Text> shardsAdded = new HashSet<Text>();
- Set<Range> returnSet = new HashSet<Range>();
- for (Range r : ranges.get(DEFAULT_KEY)) {
- if (!shardsAdded.contains(r.getStartKey().getRow())) {
- // Only add ranges with a start key for the entire partition.
- if (r.getStartKey().getColumnFamily() == null) {
- shardsAdded.add(r.getStartKey().getRow());
- }
- returnSet.add(r);
- } else {
- // if (log.isTraceEnabled())
- log.info("Skipping event specific range: " + r.toString() + " because range has already been added: "
- + shardsAdded.contains(r.getStartKey().getRow()));
- }
- }
- return returnSet;
- }
-
- public void add(String term, Range r) {
- ranges.get(DEFAULT_KEY).add(r);
- }
- }
-
- private String metadataTableName;
- private String indexTableName;
- private String reverseIndexTableName;
- private String tableName;
- private int queryThreads = 8;
- private String readAheadQueueSize;
- private String readAheadTimeOut;
- private boolean useReadAheadIterator;
- private Kryo kryo = new Kryo();
- private EventFields eventFields = new EventFields();
- private List<String> unevaluatedFields = null;
- private Map<Class<? extends Normalizer>,Normalizer> normalizerCacheMap = new HashMap<Class<? extends Normalizer>,Normalizer>();
- private static final String NULL_BYTE = "\u0000";
-
- public AbstractQueryLogic() {
- super();
- EventFields.initializeKryo(kryo);
- }
-
- /**
- * Queries metadata table to determine which terms are indexed.
- *
- * @param c
- * @param auths
- * @param queryLiterals
- * @param datatypes
- * - optional list of types
- * @return map of indexed field names to types to normalizers used in this date range
- * @throws TableNotFoundException
- * @throws IllegalAccessException
- * @throws InstantiationException
- */
- protected Map<String,Multimap<String,Class<? extends Normalizer>>> findIndexedTerms(Connector c, Authorizations auths, Set<String> queryLiterals,
- Set<String> datatypes) throws TableNotFoundException, InstantiationException, IllegalAccessException {
-
- Map<String,Multimap<String,Class<? extends Normalizer>>> results = new HashMap<String,Multimap<String,Class<? extends Normalizer>>>();
-
- for (String literal : queryLiterals) {
- if (log.isDebugEnabled())
- log.debug("Querying " + this.getMetadataTableName() + " table for " + literal);
- Range range = new Range(literal.toUpperCase());
- Scanner scanner = c.createScanner(this.getMetadataTableName(), auths);
- scanner.setRange(range);
- scanner.fetchColumnFamily(new Text(WikipediaMapper.METADATA_INDEX_COLUMN_FAMILY));
- for (Entry<Key,Value> entry : scanner) {
- if (!results.containsKey(literal)) {
- Multimap<String,Class<? extends Normalizer>> m = HashMultimap.create();
- results.put(literal, m);
- }
- // Get the column qualifier from the key. It contains the datatype and normalizer class
- String colq = entry.getKey().getColumnQualifier().toString();
- if (null != colq && colq.contains("\0")) {
- int idx = colq.indexOf("\0");
- if (idx != -1) {
- String type = colq.substring(0, idx);
- // If types are specified and this type is not in the list then skip it.
- if (null != datatypes && !datatypes.contains(type))
- continue;
- try {
- @SuppressWarnings("unchecked")
- Class<? extends Normalizer> clazz = (Class<? extends Normalizer>) Class.forName(colq.substring(idx + 1));
- if (!normalizerCacheMap.containsKey(clazz))
- normalizerCacheMap.put(clazz, clazz.newInstance());
- results.get(literal).put(type, clazz);
- } catch (ClassNotFoundException e) {
- log.error("Unable to find normalizer on class path: " + colq.substring(idx + 1), e);
- results.get(literal).put(type, LcNoDiacriticsNormalizer.class);
- }
- } else {
- log.warn("EventMetadata entry did not contain NULL byte: " + entry.getKey().toString());
- }
- } else {
- log.warn("ColumnQualifier null in EventMetadata for key: " + entry.getKey().toString());
- }
- }
- }
- if (log.isDebugEnabled())
- log.debug("METADATA RESULTS: " + results.toString());
- return results;
- }
-
- /**
- * Performs a lookup in the global index for a single non-fielded term.
- *
- * @param c
- * @param auths
- * @param value
- * @param datatypes
- * - optional list of types
- * @return ranges that fit into the date range.
- */
- protected abstract IndexRanges getTermIndexInformation(Connector c, Authorizations auths, String value, Set<String> datatypes) throws TableNotFoundException;
-
- /**
- * Performs a lookup in the global index / reverse index and returns a RangeCalculator
- *
- * @param c
- * Accumulo connection
- * @param auths
- * authset for queries
- * @param indexedTerms
- * multimap of indexed field name and Normalizers used
- * @param terms
- * multimap of field name and QueryTerm object
- * @param indexTableName
- * @param reverseIndexTableName
- * @param queryString
- * original query string
- * @param queryThreads
- * @param datatypes
- * - optional list of types
- * @return range calculator
- * @throws TableNotFoundException
- */
- protected abstract RangeCalculator getTermIndexInformation(Connector c, Authorizations auths, Multimap<String,Normalizer> indexedTerms,
- Multimap<String,QueryTerm> terms, String indexTableName, String reverseIndexTableName, String queryString, int queryThreads, Set<String> datatypes)
- throws TableNotFoundException, org.apache.commons.jexl2.parser.ParseException;
-
- protected abstract Collection<Range> getFullScanRange(Date begin, Date end, Multimap<String,QueryTerm> terms);
-
- public String getMetadataTableName() {
- return metadataTableName;
- }
-
- public String getIndexTableName() {
- return indexTableName;
- }
-
- public String getTableName() {
- return tableName;
- }
-
- public void setMetadataTableName(String metadataTableName) {
- this.metadataTableName = metadataTableName;
- }
-
- public void setIndexTableName(String indexTableName) {
- this.indexTableName = indexTableName;
- }
-
- public void setTableName(String tableName) {
- this.tableName = tableName;
- }
-
- public int getQueryThreads() {
- return queryThreads;
- }
-
- public void setQueryThreads(int queryThreads) {
- this.queryThreads = queryThreads;
- }
-
- public String getReadAheadQueueSize() {
- return readAheadQueueSize;
- }
-
- public String getReadAheadTimeOut() {
- return readAheadTimeOut;
- }
-
- public boolean isUseReadAheadIterator() {
- return useReadAheadIterator;
- }
-
- public void setReadAheadQueueSize(String readAheadQueueSize) {
- this.readAheadQueueSize = readAheadQueueSize;
- }
-
- public void setReadAheadTimeOut(String readAheadTimeOut) {
- this.readAheadTimeOut = readAheadTimeOut;
- }
-
- public void setUseReadAheadIterator(boolean useReadAheadIterator) {
- this.useReadAheadIterator = useReadAheadIterator;
- }
-
- public String getReverseIndexTableName() {
- return reverseIndexTableName;
- }
-
- public void setReverseIndexTableName(String reverseIndexTableName) {
- this.reverseIndexTableName = reverseIndexTableName;
- }
-
- public List<String> getUnevaluatedFields() {
- return unevaluatedFields;
- }
-
- public void setUnevaluatedFields(List<String> unevaluatedFields) {
- this.unevaluatedFields = unevaluatedFields;
- }
-
- public void setUnevaluatedFields(String unevaluatedFieldList) {
- this.unevaluatedFields = new ArrayList<String>();
- for (String field : unevaluatedFieldList.split(","))
- this.unevaluatedFields.add(field);
- }
-
- public Document createDocument(Key key, Value value) {
- Document doc = new Document();
-
- eventFields.clear();
- ByteBuffer buf = ByteBuffer.wrap(value.get());
- eventFields.readObjectData(kryo, buf);
-
- // Set the id to the document id which is located in the colf
- String row = key.getRow().toString();
- String colf = key.getColumnFamily().toString();
- int idx = colf.indexOf(NULL_BYTE);
- String type = colf.substring(0, idx);
- String id = colf.substring(idx + 1);
- doc.setId(id);
- for (Entry<String,Collection<FieldValue>> entry : eventFields.asMap().entrySet()) {
- for (FieldValue fv : entry.getValue()) {
- Field val = new Field();
- val.setFieldName(entry.getKey());
- val.setFieldValue(new String(fv.getValue(), Charset.forName("UTF-8")));
- doc.getFields().add(val);
- }
- }
-
- // Add the pointer for the content.
- Field docPointer = new Field();
- docPointer.setFieldName("DOCUMENT");
- docPointer.setFieldValue("DOCUMENT:" + row + "/" + type + "/" + id);
- doc.getFields().add(docPointer);
-
- return doc;
- }
-
- public String getResultsKey(Entry<Key,Value> key) {
- // Use the colf from the table, it contains the uuid and datatype
- return key.getKey().getColumnFamily().toString();
- }
-
- public Results runQuery(Connector connector, List<String> authorizations, String query, Date beginDate, Date endDate, Set<String> types) {
-
- if (StringUtils.isEmpty(query)) {
- throw new IllegalArgumentException("NULL QueryNode reference passed to " + this.getClass().getSimpleName());
- }
-
- Set<Range> ranges = new HashSet<Range>();
- Set<String> typeFilter = types;
- String array[] = authorizations.toArray(new String[0]);
- Authorizations auths = new Authorizations(array);
- Results results = new Results();
-
- // Get the query string
- String queryString = query;
-
- StopWatch abstractQueryLogic = new StopWatch();
- StopWatch optimizedQuery = new StopWatch();
- StopWatch queryGlobalIndex = new StopWatch();
- StopWatch optimizedEventQuery = new StopWatch();
- StopWatch fullScanQuery = new StopWatch();
- StopWatch processResults = new StopWatch();
-
- abstractQueryLogic.start();
-
- StopWatch parseQuery = new StopWatch();
- parseQuery.start();
-
- QueryParser parser;
- try {
- if (log.isDebugEnabled()) {
- log.debug("ShardQueryLogic calling QueryParser.execute");
- }
- parser = new QueryParser();
- parser.execute(queryString);
- } catch (org.apache.commons.jexl2.parser.ParseException e1) {
- throw new IllegalArgumentException("Error parsing query", e1);
- }
- int hash = parser.getHashValue();
- parseQuery.stop();
- if (log.isDebugEnabled()) {
- log.debug(hash + " Query: " + queryString);
- }
-
- Set<String> fields = new HashSet<String>();
- for (String f : parser.getQueryIdentifiers()) {
- fields.add(f);
- }
- if (log.isDebugEnabled()) {
- log.debug("getQueryIdentifiers: " + parser.getQueryIdentifiers().toString());
- }
- // Remove any negated fields from the fields list, we don't want to lookup negated fields
- // in the index.
- fields.removeAll(parser.getNegatedTermsForOptimizer());
-
- if (log.isDebugEnabled()) {
- log.debug("getQueryIdentifiers: " + parser.getQueryIdentifiers().toString());
- }
- // Get the mapping of field name to QueryTerm object from the query. The query term object
- // contains the operator, whether its negated or not, and the literal to test against.
- Multimap<String,QueryTerm> terms = parser.getQueryTerms();
-
- // Find out which terms are indexed
- // TODO: Should we cache indexed terms or does that not make sense since we are always
- // loading data.
- StopWatch queryMetadata = new StopWatch();
- queryMetadata.start();
- Map<String,Multimap<String,Class<? extends Normalizer>>> metadataResults;
- try {
- metadataResults = findIndexedTerms(connector, auths, fields, typeFilter);
- } catch (Exception e1) {
- throw new RuntimeException("Error in metadata lookup", e1);
- }
-
- // Create a map of indexed term to set of normalizers for it
- Multimap<String,Normalizer> indexedTerms = HashMultimap.create();
- for (Entry<String,Multimap<String,Class<? extends Normalizer>>> entry : metadataResults.entrySet()) {
- // Get the normalizer from the normalizer cache
- for (Class<? extends Normalizer> clazz : entry.getValue().values()) {
- indexedTerms.put(entry.getKey(), normalizerCacheMap.get(clazz));
- }
- }
- queryMetadata.stop();
- if (log.isDebugEnabled()) {
- log.debug(hash + " Indexed Terms: " + indexedTerms.toString());
- }
-
- Set<String> orTerms = parser.getOrTermsForOptimizer();
-
- // Iterate over the query terms to get the operators specified in the query.
- ArrayList<String> unevaluatedExpressions = new ArrayList<String>();
- boolean unsupportedOperatorSpecified = false;
- for (Entry<String,QueryTerm> entry : terms.entries()) {
- if (null == entry.getValue()) {
- continue;
- }
-
- if (null != this.unevaluatedFields && this.unevaluatedFields.contains(entry.getKey().trim())) {
- unevaluatedExpressions.add(entry.getKey().trim() + " " + entry.getValue().getOperator() + " " + entry.getValue().getValue());
- }
-
- int operator = JexlOperatorConstants.getJJTNodeType(entry.getValue().getOperator());
- if (!(operator == ParserTreeConstants.JJTEQNODE || operator == ParserTreeConstants.JJTNENODE || operator == ParserTreeConstants.JJTLENODE
- || operator == ParserTreeConstants.JJTLTNODE || operator == ParserTreeConstants.JJTGENODE || operator == ParserTreeConstants.JJTGTNODE || operator == ParserTreeConstants.JJTERNODE)) {
- unsupportedOperatorSpecified = true;
- break;
- }
- }
- if (null != unevaluatedExpressions)
- unevaluatedExpressions.trimToSize();
- if (log.isDebugEnabled()) {
- log.debug(hash + " unsupportedOperators: " + unsupportedOperatorSpecified + " indexedTerms: " + indexedTerms.toString() + " orTerms: "
- + orTerms.toString() + " unevaluatedExpressions: " + unevaluatedExpressions.toString());
- }
-
- // We can use the intersecting iterator over the field index as an optimization under the
- // following conditions
- //
- // 1. No unsupported operators in the query.
- // 2. No 'or' operators and at least one term indexed
- // or
- // 1. No unsupported operators in the query.
- // 2. and all terms indexed
- // or
- // 1. All or'd terms are indexed. NOTE, this will potentially skip some queries and push to a full table scan
- // // WE should look into finding a better way to handle whether we do an optimized query or not.
- boolean optimizationSucceeded = false;
- boolean orsAllIndexed = false;
- if (orTerms.isEmpty()) {
- orsAllIndexed = false;
- } else {
- orsAllIndexed = indexedTerms.keySet().containsAll(orTerms);
- }
-
- if (log.isDebugEnabled()) {
- log.debug("All or terms are indexed");
- }
-
- if (!unsupportedOperatorSpecified
- && (((null == orTerms || orTerms.isEmpty()) && indexedTerms.size() > 0) || (fields.size() > 0 && indexedTerms.size() == fields.size()) || orsAllIndexed)) {
- optimizedQuery.start();
- // Set up intersecting iterator over field index.
-
- // Get information from the global index for the indexed terms. The results object will contain the term
- // mapped to an object that contains the total count, and partitions where this term is located.
-
- // TODO: Should we cache indexed term information or does that not make sense since we are always loading data
- queryGlobalIndex.start();
- IndexRanges termIndexInfo;
- try {
- // If fields is null or zero, then it's probably the case that the user entered a value
- // to search for with no fields. Check for the value in index.
- if (fields.isEmpty()) {
- termIndexInfo = this.getTermIndexInformation(connector, auths, queryString, typeFilter);
- if (null != termIndexInfo && termIndexInfo.getRanges().isEmpty()) {
- // Then we didn't find anything in the index for this query. This may happen for an indexed term that has wildcards
- // in unhandled locations.
- // Break out of here by throwing a named exception and do full scan
- throw new DoNotPerformOptimizedQueryException();
- }
- // We need to rewrite the query string here so that it's valid.
- if (termIndexInfo instanceof UnionIndexRanges) {
- UnionIndexRanges union = (UnionIndexRanges) termIndexInfo;
- StringBuilder buf = new StringBuilder();
- String sep = "";
- for (String fieldName : union.getFieldNamesAndValues().keySet()) {
- buf.append(sep).append(fieldName).append(" == ");
- if (!(queryString.startsWith("'") && queryString.endsWith("'"))) {
- buf.append("'").append(queryString).append("'");
- } else {
- buf.append(queryString);
- }
- sep = " or ";
- }
- if (log.isDebugEnabled()) {
- log.debug("Rewrote query for non-fielded single term query: " + queryString + " to " + buf.toString());
- }
- queryString = buf.toString();
- } else {
- throw new RuntimeException("Unexpected IndexRanges implementation");
- }
- } else {
- RangeCalculator calc = this.getTermIndexInformation(connector, auths, indexedTerms, terms, this.getIndexTableName(), this.getReverseIndexTableName(),
- queryString, this.queryThreads, typeFilter);
- if (null == calc.getResult() || calc.getResult().isEmpty()) {
- // Then we didn't find anything in the index for this query. This may happen for an indexed term that has wildcards
- // in unhandled locations.
- // Break out of here by throwing a named exception and do full scan
- throw new DoNotPerformOptimizedQueryException();
- }
- termIndexInfo = new UnionIndexRanges();
- termIndexInfo.setIndexValuesToOriginalValues(calc.getIndexValues());
- termIndexInfo.setFieldNamesAndValues(calc.getIndexEntries());
- termIndexInfo.getTermCardinality().putAll(calc.getTermCardinalities());
- for (Range r : calc.getResult()) {
- // foo is a placeholder and is ignored.
- termIndexInfo.add("foo", r);
- }
- }
- } catch (TableNotFoundException e) {
- log.error(this.getIndexTableName() + "not found", e);
- throw new RuntimeException(this.getIndexTableName() + "not found", e);
- } catch (org.apache.commons.jexl2.parser.ParseException e) {
- throw new RuntimeException("Error determining ranges for query: " + queryString, e);
- } catch (DoNotPerformOptimizedQueryException e) {
- log.info("Indexed fields not found in index, performing full scan");
- termIndexInfo = null;
- }
- queryGlobalIndex.stop();
-
- // Determine if we should proceed with optimized query based on results from the global index
- boolean proceed = false;
- if (null == termIndexInfo || termIndexInfo.getFieldNamesAndValues().values().size() == 0) {
- proceed = false;
- } else if (null != orTerms && orTerms.size() > 0 && (termIndexInfo.getFieldNamesAndValues().values().size() == indexedTerms.size())) {
- proceed = true;
- } else if (termIndexInfo.getFieldNamesAndValues().values().size() > 0) {
- proceed = true;
- } else if (orsAllIndexed) {
- proceed = true;
- } else {
- proceed = false;
- }
- if (log.isDebugEnabled()) {
- log.debug("Proceed with optimized query: " + proceed);
- if (null != termIndexInfo)
- log.debug("termIndexInfo.getTermsFound().size(): " + termIndexInfo.getFieldNamesAndValues().values().size() + " indexedTerms.size: "
- + indexedTerms.size() + " fields.size: " + fields.size());
- }
- if (proceed) {
-
- if (log.isDebugEnabled()) {
- log.debug(hash + " Performing optimized query");
- }
- // Use the scan ranges from the GlobalIndexRanges object as the ranges for the batch scanner
- ranges = termIndexInfo.getRanges();
- if (log.isDebugEnabled()) {
- log.info(hash + " Ranges: count: " + ranges.size() + ", " + ranges.toString());
- }
-
- // Create BatchScanner, set the ranges, and setup the iterators.
- optimizedEventQuery.start();
- BatchScanner bs = null;
- try {
- bs = connector.createBatchScanner(this.getTableName(), auths, queryThreads);
- bs.setRanges(ranges);
- IteratorSetting si = new IteratorSetting(21, "eval", OptimizedQueryIterator.class);
-
- if (log.isDebugEnabled()) {
- log.debug("Setting scan option: " + EvaluatingIterator.QUERY_OPTION + " to " + queryString);
- }
- // Set the query option
- si.addOption(EvaluatingIterator.QUERY_OPTION, queryString);
- // Set the Indexed Terms List option. This is the field name and normalized field value pair separated
- // by a comma.
- StringBuilder buf = new StringBuilder();
- String sep = "";
- for (Entry<String,String> entry : termIndexInfo.getFieldNamesAndValues().entries()) {
- buf.append(sep);
- buf.append(entry.getKey());
- buf.append(":");
- buf.append(termIndexInfo.getIndexValuesToOriginalValues().get(entry.getValue()));
- buf.append(":");
- buf.append(entry.getValue());
- if (sep.equals("")) {
- sep = ";";
- }
- }
- if (log.isDebugEnabled()) {
- log.debug("Setting scan option: " + FieldIndexQueryReWriter.INDEXED_TERMS_LIST + " to " + buf.toString());
- }
- FieldIndexQueryReWriter rewriter = new FieldIndexQueryReWriter();
- String q = "";
- try {
- q = queryString;
- q = rewriter.applyCaseSensitivity(q, true, false);// Set upper/lower case for fieldname/fieldvalue
- Map<String,String> opts = new HashMap<String,String>();
- opts.put(FieldIndexQueryReWriter.INDEXED_TERMS_LIST, buf.toString());
- q = rewriter.removeNonIndexedTermsAndInvalidRanges(q, opts);
- q = rewriter.applyNormalizedTerms(q, opts);
- if (log.isDebugEnabled()) {
- log.debug("runServerQuery, FieldIndex Query: " + q);
- }
- } catch (org.apache.commons.jexl2.parser.ParseException ex) {
- log.error("Could not parse query, Jexl ParseException: " + ex);
- } catch (Exception ex) {
- log.error("Problem rewriting query, Exception: " + ex.getMessage());
- }
- si.addOption(BooleanLogicIterator.FIELD_INDEX_QUERY, q);
-
- // Set the term cardinality option
- sep = "";
- buf.delete(0, buf.length());
- for (Entry<String,Long> entry : termIndexInfo.getTermCardinality().entrySet()) {
- buf.append(sep);
- buf.append(entry.getKey());
- buf.append(":");
- buf.append(entry.getValue());
- sep = ",";
- }
- if (log.isDebugEnabled())
- log.debug("Setting scan option: " + BooleanLogicIterator.TERM_CARDINALITIES + " to " + buf.toString());
- si.addOption(BooleanLogicIterator.TERM_CARDINALITIES, buf.toString());
- if (this.useReadAheadIterator) {
- if (log.isDebugEnabled()) {
- log.debug("Enabling read ahead iterator with queue size: " + this.readAheadQueueSize + " and timeout: " + this.readAheadTimeOut);
- }
- si.addOption(ReadAheadIterator.QUEUE_SIZE, this.readAheadQueueSize);
- si.addOption(ReadAheadIterator.TIMEOUT, this.readAheadTimeOut);
-
- }
-
- if (null != unevaluatedExpressions) {
- StringBuilder unevaluatedExpressionList = new StringBuilder();
- String sep2 = "";
- for (String exp : unevaluatedExpressions) {
- unevaluatedExpressionList.append(sep2).append(exp);
- sep2 = ",";
- }
- if (log.isDebugEnabled())
- log.debug("Setting scan option: " + EvaluatingIterator.UNEVALUTED_EXPRESSIONS + " to " + unevaluatedExpressionList.toString());
- si.addOption(EvaluatingIterator.UNEVALUTED_EXPRESSIONS, unevaluatedExpressionList.toString());
- }
-
- bs.addScanIterator(si);
-
- processResults.start();
- processResults.suspend();
- long count = 0;
- for (Entry<Key,Value> entry : bs) {
- count++;
- // The key that is returned by the EvaluatingIterator is not the same key that is in
- // the table. The value that is returned by the EvaluatingIterator is a kryo
- // serialized EventFields object.
- processResults.resume();
- Document d = this.createDocument(entry.getKey(), entry.getValue());
- results.getResults().add(d);
- processResults.suspend();
- }
- log.info(count + " matching entries found in optimized query.");
- optimizationSucceeded = true;
- processResults.stop();
- } catch (TableNotFoundException e) {
- log.error(this.getTableName() + "not found", e);
- throw new RuntimeException(this.getIndexTableName() + "not found", e);
- } finally {
- if (bs != null) {
- bs.close();
- }
- }
- optimizedEventQuery.stop();
- }
- optimizedQuery.stop();
- }
-
- // WE should look into finding a better way to handle whether we do an optimized query or not.
- // We are not setting up an else condition here because we may have aborted the logic early in the if statement.
- if (!optimizationSucceeded || ((null != orTerms && orTerms.size() > 0) && (indexedTerms.size() != fields.size()) && !orsAllIndexed)) {
- // if (!optimizationSucceeded || ((null != orTerms && orTerms.size() > 0) && (indexedTerms.size() != fields.size()))) {
- fullScanQuery.start();
- if (log.isDebugEnabled()) {
- log.debug(hash + " Performing full scan query");
- }
-
- // Set up a full scan using the date ranges from the query
- // Create BatchScanner, set the ranges, and setup the iterators.
- BatchScanner bs = null;
- try {
- // The ranges are the start and end dates
- Collection<Range> r = getFullScanRange(beginDate, endDate, terms);
- ranges.addAll(r);
-
- if (log.isDebugEnabled()) {
- log.debug(hash + " Ranges: count: " + ranges.size() + ", " + ranges.toString());
- }
-
- bs = connector.createBatchScanner(this.getTableName(), auths, queryThreads);
- bs.setRanges(ranges);
- IteratorSetting si = new IteratorSetting(22, "eval", EvaluatingIterator.class);
- // Create datatype regex if needed
- if (null != typeFilter) {
- StringBuilder buf = new StringBuilder();
- String s = "";
- for (String type : typeFilter) {
- buf.append(s).append(type).append(".*");
- s = "|";
- }
- if (log.isDebugEnabled())
- log.debug("Setting colf regex iterator to: " + buf.toString());
- IteratorSetting ri = new IteratorSetting(21, "typeFilter", RegExFilter.class);
- RegExFilter.setRegexs(ri, null, buf.toString(), null, null, false);
- bs.addScanIterator(ri);
- }
- if (log.isDebugEnabled()) {
- log.debug("Setting scan option: " + EvaluatingIterator.QUERY_OPTION + " to " + queryString);
- }
- si.addOption(EvaluatingIterator.QUERY_OPTION, queryString);
- if (null != unevaluatedExpressions) {
- StringBuilder unevaluatedExpressionList = new StringBuilder();
- String sep2 = "";
- for (String exp : unevaluatedExpressions) {
- unevaluatedExpressionList.append(sep2).append(exp);
- sep2 = ",";
- }
- if (log.isDebugEnabled())
- log.debug("Setting scan option: " + EvaluatingIterator.UNEVALUTED_EXPRESSIONS + " to " + unevaluatedExpressionList.toString());
- si.addOption(EvaluatingIterator.UNEVALUTED_EXPRESSIONS, unevaluatedExpressionList.toString());
- }
- bs.addScanIterator(si);
- long count = 0;
- processResults.start();
- processResults.suspend();
- for (Entry<Key,Value> entry : bs) {
- count++;
- // The key that is returned by the EvaluatingIterator is not the same key that is in
- // the partition table. The value that is returned by the EvaluatingIterator is a kryo
- // serialized EventFields object.
- processResults.resume();
- Document d = this.createDocument(entry.getKey(), entry.getValue());
- results.getResults().add(d);
- processResults.suspend();
- }
- processResults.stop();
- log.info(count + " matching entries found in full scan query.");
- } catch (TableNotFoundException e) {
- log.error(this.getTableName() + "not found", e);
- } finally {
- if (bs != null) {
- bs.close();
- }
- }
- fullScanQuery.stop();
- }
-
- log.info("AbstractQueryLogic: " + queryString + " " + timeString(abstractQueryLogic.getTime()));
- log.info(" 1) parse query " + timeString(parseQuery.getTime()));
- log.info(" 2) query metadata " + timeString(queryMetadata.getTime()));
- log.info(" 3) full scan query " + timeString(fullScanQuery.getTime()));
- log.info(" 3) optimized query " + timeString(optimizedQuery.getTime()));
- log.info(" 1) process results " + timeString(processResults.getTime()));
- log.info(" 1) query global index " + timeString(queryGlobalIndex.getTime()));
- log.info(hash + " Query completed.");
-
- return results;
- }
-
- private static String timeString(long millis) {
- return String.format("%4.2f", millis / 1000.);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/logic/ContentLogic.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/logic/ContentLogic.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/logic/ContentLogic.java
deleted file mode 100644
index 2e65a38..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/logic/ContentLogic.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.logic;
-
-import java.util.List;
-import java.util.Map.Entry;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.examples.wikisearch.ingest.WikipediaMapper;
-import org.apache.accumulo.examples.wikisearch.sample.Document;
-import org.apache.accumulo.examples.wikisearch.sample.Field;
-import org.apache.accumulo.examples.wikisearch.sample.Results;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.lang.StringUtils;
-import org.apache.log4j.Logger;
-
-
-/**
- * This query table implementation returns a Results object that contains documents from the wiki table. The query will contain the partition id, wikitype, and
- * UID so that we can seek directly to the document. The document is stored as base64 compressed binary in the Accumulo table. We will decompress the data so
- * that it is base64 encoded binary data in the Results object.
- *
- * The query that needs to be passed to the web service is: DOCUMENT:partitionId/wikitype/uid.
- *
- */
-public class ContentLogic {
-
- private static final Logger log = Logger.getLogger(ContentLogic.class);
-
- private static final String NULL_BYTE = "\u0000";
-
- private String tableName = null;
-
- private Pattern queryPattern = Pattern.compile("^DOCUMENT:(.*)/(.*)/(.*)$");
-
- public String getTableName() {
- return tableName;
- }
-
- public void setTableName(String tableName) {
- this.tableName = tableName;
- }
-
- public Results runQuery(Connector connector, String query, List<String> authorizations) {
-
- Results results = new Results();
- Authorizations auths = new Authorizations(StringUtils.join(authorizations, "|"));
-
- Matcher match = queryPattern.matcher(query);
- if (!match.matches()) {
- throw new IllegalArgumentException("Query does not match the pattern: DOCUMENT:partitionId/wikitype/uid, your query: " + query.toString());
- } else {
- String partitionId = match.group(1);
- String wikitype = match.group(2);
- String id = match.group(3);
-
- log.debug("Received pieces: " + partitionId + ", " + wikitype + ", " + id);
-
- // Create the Range
- Key startKey = new Key(partitionId, WikipediaMapper.DOCUMENT_COLUMN_FAMILY, wikitype + NULL_BYTE + id);
- Key endKey = new Key(partitionId, WikipediaMapper.DOCUMENT_COLUMN_FAMILY, wikitype + NULL_BYTE + id + NULL_BYTE);
- Range r = new Range(startKey, true, endKey, false);
-
- log.debug("Setting range: " + r);
-
- try {
- Scanner scanner = connector.createScanner(this.getTableName(), auths);
- scanner.setRange(r);
- // This should in theory only match one thing.
- for (Entry<Key,Value> entry : scanner) {
- Document doc = new Document();
- doc.setId(id);
- Field val = new Field();
- val.setFieldName("DOCUMENT");
- val.setFieldValue(new String(Base64.decodeBase64(entry.getValue().toString())));
- doc.getFields().add(val);
- results.getResults().add(doc);
- }
- } catch (TableNotFoundException e) {
- throw new RuntimeException("Table not found: " + this.getTableName(), e);
- }
-
- }
- return results;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/logic/QueryLogic.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/logic/QueryLogic.java b/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/logic/QueryLogic.java
deleted file mode 100644
index bcfeb70..0000000
--- a/src/examples/wikisearch/query/src/main/java/org/apache/accumulo/examples/wikisearch/logic/QueryLogic.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.logic;
-
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.examples.wikisearch.iterator.EvaluatingIterator;
-import org.apache.accumulo.examples.wikisearch.normalizer.LcNoDiacriticsNormalizer;
-import org.apache.accumulo.examples.wikisearch.normalizer.Normalizer;
-import org.apache.accumulo.examples.wikisearch.parser.QueryParser.QueryTerm;
-import org.apache.accumulo.examples.wikisearch.parser.RangeCalculator;
-import org.apache.accumulo.examples.wikisearch.protobuf.Uid;
-import org.apache.accumulo.examples.wikisearch.util.TextUtil;
-import org.apache.hadoop.io.Text;
-import org.apache.log4j.Logger;
-
-import com.google.common.collect.Multimap;
-import com.google.protobuf.InvalidProtocolBufferException;
-
-/**
- * <pre>
- * <h2>Overview</h2>
- * QueryTable implementation that works with the JEXL grammar. This QueryTable
- * uses the metadata, global index, and partitioned table to return
- * results based on the query. Example queries:
- *
- * <b>Single Term Query</b>
- * 'foo' - looks in global index for foo, and if any entries are found, then the query
- * is rewritten to be field1 == 'foo' or field2 == 'foo', etc. This is then passed
- * down the optimized query path which uses the intersecting iterators on the shard
- * table.
- *
- * <b>Boolean expression</b>
- * field == 'foo' - For fielded queries, those that contain a field, an operator, and a literal (string or number),
- * the query is parsed and the set of eventFields in the query that are indexed is determined by
- * querying the metadata table. Depending on the conjunctions in the query (or, and, not) and the
- * eventFields that are indexed, the query may be sent down the optimized path or the full scan path.
- *
- * We are not supporting all of the operators that JEXL supports at this time. We are supporting the following operators:
- *
- * ==, !=, >, ≥, <, ≤, =~, and !~
- *
- * Custom functions can be created and registered with the Jexl engine. The functions can be used in the queries in conjunction
- * with other supported operators. A sample function has been created, called between, and is bound to the 'f' namespace. An
- * example using this function is : "f:between(LATITUDE,60.0, 70.0)"
- *
- * <h2>Constraints on Query Structure</h2>
- * Queries that are sent to this class need to be formatted such that there is a space on either side of the operator. We are
- * rewriting the query in some cases and the current implementation is expecting a space on either side of the operator. Users
- * should also be aware that the literals used in the query need to match the data in the table. If an error occurs in the evaluation
- * we are skipping the event.
- *
- * <h2>Notes on Optimization</h2>
- * Queries that meet any of the following criteria will perform a full scan of the events in the partitioned table:
- *
- * 1. An 'or' conjunction exists in the query but not all of the terms are indexed.
- * 2. No indexed terms exist in the query
- * 3. An unsupported operator exists in the query
- *
- * </pre>
- *
- */
-public class QueryLogic extends AbstractQueryLogic {
-
- protected static Logger log = Logger.getLogger(QueryLogic.class);
-
- public QueryLogic() {
- super();
- }
-
- @Override
- protected RangeCalculator getTermIndexInformation(Connector c, Authorizations auths, Multimap<String,Normalizer> indexedTerms,
- Multimap<String,QueryTerm> terms, String indexTableName, String reverseIndexTableName, String queryString, int queryThreads, Set<String> typeFilter)
- throws TableNotFoundException, org.apache.commons.jexl2.parser.ParseException {
- RangeCalculator calc = new RangeCalculator();
- calc.execute(c, auths, indexedTerms, terms, queryString, this, typeFilter);
- return calc;
- }
-
- protected Collection<Range> getFullScanRange(Date begin, Date end, Multimap<String,QueryTerm> terms) {
- return Collections.singletonList(new Range());
- }
-
- @Override
- protected IndexRanges getTermIndexInformation(Connector c, Authorizations auths, String value, Set<String> typeFilter) throws TableNotFoundException {
- final String dummyTermName = "DUMMY";
- UnionIndexRanges indexRanges = new UnionIndexRanges();
-
- // The entries in the index are normalized, since we don't have a field, just try using the LcNoDiacriticsNormalizer.
- String normalizedFieldValue = new LcNoDiacriticsNormalizer().normalizeFieldValue("", value);
- // Remove the begin and end ' marks
- if (normalizedFieldValue.startsWith("'") && normalizedFieldValue.endsWith("'")) {
- normalizedFieldValue = normalizedFieldValue.substring(1, normalizedFieldValue.length() - 1);
- }
- Text fieldValue = new Text(normalizedFieldValue);
- if (log.isDebugEnabled()) {
- log.debug("Querying index table : " + this.getIndexTableName() + " for normalized indexed term: " + fieldValue);
- }
- Scanner scanner = c.createScanner(this.getIndexTableName(), auths);
- Range r = new Range(fieldValue);
- scanner.setRange(r);
- if (log.isDebugEnabled()) {
- log.debug("Range for index query: " + r.toString());
- }
- for (Entry<Key,Value> entry : scanner) {
- if (log.isDebugEnabled()) {
- log.debug("Index entry: " + entry.getKey().toString());
- }
- // Get the shard id and datatype from the colq
- String fieldName = entry.getKey().getColumnFamily().toString();
- String colq = entry.getKey().getColumnQualifier().toString();
- int separator = colq.indexOf(EvaluatingIterator.NULL_BYTE_STRING);
- String shardId = null;
- String datatype = null;
- if (separator != -1) {
- shardId = colq.substring(0, separator);
- datatype = colq.substring(separator + 1);
- } else {
- shardId = colq;
- }
- // Skip this entry if the type is not correct
- if (null != datatype && null != typeFilter && !typeFilter.contains(datatype))
- continue;
- // Parse the UID.List object from the value
- Uid.List uidList = null;
- try {
- uidList = Uid.List.parseFrom(entry.getValue().get());
- } catch (InvalidProtocolBufferException e) {
- // Don't add UID information, at least we know what shards
- // it is located in.
- }
-
- // Add the count for this shard to the total count for the term.
- long count = 0;
- Long storedCount = indexRanges.getTermCardinality().get(dummyTermName);
- if (null == storedCount) {
- count = uidList.getCOUNT();
- } else {
- count = uidList.getCOUNT() + storedCount;
- }
- indexRanges.getTermCardinality().put(dummyTermName, count);
- // Add the field name
- indexRanges.getFieldNamesAndValues().put(fieldName, normalizedFieldValue);
-
- // Create the keys
- Text shard = new Text(shardId);
- if (uidList.getIGNORE()) {
- // Then we create a scan range that is the entire shard
- indexRanges.add(dummyTermName, new Range(shard));
- } else {
- // We should have UUIDs, create event ranges
- for (String uuid : uidList.getUIDList()) {
- Text cf = new Text(datatype);
- TextUtil.textAppend(cf, uuid);
- Key startKey = new Key(shard, cf);
- Key endKey = new Key(shard, new Text(cf.toString() + EvaluatingIterator.NULL_BYTE_STRING));
- Range eventRange = new Range(startKey, true, endKey, false);
- indexRanges.add(dummyTermName, eventRange);
- }
- }
- }
- if (log.isDebugEnabled()) {
- log.debug("Found " + indexRanges.getRanges().size() + " entries in the index for field value: " + normalizedFieldValue);
- }
- return indexRanges;
-
- }
-
-}
[13/39] ACCUMULO-600 removed wikisearch from trunk
Posted by ec...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/protobuf/Uid.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/protobuf/Uid.java b/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/protobuf/Uid.java
deleted file mode 100644
index c469748..0000000
--- a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/protobuf/Uid.java
+++ /dev/null
@@ -1,470 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-// Generated by the protocol buffer compiler. DO NOT EDIT!
-// source: Uid.proto
-
-package org.apache.accumulo.examples.wikisearch.protobuf;
-
-public final class Uid {
- private Uid() {}
-
- public static void registerAllExtensions(com.google.protobuf.ExtensionRegistry registry) {}
-
- public static final class List extends com.google.protobuf.GeneratedMessage {
- // Use List.newBuilder() to construct.
- private List() {
- initFields();
- }
-
- private List(boolean noInit) {}
-
- private static final List defaultInstance;
-
- public static List getDefaultInstance() {
- return defaultInstance;
- }
-
- public List getDefaultInstanceForType() {
- return defaultInstance;
- }
-
- public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
- return org.apache.accumulo.examples.wikisearch.protobuf.Uid.internal_static_protobuf_List_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable internalGetFieldAccessorTable() {
- return org.apache.accumulo.examples.wikisearch.protobuf.Uid.internal_static_protobuf_List_fieldAccessorTable;
- }
-
- // required bool IGNORE = 1;
- public static final int IGNORE_FIELD_NUMBER = 1;
- private boolean hasIGNORE;
- private boolean iGNORE_ = false;
-
- public boolean hasIGNORE() {
- return hasIGNORE;
- }
-
- public boolean getIGNORE() {
- return iGNORE_;
- }
-
- // required uint64 COUNT = 2;
- public static final int COUNT_FIELD_NUMBER = 2;
- private boolean hasCOUNT;
- private long cOUNT_ = 0L;
-
- public boolean hasCOUNT() {
- return hasCOUNT;
- }
-
- public long getCOUNT() {
- return cOUNT_;
- }
-
- // repeated string UID = 3;
- public static final int UID_FIELD_NUMBER = 3;
- private java.util.List<java.lang.String> uID_ = java.util.Collections.emptyList();
-
- public java.util.List<java.lang.String> getUIDList() {
- return uID_;
- }
-
- public int getUIDCount() {
- return uID_.size();
- }
-
- public java.lang.String getUID(int index) {
- return uID_.get(index);
- }
-
- private void initFields() {}
-
- public final boolean isInitialized() {
- if (!hasIGNORE)
- return false;
- if (!hasCOUNT)
- return false;
- return true;
- }
-
- public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException {
- getSerializedSize();
- if (hasIGNORE()) {
- output.writeBool(1, getIGNORE());
- }
- if (hasCOUNT()) {
- output.writeUInt64(2, getCOUNT());
- }
- for (java.lang.String element : getUIDList()) {
- output.writeString(3, element);
- }
- getUnknownFields().writeTo(output);
- }
-
- private int memoizedSerializedSize = -1;
-
- public int getSerializedSize() {
- int size = memoizedSerializedSize;
- if (size != -1)
- return size;
-
- size = 0;
- if (hasIGNORE()) {
- size += com.google.protobuf.CodedOutputStream.computeBoolSize(1, getIGNORE());
- }
- if (hasCOUNT()) {
- size += com.google.protobuf.CodedOutputStream.computeUInt64Size(2, getCOUNT());
- }
- {
- int dataSize = 0;
- for (java.lang.String element : getUIDList()) {
- dataSize += com.google.protobuf.CodedOutputStream.computeStringSizeNoTag(element);
- }
- size += dataSize;
- size += 1 * getUIDList().size();
- }
- size += getUnknownFields().getSerializedSize();
- memoizedSerializedSize = size;
- return size;
- }
-
- public static org.apache.accumulo.examples.wikisearch.protobuf.Uid.List parseFrom(com.google.protobuf.ByteString data) throws com.google.protobuf.InvalidProtocolBufferException {
- return newBuilder().mergeFrom(data).buildParsed();
- }
-
- public static org.apache.accumulo.examples.wikisearch.protobuf.Uid.List parseFrom(com.google.protobuf.ByteString data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return newBuilder().mergeFrom(data, extensionRegistry).buildParsed();
- }
-
- public static org.apache.accumulo.examples.wikisearch.protobuf.Uid.List parseFrom(byte[] data) throws com.google.protobuf.InvalidProtocolBufferException {
- return newBuilder().mergeFrom(data).buildParsed();
- }
-
- public static org.apache.accumulo.examples.wikisearch.protobuf.Uid.List parseFrom(byte[] data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return newBuilder().mergeFrom(data, extensionRegistry).buildParsed();
- }
-
- public static org.apache.accumulo.examples.wikisearch.protobuf.Uid.List parseFrom(java.io.InputStream input) throws java.io.IOException {
- return newBuilder().mergeFrom(input).buildParsed();
- }
-
- public static org.apache.accumulo.examples.wikisearch.protobuf.Uid.List parseFrom(java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return newBuilder().mergeFrom(input, extensionRegistry).buildParsed();
- }
-
- public static org.apache.accumulo.examples.wikisearch.protobuf.Uid.List parseDelimitedFrom(java.io.InputStream input) throws java.io.IOException {
- Builder builder = newBuilder();
- if (builder.mergeDelimitedFrom(input)) {
- return builder.buildParsed();
- } else {
- return null;
- }
- }
-
- public static org.apache.accumulo.examples.wikisearch.protobuf.Uid.List parseDelimitedFrom(java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- Builder builder = newBuilder();
- if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
- return builder.buildParsed();
- } else {
- return null;
- }
- }
-
- public static org.apache.accumulo.examples.wikisearch.protobuf.Uid.List parseFrom(com.google.protobuf.CodedInputStream input) throws java.io.IOException {
- return newBuilder().mergeFrom(input).buildParsed();
- }
-
- public static org.apache.accumulo.examples.wikisearch.protobuf.Uid.List parseFrom(com.google.protobuf.CodedInputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return newBuilder().mergeFrom(input, extensionRegistry).buildParsed();
- }
-
- public static Builder newBuilder() {
- return Builder.create();
- }
-
- public Builder newBuilderForType() {
- return newBuilder();
- }
-
- public static Builder newBuilder(org.apache.accumulo.examples.wikisearch.protobuf.Uid.List prototype) {
- return newBuilder().mergeFrom(prototype);
- }
-
- public Builder toBuilder() {
- return newBuilder(this);
- }
-
- public static final class Builder extends com.google.protobuf.GeneratedMessage.Builder<Builder> {
- private org.apache.accumulo.examples.wikisearch.protobuf.Uid.List result;
-
- // Construct using protobuf.Uid.List.newBuilder()
- private Builder() {}
-
- private static Builder create() {
- Builder builder = new Builder();
- builder.result = new org.apache.accumulo.examples.wikisearch.protobuf.Uid.List();
- return builder;
- }
-
- protected org.apache.accumulo.examples.wikisearch.protobuf.Uid.List internalGetResult() {
- return result;
- }
-
- public Builder clear() {
- if (result == null) {
- throw new IllegalStateException("Cannot call clear() after build().");
- }
- result = new org.apache.accumulo.examples.wikisearch.protobuf.Uid.List();
- return this;
- }
-
- public Builder clone() {
- return create().mergeFrom(result);
- }
-
- public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() {
- return org.apache.accumulo.examples.wikisearch.protobuf.Uid.List.getDescriptor();
- }
-
- public org.apache.accumulo.examples.wikisearch.protobuf.Uid.List getDefaultInstanceForType() {
- return org.apache.accumulo.examples.wikisearch.protobuf.Uid.List.getDefaultInstance();
- }
-
- public boolean isInitialized() {
- return result.isInitialized();
- }
-
- public org.apache.accumulo.examples.wikisearch.protobuf.Uid.List build() {
- if (result != null && !isInitialized()) {
- throw newUninitializedMessageException(result);
- }
- return buildPartial();
- }
-
- private org.apache.accumulo.examples.wikisearch.protobuf.Uid.List buildParsed() throws com.google.protobuf.InvalidProtocolBufferException {
- if (!isInitialized()) {
- throw newUninitializedMessageException(result).asInvalidProtocolBufferException();
- }
- return buildPartial();
- }
-
- public org.apache.accumulo.examples.wikisearch.protobuf.Uid.List buildPartial() {
- if (result == null) {
- throw new IllegalStateException("build() has already been called on this Builder.");
- }
- if (result.uID_ != java.util.Collections.EMPTY_LIST) {
- result.uID_ = java.util.Collections.unmodifiableList(result.uID_);
- }
- org.apache.accumulo.examples.wikisearch.protobuf.Uid.List returnMe = result;
- result = null;
- return returnMe;
- }
-
- public Builder mergeFrom(com.google.protobuf.Message other) {
- if (other instanceof org.apache.accumulo.examples.wikisearch.protobuf.Uid.List) {
- return mergeFrom((org.apache.accumulo.examples.wikisearch.protobuf.Uid.List) other);
- } else {
- super.mergeFrom(other);
- return this;
- }
- }
-
- public Builder mergeFrom(org.apache.accumulo.examples.wikisearch.protobuf.Uid.List other) {
- if (other == org.apache.accumulo.examples.wikisearch.protobuf.Uid.List.getDefaultInstance())
- return this;
- if (other.hasIGNORE()) {
- setIGNORE(other.getIGNORE());
- }
- if (other.hasCOUNT()) {
- setCOUNT(other.getCOUNT());
- }
- if (!other.uID_.isEmpty()) {
- if (result.uID_.isEmpty()) {
- result.uID_ = new java.util.ArrayList<java.lang.String>();
- }
- result.uID_.addAll(other.uID_);
- }
- this.mergeUnknownFields(other.getUnknownFields());
- return this;
- }
-
- public Builder mergeFrom(com.google.protobuf.CodedInputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet.newBuilder(this.getUnknownFields());
- while (true) {
- int tag = input.readTag();
- switch (tag) {
- case 0:
- this.setUnknownFields(unknownFields.build());
- return this;
- default: {
- if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) {
- this.setUnknownFields(unknownFields.build());
- return this;
- }
- break;
- }
- case 8: {
- setIGNORE(input.readBool());
- break;
- }
- case 16: {
- setCOUNT(input.readUInt64());
- break;
- }
- case 26: {
- addUID(input.readString());
- break;
- }
- }
- }
- }
-
- // required bool IGNORE = 1;
- public boolean hasIGNORE() {
- return result.hasIGNORE();
- }
-
- public boolean getIGNORE() {
- return result.getIGNORE();
- }
-
- public Builder setIGNORE(boolean value) {
- result.hasIGNORE = true;
- result.iGNORE_ = value;
- return this;
- }
-
- public Builder clearIGNORE() {
- result.hasIGNORE = false;
- result.iGNORE_ = false;
- return this;
- }
-
- // required uint64 COUNT = 2;
- public boolean hasCOUNT() {
- return result.hasCOUNT();
- }
-
- public long getCOUNT() {
- return result.getCOUNT();
- }
-
- public Builder setCOUNT(long value) {
- result.hasCOUNT = true;
- result.cOUNT_ = value;
- return this;
- }
-
- public Builder clearCOUNT() {
- result.hasCOUNT = false;
- result.cOUNT_ = 0L;
- return this;
- }
-
- // repeated string UID = 3;
- public java.util.List<java.lang.String> getUIDList() {
- return java.util.Collections.unmodifiableList(result.uID_);
- }
-
- public int getUIDCount() {
- return result.getUIDCount();
- }
-
- public java.lang.String getUID(int index) {
- return result.getUID(index);
- }
-
- public Builder setUID(int index, java.lang.String value) {
- if (value == null) {
- throw new NullPointerException();
- }
- result.uID_.set(index, value);
- return this;
- }
-
- public Builder addUID(java.lang.String value) {
- if (value == null) {
- throw new NullPointerException();
- }
- if (result.uID_.isEmpty()) {
- result.uID_ = new java.util.ArrayList<java.lang.String>();
- }
- result.uID_.add(value);
- return this;
- }
-
- public Builder addAllUID(java.lang.Iterable<? extends java.lang.String> values) {
- if (result.uID_.isEmpty()) {
- result.uID_ = new java.util.ArrayList<java.lang.String>();
- }
- super.addAll(values, result.uID_);
- return this;
- }
-
- public Builder clearUID() {
- result.uID_ = java.util.Collections.emptyList();
- return this;
- }
-
- // @@protoc_insertion_point(builder_scope:protobuf.List)
- }
-
- static {
- defaultInstance = new List(true);
- org.apache.accumulo.examples.wikisearch.protobuf.Uid.internalForceInit();
- defaultInstance.initFields();
- }
-
- // @@protoc_insertion_point(class_scope:protobuf.List)
- }
-
- private static com.google.protobuf.Descriptors.Descriptor internal_static_protobuf_List_descriptor;
- private static com.google.protobuf.GeneratedMessage.FieldAccessorTable internal_static_protobuf_List_fieldAccessorTable;
-
- public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() {
- return descriptor;
- }
-
- private static com.google.protobuf.Descriptors.FileDescriptor descriptor;
- static {
- java.lang.String[] descriptorData = {"\n\tUid.proto\022\010protobuf\"2\n\004List\022\016\n\006IGNORE\030"
- + "\001 \002(\010\022\r\n\005COUNT\030\002 \002(\004\022\013\n\003UID\030\003 \003(\tB\014\n\010pro" + "tobufH\001"};
- com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
- public com.google.protobuf.ExtensionRegistry assignDescriptors(com.google.protobuf.Descriptors.FileDescriptor root) {
- descriptor = root;
- internal_static_protobuf_List_descriptor = getDescriptor().getMessageTypes().get(0);
- internal_static_protobuf_List_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable(
- internal_static_protobuf_List_descriptor, new java.lang.String[] {"IGNORE", "COUNT", "UID",}, org.apache.accumulo.examples.wikisearch.protobuf.Uid.List.class,
- org.apache.accumulo.examples.wikisearch.protobuf.Uid.List.Builder.class);
- return null;
- }
- };
- com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom(descriptorData, new com.google.protobuf.Descriptors.FileDescriptor[] {},
- assigner);
- }
-
- public static void internalForceInit() {}
-
- // @@protoc_insertion_point(outer_class_scope)
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/reader/AggregatingRecordReader.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/reader/AggregatingRecordReader.java b/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/reader/AggregatingRecordReader.java
deleted file mode 100644
index 09755c0..0000000
--- a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/reader/AggregatingRecordReader.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.reader;
-
-
-import java.io.IOException;
-
-import org.apache.accumulo.examples.wikisearch.ingest.WikipediaConfiguration;
-import org.apache.accumulo.examples.wikisearch.ingest.WikipediaInputFormat.WikipediaInputSplit;
-import org.apache.accumulo.examples.wikisearch.util.TextUtil;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-
-
-/**
- * This class aggregates Text values based on a start and end filter. An example use case for this would be XML data. This will not work with data that has
- * nested start and stop tokens.
- *
- */
-public class AggregatingRecordReader extends LongLineRecordReader {
-
- public static final String START_TOKEN = "aggregating.token.start";
- public static final String END_TOKEN = "aggregating.token.end";
- public static final String RETURN_PARTIAL_MATCHES = "aggregating.allow.partial";
-
- private LongWritable key = new LongWritable();
- private String startToken = null;
- private String endToken = null;
- private long counter = 0;
- private Text aggValue = new Text();
- private boolean startFound = false;
- private StringBuilder remainder = new StringBuilder(0);
- private boolean returnPartialMatches = false;
-
- @Override
- public LongWritable getCurrentKey() {
- key.set(counter);
- return key;
- }
-
- @Override
- public Text getCurrentValue() {
- return aggValue;
- }
-
- @Override
- public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException {
- super.initialize(((WikipediaInputSplit)genericSplit).getFileSplit(), context);
- this.startToken = WikipediaConfiguration.isNull(context.getConfiguration(), START_TOKEN, String.class);
- this.endToken = WikipediaConfiguration.isNull(context.getConfiguration(), END_TOKEN, String.class);
- this.returnPartialMatches = context.getConfiguration().getBoolean(RETURN_PARTIAL_MATCHES, false);
-
- /*
- * Text-appending works almost exactly like the + operator on Strings- it creates a byte array exactly the size of [prefix + suffix] and dumps the bytes
- * into the new array. This module works by doing lots of little additions, one line at a time. With most XML, the documents are partitioned on line
- * boundaries, so we will generally have lots of additions. Setting a large default byte array for a text object can avoid this and give us
- * StringBuilder-like functionality for Text objects.
- */
- byte[] txtBuffer = new byte[2048];
- aggValue.set(txtBuffer);
- }
-
- @Override
- public boolean nextKeyValue() throws IOException {
- aggValue.clear();
- boolean hasNext = false;
- boolean finished = false;
- // Find the start token
- while (!finished && (((hasNext = super.nextKeyValue()) == true) || remainder.length() > 0)) {
- if (hasNext)
- finished = process(super.getCurrentValue());
- else
- finished = process(null);
- if (finished) {
- startFound = false;
- counter++;
- return true;
- }
- }
- // If we have anything loaded in the agg value (and we found a start)
- // then we ran out of data before finding the end. Just return the
- // data we have and if it's not valid, downstream parsing of the data
- // will fail.
- if (returnPartialMatches && startFound && aggValue.getLength() > 0) {
- startFound = false;
- counter++;
- return true;
- }
- return false;
- }
-
- /**
- * Populates aggValue with the contents of the Text object.
- *
- * @param t
- * @return true if aggValue is complete, else false and needs more data.
- */
- private boolean process(Text t) {
-
- if (null != t)
- remainder.append(t.toString());
- while (remainder.length() > 0) {
- if (!startFound) {
- // If found, then begin aggregating at the start offset
- int start = remainder.indexOf(startToken);
- if (-1 != start) {
- // Append the start token to the aggregate value
- TextUtil.textAppendNoNull(aggValue, remainder.substring(start, start + startToken.length()), false);
- // Remove to the end of the start token from the remainder
- remainder.delete(0, start + startToken.length());
- startFound = true;
- } else {
- // If we are looking for the start and have not found it, then remove
- // the bytes
- remainder.delete(0, remainder.length());
- }
- } else {
- // Try to find the end
- int end = remainder.indexOf(endToken);
- // Also try to find the start
- int start = remainder.indexOf(startToken);
- if (-1 == end) {
- if (returnPartialMatches && start >= 0) {
- // End token not found, but another start token was found...
- // The amount to copy is up to the beginning of the next start token
- TextUtil.textAppendNoNull(aggValue, remainder.substring(0, start), false);
- remainder.delete(0, start);
- return true;
- } else {
- // Not found, aggregate the entire remainder
- TextUtil.textAppendNoNull(aggValue, remainder.toString(), false);
- // Delete all chars from remainder
- remainder.delete(0, remainder.length());
- }
- } else {
- if (returnPartialMatches && start >= 0 && start < end) {
- // We found the end token, but found another start token first, so
- // deal with that.
- TextUtil.textAppendNoNull(aggValue, remainder.substring(0, start), false);
- remainder.delete(0, start);
- return true;
- } else {
- // END_TOKEN was found. Extract to the end of END_TOKEN
- TextUtil.textAppendNoNull(aggValue, remainder.substring(0, end + endToken.length()), false);
- // Remove from remainder up to the end of END_TOKEN
- remainder.delete(0, end + endToken.length());
- return true;
- }
- }
- }
- }
- return false;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/reader/LfLineReader.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/reader/LfLineReader.java b/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/reader/LfLineReader.java
deleted file mode 100644
index a4da0ad..0000000
--- a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/reader/LfLineReader.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.reader;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
-
-/**
- * A class that provides a line reader from an input stream.
- */
-public class LfLineReader {
- private static final int DEFAULT_BUFFER_SIZE = 64 * 1024;
- private int bufferSize = DEFAULT_BUFFER_SIZE;
- private InputStream in;
- private byte[] buffer;
- // the number of bytes of real data in the buffer
- private int bufferLength = 0;
- // the current position in the buffer
- private int bufferPosn = 0;
-
- private static final byte LF = '\n';
-
- /**
- * Create a line reader that reads from the given stream using the default buffer-size (64k).
- *
- * @param in
- * The input stream
- * @throws IOException
- */
- public LfLineReader(InputStream in) {
- this(in, DEFAULT_BUFFER_SIZE);
- }
-
- /**
- * Create a line reader that reads from the given stream using the given buffer-size.
- *
- * @param in
- * The input stream
- * @param bufferSize
- * Size of the read buffer
- * @throws IOException
- */
- public LfLineReader(InputStream in, int bufferSize) {
- this.in = in;
- this.bufferSize = bufferSize;
- this.buffer = new byte[this.bufferSize];
- }
-
- /**
- * Create a line reader that reads from the given stream using the <code>io.file.buffer.size</code> specified in the given <code>Configuration</code>.
- *
- * @param in
- * input stream
- * @param conf
- * configuration
- * @throws IOException
- */
- public LfLineReader(InputStream in, Configuration conf) throws IOException {
- this(in, conf.getInt("io.file.buffer.size", DEFAULT_BUFFER_SIZE));
- }
-
- /**
- * Close the underlying stream.
- *
- * @throws IOException
- */
- public void close() throws IOException {
- in.close();
- }
-
- /**
- * Read one line from the InputStream into the given Text. A line can be terminated by '\n' (LF). EOF also terminates an otherwise unterminated line.
- *
- * @param str
- * the object to store the given line (without newline)
- * @param maxLineLength
- * the maximum number of bytes to store into str; the rest of the line is silently discarded.
- * @param maxBytesToConsume
- * the maximum number of bytes to consume in this call. This is only a hint, because if the line cross this threshold, we allow it to happen. It can
- * overshoot potentially by as much as one buffer length.
- *
- * @return the number of bytes read including the (longest) newline found.
- *
- * @throws IOException
- * if the underlying stream throws
- */
- public int readLine(Text str, int maxLineLength, int maxBytesToConsume) throws IOException {
- /*
- * We're reading data from in, but the head of the stream may be already buffered in buffer, so we have several cases: 1. No newline characters are in the
- * buffer, so we need to copy everything and read another buffer from the stream. 2. An unambiguously terminated line is in buffer, so we just copy to str.
- */
- str.clear();
- int txtLength = 0; // tracks str.getLength(), as an optimization
- int newlineLength = 0; // length of terminating newline
- long bytesConsumed = 0;
- do {
- int startPosn = bufferPosn; // starting from where we left off the last time
- if (bufferPosn >= bufferLength) {
- startPosn = bufferPosn = 0;
- bufferLength = in.read(buffer);
- if (bufferLength <= 0)
- break; // EOF
- }
- for (; bufferPosn < bufferLength; ++bufferPosn) { // search for newline
- if (buffer[bufferPosn] == LF) {
- newlineLength = 1;
- ++bufferPosn; // at next invocation proceed from following byte
- break;
- }
- }
- int readLength = bufferPosn - startPosn;
- bytesConsumed += readLength;
- int appendLength = readLength - newlineLength;
- if (appendLength > maxLineLength - txtLength) {
- appendLength = maxLineLength - txtLength;
- }
- if (appendLength > 0) {
- str.append(buffer, startPosn, appendLength);
- txtLength += appendLength;
- }
- } while (newlineLength == 0 && bytesConsumed < maxBytesToConsume);
-
- if (bytesConsumed > Integer.MAX_VALUE)
- throw new IOException("Too many bytes before newline: " + bytesConsumed);
- return (int) bytesConsumed;
- }
-
- /**
- * Read from the InputStream into the given Text.
- *
- * @param str
- * the object to store the given line
- * @param maxLineLength
- * the maximum number of bytes to store into str.
- * @return the number of bytes read including the newline
- * @throws IOException
- * if the underlying stream throws
- */
- public int readLine(Text str, int maxLineLength) throws IOException {
- return readLine(str, maxLineLength, Integer.MAX_VALUE);
- }
-
- /**
- * Read from the InputStream into the given Text.
- *
- * @param str
- * the object to store the given line
- * @return the number of bytes read including the newline
- * @throws IOException
- * if the underlying stream throws
- */
- public int readLine(Text str) throws IOException {
- return readLine(str, Integer.MAX_VALUE, Integer.MAX_VALUE);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/reader/LongLineRecordReader.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/reader/LongLineRecordReader.java b/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/reader/LongLineRecordReader.java
deleted file mode 100644
index f36c373..0000000
--- a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/reader/LongLineRecordReader.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.reader;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.hadoop.io.compress.CompressionCodecFactory;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.lib.input.FileSplit;
-import org.apache.hadoop.mapreduce.lib.input.LineRecordReader;
-import org.apache.hadoop.util.LineReader;
-
-/**
- * A copy of {@link LineRecordReader} which does not discard lines longer than "mapred.linerecordreader.maxlength". Instead, it returns them, leaving it to the
- * mapper to decide what to do with it. It also does not treat '\r' (CR) characters as new lines -- it uses {@link LfLineReader} instead of {@link LineReader}
- * to read lines.
- */
-public class LongLineRecordReader extends RecordReader<LongWritable,Text> {
- private CompressionCodecFactory compressionCodecs = null;
- private long start;
- private long pos;
- private long end;
- private LfLineReader in;
- private int maxLineLength;
- private LongWritable key = null;
- private Text value = null;
-
- @Override
- public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException {
- FileSplit split = (FileSplit) genericSplit;
- Configuration job = context.getConfiguration();
- this.maxLineLength = job.getInt("mapred.linerecordreader.maxlength", Integer.MAX_VALUE);
- start = split.getStart();
- end = start + split.getLength();
- final Path file = split.getPath();
- compressionCodecs = new CompressionCodecFactory(job);
- final CompressionCodec codec = compressionCodecs.getCodec(file);
-
- // open the file and seek to the start of the split
- FileSystem fs = file.getFileSystem(job);
- FSDataInputStream fileIn = fs.open(split.getPath());
- boolean skipFirstLine = false;
- if (codec != null) {
- in = new LfLineReader(codec.createInputStream(fileIn), job);
- end = Long.MAX_VALUE;
- } else {
- if (start != 0) {
- skipFirstLine = true;
- --start;
- fileIn.seek(start);
- }
- in = new LfLineReader(fileIn, job);
- }
- if (skipFirstLine) { // skip first line and re-establish "start".
- start += in.readLine(new Text(), 0, (int) Math.min(Integer.MAX_VALUE, end - start));
- }
- this.pos = start;
- }
-
- @Override
- public boolean nextKeyValue() throws IOException {
- if (key == null) {
- key = new LongWritable();
- }
- key.set(pos);
- if (value == null) {
- value = new Text();
- }
- int newSize = 0;
- if (pos < end) {
- newSize = in.readLine(value, maxLineLength, Math.max((int) Math.min(Integer.MAX_VALUE, end - pos), maxLineLength));
- if (newSize != 0) {
- pos += newSize;
- }
- }
- if (newSize == 0) {
- key = null;
- value = null;
- return false;
- } else {
- return true;
- }
- }
-
- @Override
- public LongWritable getCurrentKey() {
- return key;
- }
-
- @Override
- public Text getCurrentValue() {
- return value;
- }
-
- /**
- * Get the progress within the split
- */
- @Override
- public float getProgress() {
- if (start == end) {
- return 0.0f;
- } else {
- return Math.min(1.0f, (pos - start) / (float) (end - start));
- }
- }
-
- @Override
- public synchronized void close() throws IOException {
- if (in != null) {
- in.close();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/util/TextUtil.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/util/TextUtil.java b/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/util/TextUtil.java
deleted file mode 100644
index 1623d55..0000000
--- a/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/util/TextUtil.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.util;
-
-import java.nio.ByteBuffer;
-import java.nio.charset.CharacterCodingException;
-
-import org.apache.accumulo.core.iterators.user.SummingCombiner;
-import org.apache.hadoop.io.Text;
-
-public class TextUtil {
-
- /**
- * Appends a null byte followed by the UTF-8 bytes of the given string to the given {@link Text}
- *
- * @param text
- * the Text to which to append
- * @param string
- * the String to append
- */
- public static void textAppend(Text text, String string) {
- appendNullByte(text);
- textAppendNoNull(text, string);
- }
-
- public static void textAppend(Text text, String string, boolean replaceBadChar) {
- appendNullByte(text);
- textAppendNoNull(text, string, replaceBadChar);
- }
-
- public static void textAppend(Text t, long s) {
- t.append(nullByte, 0, 1);
- t.append(SummingCombiner.FIXED_LEN_ENCODER.encode(s), 0, 8);
- }
-
- private static final byte[] nullByte = {0};
-
- /**
- * Appends a null byte to the given text
- *
- * @param text
- * the text to which to append the null byte
- */
- public static void appendNullByte(Text text) {
- text.append(nullByte, 0, nullByte.length);
- }
-
- /**
- * Appends the UTF-8 bytes of the given string to the given {@link Text}
- *
- * @param t
- * the Text to which to append
- * @param s
- * the String to append
- */
- public static void textAppendNoNull(Text t, String s) {
- textAppendNoNull(t, s, false);
- }
-
- /**
- * Appends the UTF-8 bytes of the given string to the given {@link Text}
- *
- * @param t
- * @param s
- * @param replaceBadChar
- */
- public static void textAppendNoNull(Text t, String s, boolean replaceBadChar) {
- try {
- ByteBuffer buffer = Text.encode(s, replaceBadChar);
- t.append(buffer.array(), 0, buffer.limit());
- } catch (CharacterCodingException cce) {
- throw new IllegalArgumentException(cce);
- }
- }
-
- /**
- * Converts the given string its UTF-8 bytes. This uses Hadoop's method for converting string to UTF-8 and is much faster than calling
- * {@link String#getBytes(String)}.
- *
- * @param string
- * the string to convert
- * @return the UTF-8 representation of the string
- */
- public static byte[] toUtf8(String string) {
- ByteBuffer buffer;
- try {
- buffer = Text.encode(string, false);
- } catch (CharacterCodingException cce) {
- throw new IllegalArgumentException(cce);
- }
- byte[] bytes = new byte[buffer.limit()];
- System.arraycopy(buffer.array(), 0, bytes, 0, bytes.length);
- return bytes;
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/main/protobuf/TermWeight.proto
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/main/protobuf/TermWeight.proto b/src/examples/wikisearch/ingest/src/main/protobuf/TermWeight.proto
deleted file mode 100644
index 41ae188..0000000
--- a/src/examples/wikisearch/ingest/src/main/protobuf/TermWeight.proto
+++ /dev/null
@@ -1,28 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// compile with protoc --java_out ../java
-// compile extra builder util with java accumulo.data.protobuf.builder.ProtoBufBuilder -d ../java accumulo.data.protobuf.UidList
-// classpath for compile command should include ../../../target/classes and protobuf-java-2.2.0.jar
-
-package protobuf;
-
-option java_package = "protobuf";
-option optimize_for = SPEED;
-
-message Info {
- required float normalizedTermFrequency = 1;
- repeated uint32 wordOffset = 2;
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/main/protobuf/Uid.proto
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/main/protobuf/Uid.proto b/src/examples/wikisearch/ingest/src/main/protobuf/Uid.proto
deleted file mode 100644
index 30aa446..0000000
--- a/src/examples/wikisearch/ingest/src/main/protobuf/Uid.proto
+++ /dev/null
@@ -1,29 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// compile with protoc --java_out ../java
-// compile extra builder util with java accumulo.data.protobuf.builder.ProtoBufBuilder -d ../java accumulo.data.protobuf.UidList
-// classpath for compile command should include ../../../target/classes and protobuf-java-2.2.0.jar
-
-package protobuf;
-
-option java_package = "protobuf";
-option optimize_for = SPEED;
-
-message List {
- required bool IGNORE = 1;
- required uint64 COUNT = 2;
- repeated string UID = 3;
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/main/protobuf/compile_protos.sh
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/main/protobuf/compile_protos.sh b/src/examples/wikisearch/ingest/src/main/protobuf/compile_protos.sh
deleted file mode 100755
index 6702998..0000000
--- a/src/examples/wikisearch/ingest/src/main/protobuf/compile_protos.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-for PROTO in `ls -1 *proto`; do protoc --java_out ../java $PROTO; done
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/StandaloneStatusReporter.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/StandaloneStatusReporter.java b/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/StandaloneStatusReporter.java
deleted file mode 100644
index 6af1e9b..0000000
--- a/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/StandaloneStatusReporter.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.ingest;
-
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.mapreduce.Counters;
-import org.apache.hadoop.mapreduce.StatusReporter;
-
-public class StandaloneStatusReporter extends StatusReporter {
-
- private Counters c = new Counters();
-
- private long filesProcessed = 0;
- private long recordsProcessed = 0;
-
- public Counters getCounters() {
- return c;
- }
-
- @Override
- public Counter getCounter(Enum<?> name) {
- return c.findCounter(name);
- }
-
- @Override
- public Counter getCounter(String group, String name) {
- return c.findCounter(group, name);
- }
-
- @Override
- public void progress() {
- // do nothing
- }
-
- @Override
- public void setStatus(String status) {
- // do nothing
- }
-
- public long getFilesProcessed() {
- return filesProcessed;
- }
-
- public long getRecordsProcessed() {
- return recordsProcessed;
- }
-
- public void incrementFilesProcessed() {
- filesProcessed++;
- recordsProcessed = 0;
- }
-
- public void incrementRecordsProcessed() {
- recordsProcessed++;
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaInputSplitTest.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaInputSplitTest.java b/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaInputSplitTest.java
deleted file mode 100644
index f6b2791..0000000
--- a/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaInputSplitTest.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.ingest;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInput;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-
-import junit.framework.Assert;
-
-import org.apache.accumulo.examples.wikisearch.ingest.WikipediaInputFormat.WikipediaInputSplit;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.lib.input.FileSplit;
-import org.junit.Test;
-
-public class WikipediaInputSplitTest {
- @Test
- public void testSerialization() throws IOException {
- Path testPath = new Path("/foo/bar");
- String[] hosts = new String[2];
- hosts[0] = "abcd";
- hosts[1] = "efgh";
- FileSplit fSplit = new FileSplit(testPath, 1, 2, hosts);
- WikipediaInputSplit split = new WikipediaInputSplit(fSplit, 7);
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- ObjectOutputStream out = new ObjectOutputStream(baos);
- split.write(out);
- out.close();
- baos.close();
-
- ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
- DataInput in = new ObjectInputStream(bais);
-
- WikipediaInputSplit split2 = new WikipediaInputSplit();
- split2.readFields(in);
- Assert.assertTrue(bais.available() == 0);
- bais.close();
-
- Assert.assertTrue(split.getPartition() == split2.getPartition());
-
- FileSplit fSplit2 = split2.getFileSplit();
- Assert.assertTrue(fSplit.getPath().equals(fSplit2.getPath()));
- Assert.assertTrue(fSplit.getStart() == fSplit2.getStart());
- Assert.assertTrue(fSplit.getLength() == fSplit2.getLength());
-
- String[] hosts2 = fSplit2.getLocations();
- Assert.assertEquals(hosts.length, hosts2.length);
- for (int i = 0; i < hosts.length; i++) {
- Assert.assertEquals(hosts[i], hosts2[i]);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaMapperTest.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaMapperTest.java b/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaMapperTest.java
deleted file mode 100644
index c659ec4..0000000
--- a/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaMapperTest.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.ingest;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URL;
-import java.util.HashMap;
-import java.util.Map.Entry;
-
-import junit.framework.Assert;
-
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.MutationsRejectedException;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.mock.MockInstance;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.util.ContextFactory;
-import org.apache.accumulo.examples.wikisearch.reader.AggregatingRecordReader;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.RecordWriter;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.lib.input.FileSplit;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
-import org.junit.Before;
-
-/**
- * Load some data into mock accumulo
- */
-public class WikipediaMapperTest {
-
- private static final String METADATA_TABLE_NAME = "wikiMetadata";
-
- private static final String TABLE_NAME = "wiki";
-
- private static final String INDEX_TABLE_NAME = "wikiIndex";
-
- private static final String RINDEX_TABLE_NAME = "wikiReverseIndex";
-
- private class MockAccumuloRecordWriter extends RecordWriter<Text,Mutation> {
- @Override
- public void write(Text key, Mutation value) throws IOException, InterruptedException {
- try {
- writerMap.get(key).addMutation(value);
- } catch (MutationsRejectedException e) {
- throw new IOException("Error adding mutation", e);
- }
- }
-
- @Override
- public void close(TaskAttemptContext context) throws IOException, InterruptedException {
- try {
- for (BatchWriter w : writerMap.values()) {
- w.flush();
- w.close();
- }
- } catch (MutationsRejectedException e) {
- throw new IOException("Error closing Batch Writer", e);
- }
- }
-
- }
-
- private Connector c = null;
- private Configuration conf = new Configuration();
- private HashMap<Text,BatchWriter> writerMap = new HashMap<Text,BatchWriter>();
-
- @Before
- public void setup() throws Exception {
-
- conf.set(AggregatingRecordReader.START_TOKEN, "<page>");
- conf.set(AggregatingRecordReader.END_TOKEN, "</page>");
- conf.set(WikipediaConfiguration.TABLE_NAME, TABLE_NAME);
- conf.set(WikipediaConfiguration.NUM_PARTITIONS, "1");
- conf.set(WikipediaConfiguration.NUM_GROUPS, "1");
-
- MockInstance i = new MockInstance();
- c = i.getConnector("root", "pass");
- c.tableOperations().delete(METADATA_TABLE_NAME);
- c.tableOperations().delete(TABLE_NAME);
- c.tableOperations().delete(INDEX_TABLE_NAME);
- c.tableOperations().delete(RINDEX_TABLE_NAME);
- c.tableOperations().create(METADATA_TABLE_NAME);
- c.tableOperations().create(TABLE_NAME);
- c.tableOperations().create(INDEX_TABLE_NAME);
- c.tableOperations().create(RINDEX_TABLE_NAME);
-
- writerMap.put(new Text(METADATA_TABLE_NAME), c.createBatchWriter(METADATA_TABLE_NAME, 1000L, 1000L, 1));
- writerMap.put(new Text(TABLE_NAME), c.createBatchWriter(TABLE_NAME, 1000L, 1000L, 1));
- writerMap.put(new Text(INDEX_TABLE_NAME), c.createBatchWriter(INDEX_TABLE_NAME, 1000L, 1000L, 1));
- writerMap.put(new Text(RINDEX_TABLE_NAME), c.createBatchWriter(RINDEX_TABLE_NAME, 1000L, 1000L, 1));
-
- TaskAttemptContext context = ContextFactory.createTaskAttemptContext(conf);
-
- RawLocalFileSystem fs = new RawLocalFileSystem();
- fs.setConf(conf);
-
- URL url = ClassLoader.getSystemResource("enwiki-20110901-001.xml");
- Assert.assertNotNull(url);
- File data = new File(url.toURI());
- Path tmpFile = new Path(data.getAbsolutePath());
-
- // Setup the Mapper
- InputSplit split = new FileSplit(tmpFile, 0, fs.pathToFile(tmpFile).length(), null);
- AggregatingRecordReader rr = new AggregatingRecordReader();
- Path ocPath = new Path(tmpFile, "oc");
- OutputCommitter oc = new FileOutputCommitter(ocPath, context);
- fs.deleteOnExit(ocPath);
- StandaloneStatusReporter sr = new StandaloneStatusReporter();
- rr.initialize(split, context);
- MockAccumuloRecordWriter rw = new MockAccumuloRecordWriter();
- WikipediaMapper mapper = new WikipediaMapper();
-
- // Load data into Mock Accumulo
- Mapper<LongWritable,Text,Text,Mutation>.Context con = ContextFactory.createMapContext(mapper, context, rr, rw, oc, sr, split);
- mapper.run(con);
-
- // Flush and close record writers.
- rw.close(context);
-
- }
-
- private void debugQuery(String tableName) throws Exception {
- Scanner s = c.createScanner(tableName, new Authorizations("all"));
- Range r = new Range();
- s.setRange(r);
- for (Entry<Key,Value> entry : s)
- System.out.println(entry.getKey().toString() + " " + entry.getValue().toString());
- }
-
- public void testViewAllData() throws Exception {
- debugQuery(METADATA_TABLE_NAME);
- debugQuery(TABLE_NAME);
- debugQuery(INDEX_TABLE_NAME);
- debugQuery(RINDEX_TABLE_NAME);
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/GlobalIndexUidTest.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/GlobalIndexUidTest.java b/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/GlobalIndexUidTest.java
deleted file mode 100644
index 6619ede..0000000
--- a/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/GlobalIndexUidTest.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.iterator;
-
-import static org.junit.Assert.assertTrue;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.UUID;
-
-import org.apache.accumulo.core.client.IteratorSetting;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.Combiner;
-import org.apache.accumulo.examples.wikisearch.protobuf.Uid;
-import org.apache.accumulo.examples.wikisearch.protobuf.Uid.List.Builder;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.junit.Before;
-import org.junit.Test;
-
-public class GlobalIndexUidTest {
- private GlobalIndexUidCombiner combiner;
- private List<Value> values;
-
- @Before
- public void setup() throws Exception {
- combiner = new GlobalIndexUidCombiner();
- combiner.init(null, Collections.singletonMap("all", "true"), null);
- values = new ArrayList<Value>();
- }
-
- private Uid.List.Builder createNewUidList() {
- return Uid.List.newBuilder();
- }
-
- @Test
- public void testSingleUid() {
- Builder b = createNewUidList();
- b.setCOUNT(1);
- b.setIGNORE(false);
- b.addUID(UUID.randomUUID().toString());
- Uid.List uidList = b.build();
- Value val = new Value(uidList.toByteArray());
- values.add(val);
- Value result = combiner.reduce(new Key(), values.iterator());
- assertTrue(val.compareTo(result.get()) == 0);
- }
-
- @Test
- public void testLessThanMax() throws Exception {
- List<String> savedUUIDs = new ArrayList<String>();
- for (int i = 0; i < GlobalIndexUidCombiner.MAX - 1; i++) {
- Builder b = createNewUidList();
- b.setIGNORE(false);
- String uuid = UUID.randomUUID().toString();
- savedUUIDs.add(uuid);
- b.setCOUNT(i);
- b.addUID(uuid);
- Uid.List uidList = b.build();
- Value val = new Value(uidList.toByteArray());
- values.add(val);
- }
- Value result = combiner.reduce(new Key(), values.iterator());
- Uid.List resultList = Uid.List.parseFrom(result.get());
- assertTrue(resultList.getIGNORE() == false);
- assertTrue(resultList.getUIDCount() == (GlobalIndexUidCombiner.MAX - 1));
- List<String> resultListUUIDs = resultList.getUIDList();
- for (String s : savedUUIDs)
- assertTrue(resultListUUIDs.contains(s));
- }
-
- @Test
- public void testEqualsMax() throws Exception {
- List<String> savedUUIDs = new ArrayList<String>();
- for (int i = 0; i < GlobalIndexUidCombiner.MAX; i++) {
- Builder b = createNewUidList();
- b.setIGNORE(false);
- String uuid = UUID.randomUUID().toString();
- savedUUIDs.add(uuid);
- b.setCOUNT(i);
- b.addUID(uuid);
- Uid.List uidList = b.build();
- Value val = new Value(uidList.toByteArray());
- values.add(val);
- }
- Value result = combiner.reduce(new Key(), values.iterator());
- Uid.List resultList = Uid.List.parseFrom(result.get());
- assertTrue(resultList.getIGNORE() == false);
- assertTrue(resultList.getUIDCount() == (GlobalIndexUidCombiner.MAX));
- List<String> resultListUUIDs = resultList.getUIDList();
- for (String s : savedUUIDs)
- assertTrue(resultListUUIDs.contains(s));
- }
-
- @Test
- public void testMoreThanMax() throws Exception {
- List<String> savedUUIDs = new ArrayList<String>();
- for (int i = 0; i < GlobalIndexUidCombiner.MAX + 10; i++) {
- Builder b = createNewUidList();
- b.setIGNORE(false);
- String uuid = UUID.randomUUID().toString();
- savedUUIDs.add(uuid);
- b.setCOUNT(1);
- b.addUID(uuid);
- Uid.List uidList = b.build();
- Value val = new Value(uidList.toByteArray());
- values.add(val);
- }
- Value result = combiner.reduce(new Key(), values.iterator());
- Uid.List resultList = Uid.List.parseFrom(result.get());
- assertTrue(resultList.getIGNORE() == true);
- assertTrue(resultList.getUIDCount() == 0);
- assertTrue(resultList.getCOUNT() == (GlobalIndexUidCombiner.MAX + 10));
- }
-
- @Test
- public void testSeenIgnore() throws Exception {
- Builder b = createNewUidList();
- b.setIGNORE(true);
- b.setCOUNT(0);
- Uid.List uidList = b.build();
- Value val = new Value(uidList.toByteArray());
- values.add(val);
- b = createNewUidList();
- b.setIGNORE(false);
- b.setCOUNT(1);
- b.addUID(UUID.randomUUID().toString());
- uidList = b.build();
- val = new Value(uidList.toByteArray());
- values.add(val);
- Value result = combiner.reduce(new Key(), values.iterator());
- Uid.List resultList = Uid.List.parseFrom(result.get());
- assertTrue(resultList.getIGNORE() == true);
- assertTrue(resultList.getUIDCount() == 0);
- assertTrue(resultList.getCOUNT() == 1);
- }
-
- @Test
- public void testInvalidValueType() throws Exception {
- Combiner comb = new GlobalIndexUidCombiner();
- IteratorSetting setting = new IteratorSetting(1, GlobalIndexUidCombiner.class);
- GlobalIndexUidCombiner.setCombineAllColumns(setting, true);
- GlobalIndexUidCombiner.setLossyness(setting, true);
- comb.init(null, setting.getOptions(), null);
- Logger.getLogger(GlobalIndexUidCombiner.class).setLevel(Level.OFF);
- Value val = new Value(UUID.randomUUID().toString().getBytes());
- values.add(val);
- Value result = comb.reduce(new Key(), values.iterator());
- Uid.List resultList = Uid.List.parseFrom(result.get());
- assertTrue(resultList.getIGNORE() == false);
- assertTrue(resultList.getUIDCount() == 0);
- assertTrue(resultList.getCOUNT() == 0);
- }
-
- @Test
- public void testCount() throws Exception {
- UUID uuid = UUID.randomUUID();
- // Collect the same UUID five times.
- for (int i = 0; i < 5; i++) {
- Builder b = createNewUidList();
- b.setCOUNT(1);
- b.setIGNORE(false);
- b.addUID(uuid.toString());
- Uid.List uidList = b.build();
- Value val = new Value(uidList.toByteArray());
- values.add(val);
- }
- Value result = combiner.reduce(new Key(), values.iterator());
- Uid.List resultList = Uid.List.parseFrom(result.get());
- assertTrue(resultList.getIGNORE() == false);
- assertTrue(resultList.getUIDCount() == 1);
- assertTrue(resultList.getCOUNT() == 5);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/TextIndexTest.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/TextIndexTest.java b/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/TextIndexTest.java
deleted file mode 100644
index 7297b5a..0000000
--- a/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/TextIndexTest.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.iterator;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import junit.framework.Assert;
-
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.examples.wikisearch.protobuf.TermWeight;
-import org.apache.accumulo.examples.wikisearch.protobuf.TermWeight.Info.Builder;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.google.protobuf.InvalidProtocolBufferException;
-
-public class TextIndexTest {
- private TextIndexCombiner combiner;
- private List<Value> values;
-
- @Before
- public void setup() throws Exception {
- combiner = new TextIndexCombiner();
- combiner.init(null, Collections.singletonMap("all", "true"), null);
- values = new ArrayList<Value>();
- }
-
- @After
- public void cleanup() {
-
- }
-
- private TermWeight.Info.Builder createBuilder() {
- return TermWeight.Info.newBuilder();
- }
-
- @Test
- public void testSingleValue() throws InvalidProtocolBufferException {
- Builder builder = createBuilder();
- builder.addWordOffset(1);
- builder.addWordOffset(5);
- builder.setNormalizedTermFrequency(0.1f);
-
- values.add(new Value(builder.build().toByteArray()));
-
- Value result = combiner.reduce(new Key(), values.iterator());
-
- TermWeight.Info info = TermWeight.Info.parseFrom(result.get());
-
- Assert.assertTrue(info.getNormalizedTermFrequency() == 0.1f);
-
- List<Integer> offsets = info.getWordOffsetList();
- Assert.assertTrue(offsets.size() == 2);
- Assert.assertTrue(offsets.get(0) == 1);
- Assert.assertTrue(offsets.get(1) == 5);
- }
-
- @Test
- public void testAggregateTwoValues() throws InvalidProtocolBufferException {
- Builder builder = createBuilder();
- builder.addWordOffset(1);
- builder.addWordOffset(5);
- builder.setNormalizedTermFrequency(0.1f);
-
- values.add(new Value(builder.build().toByteArray()));
-
- builder = createBuilder();
- builder.addWordOffset(3);
- builder.setNormalizedTermFrequency(0.05f);
-
- values.add(new Value(builder.build().toByteArray()));
-
- Value result = combiner.reduce(new Key(), values.iterator());
-
- TermWeight.Info info = TermWeight.Info.parseFrom(result.get());
-
- Assert.assertTrue(info.getNormalizedTermFrequency() == 0.15f);
-
- List<Integer> offsets = info.getWordOffsetList();
- Assert.assertTrue(offsets.size() == 3);
- Assert.assertTrue(offsets.get(0) == 1);
- Assert.assertTrue(offsets.get(1) == 3);
- Assert.assertTrue(offsets.get(2) == 5);
- }
-
- @Test
- public void testAggregateManyValues() throws InvalidProtocolBufferException {
- Builder builder = createBuilder();
- builder.addWordOffset(13);
- builder.addWordOffset(15);
- builder.addWordOffset(19);
- builder.setNormalizedTermFrequency(0.12f);
-
- values.add(new Value(builder.build().toByteArray()));
-
- builder = createBuilder();
- builder.addWordOffset(1);
- builder.addWordOffset(5);
- builder.setNormalizedTermFrequency(0.1f);
-
- values.add(new Value(builder.build().toByteArray()));
-
- builder = createBuilder();
- builder.addWordOffset(3);
- builder.setNormalizedTermFrequency(0.05f);
-
- values.add(new Value(builder.build().toByteArray()));
-
- Value result = combiner.reduce(new Key(), values.iterator());
-
- TermWeight.Info info = TermWeight.Info.parseFrom(result.get());
-
- Assert.assertTrue(info.getNormalizedTermFrequency() == 0.27f);
-
- List<Integer> offsets = info.getWordOffsetList();
- Assert.assertTrue(offsets.size() == 6);
- Assert.assertTrue(offsets.get(0) == 1);
- Assert.assertTrue(offsets.get(1) == 3);
- Assert.assertTrue(offsets.get(2) == 5);
- Assert.assertTrue(offsets.get(3) == 13);
- Assert.assertTrue(offsets.get(4) == 15);
- Assert.assertTrue(offsets.get(5) == 19);
- }
-
- @Test
- public void testEmptyValue() throws InvalidProtocolBufferException {
- Builder builder = createBuilder();
- builder.addWordOffset(13);
- builder.addWordOffset(15);
- builder.addWordOffset(19);
- builder.setNormalizedTermFrequency(0.12f);
-
- values.add(new Value("".getBytes()));
- values.add(new Value(builder.build().toByteArray()));
- values.add(new Value("".getBytes()));
-
- builder = createBuilder();
- builder.addWordOffset(1);
- builder.addWordOffset(5);
- builder.setNormalizedTermFrequency(0.1f);
-
- values.add(new Value(builder.build().toByteArray()));
- values.add(new Value("".getBytes()));
-
- builder = createBuilder();
- builder.addWordOffset(3);
- builder.setNormalizedTermFrequency(0.05f);
-
- values.add(new Value(builder.build().toByteArray()));
- values.add(new Value("".getBytes()));
-
- Value result = combiner.reduce(new Key(), values.iterator());
-
- TermWeight.Info info = TermWeight.Info.parseFrom(result.get());
-
- Assert.assertTrue(info.getNormalizedTermFrequency() == 0.27f);
-
- List<Integer> offsets = info.getWordOffsetList();
- Assert.assertTrue(offsets.size() == 6);
- Assert.assertTrue(offsets.get(0) == 1);
- Assert.assertTrue(offsets.get(1) == 3);
- Assert.assertTrue(offsets.get(2) == 5);
- Assert.assertTrue(offsets.get(3) == 13);
- Assert.assertTrue(offsets.get(4) == 15);
- Assert.assertTrue(offsets.get(5) == 19);
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8db62992/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/normalizer/testNumberNormalizer.java
----------------------------------------------------------------------
diff --git a/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/normalizer/testNumberNormalizer.java b/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/normalizer/testNumberNormalizer.java
deleted file mode 100644
index 470633c..0000000
--- a/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/normalizer/testNumberNormalizer.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.wikisearch.normalizer;
-
-import static org.junit.Assert.assertTrue;
-
-import org.apache.accumulo.examples.wikisearch.normalizer.NumberNormalizer;
-import org.junit.Test;
-
-public class testNumberNormalizer {
-
- @Test
- public void test1() throws Exception {
- NumberNormalizer nn = new NumberNormalizer();
-
- String n1 = nn.normalizeFieldValue(null, "1");
- String n2 = nn.normalizeFieldValue(null, "1.00000000");
-
- assertTrue(n1.compareTo(n2) < 0);
-
- }
-
- @Test
- public void test2() {
- NumberNormalizer nn = new NumberNormalizer();
-
- String n1 = nn.normalizeFieldValue(null, "-1.0");
- String n2 = nn.normalizeFieldValue(null, "1.0");
-
- assertTrue(n1.compareTo(n2) < 0);
-
- }
-
- @Test
- public void test3() {
- NumberNormalizer nn = new NumberNormalizer();
- String n1 = nn.normalizeFieldValue(null, "-0.0001");
- String n2 = nn.normalizeFieldValue(null, "0");
- String n3 = nn.normalizeFieldValue(null, "0.00001");
-
- assertTrue((n1.compareTo(n2) < 0) && (n2.compareTo(n3) < 0));
- }
-
- @Test
- public void test4() {
- NumberNormalizer nn = new NumberNormalizer();
- String nn1 = nn.normalizeFieldValue(null, Integer.toString(Integer.MAX_VALUE));
- String nn2 = nn.normalizeFieldValue(null, Integer.toString(Integer.MAX_VALUE - 1));
-
- assertTrue((nn2.compareTo(nn1) < 0));
-
- }
-
- @Test
- public void test5() {
- NumberNormalizer nn = new NumberNormalizer();
- String nn1 = nn.normalizeFieldValue(null, "-0.001");
- String nn2 = nn.normalizeFieldValue(null, "-0.0009");
- String nn3 = nn.normalizeFieldValue(null, "-0.00090");
-
- assertTrue((nn3.compareTo(nn2) == 0) && (nn2.compareTo(nn1) > 0));
-
- }
-
- @Test
- public void test6() {
- NumberNormalizer nn = new NumberNormalizer();
- String nn1 = nn.normalizeFieldValue(null, "00.0");
- String nn2 = nn.normalizeFieldValue(null, "0");
- String nn3 = nn.normalizeFieldValue(null, "0.0");
-
- assertTrue((nn3.compareTo(nn2) == 0) && (nn2.compareTo(nn1) == 0));
-
- }
-
-}
[39/39] git commit: Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by ec...@apache.org.
Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6c80fa50
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6c80fa50
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6c80fa50
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 6c80fa5038c906ee2f9875c0f3a9c16b9248ecba
Parents: f478737 ad9ee5e
Author: Eric Newton <er...@gmail.com>
Authored: Tue Nov 26 10:48:03 2013 -0500
Committer: Eric Newton <er...@gmail.com>
Committed: Tue Nov 26 10:48:03 2013 -0500
----------------------------------------------------------------------
----------------------------------------------------------------------