You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by uj...@apache.org on 2014/03/19 17:08:26 UTC
[16/50] [abbrv] git commit: ACCUMULO-469 added license headers
ACCUMULO-469 added license headers
git-svn-id: https://svn.apache.org/repos/asf/incubator/accumulo/branches/1.4@1302533 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/commit/66bb45c9
Tree: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/tree/66bb45c9
Diff: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/diff/66bb45c9
Branch: refs/heads/master
Commit: 66bb45c9b23bb4a91fa2c2aa8d12b320a5995016
Parents: e1dfeb6
Author: Billie Rinaldi <bi...@apache.org>
Authored: Mon Mar 19 16:34:43 2012 +0000
Committer: Billie Rinaldi <bi...@apache.org>
Committed: Mon Mar 19 16:34:43 2012 +0000
----------------------------------------------------------------------
.../wikisearch/ingest/LRUOutputCombiner.java | 24 ++++++++--
.../output/BufferingRFileRecordWriter.java | 50 ++++++++++++--------
.../output/SortingRFileOutputFormat.java | 20 +++++++-
.../ingest/WikipediaInputSplitTest.java | 39 ++++++++++-----
4 files changed, 93 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/66bb45c9/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/LRUOutputCombiner.java
----------------------------------------------------------------------
diff --git a/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/LRUOutputCombiner.java b/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/LRUOutputCombiner.java
index e641f36..7d7b6dc 100644
--- a/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/LRUOutputCombiner.java
+++ b/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/LRUOutputCombiner.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.accumulo.examples.wikisearch.ingest;
import java.util.LinkedHashMap;
@@ -6,14 +22,12 @@ import java.util.Map;
public class LRUOutputCombiner<Key,Value> extends LinkedHashMap<Key,Value> {
private static final long serialVersionUID = 1L;
-
- public static abstract class Fold <Value>
- {
+
+ public static abstract class Fold<Value> {
public abstract Value fold(Value oldValue, Value newValue);
}
- public static abstract class Output<Key,Value>
- {
+ public static abstract class Output<Key,Value> {
public abstract void output(Key key, Value value);
}
http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/66bb45c9/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/output/BufferingRFileRecordWriter.java
----------------------------------------------------------------------
diff --git a/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/output/BufferingRFileRecordWriter.java b/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/output/BufferingRFileRecordWriter.java
index 579bbe1..9b663de 100644
--- a/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/output/BufferingRFileRecordWriter.java
+++ b/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/output/BufferingRFileRecordWriter.java
@@ -1,10 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.accumulo.examples.wikisearch.output;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
-import java.util.TreeMap;
import java.util.Map.Entry;
+import java.util.TreeMap;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.data.ColumnUpdate;
@@ -31,12 +47,10 @@ final class BufferingRFileRecordWriter extends RecordWriter<Text,Mutation> {
private Map<Text,TreeMap<Key,Value>> buffers = new HashMap<Text,TreeMap<Key,Value>>();
private Map<Text,Long> bufferSizes = new HashMap<Text,Long>();
-
- private TreeMap<Key,Value> getBuffer(Text tablename)
- {
+
+ private TreeMap<Key,Value> getBuffer(Text tablename) {
TreeMap<Key,Value> buffer = buffers.get(tablename);
- if(buffer == null)
- {
+ if (buffer == null) {
buffer = new TreeMap<Key,Value>();
buffers.put(tablename, buffer);
bufferSizes.put(tablename, 0l);
@@ -44,14 +58,11 @@ final class BufferingRFileRecordWriter extends RecordWriter<Text,Mutation> {
return buffer;
}
- private Text getLargestTablename()
- {
+ private Text getLargestTablename() {
long max = 0;
Text table = null;
- for(Entry<Text,Long> e:bufferSizes.entrySet())
- {
- if(e.getValue() > max)
- {
+ for (Entry<Text,Long> e : bufferSizes.entrySet()) {
+ if (e.getValue() > max) {
max = e.getValue();
table = e.getKey();
}
@@ -59,10 +70,9 @@ final class BufferingRFileRecordWriter extends RecordWriter<Text,Mutation> {
return table;
}
- private void flushLargestTable() throws IOException
- {
+ private void flushLargestTable() throws IOException {
Text tablename = getLargestTablename();
- if(tablename == null)
+ if (tablename == null)
return;
long bufferSize = bufferSizes.get(tablename);
TreeMap<Key,Value> buffer = buffers.get(tablename);
@@ -98,7 +108,7 @@ final class BufferingRFileRecordWriter extends RecordWriter<Text,Mutation> {
@Override
public void close(TaskAttemptContext arg0) throws IOException, InterruptedException {
- while(size > 0)
+ while (size > 0)
flushLargestTable();
}
@@ -106,9 +116,9 @@ final class BufferingRFileRecordWriter extends RecordWriter<Text,Mutation> {
public void write(Text table, Mutation mutation) throws IOException, InterruptedException {
TreeMap<Key,Value> buffer = getBuffer(table);
int mutationSize = 0;
- for(ColumnUpdate update: mutation.getUpdates())
- {
- Key k = new Key(mutation.getRow(),update.getColumnFamily(),update.getColumnQualifier(),update.getColumnVisibility(),update.getTimestamp(),update.isDeleted());
+ for (ColumnUpdate update : mutation.getUpdates()) {
+ Key k = new Key(mutation.getRow(), update.getColumnFamily(), update.getColumnQualifier(), update.getColumnVisibility(), update.getTimestamp(),
+ update.isDeleted());
Value v = new Value(update.getValue());
// TODO account for object overhead
mutationSize += k.getSize();
@@ -121,7 +131,7 @@ final class BufferingRFileRecordWriter extends RecordWriter<Text,Mutation> {
// TODO use a MutableLong instead
bufferSize += mutationSize;
bufferSizes.put(table, bufferSize);
-
+
while (size >= maxSize) {
flushLargestTable();
}
http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/66bb45c9/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/output/SortingRFileOutputFormat.java
----------------------------------------------------------------------
diff --git a/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/output/SortingRFileOutputFormat.java b/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/output/SortingRFileOutputFormat.java
index 2738e2c..1fa8fdc 100644
--- a/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/output/SortingRFileOutputFormat.java
+++ b/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/output/SortingRFileOutputFormat.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.accumulo.examples.wikisearch.output;
import java.io.IOException;
@@ -14,9 +30,9 @@ import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
public class SortingRFileOutputFormat extends OutputFormat<Text,Mutation> {
-
+
// private static final Logger log = Logger.getLogger(SortingRFileOutputFormat.class);
-
+
public static final String PATH_NAME = "sortingrfileoutputformat.path";
public static final String MAX_BUFFER_SIZE = "sortingrfileoutputformat.max.buffer.size";
http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/66bb45c9/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaInputSplitTest.java
----------------------------------------------------------------------
diff --git a/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaInputSplitTest.java b/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaInputSplitTest.java
index 3da5a62..f6b2791 100644
--- a/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaInputSplitTest.java
+++ b/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaInputSplitTest.java
@@ -1,9 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.accumulo.examples.wikisearch.ingest;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInput;
-import java.io.DataOutput;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
@@ -17,20 +32,19 @@ import org.junit.Test;
public class WikipediaInputSplitTest {
@Test
- public void testSerialization() throws IOException
- {
+ public void testSerialization() throws IOException {
Path testPath = new Path("/foo/bar");
- String [] hosts = new String [2];
+ String[] hosts = new String[2];
hosts[0] = "abcd";
hosts[1] = "efgh";
- FileSplit fSplit = new FileSplit(testPath,1,2,hosts);
- WikipediaInputSplit split = new WikipediaInputSplit(fSplit,7);
+ FileSplit fSplit = new FileSplit(testPath, 1, 2, hosts);
+ WikipediaInputSplit split = new WikipediaInputSplit(fSplit, 7);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
ObjectOutputStream out = new ObjectOutputStream(baos);
split.write(out);
- out.close();
+ out.close();
baos.close();
-
+
ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
DataInput in = new ObjectInputStream(bais);
@@ -45,12 +59,11 @@ public class WikipediaInputSplitTest {
Assert.assertTrue(fSplit.getPath().equals(fSplit2.getPath()));
Assert.assertTrue(fSplit.getStart() == fSplit2.getStart());
Assert.assertTrue(fSplit.getLength() == fSplit2.getLength());
-
- String [] hosts2 = fSplit2.getLocations();
+
+ String[] hosts2 = fSplit2.getLocations();
Assert.assertEquals(hosts.length, hosts2.length);
- for(int i = 0; i < hosts.length; i++)
- {
- Assert.assertEquals(hosts[i],hosts2[i]);
+ for (int i = 0; i < hosts.length; i++) {
+ Assert.assertEquals(hosts[i], hosts2[i]);
}
}
}