You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by uj...@apache.org on 2014/02/06 06:39:49 UTC
[10/50] git commit: ACCUMULO-375 added compression and increased the
minimum split size
ACCUMULO-375 added compression and increased the minimum split size
git-svn-id: https://svn.apache.org/repos/asf/incubator/accumulo/branches/1.4@1241940 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/commit/57bf9cf3
Tree: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/tree/57bf9cf3
Diff: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/diff/57bf9cf3
Branch: refs/heads/1.4.5-SNAPSHOT
Commit: 57bf9cf34b891c12d4361e8d5224a2aa38b63a6c
Parents: 266455b
Author: Adam Fuchs <af...@apache.org>
Authored: Wed Feb 8 15:37:04 2012 +0000
Committer: Adam Fuchs <af...@apache.org>
Committed: Wed Feb 8 15:37:04 2012 +0000
----------------------------------------------------------------------
.../examples/wikisearch/ingest/WikipediaPartitionedIngester.java | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/57bf9cf3/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaPartitionedIngester.java
----------------------------------------------------------------------
diff --git a/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaPartitionedIngester.java b/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaPartitionedIngester.java
index 43f5e29..5571290 100644
--- a/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaPartitionedIngester.java
+++ b/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaPartitionedIngester.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
@@ -178,6 +179,8 @@ public class WikipediaPartitionedIngester extends Configured implements Tool {
partitionerJob.setOutputFormatClass(SequenceFileOutputFormat.class);
Path outputDir = WikipediaConfiguration.getPartitionedArticlesPath(partitionerConf);
SequenceFileOutputFormat.setOutputPath(partitionerJob, outputDir);
+ SequenceFileOutputFormat.setCompressOutput(partitionerJob, true);
+ SequenceFileOutputFormat.setOutputCompressionType(partitionerJob, CompressionType.RECORD);
return partitionerJob.waitForCompletion(true) ? 0 : 1;
}
@@ -209,6 +212,7 @@ public class WikipediaPartitionedIngester extends Configured implements Tool {
// setup input format
ingestJob.setInputFormatClass(SequenceFileInputFormat.class);
SequenceFileInputFormat.setInputPaths(ingestJob, WikipediaConfiguration.getPartitionedArticlesPath(ingestConf));
+ SequenceFileInputFormat.setMinInputSplitSize(ingestJob, 1l << 28);
// setup output format
ingestJob.setMapOutputKeyClass(Text.class);