You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jena.apache.org by rv...@apache.org on 2014/10/20 16:47:49 UTC
[27/50] [abbrv] Get Jena Hadoop RDF Tools building
http://git-wip-us.apache.org/repos/asf/jena/blob/92fb810a/jena-hadoop-rdf/hadoop-rdf-stats/src/main/java/org/apache/jena/hadoop/rdf/stats/jobs/JobFactory.java
----------------------------------------------------------------------
diff --git a/jena-hadoop-rdf/hadoop-rdf-stats/src/main/java/org/apache/jena/hadoop/rdf/stats/jobs/JobFactory.java b/jena-hadoop-rdf/hadoop-rdf-stats/src/main/java/org/apache/jena/hadoop/rdf/stats/jobs/JobFactory.java
new file mode 100644
index 0000000..10ba65d
--- /dev/null
+++ b/jena-hadoop-rdf/hadoop-rdf-stats/src/main/java/org/apache/jena/hadoop/rdf/stats/jobs/JobFactory.java
@@ -0,0 +1,757 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.hadoop.rdf.stats.jobs;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.SequenceFile.CompressionType;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.compress.BZip2Codec;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.NLineInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.jena.hadoop.rdf.io.input.NQuadsInputFormat;
+import org.apache.jena.hadoop.rdf.io.input.NTriplesInputFormat;
+import org.apache.jena.hadoop.rdf.io.input.QuadsInputFormat;
+import org.apache.jena.hadoop.rdf.io.input.TriplesInputFormat;
+import org.apache.jena.hadoop.rdf.io.input.TriplesOrQuadsInputFormat;
+import org.apache.jena.hadoop.rdf.io.output.NQuadsOutputFormat;
+import org.apache.jena.hadoop.rdf.io.output.NTriplesNodeOutputFormat;
+import org.apache.jena.hadoop.rdf.io.output.NTriplesOutputFormat;
+import org.apache.jena.hadoop.rdf.mapreduce.KeyMapper;
+import org.apache.jena.hadoop.rdf.mapreduce.RdfMapReduceConstants;
+import org.apache.jena.hadoop.rdf.mapreduce.TextCountReducer;
+import org.apache.jena.hadoop.rdf.mapreduce.characteristics.CharacteristicSetReducer;
+import org.apache.jena.hadoop.rdf.mapreduce.characteristics.QuadCharacteristicSetGeneratingReducer;
+import org.apache.jena.hadoop.rdf.mapreduce.characteristics.TripleCharacteristicSetGeneratingReducer;
+import org.apache.jena.hadoop.rdf.mapreduce.count.NodeCountReducer;
+import org.apache.jena.hadoop.rdf.mapreduce.count.QuadNodeCountMapper;
+import org.apache.jena.hadoop.rdf.mapreduce.count.TripleNodeCountMapper;
+import org.apache.jena.hadoop.rdf.mapreduce.count.datatypes.QuadDataTypeCountMapper;
+import org.apache.jena.hadoop.rdf.mapreduce.count.datatypes.TripleDataTypeCountMapper;
+import org.apache.jena.hadoop.rdf.mapreduce.count.namespaces.QuadNamespaceCountMapper;
+import org.apache.jena.hadoop.rdf.mapreduce.count.namespaces.TripleNamespaceCountMapper;
+import org.apache.jena.hadoop.rdf.mapreduce.count.positional.QuadObjectCountMapper;
+import org.apache.jena.hadoop.rdf.mapreduce.count.positional.TripleObjectCountMapper;
+import org.apache.jena.hadoop.rdf.mapreduce.filter.positional.QuadFilterByPredicateMapper;
+import org.apache.jena.hadoop.rdf.mapreduce.filter.positional.TripleFilterByPredicateUriMapper;
+import org.apache.jena.hadoop.rdf.mapreduce.group.QuadGroupBySubjectMapper;
+import org.apache.jena.hadoop.rdf.mapreduce.group.TripleGroupBySubjectMapper;
+import org.apache.jena.hadoop.rdf.types.CharacteristicSetWritable;
+import org.apache.jena.hadoop.rdf.types.NodeWritable;
+import org.apache.jena.hadoop.rdf.types.QuadWritable;
+import org.apache.jena.hadoop.rdf.types.TripleWritable;
+
+import com.hp.hpl.jena.vocabulary.RDF;
+
+/**
+ * Factory that can produce {@link Job} instances for computing various RDF
+ * statistics
+ *
+ *
+ *
+ */
+public class JobFactory {
+
+ /**
+ * Private constructor prevents instantiation
+ */
+ private JobFactory() {
+ }
+
+ /**
+ * Gets a job for computing node counts on RDF triple inputs
+ *
+ * @param config
+ * Configuration
+ * @param inputPaths
+ * Input paths
+ * @param outputPath
+ * Output path
+ * @return Job
+ * @throws IOException
+ */
+ public static Job getTripleNodeCountJob(Configuration config, String[] inputPaths, String outputPath) throws IOException {
+ Job job = Job.getInstance(config);
+ job.setJarByClass(JobFactory.class);
+ job.setJobName("RDF Triples Node Usage Count");
+
+ // Map/Reduce classes
+ job.setMapperClass(TripleNodeCountMapper.class);
+ job.setMapOutputKeyClass(NodeWritable.class);
+ job.setMapOutputValueClass(LongWritable.class);
+ job.setReducerClass(NodeCountReducer.class);
+
+ // Input and Output
+ job.setInputFormatClass(TriplesInputFormat.class);
+ job.setOutputFormatClass(NTriplesNodeOutputFormat.class);
+ FileInputFormat.setInputPaths(job, StringUtils.arrayToString(inputPaths));
+ FileOutputFormat.setOutputPath(job, new Path(outputPath));
+
+ return job;
+ }
+
+ /**
+ * Gets a job for computing node counts on RDF quad inputs
+ *
+ * @param config
+ * Configuration
+ * @param inputPaths
+ * Input paths
+ * @param outputPath
+ * Output path
+ * @return Job
+ * @throws IOException
+ */
+ public static Job getQuadNodeCountJob(Configuration config, String[] inputPaths, String outputPath) throws IOException {
+ Job job = Job.getInstance(config);
+ job.setJarByClass(JobFactory.class);
+ job.setJobName("RDF Quads Node Usage Count");
+
+ // Map/Reduce classes
+ job.setMapperClass(QuadNodeCountMapper.class);
+ job.setMapOutputKeyClass(NodeWritable.class);
+ job.setMapOutputValueClass(LongWritable.class);
+ job.setReducerClass(NodeCountReducer.class);
+
+ // Input and Output
+ job.setInputFormatClass(QuadsInputFormat.class);
+ job.setOutputFormatClass(NTriplesNodeOutputFormat.class);
+ FileInputFormat.setInputPaths(job, StringUtils.arrayToString(inputPaths));
+ FileOutputFormat.setOutputPath(job, new Path(outputPath));
+
+ return job;
+ }
+
+ /**
+ * Gets a job for computing node counts on RDF triple and/or quad inputs
+ *
+ * @param config
+ * Configuration
+ * @param inputPaths
+ * Input paths
+ * @param outputPath
+ * Output path
+ * @return Job
+ * @throws IOException
+ */
+ public static Job getNodeCountJob(Configuration config, String[] inputPaths, String outputPath) throws IOException {
+ Job job = Job.getInstance(config);
+ job.setJarByClass(JobFactory.class);
+ job.setJobName("RDF Node Usage Count");
+
+ // Map/Reduce classes
+ job.setMapperClass(QuadNodeCountMapper.class);
+ job.setMapOutputKeyClass(NodeWritable.class);
+ job.setMapOutputValueClass(LongWritable.class);
+ job.setReducerClass(NodeCountReducer.class);
+
+ // Input and Output
+ job.setInputFormatClass(TriplesOrQuadsInputFormat.class);
+ job.setOutputFormatClass(NTriplesNodeOutputFormat.class);
+ FileInputFormat.setInputPaths(job, StringUtils.arrayToString(inputPaths));
+ FileOutputFormat.setOutputPath(job, new Path(outputPath));
+
+ return job;
+ }
+
+ /**
+ * Gets a sequence of jobs that can be used to compute characteristic sets
+ * for RDF triples
+ *
+ * @param config
+ * Configuration
+ * @param inputPaths
+ * Input paths
+ * @param intermediateOutputPath
+ * Intermediate output path
+ * @param outputPath
+ * Final output path
+ * @return Sequence of jobs
+ * @throws IOException
+ */
+ public static Job[] getTripleCharacteristicSetJobs(Configuration config, String[] inputPaths, String intermediateOutputPath,
+ String outputPath) throws IOException {
+ Job[] jobs = new Job[2];
+
+ Job job = Job.getInstance(config);
+ job.setJarByClass(JobFactory.class);
+ job.setJobName("RDF Triples Characteristic Set (Generation)");
+
+ // Map/Reduce classes
+ job.setMapperClass(TripleGroupBySubjectMapper.class);
+ job.setMapOutputKeyClass(NodeWritable.class);
+ job.setMapOutputValueClass(TripleWritable.class);
+ job.setReducerClass(TripleCharacteristicSetGeneratingReducer.class);
+ job.setOutputKeyClass(CharacteristicSetWritable.class);
+ job.setOutputValueClass(NullWritable.class);
+
+ // Input and Output
+ job.setInputFormatClass(TriplesInputFormat.class);
+ job.setOutputFormatClass(SequenceFileOutputFormat.class);
+ FileInputFormat.setInputPaths(job, StringUtils.arrayToString(inputPaths));
+ FileOutputFormat.setOutputPath(job, new Path(intermediateOutputPath));
+ SequenceFileOutputFormat.setCompressOutput(job, true);
+ FileOutputFormat.setOutputCompressorClass(job, BZip2Codec.class);
+ SequenceFileOutputFormat.setOutputCompressionType(job, CompressionType.BLOCK);
+
+ jobs[0] = job;
+
+ job = Job.getInstance(config);
+ job.setJarByClass(JobFactory.class);
+ job.setJobName("RDF Triples Characteristic Set (Reduction)");
+
+ // Map/Reduce classes
+ job.setMapperClass(KeyMapper.class);
+ job.setMapOutputKeyClass(CharacteristicSetWritable.class);
+ job.setMapOutputValueClass(CharacteristicSetWritable.class);
+ job.setReducerClass(CharacteristicSetReducer.class);
+ job.setOutputKeyClass(CharacteristicSetWritable.class);
+ job.setOutputValueClass(CharacteristicSetWritable.class);
+
+ // Input and Output
+ job.setInputFormatClass(SequenceFileInputFormat.class);
+ job.setOutputFormatClass(TextOutputFormat.class);
+ FileInputFormat.setInputPaths(job, intermediateOutputPath);
+ FileOutputFormat.setOutputPath(job, new Path(outputPath));
+
+ jobs[1] = job;
+ return jobs;
+ }
+
+ /**
+ * Gets a sequence of jobs that can be used to compute characteristic sets
+ * for RDF quads
+ *
+ * @param config
+ * Configuration
+ * @param inputPaths
+ * Input paths
+ * @param intermediateOutputPath
+ * Intermediate output path
+ * @param outputPath
+ * Final output path
+ * @return Sequence of jobs
+ * @throws IOException
+ */
+ public static Job[] getQuadCharacteristicSetJobs(Configuration config, String[] inputPaths, String intermediateOutputPath,
+ String outputPath) throws IOException {
+ Job[] jobs = new Job[2];
+
+ Job job = Job.getInstance(config);
+ job.setJarByClass(JobFactory.class);
+ job.setJobName("RDF Quads Characteristic Set (Generation)");
+
+ // Map/Reduce classes
+ job.setMapperClass(QuadGroupBySubjectMapper.class);
+ job.setMapOutputKeyClass(NodeWritable.class);
+ job.setMapOutputValueClass(QuadWritable.class);
+ job.setReducerClass(QuadCharacteristicSetGeneratingReducer.class);
+ job.setOutputKeyClass(CharacteristicSetWritable.class);
+ job.setOutputValueClass(NullWritable.class);
+
+ // Input and Output
+ job.setInputFormatClass(QuadsInputFormat.class);
+ job.setOutputFormatClass(SequenceFileOutputFormat.class);
+ FileInputFormat.setInputPaths(job, StringUtils.arrayToString(inputPaths));
+ FileOutputFormat.setOutputPath(job, new Path(intermediateOutputPath));
+ SequenceFileOutputFormat.setCompressOutput(job, true);
+ FileOutputFormat.setOutputCompressorClass(job, BZip2Codec.class);
+ SequenceFileOutputFormat.setOutputCompressionType(job, CompressionType.BLOCK);
+
+ jobs[0] = job;
+
+ job = Job.getInstance(config);
+ job.setJarByClass(JobFactory.class);
+ job.setJobName("RDF Quads Characteristic Set (Reduction)");
+
+ // Map/Reduce classes
+ job.setMapperClass(KeyMapper.class);
+ job.setMapOutputKeyClass(CharacteristicSetWritable.class);
+ job.setMapOutputValueClass(CharacteristicSetWritable.class);
+ job.setReducerClass(CharacteristicSetReducer.class);
+ job.setOutputKeyClass(CharacteristicSetWritable.class);
+ job.setOutputValueClass(CharacteristicSetWritable.class);
+
+ // Input and Output
+ job.setInputFormatClass(SequenceFileInputFormat.class);
+ job.setOutputFormatClass(TextOutputFormat.class);
+ FileInputFormat.setInputPaths(job, intermediateOutputPath);
+ FileOutputFormat.setOutputPath(job, new Path(outputPath));
+
+ jobs[1] = job;
+ return jobs;
+ }
+
+ /**
+ * Gets a sequence of jobs that can be used to compute characteristic sets
+ * for RDF triple and/or quad inputs
+ *
+ * @param config
+ * Configuration
+ * @param inputPaths
+ * Input paths
+ * @param intermediateOutputPath
+ * Intermediate output path
+ * @param outputPath
+ * Final output path
+ * @return Sequence of jobs
+ * @throws IOException
+ */
+ public static Job[] getCharacteristicSetJobs(Configuration config, String[] inputPaths, String intermediateOutputPath,
+ String outputPath) throws IOException {
+ Job[] jobs = new Job[2];
+
+ Job job = Job.getInstance(config);
+ job.setJarByClass(JobFactory.class);
+ job.setJobName("RDF Characteristic Set (Generation)");
+
+ // Map/Reduce classes
+ job.setMapperClass(QuadGroupBySubjectMapper.class);
+ job.setMapOutputKeyClass(NodeWritable.class);
+ job.setMapOutputValueClass(QuadWritable.class);
+ job.setReducerClass(QuadCharacteristicSetGeneratingReducer.class);
+ job.setOutputKeyClass(CharacteristicSetWritable.class);
+ job.setOutputValueClass(NullWritable.class);
+
+ // Input and Output
+ job.setInputFormatClass(TriplesOrQuadsInputFormat.class);
+ job.setOutputFormatClass(SequenceFileOutputFormat.class);
+ FileInputFormat.setInputPaths(job, StringUtils.arrayToString(inputPaths));
+ FileOutputFormat.setOutputPath(job, new Path(intermediateOutputPath));
+ SequenceFileOutputFormat.setCompressOutput(job, true);
+ FileOutputFormat.setOutputCompressorClass(job, BZip2Codec.class);
+ SequenceFileOutputFormat.setOutputCompressionType(job, CompressionType.BLOCK);
+
+ jobs[0] = job;
+
+ job = Job.getInstance(config);
+ job.setJarByClass(JobFactory.class);
+ job.setJobName("RDF Characteristic Set (Reduction)");
+
+ // Map/Reduce classes
+ job.setMapperClass(KeyMapper.class);
+ job.setMapOutputKeyClass(CharacteristicSetWritable.class);
+ job.setMapOutputValueClass(CharacteristicSetWritable.class);
+ job.setReducerClass(CharacteristicSetReducer.class);
+ job.setOutputKeyClass(CharacteristicSetWritable.class);
+ job.setOutputValueClass(CharacteristicSetWritable.class);
+
+ // Input and Output
+ job.setInputFormatClass(SequenceFileInputFormat.class);
+ job.setOutputFormatClass(TextOutputFormat.class);
+ FileInputFormat.setInputPaths(job, intermediateOutputPath);
+ FileOutputFormat.setOutputPath(job, new Path(outputPath));
+
+ jobs[1] = job;
+ return jobs;
+ }
+
+ /**
+ * Gets a job for computing type counts on RDF triple inputs
+ *
+ * @param config
+ * Configuration
+ * @param inputPaths
+ * Input paths
+ * @param intermediateOutputPath
+ * Path for intermediate output which will be all the type
+ * declaration triples present in the inputs
+ * @param outputPath
+ * Output path
+ * @return Job
+ * @throws IOException
+ */
+ public static Job[] getTripleTypeCountJobs(Configuration config, String[] inputPaths, String intermediateOutputPath,
+ String outputPath) throws IOException {
+ Job[] jobs = new Job[2];
+
+ Job job = Job.getInstance(config);
+ job.setJarByClass(JobFactory.class);
+ job.setJobName("RDF Type Triples Extraction");
+
+ // Map/Reduce classes
+ job.getConfiguration().setStrings(RdfMapReduceConstants.FILTER_PREDICATE_URIS, RDF.type.getURI());
+ job.setMapperClass(TripleFilterByPredicateUriMapper.class);
+ job.setMapOutputKeyClass(LongWritable.class);
+ job.setMapOutputValueClass(TripleWritable.class);
+
+ // Input and Output Format
+ job.setInputFormatClass(TriplesInputFormat.class);
+ job.setOutputFormatClass(NTriplesOutputFormat.class);
+ FileInputFormat.setInputPaths(job, StringUtils.arrayToString(inputPaths));
+ FileOutputFormat.setOutputPath(job, new Path(intermediateOutputPath));
+
+ jobs[0] = job;
+
+ // Object Node Usage count job
+ job = Job.getInstance(config);
+ job.setJarByClass(JobFactory.class);
+ job.setJobName("RDF Triples Type Usage Count");
+
+ // Map/Reduce classes
+ job.setMapperClass(TripleObjectCountMapper.class);
+ job.setMapOutputKeyClass(NodeWritable.class);
+ job.setMapOutputValueClass(LongWritable.class);
+ job.setReducerClass(NodeCountReducer.class);
+
+ // Input and Output
+ job.setInputFormatClass(NTriplesInputFormat.class);
+ NLineInputFormat.setNumLinesPerSplit(job, 10000); // TODO Would be
+ // better if this was
+ // intelligently
+ // configured
+ job.setOutputFormatClass(NTriplesNodeOutputFormat.class);
+ FileInputFormat.setInputPaths(job, intermediateOutputPath);
+ FileOutputFormat.setOutputPath(job, new Path(outputPath));
+
+ jobs[1] = job;
+
+ return jobs;
+ }
+
+ /**
+ * Gets a job for computing type counts on RDF quad inputs
+ *
+ * @param config
+ * Configuration
+ * @param inputPaths
+ * Input paths
+ * @param intermediateOutputPath
+ * Path for intermediate output which will be all the type
+ * declaration quads present in the inputs
+ * @param outputPath
+ * Output path
+ * @return Job
+ * @throws IOException
+ */
+ public static Job[] getQuadTypeCountJobs(Configuration config, String[] inputPaths, String intermediateOutputPath,
+ String outputPath) throws IOException {
+ Job[] jobs = new Job[2];
+
+ Job job = Job.getInstance(config);
+ job.setJarByClass(JobFactory.class);
+ job.setJobName("RDF Type Quads Extraction");
+
+ // Map/Reduce classes
+ job.getConfiguration().setStrings(RdfMapReduceConstants.FILTER_PREDICATE_URIS, RDF.type.getURI());
+ job.setMapperClass(QuadFilterByPredicateMapper.class);
+ job.setMapOutputKeyClass(LongWritable.class);
+ job.setMapOutputValueClass(QuadWritable.class);
+
+ // Input and Output Format
+ job.setInputFormatClass(QuadsInputFormat.class);
+ job.setOutputFormatClass(NQuadsOutputFormat.class);
+ FileInputFormat.setInputPaths(job, StringUtils.arrayToString(inputPaths));
+ FileOutputFormat.setOutputPath(job, new Path(intermediateOutputPath));
+
+ jobs[0] = job;
+
+ // Object Node Usage count job
+ job = Job.getInstance(config);
+ job.setJarByClass(JobFactory.class);
+ job.setJobName("RDF Quads Type Usage Count");
+
+ // Map/Reduce classes
+ job.setMapperClass(QuadObjectCountMapper.class);
+ job.setMapOutputKeyClass(NodeWritable.class);
+ job.setMapOutputValueClass(LongWritable.class);
+ job.setReducerClass(NodeCountReducer.class);
+
+ // Input and Output
+ job.setInputFormatClass(NQuadsInputFormat.class);
+ NLineInputFormat.setNumLinesPerSplit(job, 10000); // TODO Would be
+ // better if this was
+ // intelligently
+ // configured
+ job.setOutputFormatClass(NTriplesNodeOutputFormat.class);
+ FileInputFormat.setInputPaths(job, intermediateOutputPath);
+ FileOutputFormat.setOutputPath(job, new Path(outputPath));
+
+ jobs[1] = job;
+
+ return jobs;
+ }
+
+ /**
+ * Gets a job for computing type counts on RDF triple and/or quad inputs
+ *
+ * @param config
+ * Configuration
+ * @param inputPaths
+ * Input paths
+ * @param intermediateOutputPath
+ * Path for intermediate output which will be all the type
+ * declaration quads present in the inputs
+ * @param outputPath
+ * Output path
+ * @return Job
+ * @throws IOException
+ */
+ public static Job[] getTypeCountJobs(Configuration config, String[] inputPaths, String intermediateOutputPath,
+ String outputPath) throws IOException {
+ Job[] jobs = new Job[2];
+
+ Job job = Job.getInstance(config);
+ job.setJarByClass(JobFactory.class);
+ job.setJobName("RDF Type Extraction");
+
+ // Map/Reduce classes
+ job.getConfiguration().setStrings(RdfMapReduceConstants.FILTER_PREDICATE_URIS, RDF.type.getURI());
+ job.setMapperClass(QuadFilterByPredicateMapper.class);
+ job.setMapOutputKeyClass(LongWritable.class);
+ job.setMapOutputValueClass(QuadWritable.class);
+
+ // Input and Output Format
+ job.setInputFormatClass(TriplesOrQuadsInputFormat.class);
+ job.setOutputFormatClass(NQuadsOutputFormat.class);
+ FileInputFormat.setInputPaths(job, StringUtils.arrayToString(inputPaths));
+ FileOutputFormat.setOutputPath(job, new Path(intermediateOutputPath));
+
+ jobs[0] = job;
+
+ // Object Node Usage count job
+ job = Job.getInstance(config);
+ job.setJarByClass(JobFactory.class);
+ job.setJobName("RDF Type Usage Count");
+
+ // Map/Reduce classes
+ job.setMapperClass(QuadObjectCountMapper.class);
+ job.setMapOutputKeyClass(NodeWritable.class);
+ job.setMapOutputValueClass(LongWritable.class);
+ job.setReducerClass(NodeCountReducer.class);
+
+ // Input and Output
+ job.setInputFormatClass(NQuadsInputFormat.class);
+ NLineInputFormat.setNumLinesPerSplit(job, 10000); // TODO Would be
+ // better if this was
+ // intelligently
+ // configured
+ job.setOutputFormatClass(NTriplesNodeOutputFormat.class);
+ FileInputFormat.setInputPaths(job, intermediateOutputPath);
+ FileOutputFormat.setOutputPath(job, new Path(outputPath));
+
+ jobs[1] = job;
+
+ return jobs;
+ }
+
+ /**
+ * Gets a job for computing literal data type counts on RDF triple inputs
+ *
+ * @param config
+ * Configuration
+ * @param inputPaths
+ * Input paths
+ * @param outputPath
+ * Output path
+ * @return Job
+ * @throws IOException
+ */
+ public static Job getTripleDataTypeCountJob(Configuration config, String[] inputPaths, String outputPath) throws IOException {
+ Job job = Job.getInstance(config);
+ job.setJarByClass(JobFactory.class);
+ job.setJobName("RDF Triples Literal Data Type Usage Count");
+
+ // Map/Reduce classes
+ job.setMapperClass(TripleDataTypeCountMapper.class);
+ job.setMapOutputKeyClass(NodeWritable.class);
+ job.setMapOutputValueClass(LongWritable.class);
+ job.setReducerClass(NodeCountReducer.class);
+
+ // Input and Output
+ job.setInputFormatClass(TriplesInputFormat.class);
+ job.setOutputFormatClass(NTriplesNodeOutputFormat.class);
+ FileInputFormat.setInputPaths(job, StringUtils.arrayToString(inputPaths));
+ FileOutputFormat.setOutputPath(job, new Path(outputPath));
+
+ return job;
+ }
+
+ /**
+ * Gets a job for computing literal data type counts on RDF quad inputs
+ *
+ * @param config
+ * Configuration
+ * @param inputPaths
+ * Input paths
+ * @param outputPath
+ * Output path
+ * @return Job
+ * @throws IOException
+ */
+ public static Job getQuadDataTypeCountJob(Configuration config, String[] inputPaths, String outputPath) throws IOException {
+ Job job = Job.getInstance(config);
+ job.setJarByClass(JobFactory.class);
+ job.setJobName("RDF Quads Literal Data Type Usage Count");
+
+ // Map/Reduce classes
+ job.setMapperClass(QuadDataTypeCountMapper.class);
+ job.setMapOutputKeyClass(NodeWritable.class);
+ job.setMapOutputValueClass(LongWritable.class);
+ job.setReducerClass(NodeCountReducer.class);
+
+ // Input and Output
+ job.setInputFormatClass(QuadsInputFormat.class);
+ job.setOutputFormatClass(NTriplesNodeOutputFormat.class);
+ FileInputFormat.setInputPaths(job, StringUtils.arrayToString(inputPaths));
+ FileOutputFormat.setOutputPath(job, new Path(outputPath));
+
+ return job;
+ }
+
+ /**
+ * Gets a job for computing literal data type counts on RDF triple and/or
+ * quad inputs
+ *
+ * @param config
+ * Configuration
+ * @param inputPaths
+ * Input paths
+ * @param outputPath
+ * Output path
+ * @return Job
+ * @throws IOException
+ */
+ public static Job getDataTypeCountJob(Configuration config, String[] inputPaths, String outputPath) throws IOException {
+ Job job = Job.getInstance(config);
+ job.setJarByClass(JobFactory.class);
+ job.setJobName("RDF Literal Data Type Usage Count");
+
+ // Map/Reduce classes
+ job.setMapperClass(QuadDataTypeCountMapper.class);
+ job.setMapOutputKeyClass(NodeWritable.class);
+ job.setMapOutputValueClass(LongWritable.class);
+ job.setReducerClass(NodeCountReducer.class);
+
+ // Input and Output
+ job.setInputFormatClass(TriplesOrQuadsInputFormat.class);
+ job.setOutputFormatClass(NTriplesNodeOutputFormat.class);
+ FileInputFormat.setInputPaths(job, StringUtils.arrayToString(inputPaths));
+ FileOutputFormat.setOutputPath(job, new Path(outputPath));
+
+ return job;
+ }
+
+ /**
+ * Gets a job for computing literal data type counts on RDF triple inputs
+ *
+ * @param config
+ * Configuration
+ * @param inputPaths
+ * Input paths
+ * @param outputPath
+ * Output path
+ * @return Job
+ * @throws IOException
+ */
+ public static Job getTripleNamespaceCountJob(Configuration config, String[] inputPaths, String outputPath) throws IOException {
+ Job job = Job.getInstance(config);
+ job.setJarByClass(JobFactory.class);
+ job.setJobName("RDF Triples Namespace Usage Count");
+
+ // Map/Reduce classes
+ job.setMapperClass(TripleNamespaceCountMapper.class);
+ job.setMapOutputKeyClass(Text.class);
+ job.setMapOutputValueClass(LongWritable.class);
+ job.setReducerClass(TextCountReducer.class);
+
+ // Input and Output
+ job.setInputFormatClass(TriplesInputFormat.class);
+ job.setOutputFormatClass(TextOutputFormat.class);
+ FileInputFormat.setInputPaths(job, StringUtils.arrayToString(inputPaths));
+ FileOutputFormat.setOutputPath(job, new Path(outputPath));
+
+ return job;
+ }
+
+ /**
+ * Gets a job for computing literal data type counts on RDF quad inputs
+ *
+ * @param config
+ * Configuration
+ * @param inputPaths
+ * Input paths
+ * @param outputPath
+ * Output path
+ * @return Job
+ * @throws IOException
+ */
+ public static Job getQuadNamespaceCountJob(Configuration config, String[] inputPaths, String outputPath) throws IOException {
+ Job job = Job.getInstance(config);
+ job.setJarByClass(JobFactory.class);
+ job.setJobName("RDF Quads Namespace Usage Count");
+
+ // Map/Reduce classes
+ job.setMapperClass(QuadNamespaceCountMapper.class);
+ job.setMapOutputKeyClass(Text.class);
+ job.setMapOutputValueClass(LongWritable.class);
+ job.setReducerClass(TextCountReducer.class);
+
+ // Input and Output
+ job.setInputFormatClass(QuadsInputFormat.class);
+ job.setOutputFormatClass(TextOutputFormat.class);
+ FileInputFormat.setInputPaths(job, StringUtils.arrayToString(inputPaths));
+ FileOutputFormat.setOutputPath(job, new Path(outputPath));
+
+ return job;
+ }
+
+ /**
+ * Gets a job for computing literal data type counts on RDF triple and/or
+ * quad inputs
+ *
+ * @param config
+ * Configuration
+ * @param inputPaths
+ * Input paths
+ * @param outputPath
+ * Output path
+ * @return Job
+ * @throws IOException
+ */
+ public static Job getNamespaceCountJob(Configuration config, String[] inputPaths, String outputPath) throws IOException {
+ Job job = Job.getInstance(config);
+ job.setJarByClass(JobFactory.class);
+ job.setJobName("RDF Namespace Usage Count");
+
+ // Map/Reduce classes
+ job.setMapperClass(QuadNamespaceCountMapper.class);
+ job.setMapOutputKeyClass(Text.class);
+ job.setMapOutputValueClass(LongWritable.class);
+ job.setReducerClass(TextCountReducer.class);
+
+ // Input and Output
+ job.setInputFormatClass(TriplesOrQuadsInputFormat.class);
+ job.setOutputFormatClass(TextOutputFormat.class);
+ FileInputFormat.setInputPaths(job, StringUtils.arrayToString(inputPaths));
+ FileOutputFormat.setOutputPath(job, new Path(outputPath));
+
+ return job;
+ }
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/92fb810a/jena-hadoop-rdf/pom.xml
----------------------------------------------------------------------
diff --git a/jena-hadoop-rdf/pom.xml b/jena-hadoop-rdf/pom.xml
new file mode 100644
index 0000000..dd897d7
--- /dev/null
+++ b/jena-hadoop-rdf/pom.xml
@@ -0,0 +1,105 @@
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>jena-hadoop-rdf</artifactId>
+ <version>0.9.0-SNAPSHOT</version>
+ <packaging>pom</packaging>
+
+ <parent>
+ <groupId>org.apache.jena</groupId>
+ <artifactId>jena-parent</artifactId>
+ <version>12-SNAPSHOT</version>
+ <relativePath>../jena-parent</relativePath>
+ </parent>
+
+ <name>Apache Jena - RDF Tools for Hadoop</name>
+ <description>A collection of tools for working with RDF on the Hadoop platform</description>
+
+ <modules>
+ <module>hadoop-rdf-io</module>
+ <module>hadoop-rdf-common</module>
+ <module>hadoop-rdf-mapreduce</module>
+ <module>hadoop-rdf-stats</module>
+ </modules>
+
+ <!-- Properties common across all profiles -->
+ <properties>
+ <plugin.compiler.version>2.5.1</plugin.compiler.version>
+ <arq.version>2.12.1</arq.version>
+ <junit.version>4.11</junit.version>
+ <mrunit.version>1.0.0</mrunit.version>
+ </properties>
+
+ <!-- Profiles to allow building for different Hadoop versions -->
+ <profiles>
+ <!-- Hadoop 2.x Stable -->
+ <profile>
+ <id>hadoop_2x</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
+ <properties>
+ <hadoop.version>2.5.1</hadoop.version>
+ </properties>
+ </profile>
+
+ <!-- Hadoop 0.23 -->
+ <profile>
+ <id>hadoop_023x</id>
+ <properties>
+ <hadoop.version>0.23.11</hadoop.version>
+ </properties>
+ </profile>
+ </profiles>
+
+ <dependencyManagement>
+ <dependencies>
+ <!-- Hadoop Dependencies -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-common</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <!-- Jena Dependencies -->
+ <dependency>
+ <groupId>org.apache.jena</groupId>
+ <artifactId>jena-arq</artifactId>
+ <version>${arq.version}</version>
+ </dependency>
+
+ <!-- Test Dependencies -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junit.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.mrunit</groupId>
+ <artifactId>mrunit</artifactId>
+ <version>${mrunit.version}</version>
+ <classifier>hadoop2</classifier>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+</project>
http://git-wip-us.apache.org/repos/asf/jena/blob/92fb810a/jena-parent/pom.xml
----------------------------------------------------------------------
diff --git a/jena-parent/pom.xml b/jena-parent/pom.xml
index 1ff14ca..696292e 100644
--- a/jena-parent/pom.xml
+++ b/jena-parent/pom.xml
@@ -16,7 +16,8 @@
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.jena</groupId>
@@ -71,19 +72,19 @@
<!-- Turn of doclint for java8 and later -->
<id>doclint-java8-disable</id>
<activation>
- <jdk>[1.8,)</jdk>
+ <jdk>[1.8,)</jdk>
</activation>
<build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <configuration>
- <additionalparam>-Xdoclint:none</additionalparam>
- </configuration>
- </plugin>
- </plugins>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <configuration>
+ <additionalparam>-Xdoclint:none</additionalparam>
+ </configuration>
+ </plugin>
+ </plugins>
</build>
</profile>
</profiles>
@@ -295,54 +296,32 @@
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
- <id>enforce-java</id>
+ <id>enforce</id>
<goals>
<goal>enforce</goal>
</goals>
- <configuration>
+ <configuration combine.self="override">
<rules>
<requireJavaVersion>
<version>1.7.0</version>
</requireJavaVersion>
- </rules>
+ <requirePluginVersions>
+ <message>Best practice is to always define plugin
+ versions!</message>
+ <banLatest>true</banLatest>
+ <banRelease>true</banRelease>
+ </requirePluginVersions>
+ <requireReleaseDeps>
+ <message>No SNAPSHOT dependencies are allowed!</message>
+ <onlyWhenRelease>true</onlyWhenRelease>
+ </requireReleaseDeps>
+ </rules>
+ <fail>true</fail>
+ <failFast>true</failFast>
</configuration>
</execution>
</executions>
</plugin>
-
- <!-- Run the RAT Plugin -->
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <version>0.11</version>
- <executions>
- <execution>
- <id>rat-checks</id>
- <phase>verify</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <excludeSubProjects>true</excludeSubProjects>
- <excludes>
- <exclude>**/DEPENDENCIES</exclude>
- <exclude>**/*.txt</exclude>
-
- <!-- These might be too extreme an excludes -->
- <!--
- Unfortunately Jena historically has large numbers of small
- test files with no license headers. Since Jena was donated to the
- ASF under a Software Grant it is assumed that all rights for the
-
- -->
- <exclude>**/src/test/resources/**/*</exclude>
- <exclude>**/testing/**/*</exclude>
-
- </excludes>
- </configuration>
- </plugin>
</plugins>
<!-- Plugin version list: http://maven.apache.org/plugins/index.html -->
@@ -475,18 +454,18 @@
<!-- Temp fix for remote-resources while waiting for remote-resources >1.5
1.6 should fix this - SNAPSHOT already does.
-->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-remote-resources-plugin</artifactId>
- <version>1.5</version>
- <dependencies>
- <dependency>
- <groupId>org.apache.maven.shared</groupId>
- <artifactId>maven-filtering</artifactId>
- <version>1.2</version>
- </dependency>
- </dependencies>
- </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-remote-resources-plugin</artifactId>
+ <version>1.5</version>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.maven.shared</groupId>
+ <artifactId>maven-filtering</artifactId>
+ <version>1.2</version>
+ </dependency>
+ </dependencies>
+ </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -523,7 +502,8 @@
<configuration>
<rules>
<requirePluginVersions>
- <message>Best practice is to always define plugin versions!</message>
+ <message>Best practice is to always define plugin
+ versions!</message>
<banLatest>true</banLatest>
<banRelease>true</banRelease>
</requirePluginVersions>
http://git-wip-us.apache.org/repos/asf/jena/blob/92fb810a/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 34e5918..4b91882 100644
--- a/pom.xml
+++ b/pom.xml
@@ -81,6 +81,7 @@
<!-- <module>jena-maven-tools</module> -->
<!-- <module>apache-jena-libs</module> -->
<!-- <module>apache-jena</module> -->
+ <!-- <module>jena-hadoop-rdf</module> -->
</modules>
</profile>
@@ -117,6 +118,8 @@
<module>jena-jdbc</module>
<module>jena-maven-tools</module>
+
+ <module>jena-hadoop-rdf</module>
<module>apache-jena-libs</module>
<module>apache-jena</module>