You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2016/11/25 20:24:54 UTC
[13/42] incubator-streams git commit: STREAMS-440: custom
checkstyle.xml, address compliance
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java
index 1485023..ea512b3 100644
--- a/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java
+++ b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java
@@ -16,6 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.apache.streams.plugins.elasticsearch;
import org.apache.maven.plugin.AbstractMojo;
@@ -33,58 +34,59 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.List;
-@Mojo( name = "generate-resources",
- defaultPhase = LifecyclePhase.GENERATE_RESOURCES
-)
-@Execute( goal = "generate-resources",
- phase = LifecyclePhase.GENERATE_RESOURCES
-)
+@Mojo ( name = "generate-resources",
+ defaultPhase = LifecyclePhase.GENERATE_RESOURCES
+ )
+@Execute ( goal = "generate-resources",
+ phase = LifecyclePhase.GENERATE_RESOURCES
+ )
public class StreamsElasticsearchResourceGeneratorMojo extends AbstractMojo {
- private final static Logger LOGGER = LoggerFactory.getLogger(StreamsElasticsearchResourceGeneratorMojo.class);
+ private static final Logger LOGGER = LoggerFactory.getLogger(StreamsElasticsearchResourceGeneratorMojo.class);
- private volatile MojoFailureException mojoFailureException;
+ private volatile MojoFailureException mojoFailureException;
- @Component
- private MavenProject project;
+ @Component
+ private MavenProject project;
-// @Component
-// private Settings settings;
-//
-// @Parameter( defaultValue = "${localRepository}", readonly = true, required = true )
-// protected ArtifactRepository localRepository;
-//
-// @Parameter( defaultValue = "${plugin}", readonly = true ) // Maven 3 only
-// private PluginDescriptor plugin;
-//
- @Parameter( defaultValue = "${project.basedir}", readonly = true )
- private File basedir;
+ // @Component
+ // private Settings settings;
+ //
+ // @Parameter( defaultValue = "${localRepository}", readonly = true, required = true )
+ // protected ArtifactRepository localRepository;
+ //
+ // @Parameter( defaultValue = "${plugin}", readonly = true ) // Maven 3 only
+ // private PluginDescriptor plugin;
+ //
+ @Parameter( defaultValue = "${project.basedir}", readonly = true )
+ private File basedir;
- @Parameter( defaultValue = "src/main/jsonschema", readonly = true ) // Maven 3 only
- public String sourceDirectory;
+ @Parameter( defaultValue = "src/main/jsonschema", readonly = true ) // Maven 3 only
+ public String sourceDirectory;
- @Parameter( readonly = true ) // Maven 3 only
- public List<String> sourcePaths;
+ @Parameter( readonly = true ) // Maven 3 only
+ public List<String> sourcePaths;
- @Parameter(defaultValue = "./target/generated-resources/streams-plugin-elasticsearch", readonly = true)
- public String targetDirectory;
+ @Parameter(defaultValue = "./target/generated-resources/streams-plugin-elasticsearch", readonly = true)
+ public String targetDirectory;
- public void execute() throws MojoExecutionException, MojoFailureException {
+ public void execute() throws MojoExecutionException, MojoFailureException {
- //addProjectDependenciesToClasspath();
+ //addProjectDependenciesToClasspath();
- StreamsElasticsearchGenerationConfig config = new StreamsElasticsearchGenerationConfig();
+ StreamsElasticsearchGenerationConfig config = new StreamsElasticsearchGenerationConfig();
- if( sourcePaths != null && sourcePaths.size() > 0)
- config.setSourcePaths(sourcePaths);
- else
- config.setSourceDirectory(sourceDirectory);
- config.setTargetDirectory(targetDirectory);
+ if ( sourcePaths != null && sourcePaths.size() > 0) {
+ config.setSourcePaths(sourcePaths);
+ } else {
+ config.setSourceDirectory(sourceDirectory);
+ }
+ config.setTargetDirectory(targetDirectory);
- StreamsElasticsearchResourceGenerator streamsElasticsearchResourceGenerator = new StreamsElasticsearchResourceGenerator(config);
+ StreamsElasticsearchResourceGenerator streamsElasticsearchResourceGenerator = new StreamsElasticsearchResourceGenerator(config);
- streamsElasticsearchResourceGenerator.run();
+ streamsElasticsearchResourceGenerator.run();
- }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorCLITest.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorCLITest.java b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorCLITest.java
index 809b9a0..887461c 100644
--- a/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorCLITest.java
+++ b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorCLITest.java
@@ -16,11 +16,13 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.apache.streams.plugins.elasticsearch.test;
+import org.apache.streams.plugins.elasticsearch.StreamsElasticsearchResourceGenerator;
+
import com.google.common.collect.Lists;
import com.google.common.io.Files;
-import org.apache.streams.plugins.elasticsearch.StreamsElasticsearchResourceGenerator;
import org.junit.Test;
import java.io.File;
@@ -30,28 +32,28 @@ import java.util.List;
import static org.apache.streams.plugins.elasticsearch.test.StreamsElasticsearchResourceGeneratorTest.jsonFilter;
/**
- * Created by sblackmon on 5/5/16.
+ * Test that StreamsElasticsearchResourceGeneratorCLI generates resources.
*/
public class StreamsElasticsearchResourceGeneratorCLITest {
- @Test
- public void testStreamsElasticsearchResourceGeneratorCLI() throws Exception {
+ @Test
+ public void testStreamsElasticsearchResourceGeneratorCLI() throws Exception {
- String sourceDirectory = "target/test-classes/activitystreams-schemas";
- String targetDirectory = "target/generated-resources/elasticsearch-cli";
+ String sourceDirectory = "target/test-classes/activitystreams-schemas";
+ String targetDirectory = "target/generated-resources/elasticsearch-cli";
- List<String> argsList = Lists.newArrayList(sourceDirectory, targetDirectory);
- StreamsElasticsearchResourceGenerator.main(argsList.toArray(new String[0]));
+ List<String> argsList = Lists.newArrayList(sourceDirectory, targetDirectory);
+ StreamsElasticsearchResourceGenerator.main(argsList.toArray(new String[0]));
- File testOutput = new File(targetDirectory);
+ File testOutput = new File(targetDirectory);
- assert( testOutput != null );
- assert( testOutput.exists() == true );
- assert( testOutput.isDirectory() == true );
+ assert ( testOutput != null );
+ assert ( testOutput.exists() == true );
+ assert ( testOutput.isDirectory() == true );
- Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput)
- .filter(jsonFilter);
- Collection<File> outputCollection = Lists.newArrayList(outputIterator);
- assert( outputCollection.size() == 133 );
- }
+ Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput)
+ .filter(jsonFilter);
+ Collection<File> outputCollection = Lists.newArrayList(outputIterator);
+ assert ( outputCollection.size() == 133 );
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorMojoIT.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorMojoIT.java b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorMojoIT.java
index 2a24846..a0fbb99 100644
--- a/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorMojoIT.java
+++ b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorMojoIT.java
@@ -16,6 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.apache.streams.plugins.elasticsearch.test;
import com.google.common.collect.Lists;
@@ -31,38 +32,37 @@ import java.util.ArrayList;
import java.util.List;
/**
- * Tests that streams-plugin-hive running via maven generates hql resources
+ * Tests that streams-plugin-elasticsearch running via maven generates elasticsearch mapping resources.
*/
public class StreamsElasticsearchResourceGeneratorMojoIT extends TestCase {
- private final static Logger LOGGER = LoggerFactory.getLogger(StreamsElasticsearchResourceGeneratorMojoIT.class);
+ private static final Logger LOGGER = LoggerFactory.getLogger(StreamsElasticsearchResourceGeneratorMojoIT.class);
- protected void setUp() throws Exception
- {
- // required for mojo lookups to work
- super.setUp();
- }
+ protected void setUp() throws Exception {
+ // required for mojo lookups to work
+ super.setUp();
+ }
- @Test
- public void testStreamsElasticsearchResourceGeneratorMojo() throws Exception {
+ @Test
+ public void testStreamsElasticsearchResourceGeneratorMojo() throws Exception {
- File testDir = ResourceExtractor.simpleExtractResources( getClass(), "/streams-plugin-elasticsearch" );
+ File testDir = ResourceExtractor.simpleExtractResources( getClass(), "/streams-plugin-elasticsearch" );
- Verifier verifier;
+ Verifier verifier;
- verifier = new Verifier( testDir.getAbsolutePath() );
+ verifier = new Verifier( testDir.getAbsolutePath() );
- List cliOptions = new ArrayList();
- cliOptions.add( "-N" );
- verifier.executeGoals( Lists.<String>newArrayList(
- "clean",
- "dependency:unpack-dependencies",
- "generate-resources"));
+ List cliOptions = new ArrayList();
+ cliOptions.add( "-N" );
+ verifier.executeGoals( Lists.<String>newArrayList(
+ "clean",
+ "dependency:unpack-dependencies",
+ "generate-resources"));
- verifier.verifyErrorFreeLog();
+ verifier.verifyErrorFreeLog();
- verifier.resetStreams();
+ verifier.resetStreams();
- }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java
index 9c1eed4..4322b11 100644
--- a/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java
+++ b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java
@@ -16,24 +16,26 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.apache.streams.plugins.elasticsearch.test;
+import org.apache.streams.plugins.elasticsearch.StreamsElasticsearchGenerationConfig;
+import org.apache.streams.plugins.elasticsearch.StreamsElasticsearchResourceGenerator;
+
import com.google.common.base.Predicate;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.io.Files;
import org.apache.commons.io.FileUtils;
-import org.apache.streams.plugins.elasticsearch.StreamsElasticsearchGenerationConfig;
-import org.apache.streams.plugins.elasticsearch.StreamsElasticsearchResourceGenerator;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.annotation.Nullable;
import java.io.File;
import java.util.Collection;
import java.util.Iterator;
+import javax.annotation.Nullable;
import static org.apache.streams.util.schema.FileUtil.dropSourcePathPrefix;
@@ -42,87 +44,91 @@ import static org.apache.streams.util.schema.FileUtil.dropSourcePathPrefix;
*/
public class StreamsElasticsearchResourceGeneratorTest {
- private final static Logger LOGGER = LoggerFactory.getLogger(StreamsElasticsearchResourceGeneratorTest.class);
-
- public static final Predicate<File> jsonFilter = new Predicate<File>() {
- @Override
- public boolean apply(@Nullable File file) {
- if( file.getName().endsWith(".json") )
- return true;
- else return false;
- }
- };
+ private static final Logger LOGGER = LoggerFactory.getLogger(StreamsElasticsearchResourceGeneratorTest.class);
- /**
- * Test that Elasticsearch resources are generated
- *
- * @throws Exception
- */
- @Test
- public void StreamsElasticsearchResourceGenerator() throws Exception {
-
- StreamsElasticsearchGenerationConfig config = new StreamsElasticsearchGenerationConfig();
-
- String sourceDirectory = "target/test-classes/activitystreams-schemas";
-
- config.setSourceDirectory(sourceDirectory);
-
- config.setTargetDirectory("target/generated-resources/elasticsearch");
-
- config.setExclusions(Sets.newHashSet("attachments"));
+ public static final Predicate<File> jsonFilter = new Predicate<File>() {
+ @Override
+ public boolean apply(@Nullable File file) {
+ if ( file.getName().endsWith(".json") ) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+ };
- config.setMaxDepth(2);
+ /**
+ * Test that Elasticsearch resources are generated.
+ *
+ * @throws Exception Exception
+ */
+ @Test
+ public void StreamsElasticsearchResourceGenerator() throws Exception {
- StreamsElasticsearchResourceGenerator streamsElasticsearchResourceGenerator = new StreamsElasticsearchResourceGenerator(config);
- streamsElasticsearchResourceGenerator.run();
+ StreamsElasticsearchGenerationConfig config = new StreamsElasticsearchGenerationConfig();
- File testOutput = config.getTargetDirectory();
+ String sourceDirectory = "target/test-classes/activitystreams-schemas";
- Predicate<File> jsonFilter = new Predicate<File>() {
- @Override
- public boolean apply(@Nullable File file) {
- if( file.getName().endsWith(".json") )
- return true;
- else return false;
- }
- };
+ config.setSourceDirectory(sourceDirectory);
- assert( testOutput != null );
- assert( testOutput.exists() == true );
- assert( testOutput.isDirectory() == true );
+ config.setTargetDirectory("target/generated-resources/elasticsearch");
- Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput)
- .filter(jsonFilter);
- Collection<File> outputCollection = Lists.newArrayList(outputIterator);
- assert( outputCollection.size() == 133 );
+ config.setExclusions(Sets.newHashSet("attachments"));
- String expectedDirectory = "target/test-classes/expected";
- File testExpected = new File( expectedDirectory );
+ config.setMaxDepth(2);
- Iterable<File> expectedIterator = Files.fileTreeTraverser().breadthFirstTraversal(testExpected)
- .filter(jsonFilter);
- Collection<File> expectedCollection = Lists.newArrayList(expectedIterator);
+ StreamsElasticsearchResourceGenerator streamsElasticsearchResourceGenerator = new StreamsElasticsearchResourceGenerator(config);
+ streamsElasticsearchResourceGenerator.run();
- int fails = 0;
+ File testOutput = config.getTargetDirectory();
- Iterator<File> iterator = expectedCollection.iterator();
- while( iterator.hasNext() ) {
- File objectExpected = iterator.next();
- String expectedEnd = dropSourcePathPrefix(objectExpected.getAbsolutePath(), expectedDirectory);
- File objectActual = new File(config.getTargetDirectory() + "/" + expectedEnd);
- LOGGER.info("Comparing: {} and {}", objectExpected.getAbsolutePath(), objectActual.getAbsolutePath());
- assert( objectActual.exists());
- if( FileUtils.contentEquals(objectActual, objectExpected) == true ) {
- LOGGER.info("Exact Match!");
- } else {
- LOGGER.info("No Match!");
- fails++;
- }
- }
- if( fails > 0 ) {
- LOGGER.info("Fails: {}", fails);
- Assert.fail();
+ Predicate<File> jsonFilter = new Predicate<File>() {
+ @Override
+ public boolean apply(@Nullable File file) {
+ if ( file.getName().endsWith(".json") ) {
+ return true;
+ } else {
+ return false;
}
+ }
+ };
+ assert ( testOutput != null );
+ assert ( testOutput.exists() == true );
+ assert ( testOutput.isDirectory() == true );
+
+ Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput)
+ .filter(jsonFilter);
+ Collection<File> outputCollection = Lists.newArrayList(outputIterator);
+ assert ( outputCollection.size() == 133 );
+
+ String expectedDirectory = "target/test-classes/expected";
+ File testExpected = new File( expectedDirectory );
+
+ Iterable<File> expectedIterator = Files.fileTreeTraverser().breadthFirstTraversal(testExpected)
+ .filter(jsonFilter);
+ Collection<File> expectedCollection = Lists.newArrayList(expectedIterator);
+
+ int fails = 0;
+
+ Iterator<File> iterator = expectedCollection.iterator();
+ while ( iterator.hasNext() ) {
+ File objectExpected = iterator.next();
+ String expectedEnd = dropSourcePathPrefix(objectExpected.getAbsolutePath(), expectedDirectory);
+ File objectActual = new File(config.getTargetDirectory() + "/" + expectedEnd);
+ LOGGER.info("Comparing: {} and {}", objectExpected.getAbsolutePath(), objectActual.getAbsolutePath());
+ assert ( objectActual.exists());
+ if ( FileUtils.contentEquals(objectActual, objectExpected) == true ) {
+ LOGGER.info("Exact Match!");
+ } else {
+ LOGGER.info("No Match!");
+ fails++;
+ }
}
+ if ( fails > 0 ) {
+ LOGGER.info("Fails: {}", fails);
+ Assert.fail();
+ }
+
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java
index 977c3d9..d939372 100644
--- a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java
+++ b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java
@@ -20,6 +20,7 @@
package org.apache.streams.plugins.hbase;
import org.apache.streams.util.schema.GenerationConfig;
+
import org.jsonschema2pojo.DefaultGenerationConfig;
import org.jsonschema2pojo.util.URLUtil;
@@ -33,77 +34,80 @@ import java.util.List;
import java.util.Set;
/**
- * Configures StreamsHiveResourceGenerator
- *
- *
+ * Configures StreamsHiveResourceGenerator.
*/
public class StreamsHbaseGenerationConfig extends DefaultGenerationConfig implements GenerationConfig {
- public String getSourceDirectory() {
- return sourceDirectory;
+ public String getSourceDirectory() {
+ return sourceDirectory;
+ }
+
+ public List<String> getSourcePaths() {
+ return sourcePaths;
+ }
+
+ private String columnFamily;
+ private String sourceDirectory;
+ private List<String> sourcePaths = new ArrayList<String>();
+ private String targetDirectory;
+ private int maxDepth = 1;
+
+ public Set<String> getExclusions() {
+ return exclusions;
+ }
+
+ public void setExclusions(Set<String> exclusions) {
+ this.exclusions = exclusions;
+ }
+
+ private Set<String> exclusions = new HashSet<String>();
+
+ public int getMaxDepth() {
+ return maxDepth;
+ }
+
+ public void setSourceDirectory(String sourceDirectory) {
+ this.sourceDirectory = sourceDirectory;
+ }
+
+ public void setSourcePaths(List<String> sourcePaths) {
+ this.sourcePaths = sourcePaths;
+ }
+
+ public void setTargetDirectory(String targetDirectory) {
+ this.targetDirectory = targetDirectory;
+ }
+
+ public File getTargetDirectory() {
+ return new File(targetDirectory);
+ }
+
+ /**
+ * get all sources.
+ * @return Iterator of URL
+ */
+ public Iterator<URL> getSource() {
+ if (null != sourceDirectory) {
+ return Collections.singleton(URLUtil.parseURL(sourceDirectory)).iterator();
}
-
- public List<String> getSourcePaths() {
- return sourcePaths;
+ List<URL> sourceUrls = new ArrayList<URL>();
+ if ( sourcePaths != null && sourcePaths.size() > 0) {
+ for (String source : sourcePaths) {
+ sourceUrls.add(URLUtil.parseURL(source));
+ }
}
+ return sourceUrls.iterator();
+ }
- private String columnFamily;
- private String sourceDirectory;
- private List<String> sourcePaths = new ArrayList<String>();
- private String targetDirectory;
- private int maxDepth = 1;
+ public void setMaxDepth(int maxDepth) {
+ this.maxDepth = maxDepth;
+ }
- public Set<String> getExclusions() {
- return exclusions;
- }
-
- public void setExclusions(Set<String> exclusions) {
- this.exclusions = exclusions;
- }
+ public String getColumnFamily() {
+ return columnFamily;
+ }
- private Set<String> exclusions = new HashSet<String>();
-
- public int getMaxDepth() {
- return maxDepth;
- }
-
- public void setSourceDirectory(String sourceDirectory) {
- this.sourceDirectory = sourceDirectory;
- }
-
- public void setSourcePaths(List<String> sourcePaths) {
- this.sourcePaths = sourcePaths;
- }
-
- public void setTargetDirectory(String targetDirectory) {
- this.targetDirectory = targetDirectory;
- }
-
- public File getTargetDirectory() {
- return new File(targetDirectory);
- }
-
- public Iterator<URL> getSource() {
- if (null != sourceDirectory) {
- return Collections.singleton(URLUtil.parseURL(sourceDirectory)).iterator();
- }
- List<URL> sourceURLs = new ArrayList<URL>();
- if( sourcePaths != null && sourcePaths.size() > 0)
- for (String source : sourcePaths) {
- sourceURLs.add(URLUtil.parseURL(source));
- }
- return sourceURLs.iterator();
- }
-
- public void setMaxDepth(int maxDepth) {
- this.maxDepth = maxDepth;
- }
-
- public String getColumnFamily() {
- return columnFamily;
- }
-
- public void setColumnFamily(String columnFamily) {
- this.columnFamily = columnFamily;
- }
+ public void setColumnFamily(String columnFamily) {
+ this.columnFamily = columnFamily;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java
index 9f96fb8..cbea67f 100644
--- a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java
+++ b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java
@@ -19,16 +19,17 @@
package org.apache.streams.plugins.hbase;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.base.Joiner;
-import com.google.common.collect.Lists;
import org.apache.streams.util.schema.FieldType;
import org.apache.streams.util.schema.FieldUtil;
import org.apache.streams.util.schema.GenerationConfig;
import org.apache.streams.util.schema.Schema;
import org.apache.streams.util.schema.SchemaStore;
import org.apache.streams.util.schema.SchemaStoreImpl;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Joiner;
+import com.google.common.collect.Lists;
import org.jsonschema2pojo.util.URLUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,161 +53,188 @@ import static org.apache.streams.util.schema.FileUtil.writeFile;
*/
public class StreamsHbaseResourceGenerator implements Runnable {
- private final static Logger LOGGER = LoggerFactory.getLogger(StreamsHbaseResourceGenerator.class);
-
- private final static String LS = System.getProperty("line.separator");
-
- private StreamsHbaseGenerationConfig config;
-
- private SchemaStore schemaStore = new SchemaStoreImpl();
+ private static final Logger LOGGER = LoggerFactory.getLogger(StreamsHbaseResourceGenerator.class);
- private int currentDepth = 0;
+ private static final String LS = System.getProperty("line.separator");
- public static void main(String[] args) {
- StreamsHbaseGenerationConfig config = new StreamsHbaseGenerationConfig();
+ private StreamsHbaseGenerationConfig config;
- String sourceDirectory = "src/main/jsonschema";
- String targetDirectory = "target/generated-resources/hbase";
+ private SchemaStore schemaStore = new SchemaStoreImpl();
- if( args.length > 0 )
- sourceDirectory = args[0];
- if( args.length > 1 )
- targetDirectory = args[1];
+ private int currentDepth = 0;
- config.setSourceDirectory(sourceDirectory);
- config.setTargetDirectory(targetDirectory);
+ /**
+ * Run from CLI without Maven
+ *
+ * <p/>
+ * java -jar streams-plugin-hbase-jar-with-dependencies.jar StreamsHbaseResourceGenerator src/main/jsonschema target/generated-resources
+ *
+ * @param args [sourceDirectory, targetDirectory]
+ * */
+ public static void main(String[] args) {
+ StreamsHbaseGenerationConfig config = new StreamsHbaseGenerationConfig();
- StreamsHbaseResourceGenerator streamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config);
- streamsHbaseResourceGenerator.run();
+ String sourceDirectory = "src/main/jsonschema";
+ String targetDirectory = "target/generated-resources/hbase";
+ if ( args.length > 0 ) {
+ sourceDirectory = args[0];
}
-
- public StreamsHbaseResourceGenerator(StreamsHbaseGenerationConfig config) {
- this.config = config;
+ if ( args.length > 1 ) {
+ targetDirectory = args[1];
}
- public void run() {
-
- checkNotNull(config);
+ config.setSourceDirectory(sourceDirectory);
+ config.setTargetDirectory(targetDirectory);
- generate(config);
+ StreamsHbaseResourceGenerator streamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config);
+ streamsHbaseResourceGenerator.run();
- }
+ }
- public void generate(StreamsHbaseGenerationConfig config) {
+ public StreamsHbaseResourceGenerator(StreamsHbaseGenerationConfig config) {
+ this.config = config;
+ }
- LinkedList<File> sourceFiles = new LinkedList<File>();
+ @Override
+ public void run() {
- for (Iterator<URL> sources = config.getSource(); sources.hasNext();) {
- URL source = sources.next();
- sourceFiles.add(URLUtil.getFileFromURL(source));
- }
+ checkNotNull(config);
- LOGGER.info("Seeded with {} source paths:", sourceFiles.size());
+ generate(config);
- resolveRecursive((GenerationConfig)config, sourceFiles);
+ }
- LOGGER.info("Resolved {} schema files:", sourceFiles.size());
+ /**
+ * run generate using supplied StreamsHbaseGenerationConfig.
+ * @param config StreamsHbaseGenerationConfig
+ */
+ public void generate(StreamsHbaseGenerationConfig config) {
- for (Iterator<File> iterator = sourceFiles.iterator(); iterator.hasNext();) {
- File item = iterator.next();
- schemaStore.create(item.toURI());
- }
+ LinkedList<File> sourceFiles = new LinkedList<File>();
- LOGGER.info("Identified {} objects:", schemaStore.getSize());
+ for (Iterator<URL> sources = config.getSource(); sources.hasNext();) {
+ URL source = sources.next();
+ sourceFiles.add(URLUtil.getFileFromURL(source));
+ }
- for (Iterator<Schema> schemaIterator = schemaStore.getSchemaIterator(); schemaIterator.hasNext(); ) {
- Schema schema = schemaIterator.next();
- currentDepth = 0;
- if( schema.getURI().getScheme().equals("file")) {
- String inputFile = schema.getURI().getPath();
- String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory());
- for (String sourcePath : config.getSourcePaths()) {
- resourcePath = dropSourcePathPrefix(resourcePath, sourcePath);
- }
- String outputFile = config.getTargetDirectory() + "/" + swapExtension(resourcePath, "json", "txt");
+ LOGGER.info("Seeded with {} source paths:", sourceFiles.size());
- LOGGER.info("Processing {}:", resourcePath);
+ resolveRecursive((GenerationConfig)config, sourceFiles);
- String resourceId = dropExtension(resourcePath).replace("/", "_");
+ LOGGER.info("Resolved {} schema files:", sourceFiles.size());
- String resourceContent = generateResource(schema, resourceId);
+ for (Iterator<File> iterator = sourceFiles.iterator(); iterator.hasNext();) {
+ File item = iterator.next();
+ schemaStore.create(item.toURI());
+ }
- writeFile(outputFile, resourceContent);
+ LOGGER.info("Identified {} objects:", schemaStore.getSize());
- LOGGER.info("Wrote {}:", outputFile);
- }
+ for (Iterator<Schema> schemaIterator = schemaStore.getSchemaIterator(); schemaIterator.hasNext(); ) {
+ Schema schema = schemaIterator.next();
+ currentDepth = 0;
+ if ( schema.getUri().getScheme().equals("file")) {
+ String inputFile = schema.getUri().getPath();
+ String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory());
+ for (String sourcePath : config.getSourcePaths()) {
+ resourcePath = dropSourcePathPrefix(resourcePath, sourcePath);
}
+ String outputFile = config.getTargetDirectory() + "/" + swapExtension(resourcePath, "json", "txt");
- }
+ LOGGER.info("Processing {}:", resourcePath);
- public String generateResource(Schema schema, String resourceId) {
- StringBuilder resourceBuilder = new StringBuilder();
- resourceBuilder.append("CREATE ");
- resourceBuilder = appendRootObject(resourceBuilder, schema, resourceId);
- return resourceBuilder.toString();
- }
+ String resourceId = dropExtension(resourcePath).replace("/", "_");
- public StringBuilder appendRootObject(StringBuilder builder, Schema schema, String resourceId) {
- checkNotNull(builder);
- ObjectNode propertiesNode = schemaStore.resolveProperties(schema, null, resourceId);
- if( propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0) {
-
- List<String> fieldStrings = Lists.newArrayList();
-
- // table
- fieldStrings.add(hbaseEscape(schemaSymbol(schema)));
-
- // column family
- fieldStrings.add(hbaseEscape(schemaSymbol(schema)));
-
- // parent column family
- if( schema.getParent() != null )
- fieldStrings.add(hbaseEscape(schemaSymbol(schema.getParent())));
-
- // sub-object column families
- if( propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0 ) {
-
- Iterator<Map.Entry<String, JsonNode>> fields = propertiesNode.fields();
- Joiner joiner = Joiner.on(", ").skipNulls();
- for( ; fields.hasNext(); ) {
- Map.Entry<String, JsonNode> field = fields.next();
- String fieldId = field.getKey();
- if( !config.getExclusions().contains(fieldId) && field.getValue().isObject()) {
- ObjectNode fieldNode = (ObjectNode) field.getValue();
- FieldType fieldType = FieldUtil.determineFieldType(fieldNode);
- if (fieldType != null ) {
- switch (fieldType) {
- case OBJECT:
- fieldStrings.add(hbaseEscape(fieldId));
- }
- }
- }
- }
- builder.append(joiner.join(fieldStrings));
+ String resourceContent = generateResource(schema, resourceId);
- }
- }
- checkNotNull(builder);
- return builder;
- }
+ writeFile(outputFile, resourceContent);
- private static String hbaseEscape( String fieldId ) {
- return "'"+fieldId+"'";
+ LOGGER.info("Wrote {}:", outputFile);
+ }
}
- private String schemaSymbol( Schema schema ) {
- if (schema == null) return null;
- if (schema.getURI().getScheme().equals("file")) {
- String inputFile = schema.getURI().getPath();
- String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory());
- for (String sourcePath : config.getSourcePaths()) {
- resourcePath = dropSourcePathPrefix(resourcePath, sourcePath);
+ }
+
+ /**
+ * generateResource String from schema and resourceId.
+ * @param schema Schema
+ * @param resourceId String
+ * @return mapping
+ */
+ public String generateResource(Schema schema, String resourceId) {
+ StringBuilder resourceBuilder = new StringBuilder();
+ resourceBuilder.append("CREATE ");
+ resourceBuilder = appendRootObject(resourceBuilder, schema, resourceId);
+ return resourceBuilder.toString();
+ }
+
+ protected StringBuilder appendRootObject(StringBuilder builder, Schema schema, String resourceId) {
+ checkNotNull(builder);
+ ObjectNode propertiesNode = schemaStore.resolveProperties(schema, null, resourceId);
+ if ( propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0) {
+
+ List<String> fieldStrings = Lists.newArrayList();
+
+ // table
+ fieldStrings.add(hbaseEscape(schemaSymbol(schema)));
+
+ // column family
+ fieldStrings.add(hbaseEscape(schemaSymbol(schema)));
+
+ // parent column family
+ if ( schema.getParent() != null ) {
+ fieldStrings.add(hbaseEscape(schemaSymbol(schema.getParent())));
+ }
+
+ // sub-object column families
+ if ( propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0 ) {
+
+ Iterator<Map.Entry<String, JsonNode>> fields = propertiesNode.fields();
+ Joiner joiner = Joiner.on(", ").skipNulls();
+ for ( ; fields.hasNext(); ) {
+ Map.Entry<String, JsonNode> field = fields.next();
+ String fieldId = field.getKey();
+ if ( !config.getExclusions().contains(fieldId) && field.getValue().isObject()) {
+ ObjectNode fieldNode = (ObjectNode) field.getValue();
+ FieldType fieldType = FieldUtil.determineFieldType(fieldNode);
+ if (fieldType != null ) {
+ switch (fieldType) {
+ case OBJECT:
+ fieldStrings.add(hbaseEscape(fieldId));
+ break;
+ default:
+ break;
+ }
}
- return dropExtension(resourcePath).replace("/", "_");
- } else {
- return "IDK";
+ }
}
+ builder.append(joiner.join(fieldStrings));
+
+ }
+ }
+ checkNotNull(builder);
+ return builder;
+ }
+
+ private static String hbaseEscape( String fieldId ) {
+ return "'" + fieldId + "'";
+ }
+
+ private String schemaSymbol( Schema schema ) {
+ if (schema == null) {
+ return null;
+ }
+ if (schema.getUri().getScheme().equals("file")) {
+ String inputFile = schema.getUri().getPath();
+ String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory());
+ for (String sourcePath : config.getSourcePaths()) {
+ resourcePath = dropSourcePathPrefix(resourcePath, sourcePath);
+ }
+ return dropExtension(resourcePath).replace("/", "_");
+ } else {
+ return "IDK";
}
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java
index 24a6b88..addc657 100644
--- a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java
+++ b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java
@@ -34,57 +34,65 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.List;
-@Mojo( name = "generate-resources",
- defaultPhase = LifecyclePhase.GENERATE_RESOURCES
-)
-@Execute( goal = "generate-resources",
- phase = LifecyclePhase.GENERATE_RESOURCES
-)
+@Mojo (
+ name = "generate-resources",
+ defaultPhase = LifecyclePhase.GENERATE_RESOURCES
+ )
+@Execute (
+ goal = "generate-resources",
+ phase = LifecyclePhase.GENERATE_RESOURCES
+ )
public class StreamsHbaseResourceGeneratorMojo extends AbstractMojo {
- private final static Logger LOGGER = LoggerFactory.getLogger(StreamsHbaseResourceGeneratorMojo.class);
+ private static final Logger LOGGER = LoggerFactory.getLogger(StreamsHbaseResourceGeneratorMojo.class);
- private volatile MojoFailureException mojoFailureException;
+ private volatile MojoFailureException mojoFailureException;
- @Component
- private MavenProject project;
+ @Component
+ private MavenProject project;
-// @Component
-// private Settings settings;
-//
-// @Parameter( defaultValue = "${localRepository}", readonly = true, required = true )
-// protected ArtifactRepository localRepository;
-//
-// @Parameter( defaultValue = "${plugin}", readonly = true ) // Maven 3 only
-// private PluginDescriptor plugin;
-//
- @Parameter( defaultValue = "${project.basedir}", readonly = true )
- private File basedir;
+ // @Component
+ // private Settings settings;
+ //
+ // @Parameter( defaultValue = "${localRepository}", readonly = true, required = true )
+ // protected ArtifactRepository localRepository;
+ //
+ // @Parameter( defaultValue = "${plugin}", readonly = true ) // Maven 3 only
+ // private PluginDescriptor plugin;
+ //
+ @Parameter( defaultValue = "${project.basedir}", readonly = true )
+ private File basedir;
- @Parameter( defaultValue = "src/main/jsonschema", readonly = true ) // Maven 3 only
- public String sourceDirectory;
+ @Parameter( defaultValue = "src/main/jsonschema", readonly = true ) // Maven 3 only
+ public String sourceDirectory;
- @Parameter( readonly = true ) // Maven 3 only
- public List<String> sourcePaths;
+ @Parameter( readonly = true ) // Maven 3 only
+ public List<String> sourcePaths;
- @Parameter(defaultValue = "./target/generated-resources/hbase", readonly = true)
- public String targetDirectory;
+ @Parameter(defaultValue = "./target/generated-resources/hbase", readonly = true)
+ public String targetDirectory;
- public void execute() throws MojoExecutionException, MojoFailureException {
+ /**
+ * execute StreamsHbaseResourceGenerator mojo.
+ * @throws MojoExecutionException MojoExecutionException
+ * @throws MojoFailureException MojoFailureException
+ */
+ public void execute() throws MojoExecutionException, MojoFailureException {
- //addProjectDependenciesToClasspath();
+ //addProjectDependenciesToClasspath();
- StreamsHbaseGenerationConfig config = new StreamsHbaseGenerationConfig();
+ StreamsHbaseGenerationConfig config = new StreamsHbaseGenerationConfig();
- if( sourcePaths != null && sourcePaths.size() > 0)
- config.setSourcePaths(sourcePaths);
- else
- config.setSourceDirectory(sourceDirectory);
- config.setTargetDirectory(targetDirectory);
+ if ( sourcePaths != null && sourcePaths.size() > 0) {
+ config.setSourcePaths(sourcePaths);
+ } else {
+ config.setSourceDirectory(sourceDirectory);
+ }
+ config.setTargetDirectory(targetDirectory);
- StreamsHbaseResourceGenerator streamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config);
- streamsHbaseResourceGenerator.run();
+ StreamsHbaseResourceGenerator streamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config);
+ streamsHbaseResourceGenerator.run();
- }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java
index 3c223eb..254bd0e 100644
--- a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java
+++ b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java
@@ -19,13 +19,12 @@
package org.apache.streams.plugins.test;
-import com.google.common.base.Predicate;
+import org.apache.streams.plugins.hbase.StreamsHbaseResourceGenerator;
+
import com.google.common.collect.Lists;
import com.google.common.io.Files;
-import org.apache.streams.plugins.hbase.StreamsHbaseResourceGenerator;
import org.junit.Test;
-import javax.annotation.Nullable;
import java.io.File;
import java.util.Collection;
import java.util.List;
@@ -33,28 +32,28 @@ import java.util.List;
import static org.apache.streams.plugins.test.StreamsHbaseResourceGeneratorTest.txtFilter;
/**
- * Created by sblackmon on 5/5/16.
+ * Test that StreamsHbaseResourceGeneratorCLI generates resources.
*/
public class StreamsHbaseResourceGeneratorCLITest {
- @Test
- public void testStreamsHiveResourceGeneratorCLI() throws Exception {
+ @Test
+ public void testStreamsHbaseResourceGeneratorCLI() throws Exception {
- String sourceDirectory = "target/test-classes/activitystreams-schemas";
- String targetDirectory = "target/generated-resources/hbase-cli";
+ String sourceDirectory = "target/test-classes/activitystreams-schemas";
+ String targetDirectory = "target/generated-resources/hbase-cli";
- List<String> argsList = Lists.newArrayList(sourceDirectory, targetDirectory);
- StreamsHbaseResourceGenerator.main(argsList.toArray(new String[0]));
+ List<String> argsList = Lists.newArrayList(sourceDirectory, targetDirectory);
+ StreamsHbaseResourceGenerator.main(argsList.toArray(new String[0]));
- File testOutput = new File(targetDirectory);
+ File testOutput = new File(targetDirectory);
- assert( testOutput != null );
- assert( testOutput.exists() == true );
- assert( testOutput.isDirectory() == true );
+ assert ( testOutput != null );
+ assert ( testOutput.exists() == true );
+ assert ( testOutput.isDirectory() == true );
- Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput)
- .filter(txtFilter);
- Collection<File> outputCollection = Lists.newArrayList(outputIterator);
- assert( outputCollection.size() == 133 );
- }
+ Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput)
+ .filter(txtFilter);
+ Collection<File> outputCollection = Lists.newArrayList(outputIterator);
+ assert ( outputCollection.size() == 133 );
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoIT.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoIT.java b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoIT.java
index 1495bc1..ff4e49a 100644
--- a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoIT.java
+++ b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoIT.java
@@ -36,48 +36,47 @@ import java.util.List;
import static org.apache.streams.plugins.test.StreamsHbaseResourceGeneratorTest.txtFilter;
/**
- * Tests that streams-plugin-hive running via maven generates hql resources
+ * Tests that streams-plugin-hbase running via maven generates txt resources.
*/
public class StreamsHbaseResourceGeneratorMojoIT extends TestCase {
- private final static Logger LOGGER = LoggerFactory.getLogger(StreamsHbaseResourceGeneratorMojoIT.class);
+ private static final Logger LOGGER = LoggerFactory.getLogger(StreamsHbaseResourceGeneratorMojoIT.class);
- protected void setUp() throws Exception
- {
- // required for mojo lookups to work
- super.setUp();
- }
+ protected void setUp() throws Exception {
+ // required for mojo lookups to work
+ super.setUp();
+ }
- @Test
- public void testStreamsHbaseResourceGeneratorMojo() throws Exception {
+ @Test
+ public void testStreamsHbaseResourceGeneratorMojo() throws Exception {
- File testDir = ResourceExtractor.simpleExtractResources( getClass(), "/streams-plugin-hbase" );
+ File testDir = ResourceExtractor.simpleExtractResources( getClass(), "/streams-plugin-hbase" );
- Verifier verifier;
+ Verifier verifier;
- verifier = new Verifier( testDir.getAbsolutePath() );
+ verifier = new Verifier( testDir.getAbsolutePath() );
- List cliOptions = new ArrayList();
- cliOptions.add( "-N" );
- verifier.executeGoals( Lists.<String>newArrayList(
- "clean",
- "dependency:unpack-dependencies",
- "generate-resources"));
+ List cliOptions = new ArrayList();
+ cliOptions.add( "-N" );
+ verifier.executeGoals( Lists.<String>newArrayList(
+ "clean",
+ "dependency:unpack-dependencies",
+ "generate-resources"));
- verifier.verifyErrorFreeLog();
+ verifier.verifyErrorFreeLog();
- verifier.resetStreams();
+ verifier.resetStreams();
- File testOutput = new File(testDir.getAbsolutePath() + "/target/generated-resources/hbase-mojo");
+ File testOutput = new File(testDir.getAbsolutePath() + "/target/generated-resources/hbase-mojo");
- assert( testOutput != null );
- assert( testOutput.exists() == true );
- assert( testOutput.isDirectory() == true );
+ assert ( testOutput != null );
+ assert ( testOutput.exists() == true );
+ assert ( testOutput.isDirectory() == true );
- Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput)
- .filter(txtFilter);
- Collection<File> outputCollection = Lists.newArrayList(outputIterator);
- assert( outputCollection.size() == 133 );
- }
+ Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput)
+ .filter(txtFilter);
+ Collection<File> outputCollection = Lists.newArrayList(outputIterator);
+ assert ( outputCollection.size() == 133 );
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java
index 254578c..ac876d5 100644
--- a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java
+++ b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java
@@ -19,22 +19,23 @@
package org.apache.streams.plugins.test;
+import org.apache.streams.plugins.hbase.StreamsHbaseGenerationConfig;
+import org.apache.streams.plugins.hbase.StreamsHbaseResourceGenerator;
+
import com.google.common.base.Predicate;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.io.Files;
import org.apache.commons.io.FileUtils;
-import org.apache.streams.plugins.hbase.StreamsHbaseGenerationConfig;
-import org.apache.streams.plugins.hbase.StreamsHbaseResourceGenerator;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.annotation.Nullable;
import java.io.File;
import java.util.Collection;
import java.util.Iterator;
+import javax.annotation.Nullable;
import static org.apache.streams.util.schema.FileUtil.dropSourcePathPrefix;
@@ -43,79 +44,81 @@ import static org.apache.streams.util.schema.FileUtil.dropSourcePathPrefix;
*/
public class StreamsHbaseResourceGeneratorTest {
- private final static Logger LOGGER = LoggerFactory.getLogger(StreamsHbaseResourceGeneratorTest.class);
-
- public static final Predicate<File> txtFilter = new Predicate<File>() {
- @Override
- public boolean apply(@Nullable File file) {
- if( file.getName().endsWith(".txt") )
- return true;
- else return false;
- }
- };
+ private static final Logger LOGGER = LoggerFactory.getLogger(StreamsHbaseResourceGeneratorTest.class);
- /**
- * Tests that all example activities can be loaded into Activity beans
- *
- * @throws Exception
- */
- @Test
- public void StreamsHbaseResourceGenerator() throws Exception {
+ public static final Predicate<File> txtFilter = new Predicate<File>() {
+ @Override
+ public boolean apply(@Nullable File file) {
+ if ( file.getName().endsWith(".txt") ) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+ };
- StreamsHbaseGenerationConfig config = new StreamsHbaseGenerationConfig();
+ /**
+ * Tests that all example activities can be loaded into Activity beans.
+ *
+ * @throws Exception Exception
+ */
+ @Test
+ public void testStreamsHbaseResourceGenerator() throws Exception {
- String sourceDirectory = "target/test-classes/activitystreams-schemas";
+ StreamsHbaseGenerationConfig config = new StreamsHbaseGenerationConfig();
- config.setSourceDirectory(sourceDirectory);
+ String sourceDirectory = "target/test-classes/activitystreams-schemas";
- config.setTargetDirectory("target/generated-resources/hbase");
+ config.setSourceDirectory(sourceDirectory);
- config.setExclusions(Sets.newHashSet("attachments"));
+ config.setTargetDirectory("target/generated-resources/hbase");
- config.setColumnFamily("cf");
- config.setMaxDepth(2);
+ config.setExclusions(Sets.newHashSet("attachments"));
- StreamsHbaseResourceGenerator streamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config);
- streamsHbaseResourceGenerator.run();
+ config.setColumnFamily("cf");
+ config.setMaxDepth(2);
- File testOutput = config.getTargetDirectory();
+ StreamsHbaseResourceGenerator streamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config);
+ streamsHbaseResourceGenerator.run();
- assert( testOutput != null );
- assert( testOutput.exists() == true );
- assert( testOutput.isDirectory() == true );
+ File testOutput = config.getTargetDirectory();
- Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput)
- .filter(txtFilter);
- Collection<File> outputCollection = Lists.newArrayList(outputIterator);
- assert( outputCollection.size() == 133 );
+ assert ( testOutput != null );
+ assert ( testOutput.exists() == true );
+ assert ( testOutput.isDirectory() == true );
- String expectedDirectory = "target/test-classes/expected";
- File testExpected = new File( expectedDirectory );
+ Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput)
+ .filter(txtFilter);
+ Collection<File> outputCollection = Lists.newArrayList(outputIterator);
+ assert ( outputCollection.size() == 133 );
- Iterable<File> expectedIterator = Files.fileTreeTraverser().breadthFirstTraversal(testExpected)
- .filter(txtFilter);
- Collection<File> expectedCollection = Lists.newArrayList(expectedIterator);
+ String expectedDirectory = "target/test-classes/expected";
+ File testExpected = new File( expectedDirectory );
- int fails = 0;
+ Iterable<File> expectedIterator = Files.fileTreeTraverser().breadthFirstTraversal(testExpected)
+ .filter(txtFilter);
+ Collection<File> expectedCollection = Lists.newArrayList(expectedIterator);
- Iterator<File> iterator = expectedCollection.iterator();
- while( iterator.hasNext() ) {
- File objectExpected = iterator.next();
- String expectedEnd = dropSourcePathPrefix(objectExpected.getAbsolutePath(), expectedDirectory);
- File objectActual = new File(config.getTargetDirectory() + "/" + expectedEnd);
- LOGGER.info("Comparing: {} and {}", objectExpected.getAbsolutePath(), objectActual.getAbsolutePath());
- assert( objectActual.exists());
- if( FileUtils.contentEquals(objectActual, objectExpected) == true ) {
- LOGGER.info("Exact Match!");
- } else {
- LOGGER.info("No Match!");
- fails++;
- }
- }
- if( fails > 0 ) {
- LOGGER.info("Fails: {}", fails);
- Assert.fail();
- }
+ int fails = 0;
+ Iterator<File> iterator = expectedCollection.iterator();
+ while ( iterator.hasNext() ) {
+ File objectExpected = iterator.next();
+ String expectedEnd = dropSourcePathPrefix(objectExpected.getAbsolutePath(), expectedDirectory);
+ File objectActual = new File(config.getTargetDirectory() + "/" + expectedEnd);
+ LOGGER.info("Comparing: {} and {}", objectExpected.getAbsolutePath(), objectActual.getAbsolutePath());
+ assert ( objectActual.exists());
+ if ( FileUtils.contentEquals(objectActual, objectExpected) == true ) {
+ LOGGER.info("Exact Match!");
+ } else {
+ LOGGER.info("No Match!");
+ fails++;
+ }
}
+ if ( fails > 0 ) {
+ LOGGER.info("Fails: {}", fails);
+ Assert.fail();
+ }
+
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveGenerationConfig.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveGenerationConfig.java b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveGenerationConfig.java
index b09ce18..cc32255 100644
--- a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveGenerationConfig.java
+++ b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveGenerationConfig.java
@@ -20,6 +20,7 @@
package org.apache.streams.plugins.hive;
import org.apache.streams.util.schema.GenerationConfig;
+
import org.jsonschema2pojo.DefaultGenerationConfig;
import org.jsonschema2pojo.util.URLUtil;
@@ -33,70 +34,69 @@ import java.util.List;
import java.util.Set;
/**
- * Configures StreamsHiveResourceGenerator
- *
- *
+ * Configures StreamsHiveResourceGenerator.
*/
public class StreamsHiveGenerationConfig extends DefaultGenerationConfig implements GenerationConfig {
- public String getSourceDirectory() {
- return sourceDirectory;
- }
+ public String getSourceDirectory() {
+ return sourceDirectory;
+ }
- public List<String> getSourcePaths() {
- return sourcePaths;
- }
+ public List<String> getSourcePaths() {
+ return sourcePaths;
+ }
- private String sourceDirectory;
- private List<String> sourcePaths = new ArrayList<String>();
- private String targetDirectory;
- private int maxDepth = 1;
+ private String sourceDirectory;
+ private List<String> sourcePaths = new ArrayList<String>();
+ private String targetDirectory;
+ private int maxDepth = 1;
- public Set<String> getExclusions() {
- return exclusions;
- }
+ public Set<String> getExclusions() {
+ return exclusions;
+ }
- public void setExclusions(Set<String> exclusions) {
- this.exclusions = exclusions;
- }
+ public void setExclusions(Set<String> exclusions) {
+ this.exclusions = exclusions;
+ }
- private Set<String> exclusions = new HashSet<String>();
+ private Set<String> exclusions = new HashSet<String>();
- public int getMaxDepth() {
- return maxDepth;
- }
+ public int getMaxDepth() {
+ return maxDepth;
+ }
- public void setSourceDirectory(String sourceDirectory) {
- this.sourceDirectory = sourceDirectory;
- }
+ public void setSourceDirectory(String sourceDirectory) {
+ this.sourceDirectory = sourceDirectory;
+ }
- public void setSourcePaths(List<String> sourcePaths) {
- this.sourcePaths = sourcePaths;
- }
+ public void setSourcePaths(List<String> sourcePaths) {
+ this.sourcePaths = sourcePaths;
+ }
- public void setTargetDirectory(String targetDirectory) {
- this.targetDirectory = targetDirectory;
- }
+ public void setTargetDirectory(String targetDirectory) {
+ this.targetDirectory = targetDirectory;
+ }
- @Override
- public File getTargetDirectory() {
- return new File(targetDirectory);
- }
+ @Override
+ public File getTargetDirectory() {
+ return new File(targetDirectory);
+ }
- @Override
- public Iterator<URL> getSource() {
- if (null != sourceDirectory) {
- return Collections.singleton(URLUtil.parseURL(sourceDirectory)).iterator();
- }
- List<URL> sourceURLs = new ArrayList<URL>();
- if( sourcePaths != null && sourcePaths.size() > 0)
- for (String source : sourcePaths) {
- sourceURLs.add(URLUtil.parseURL(source));
- }
- return sourceURLs.iterator();
+ @Override
+ public Iterator<URL> getSource() {
+ if (null != sourceDirectory) {
+ return Collections.singleton(URLUtil.parseURL(sourceDirectory)).iterator();
}
-
- public void setMaxDepth(int maxDepth) {
- this.maxDepth = maxDepth;
+ List<URL> sourceUrls = new ArrayList<URL>();
+ if ( sourcePaths != null && sourcePaths.size() > 0) {
+ for (String source : sourcePaths) {
+ sourceUrls.add(URLUtil.parseURL(source));
+ }
}
+ return sourceUrls.iterator();
+ }
+
+ public void setMaxDepth(int maxDepth) {
+ this.maxDepth = maxDepth;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGenerator.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGenerator.java b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGenerator.java
index c270e14..e6cb54c 100644
--- a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGenerator.java
+++ b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGenerator.java
@@ -19,12 +19,6 @@
package org.apache.streams.plugins.hive;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
import org.apache.streams.util.schema.FieldType;
import org.apache.streams.util.schema.FieldUtil;
import org.apache.streams.util.schema.FileUtil;
@@ -32,17 +26,29 @@ import org.apache.streams.util.schema.GenerationConfig;
import org.apache.streams.util.schema.Schema;
import org.apache.streams.util.schema.SchemaStore;
import org.apache.streams.util.schema.SchemaStoreImpl;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
import org.jsonschema2pojo.util.URLUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.net.URL;
-import java.util.*;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
import static com.google.common.base.Preconditions.checkNotNull;
-import static org.apache.commons.lang3.StringUtils.defaultString;
-import static org.apache.streams.util.schema.FileUtil.*;
+import static org.apache.streams.util.schema.FileUtil.dropExtension;
+import static org.apache.streams.util.schema.FileUtil.dropSourcePathPrefix;
+import static org.apache.streams.util.schema.FileUtil.swapExtension;
+import static org.apache.streams.util.schema.FileUtil.writeFile;
/**
* Generates hive table definitions for using org.openx.data.jsonserde.JsonSerDe on new-line delimited json documents.
@@ -51,277 +57,300 @@ import static org.apache.streams.util.schema.FileUtil.*;
*/
public class StreamsHiveResourceGenerator implements Runnable {
- private final static Logger LOGGER = LoggerFactory.getLogger(StreamsHiveResourceGenerator.class);
-
- private final static String LS = System.getProperty("line.separator");
+ private static final Logger LOGGER = LoggerFactory.getLogger(StreamsHiveResourceGenerator.class);
- private StreamsHiveGenerationConfig config;
+ private static final String LS = System.getProperty("line.separator");
- private SchemaStore schemaStore = new SchemaStoreImpl();
+ private StreamsHiveGenerationConfig config;
- private int currentDepth = 0;
+ private SchemaStore schemaStore = new SchemaStoreImpl();
- public static void main(String[] args) {
- StreamsHiveGenerationConfig config = new StreamsHiveGenerationConfig();
+ private int currentDepth = 0;
- String sourceDirectory = "src/main/jsonschema";
- String targetDirectory = "target/generated-resources/hive";
+ /**
+ * Run from CLI without Maven
+ *
+ * <p/>
+ * java -jar streams-plugin-hive-jar-with-dependencies.jar StreamsHiveResourceGenerator src/main/jsonschema target/generated-resources
+ *
+ * @param args [sourceDirectory, targetDirectory]
+ * */
+ public static void main(String[] args) {
+ StreamsHiveGenerationConfig config = new StreamsHiveGenerationConfig();
- if( args.length > 0 )
- sourceDirectory = args[0];
- if( args.length > 1 )
- targetDirectory = args[1];
+ String sourceDirectory = "src/main/jsonschema";
+ String targetDirectory = "target/generated-resources/hive";
- config.setSourceDirectory(sourceDirectory);
- config.setTargetDirectory(targetDirectory);
-
- StreamsHiveResourceGenerator streamsHiveResourceGenerator = new StreamsHiveResourceGenerator(config);
- streamsHiveResourceGenerator.run();
+ if ( args.length > 0 ) {
+ sourceDirectory = args[0];
}
-
- public StreamsHiveResourceGenerator(StreamsHiveGenerationConfig config) {
- this.config = config;
+ if ( args.length > 1 ) {
+ targetDirectory = args[1];
}
- public void run() {
+ config.setSourceDirectory(sourceDirectory);
+ config.setTargetDirectory(targetDirectory);
- checkNotNull(config);
+ StreamsHiveResourceGenerator streamsHiveResourceGenerator = new StreamsHiveResourceGenerator(config);
+ streamsHiveResourceGenerator.run();
+ }
- generate(config);
+ public StreamsHiveResourceGenerator(StreamsHiveGenerationConfig config) {
+ this.config = config;
+ }
- }
+ @Override
+ public void run() {
- public void generate(StreamsHiveGenerationConfig config) {
+ checkNotNull(config);
- LinkedList<File> sourceFiles = new LinkedList<File>();
+ generate(config);
- for (Iterator<URL> sources = config.getSource(); sources.hasNext();) {
- URL source = sources.next();
- sourceFiles.add(URLUtil.getFileFromURL(source));
- }
+ }
- LOGGER.info("Seeded with {} source paths:", sourceFiles.size());
+ /**
+ * run generate using supplied StreamsHiveGenerationConfig.
+ * @param config StreamsHiveGenerationConfig
+ */
+ public void generate(StreamsHiveGenerationConfig config) {
- FileUtil.resolveRecursive((GenerationConfig)config, sourceFiles);
+ LinkedList<File> sourceFiles = new LinkedList<File>();
- LOGGER.info("Resolved {} schema files:", sourceFiles.size());
+ for (Iterator<URL> sources = config.getSource(); sources.hasNext();) {
+ URL source = sources.next();
+ sourceFiles.add(URLUtil.getFileFromURL(source));
+ }
- for (Iterator<File> iterator = sourceFiles.iterator(); iterator.hasNext();) {
- File item = iterator.next();
- schemaStore.create(item.toURI());
- }
+ LOGGER.info("Seeded with {} source paths:", sourceFiles.size());
- LOGGER.info("Identified {} objects:", schemaStore.getSize());
+ FileUtil.resolveRecursive((GenerationConfig)config, sourceFiles);
- for (Iterator<Schema> schemaIterator = schemaStore.getSchemaIterator(); schemaIterator.hasNext(); ) {
- Schema schema = schemaIterator.next();
- currentDepth = 0;
- if( schema.getURI().getScheme().equals("file")) {
- String inputFile = schema.getURI().getPath();
- String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory());
- for (String sourcePath : config.getSourcePaths()) {
- resourcePath = dropSourcePathPrefix(resourcePath, sourcePath);
- }
- String outputFile = config.getTargetDirectory() + "/" + swapExtension(resourcePath, "json", "hql");
+ LOGGER.info("Resolved {} schema files:", sourceFiles.size());
- LOGGER.info("Processing {}:", resourcePath);
+ for (Iterator<File> iterator = sourceFiles.iterator(); iterator.hasNext();) {
+ File item = iterator.next();
+ schemaStore.create(item.toURI());
+ }
- String resourceId = dropExtension(resourcePath).replace("/", "_");
+ LOGGER.info("Identified {} objects:", schemaStore.getSize());
- String resourceContent = generateResource(schema, resourceId);
+ for (Iterator<Schema> schemaIterator = schemaStore.getSchemaIterator(); schemaIterator.hasNext(); ) {
+ Schema schema = schemaIterator.next();
+ currentDepth = 0;
+ if ( schema.getUri().getScheme().equals("file")) {
+ String inputFile = schema.getUri().getPath();
+ String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory());
+ for (String sourcePath : config.getSourcePaths()) {
+ resourcePath = dropSourcePathPrefix(resourcePath, sourcePath);
+ }
+ String outputFile = config.getTargetDirectory() + "/" + swapExtension(resourcePath, "json", "hql");
- writeFile(outputFile, resourceContent);
+ LOGGER.info("Processing {}:", resourcePath);
- LOGGER.info("Wrote {}:", outputFile);
- }
- }
- }
+ String resourceId = dropExtension(resourcePath).replace("/", "_");
- public String generateResource(Schema schema, String resourceId) {
- StringBuilder resourceBuilder = new StringBuilder();
- resourceBuilder.append("CREATE TABLE ");
- resourceBuilder.append(hqlEscape(resourceId));
- resourceBuilder.append(LS);
- resourceBuilder.append("(");
- resourceBuilder.append(LS);
- resourceBuilder = appendRootObject(resourceBuilder, schema, resourceId, ' ');
- resourceBuilder.append(")");
- resourceBuilder.append(LS);
- resourceBuilder.append("ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'");
- resourceBuilder.append(LS);
- resourceBuilder.append("WITH SERDEPROPERTIES (\"ignore.malformed.json\" = \"true\"");
- resourceBuilder.append(LS);
- resourceBuilder.append("STORED AS TEXTFILE");
- resourceBuilder.append(LS);
- resourceBuilder.append("LOCATION '${hiveconf:path}';");
- resourceBuilder.append(LS);
- return resourceBuilder.toString();
- }
+ String resourceContent = generateResource(schema, resourceId);
- public StringBuilder appendRootObject(StringBuilder builder, Schema schema, String resourceId, Character seperator) {
- ObjectNode propertiesNode = schemaStore.resolveProperties(schema, null, resourceId);
- if( propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0) {
- builder = appendPropertiesNode(builder, schema, propertiesNode, seperator);
- }
- return builder;
- }
+ writeFile(outputFile, resourceContent);
- private StringBuilder appendValueField(StringBuilder builder, Schema schema, String fieldId, FieldType fieldType, Character seperator) {
- // safe to append nothing
- checkNotNull(builder);
- builder.append(hqlEscape(fieldId));
- builder.append(seperator);
- builder.append(hqlType(fieldType));
- return builder;
+ LOGGER.info("Wrote {}:", outputFile);
+ }
}
-
- public StringBuilder appendArrayItems(StringBuilder builder, Schema schema, String fieldId, ObjectNode itemsNode, Character seperator) {
- // not safe to append nothing
- checkNotNull(builder);
- if( itemsNode == null ) return builder;
- if( itemsNode.has("type")) {
- try {
- FieldType itemType = FieldUtil.determineFieldType(itemsNode);
- switch( itemType ) {
- case OBJECT:
- builder = appendArrayObject(builder, schema, fieldId, itemsNode, seperator);
- break;
- case ARRAY:
- ObjectNode subArrayItems = (ObjectNode) itemsNode.get("items");
- builder = appendArrayItems(builder, schema, fieldId, subArrayItems, seperator);
- break;
- default:
- builder = appendArrayField(builder, schema, fieldId, itemType, seperator);
- }
- } catch (Exception e) {
- LOGGER.warn("No item type resolvable for {}", fieldId);
- }
- }
- checkNotNull(builder);
- return builder;
+ }
+
+ /**
+ * generateResource String from schema and resourceId.
+ * @param schema Schema
+ * @param resourceId String
+ * @return CREATE TABLE ...
+ */
+ public String generateResource(Schema schema, String resourceId) {
+ StringBuilder resourceBuilder = new StringBuilder();
+ resourceBuilder.append("CREATE TABLE ");
+ resourceBuilder.append(hqlEscape(resourceId));
+ resourceBuilder.append(LS);
+ resourceBuilder.append("(");
+ resourceBuilder.append(LS);
+ resourceBuilder = appendRootObject(resourceBuilder, schema, resourceId, ' ');
+ resourceBuilder.append(")");
+ resourceBuilder.append(LS);
+ resourceBuilder.append("ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'");
+ resourceBuilder.append(LS);
+ resourceBuilder.append("WITH SERDEPROPERTIES (\"ignore.malformed.json\" = \"true\"");
+ resourceBuilder.append(LS);
+ resourceBuilder.append("STORED AS TEXTFILE");
+ resourceBuilder.append(LS);
+ resourceBuilder.append("LOCATION '${hiveconf:path}';");
+ resourceBuilder.append(LS);
+ return resourceBuilder.toString();
+ }
+
+ protected StringBuilder appendRootObject(StringBuilder builder, Schema schema, String resourceId, Character seperator) {
+ ObjectNode propertiesNode = schemaStore.resolveProperties(schema, null, resourceId);
+ if ( propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0) {
+ builder = appendPropertiesNode(builder, schema, propertiesNode, seperator);
}
-
- private StringBuilder appendArrayField(StringBuilder builder, Schema schema, String fieldId, FieldType fieldType, Character seperator) {
- // safe to append nothing
- checkNotNull(builder);
- checkNotNull(fieldId);
- builder.append(hqlEscape(fieldId));
- builder.append(seperator);
- builder.append("ARRAY<"+hqlType(fieldType)+">");
- checkNotNull(builder);
- return builder;
+ return builder;
+ }
+
+ private StringBuilder appendValueField(StringBuilder builder, Schema schema, String fieldId, FieldType fieldType, Character seperator) {
+ // safe to append nothing
+ checkNotNull(builder);
+ builder.append(hqlEscape(fieldId));
+ builder.append(seperator);
+ builder.append(hqlType(fieldType));
+ return builder;
+ }
+
+ protected StringBuilder appendArrayItems(StringBuilder builder, Schema schema, String fieldId, ObjectNode itemsNode, Character seperator) {
+ // not safe to append nothing
+ checkNotNull(builder);
+ if ( itemsNode == null ) {
+ return builder;
}
-
- private StringBuilder appendArrayObject(StringBuilder builder, Schema schema, String fieldId, ObjectNode fieldNode, Character seperator) {
- // safe to append nothing
- checkNotNull(builder);
- checkNotNull(fieldNode);
- if( !Strings.isNullOrEmpty(fieldId)) {
- builder.append(hqlEscape(fieldId));
- builder.append(seperator);
+ if ( itemsNode.has("type")) {
+ try {
+ FieldType itemType = FieldUtil.determineFieldType(itemsNode);
+ switch ( itemType ) {
+ case OBJECT:
+ builder = appendArrayObject(builder, schema, fieldId, itemsNode, seperator);
+ break;
+ case ARRAY:
+ ObjectNode subArrayItems = (ObjectNode) itemsNode.get("items");
+ builder = appendArrayItems(builder, schema, fieldId, subArrayItems, seperator);
+ break;
+ default:
+ builder = appendArrayField(builder, schema, fieldId, itemType, seperator);
}
- builder.append("ARRAY");
- builder.append(LS);
- builder.append("<");
- builder.append(LS);
- ObjectNode propertiesNode = schemaStore.resolveProperties(schema, fieldNode, fieldId);
- builder = appendStructField(builder, schema, "", propertiesNode, ':');
- builder.append(">");
- checkNotNull(builder);
- return builder;
+ } catch (Exception ex) {
+ LOGGER.warn("No item type resolvable for {}", fieldId);
+ }
}
+ checkNotNull(builder);
+ return builder;
+ }
+
+ private StringBuilder appendArrayField(StringBuilder builder, Schema schema, String fieldId, FieldType fieldType, Character seperator) {
+ // safe to append nothing
+ checkNotNull(builder);
+ checkNotNull(fieldId);
+ builder.append(hqlEscape(fieldId));
+ builder.append(seperator);
+ builder.append("ARRAY<" + hqlType(fieldType) + ">");
+ checkNotNull(builder);
+ return builder;
+ }
+
+ private StringBuilder appendArrayObject(StringBuilder builder, Schema schema, String fieldId, ObjectNode fieldNode, Character seperator) {
+ // safe to append nothing
+ checkNotNull(builder);
+ checkNotNull(fieldNode);
+ if ( !Strings.isNullOrEmpty(fieldId)) {
+ builder.append(hqlEscape(fieldId));
+ builder.append(seperator);
+ }
+ builder.append("ARRAY");
+ builder.append(LS);
+ builder.append("<");
+ builder.append(LS);
+ ObjectNode propertiesNode = schemaStore.resolveProperties(schema, fieldNode, fieldId);
+ builder = appendStructField(builder, schema, "", propertiesNode, ':');
+ builder.append(">");
+ checkNotNull(builder);
+ return builder;
+ }
+
+ private StringBuilder appendStructField(StringBuilder builder, Schema schema, String fieldId, ObjectNode propertiesNode, Character seperator) {
+ // safe to append nothing
+ checkNotNull(builder);
+ checkNotNull(propertiesNode);
+
+ if ( propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0 ) {
+
+ currentDepth += 1;
+
+ if ( !Strings.isNullOrEmpty(fieldId)) {
+ builder.append(hqlEscape(fieldId));
+ builder.append(seperator);
+ }
+ builder.append("STRUCT");
+ builder.append(LS);
+ builder.append("<");
+ builder.append(LS);
- private StringBuilder appendStructField(StringBuilder builder, Schema schema, String fieldId, ObjectNode propertiesNode, Character seperator) {
- // safe to append nothing
- checkNotNull(builder);
- checkNotNull(propertiesNode);
-
- if( propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0 ) {
-
- currentDepth += 1;
-
- if( !Strings.isNullOrEmpty(fieldId)) {
- builder.append(hqlEscape(fieldId));
- builder.append(seperator);
- }
- builder.append("STRUCT");
- builder.append(LS);
- builder.append("<");
- builder.append(LS);
-
- builder = appendPropertiesNode(builder, schema, propertiesNode, ':');
+ builder = appendPropertiesNode(builder, schema, propertiesNode, ':');
- builder.append(">");
- builder.append(LS);
+ builder.append(">");
+ builder.append(LS);
- currentDepth -= 1;
+ currentDepth -= 1;
- }
- checkNotNull(builder);
- return builder;
}
-
- private StringBuilder appendPropertiesNode(StringBuilder builder, Schema schema, ObjectNode propertiesNode, Character seperator) {
- checkNotNull(builder);
- checkNotNull(propertiesNode);
- Iterator<Map.Entry<String, JsonNode>> fields = propertiesNode.fields();
- Joiner joiner = Joiner.on(","+LS).skipNulls();
- List<String> fieldStrings = Lists.newArrayList();
- for( ; fields.hasNext(); ) {
- Map.Entry<String, JsonNode> field = fields.next();
- String fieldId = field.getKey();
- if( !config.getExclusions().contains(fieldId) && field.getValue().isObject()) {
- ObjectNode fieldNode = (ObjectNode) field.getValue();
- FieldType fieldType = FieldUtil.determineFieldType(fieldNode);
- if (fieldType != null ) {
- switch (fieldType) {
- case ARRAY:
- ObjectNode itemsNode = (ObjectNode) fieldNode.get("items");
- if( currentDepth <= config.getMaxDepth()) {
- StringBuilder arrayItemsBuilder = appendArrayItems(new StringBuilder(), schema, fieldId, itemsNode, seperator);
- if( !Strings.isNullOrEmpty(arrayItemsBuilder.toString())) {
- fieldStrings.add(arrayItemsBuilder.toString());
- }
- }
- break;
- case OBJECT:
- ObjectNode childProperties = schemaStore.resolveProperties(schema, fieldNode, fieldId);
- if( currentDepth < config.getMaxDepth()) {
- StringBuilder structFieldBuilder = appendStructField(new StringBuilder(), schema, fieldId, childProperties, seperator);
- if( !Strings.isNullOrEmpty(structFieldBuilder.toString())) {
- fieldStrings.add(structFieldBuilder.toString());
- }
- }
- break;
- default:
- StringBuilder valueFieldBuilder = appendValueField(new StringBuilder(), schema, fieldId, fieldType, seperator);
- if( !Strings.isNullOrEmpty(valueFieldBuilder.toString())) {
- fieldStrings.add(valueFieldBuilder.toString());
- }
- }
+ checkNotNull(builder);
+ return builder;
+ }
+
+ private StringBuilder appendPropertiesNode(StringBuilder builder, Schema schema, ObjectNode propertiesNode, Character seperator) {
+ checkNotNull(builder);
+ checkNotNull(propertiesNode);
+ Iterator<Map.Entry<String, JsonNode>> fields = propertiesNode.fields();
+ Joiner joiner = Joiner.on("," + LS).skipNulls();
+ List<String> fieldStrings = Lists.newArrayList();
+ for ( ; fields.hasNext(); ) {
+ Map.Entry<String, JsonNode> field = fields.next();
+ String fieldId = field.getKey();
+ if ( !config.getExclusions().contains(fieldId) && field.getValue().isObject()) {
+ ObjectNode fieldNode = (ObjectNode) field.getValue();
+ FieldType fieldType = FieldUtil.determineFieldType(fieldNode);
+ if (fieldType != null ) {
+ switch (fieldType) {
+ case ARRAY:
+ ObjectNode itemsNode = (ObjectNode) fieldNode.get("items");
+ if ( currentDepth <= config.getMaxDepth()) {
+ StringBuilder arrayItemsBuilder = appendArrayItems(new StringBuilder(), schema, fieldId, itemsNode, seperator);
+ if ( !Strings.isNullOrEmpty(arrayItemsBuilder.toString())) {
+ fieldStrings.add(arrayItemsBuilder.toString());
}
- }
- }
- builder.append(joiner.join(fieldStrings)).append(LS);
- Preconditions.checkNotNull(builder);
- return builder;
- }
-
- private static String hqlEscape( String fieldId ) {
- return "`"+fieldId+"`";
- }
-
- private static String hqlType( FieldType fieldType ) {
- switch( fieldType ) {
- case INTEGER:
- return "INT";
- case NUMBER:
- return "FLOAT";
+ }
+ break;
case OBJECT:
- return "STRUCT";
+ ObjectNode childProperties = schemaStore.resolveProperties(schema, fieldNode, fieldId);
+ if ( currentDepth < config.getMaxDepth()) {
+ StringBuilder structFieldBuilder = appendStructField(new StringBuilder(), schema, fieldId, childProperties, seperator);
+ if ( !Strings.isNullOrEmpty(structFieldBuilder.toString())) {
+ fieldStrings.add(structFieldBuilder.toString());
+ }
+ }
+ break;
default:
- return fieldType.name().toUpperCase();
+ StringBuilder valueFieldBuilder = appendValueField(new StringBuilder(), schema, fieldId, fieldType, seperator);
+ if ( !Strings.isNullOrEmpty(valueFieldBuilder.toString())) {
+ fieldStrings.add(valueFieldBuilder.toString());
+ }
+ }
}
+ }
+ }
+ builder.append(joiner.join(fieldStrings)).append(LS);
+ Preconditions.checkNotNull(builder);
+ return builder;
+ }
+
+ private static String hqlEscape( String fieldId ) {
+ return "`" + fieldId + "`";
+ }
+
+ private static String hqlType( FieldType fieldType ) {
+ switch ( fieldType ) {
+ case INTEGER:
+ return "INT";
+ case NUMBER:
+ return "FLOAT";
+ case OBJECT:
+ return "STRUCT";
+ default:
+ return fieldType.name().toUpperCase();
}
+ }
}