You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2016/06/01 17:53:50 UTC
[34/46] incubator-streams git commit: WIP for apachecon
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1bc84dbc/streams-plugins/streams-plugin-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-elasticsearch/pom.xml b/streams-plugins/streams-plugin-elasticsearch/pom.xml
new file mode 100644
index 0000000..4f0ed4a
--- /dev/null
+++ b/streams-plugins/streams-plugin-elasticsearch/pom.xml
@@ -0,0 +1,245 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.apache.streams.plugins</groupId>
+ <artifactId>streams-plugin-elasticsearch</artifactId>
+ <version>0.3-incubating-SNAPSHOT</version>
+ <packaging>maven-plugin</packaging>
+
+ <parent>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-plugins</artifactId>
+ <version>0.3-incubating-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <properties>
+ <elasticsearch.version>2.3.0</elasticsearch.version>
+ <lucene.version>5.4.0</lucene.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-config</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>commons-logging</artifactId>
+ <groupId>commons-logging</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-schemas</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections</artifactId>
+ <version>0.9.9</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-aether-provider</artifactId>
+ <version>3.3.3</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-artifact</artifactId>
+ <version>3.3.3</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-compat</artifactId>
+ <version>3.3.3</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-core</artifactId>
+ <version>3.3.3</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.sonatype.aether</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-plugin-api</artifactId>
+ <version>3.3.3</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>2.7</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.shared</groupId>
+ <artifactId>maven-verifier</artifactId>
+ <version>1.6</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.plexus</groupId>
+ <artifactId>plexus-utils</artifactId>
+ <version>3.0.15</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-plugin-plugin</artifactId>
+ <version>3.4</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>commons-logging</artifactId>
+ <groupId>commons-logging</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.plugin-tools</groupId>
+ <artifactId>maven-plugin-tools-api</artifactId>
+ <version>3.3</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.plugin-tools</groupId>
+ <artifactId>maven-plugin-annotations</artifactId>
+ <version>3.3</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.plugin-tools</groupId>
+ <artifactId>maven-plugin-tools-generators</artifactId>
+ <version>3.3</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.plugin-testing</groupId>
+ <artifactId>maven-plugin-testing-harness</artifactId>
+ <version>3.3.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>2.6</version>
+ </dependency>
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ <version>2.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.joda</groupId>
+ <artifactId>joda-convert</artifactId>
+ <version>1.8.1</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <sourceDirectory>src/main/java</sourceDirectory>
+ <testSourceDirectory>src/test/java</testSourceDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>src/test/resources</directory>
+ </testResource>
+ </testResources>
+ <plugins>
+ <plugin>
+ <artifactId>maven-plugin-plugin</artifactId>
+ <version>3.4</version>
+ <configuration>
+ <skipErrorNoDescriptorsFound>true</skipErrorNoDescriptorsFound>
+ </configuration>
+ <executions>
+ <execution>
+ <id>mojo-descriptor</id>
+ <phase>process-classes</phase>
+ <goals>
+ <goal>descriptor</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections-maven</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>reflections</goal>
+ </goals>
+ <phase>process-classes</phase>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.4</version>
+ <executions>
+ <execution>
+ <id>resource-dependencies</id>
+ <phase>process-test-resources</phase>
+ <goals>
+ <goal>unpack-dependencies</goal>
+ </goals>
+ <configuration>
+ <includeGroupIds>org.apache.streams</includeGroupIds>
+ <includeArtifactIds>streams-schemas</includeArtifactIds>
+ <includes>**/*.json</includes>
+ <outputDirectory>${project.build.directory}/test-classes/streams-schemas</outputDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1bc84dbc/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchGenerationConfig.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchGenerationConfig.java b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchGenerationConfig.java
new file mode 100644
index 0000000..ef5b9ef
--- /dev/null
+++ b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchGenerationConfig.java
@@ -0,0 +1,81 @@
+package org.apache.streams.plugins.elasticsearch;
+
+import org.apache.streams.schema.GenerationConfig;
+import org.jsonschema2pojo.DefaultGenerationConfig;
+import org.jsonschema2pojo.util.URLUtil;
+
+import java.io.File;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Configures StreamsElasticsearchResourceGenerator
+ *
+ *
+ */
+public class StreamsElasticsearchGenerationConfig extends DefaultGenerationConfig implements GenerationConfig {
+
+ public String getSourceDirectory() {
+ return sourceDirectory;
+ }
+
+ public List<String> getSourcePaths() {
+ return sourcePaths;
+ }
+
+ private String sourceDirectory;
+ private List<String> sourcePaths = new ArrayList<String>();
+ private String targetDirectory;
+ private int maxDepth = 1;
+
+ public Set<String> getExclusions() {
+ return exclusions;
+ }
+
+ public void setExclusions(Set<String> exclusions) {
+ this.exclusions = exclusions;
+ }
+
+ private Set<String> exclusions = new HashSet<String>();
+
+ public int getMaxDepth() {
+ return maxDepth;
+ }
+
+ public void setSourceDirectory(String sourceDirectory) {
+ this.sourceDirectory = sourceDirectory;
+ }
+
+ public void setSourcePaths(List<String> sourcePaths) {
+ this.sourcePaths = sourcePaths;
+ }
+
+ public void setTargetDirectory(String targetDirectory) {
+ this.targetDirectory = targetDirectory;
+ }
+
+ public File getTargetDirectory() {
+ return new File(targetDirectory);
+ }
+
+ public Iterator<URL> getSource() {
+ if (null != sourceDirectory) {
+ return Collections.singleton(URLUtil.parseURL(sourceDirectory)).iterator();
+ }
+ List<URL> sourceURLs = new ArrayList<URL>();
+ if( sourcePaths != null && sourcePaths.size() > 0)
+ for (String source : sourcePaths) {
+ sourceURLs.add(URLUtil.parseURL(source));
+ }
+ return sourceURLs.iterator();
+ }
+
+ public void setMaxDepth(int maxDepth) {
+ this.maxDepth = maxDepth;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1bc84dbc/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGenerator.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGenerator.java b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGenerator.java
new file mode 100644
index 0000000..0f506ca
--- /dev/null
+++ b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGenerator.java
@@ -0,0 +1,373 @@
+package org.apache.streams.plugins.elasticsearch;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Joiner;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+import org.apache.streams.schema.FieldType;
+import org.apache.streams.schema.FieldUtil;
+import org.apache.streams.schema.GenerationConfig;
+import org.apache.streams.schema.Schema;
+import org.apache.streams.schema.SchemaStore;
+import org.jsonschema2pojo.util.URLUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.net.URI;
+import java.net.URL;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.streams.schema.FileUtil.dropExtension;
+import static org.apache.streams.schema.FileUtil.dropSourcePathPrefix;
+import static org.apache.streams.schema.FileUtil.resolveRecursive;
+import static org.apache.streams.schema.FileUtil.writeFile;
+
+/**
+ * Created by sblackmon on 5/3/16.
+ */
+public class StreamsElasticsearchResourceGenerator implements Runnable {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(StreamsElasticsearchResourceGenerator.class);
+
+ private final static String LS = System.getProperty("line.separator");
+
+ private StreamsElasticsearchGenerationConfig config;
+
+ private SchemaStore schemaStore = new SchemaStore();
+
+ private int currentDepth = 0;
+
+ public void main(String[] args) {
+ StreamsElasticsearchGenerationConfig config = new StreamsElasticsearchGenerationConfig();
+
+ String sourceDirectory = "target/test-classes/activities";
+ String targetDirectory = "target/generated-resources/streams-plugin-elasticsearch";
+
+ if( args.length > 0 )
+ sourceDirectory = args[0];
+ if( args.length > 1 )
+ targetDirectory = args[1];
+
+ config.setSourceDirectory(sourceDirectory);
+ config.setTargetDirectory(targetDirectory);
+
+ StreamsElasticsearchResourceGenerator StreamsElasticsearchResourceGenerator = new StreamsElasticsearchResourceGenerator(config);
+ Thread thread = new Thread(StreamsElasticsearchResourceGenerator);
+ thread.start();
+ try {
+ thread.join();
+ } catch (InterruptedException e) {
+ LOGGER.error("InterruptedException", e);
+ } catch (Exception e) {
+ LOGGER.error("Exception", e);
+ }
+ return;
+ }
+
+ public StreamsElasticsearchResourceGenerator(StreamsElasticsearchGenerationConfig config) {
+ this.config = config;
+ }
+
+ public void run() {
+
+ checkNotNull(config);
+
+ generate(config);
+
+ }
+
+ public void generate(StreamsElasticsearchGenerationConfig config) {
+
+ LinkedList<File> sourceFiles = new LinkedList<File>();
+
+ for (Iterator<URL> sources = config.getSource(); sources.hasNext();) {
+ URL source = sources.next();
+ sourceFiles.add(URLUtil.getFileFromURL(source));
+ }
+
+ LOGGER.info("Seeded with {} source paths:", sourceFiles.size());
+
+ resolveRecursive((GenerationConfig)config, sourceFiles);
+
+ LOGGER.info("Resolved {} schema files:", sourceFiles.size());
+
+ for (Iterator<File> iterator = sourceFiles.iterator(); iterator.hasNext();) {
+ File item = iterator.next();
+ schemaStore.create(item.toURI());
+ }
+
+ LOGGER.info("Identified {} objects:", schemaStore.getSize());
+
+ String outputFile = config.getTargetDirectory() + "/" + "types.cql";
+ StringBuilder typesContent = new StringBuilder();
+
+ for (Iterator<Schema> schemaIterator = schemaStore.getSchemaIterator(); schemaIterator.hasNext(); ) {
+ Schema schema = schemaIterator.next();
+ currentDepth = 0;
+ if( schema.getURI().getScheme().equals("file")) {
+ String inputFile = schema.getURI().getPath();
+ String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory());
+ for (String sourcePath : config.getSourcePaths()) {
+ resourcePath = dropSourcePathPrefix(resourcePath, sourcePath);
+ }
+
+ LOGGER.info("Processing {}:", resourcePath);
+
+ String resourceId = schemaSymbol(schema);
+
+ String resourceContent = generateResource(schema, resourceId);
+
+ typesContent.append(resourceContent);
+
+ LOGGER.info("Added {}:", resourceId);
+ }
+ }
+
+ writeFile(outputFile, typesContent.toString());
+
+ }
+
+ public String generateResource(Schema schema, String resourceId) {
+ StringBuilder resourceBuilder = new StringBuilder();
+ resourceBuilder.append("CREATE TYPE ");
+ resourceBuilder.append(resourceId);
+ resourceBuilder.append(" IF NOT EXISTS (");
+ resourceBuilder.append(LS);
+ resourceBuilder = appendRootObject(resourceBuilder, schema, resourceId, ' ');
+ resourceBuilder.append(");");
+ resourceBuilder.append(LS);
+ return resourceBuilder.toString();
+ }
+
+ public StringBuilder appendRootObject(StringBuilder builder, Schema schema, String resourceId, Character seperator) {
+ ObjectNode propertiesNode = schemaStore.resolveProperties(schema, null, resourceId);
+ if( propertiesNode.get("id") != null ) {
+ builder.append("id text PRIMARY KEY,");
+ builder.append(LS);
+ propertiesNode.remove("id");
+ }
+ if( propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0) {
+ builder = appendPropertiesNode(builder, schema, propertiesNode, seperator);
+ }
+ return builder;
+ }
+
+ private StringBuilder appendValueField(StringBuilder builder, Schema schema, String fieldId, FieldType fieldType, Character seperator) {
+ // safe to append nothing
+ checkNotNull(builder);
+ builder.append(cqlEscape(fieldId));
+ builder.append(seperator);
+ builder.append(cqlType(fieldType));
+ return builder;
+ }
+
+ public StringBuilder appendArrayItems(StringBuilder builder, Schema schema, String fieldId, ObjectNode itemsNode, Character seperator) {
+ // not safe to append nothing
+ checkNotNull(builder);
+ if( itemsNode == null ) return builder;
+ if( itemsNode.has("type")) {
+ try {
+ FieldType itemType = FieldUtil.determineFieldType(itemsNode);
+ switch( itemType ) {
+ case OBJECT:
+ Schema objectSchema = null;
+ URI parentURI = null;
+ if( itemsNode.has("$ref") || itemsNode.has("extends") ) {
+ JsonNode refNode = itemsNode.get("$ref");
+ JsonNode extendsNode = itemsNode.get("extends");
+ if (refNode != null && refNode.isValueNode())
+ parentURI = URI.create(refNode.asText());
+ else if (extendsNode != null && extendsNode.isObject())
+ parentURI = URI.create(extendsNode.get("$ref").asText());
+ URI absoluteURI;
+ if (parentURI.isAbsolute())
+ absoluteURI = parentURI;
+ else {
+ absoluteURI = schema.getURI().resolve(parentURI);
+ if (!absoluteURI.isAbsolute() || (absoluteURI.isAbsolute() && !schemaStore.getByUri(absoluteURI).isPresent() ))
+ absoluteURI = schema.getParentURI().resolve(parentURI);
+ }
+ if (absoluteURI != null && absoluteURI.isAbsolute()) {
+ Optional<Schema> schemaLookup = schemaStore.getByUri(absoluteURI);
+ if (schemaLookup.isPresent()) {
+ objectSchema = schemaLookup.get();
+ }
+ }
+ }
+ // have to resolve schema here
+
+ builder = appendArrayObject(builder, objectSchema, fieldId, seperator);
+ break;
+ case ARRAY:
+ ObjectNode subArrayItems = (ObjectNode) itemsNode.get("items");
+ builder = appendArrayItems(builder, schema, fieldId, subArrayItems, seperator);
+ break;
+ default:
+ builder = appendArrayField(builder, schema, fieldId, itemType, seperator);
+ }
+ } catch (Exception e) {
+ LOGGER.warn("No item type resolvable for {}", fieldId);
+ }
+ }
+ checkNotNull(builder);
+ return builder;
+ }
+
+ private StringBuilder appendArrayField(StringBuilder builder, Schema schema, String fieldId, FieldType fieldType, Character seperator) {
+ // safe to append nothing
+ checkNotNull(builder);
+ checkNotNull(fieldId);
+ builder.append(cqlEscape(fieldId));
+ builder.append(seperator);
+ builder.append("list<"+cqlType(fieldType)+">");
+ checkNotNull(builder);
+ return builder;
+ }
+
+ private StringBuilder appendArrayObject(StringBuilder builder, Schema schema, String fieldId, Character seperator) {
+ // safe to append nothing
+ checkNotNull(builder);
+ String schemaSymbol = schemaSymbol(schema);
+ if( !Strings.isNullOrEmpty(fieldId) && schemaSymbol != null ) {
+ builder.append(cqlEscape(fieldId));
+ builder.append(seperator);
+ builder.append("list<" + schemaSymbol + ">");
+ builder.append(LS);
+ }
+ checkNotNull(builder);
+ return builder;
+ }
+
+ private StringBuilder appendSchemaField(StringBuilder builder, Schema schema, String fieldId, Character seperator) {
+ // safe to append nothing
+ checkNotNull(builder);
+ String schemaSymbol = schemaSymbol(schema);
+ if( !Strings.isNullOrEmpty(fieldId) && schemaSymbol != null ) {
+ builder.append(cqlEscape(fieldId));
+ builder.append(seperator);
+ builder.append(schemaSymbol);
+ }
+ checkNotNull(builder);
+ return builder;
+ }
+
+ /*
+ can this be moved to streams-schemas if schemastore available in scope?
+ maybe an interface?
+ lot of boilerplate / reuse between plugins
+ however treatment is way different when resolving a type symbol vs resolving and listing fields .
+ */
+ private StringBuilder appendPropertiesNode(StringBuilder builder, Schema schema, ObjectNode propertiesNode, Character seperator) {
+ checkNotNull(builder);
+ checkNotNull(propertiesNode);
+ Iterator<Map.Entry<String, JsonNode>> fields = propertiesNode.fields();
+ Joiner joiner = Joiner.on(","+LS).skipNulls();
+ List<String> fieldStrings = Lists.newArrayList();
+ for( ; fields.hasNext(); ) {
+ Map.Entry<String, JsonNode> field = fields.next();
+ String fieldId = field.getKey();
+ if( !config.getExclusions().contains(fieldId) && field.getValue().isObject()) {
+ ObjectNode fieldNode = (ObjectNode) field.getValue();
+ FieldType fieldType = FieldUtil.determineFieldType(fieldNode);
+ if (fieldType != null ) {
+ switch (fieldType) {
+ case ARRAY:
+ ObjectNode itemsNode = (ObjectNode) fieldNode.get("items");
+ if( currentDepth <= config.getMaxDepth()) {
+ StringBuilder arrayItemsBuilder = appendArrayItems(new StringBuilder(), schema, fieldId, itemsNode, seperator);
+ if( !Strings.isNullOrEmpty(arrayItemsBuilder.toString())) {
+ fieldStrings.add(arrayItemsBuilder.toString());
+ }
+ }
+ break;
+ case OBJECT:
+ Schema objectSchema = null;
+ URI parentURI = null;
+ if( fieldNode.has("$ref") || fieldNode.has("extends") ) {
+ JsonNode refNode = fieldNode.get("$ref");
+ JsonNode extendsNode = fieldNode.get("extends");
+ if (refNode != null && refNode.isValueNode())
+ parentURI = URI.create(refNode.asText());
+ else if (extendsNode != null && extendsNode.isObject())
+ parentURI = URI.create(extendsNode.get("$ref").asText());
+ URI absoluteURI;
+ if (parentURI.isAbsolute())
+ absoluteURI = parentURI;
+ else {
+ absoluteURI = schema.getURI().resolve(parentURI);
+ if (!absoluteURI.isAbsolute() || (absoluteURI.isAbsolute() && !schemaStore.getByUri(absoluteURI).isPresent() ))
+ absoluteURI = schema.getParentURI().resolve(parentURI);
+ }
+ if (absoluteURI != null && absoluteURI.isAbsolute()) {
+ Optional<Schema> schemaLookup = schemaStore.getByUri(absoluteURI);
+ if (schemaLookup.isPresent()) {
+ objectSchema = schemaLookup.get();
+ }
+ }
+ }
+ //ObjectNode childProperties = schemaStore.resolveProperties(schema, fieldNode, fieldId);
+ if( currentDepth < config.getMaxDepth()) {
+ StringBuilder structFieldBuilder = appendSchemaField(new StringBuilder(), objectSchema, fieldId, seperator);
+ if( !Strings.isNullOrEmpty(structFieldBuilder.toString())) {
+ fieldStrings.add(structFieldBuilder.toString());
+ }
+ }
+ break;
+ default:
+ StringBuilder valueFieldBuilder = appendValueField(new StringBuilder(), schema, fieldId, fieldType, seperator);
+ if( !Strings.isNullOrEmpty(valueFieldBuilder.toString())) {
+ fieldStrings.add(valueFieldBuilder.toString());
+ }
+ }
+ }
+ }
+ }
+ builder.append(joiner.join(fieldStrings)).append(LS);
+ Preconditions.checkNotNull(builder);
+ return builder;
+ }
+
+ private static String cqlEscape( String fieldId ) {
+ return "`"+fieldId+"`";
+ }
+
+ private static String cqlType( FieldType fieldType ) {
+ switch( fieldType ) {
+ case STRING:
+ return "text";
+ case INTEGER:
+ return "int";
+ case NUMBER:
+ return "double";
+ case OBJECT:
+ return "tuple";
+ case ARRAY:
+ return "list";
+ default:
+ return fieldType.name().toUpperCase();
+ }
+ }
+
+ private String schemaSymbol( Schema schema ) {
+ if (schema == null) return null;
+ // this needs to return whatever
+ if (schema.getURI().getScheme().equals("file")) {
+ String inputFile = schema.getURI().getPath();
+ String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory());
+ for (String sourcePath : config.getSourcePaths()) {
+ resourcePath = dropSourcePathPrefix(resourcePath, sourcePath);
+ }
+ return dropExtension(resourcePath).replace("/", "_");
+ } else {
+ return "IDK";
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1bc84dbc/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java
new file mode 100644
index 0000000..6969051
--- /dev/null
+++ b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java
@@ -0,0 +1,93 @@
+package org.apache.streams.plugins.elasticsearch;
+
+import org.apache.maven.plugin.AbstractMojo;
+import org.apache.maven.plugin.MojoExecutionException;
+import org.apache.maven.plugin.MojoFailureException;
+import org.apache.maven.plugins.annotations.Component;
+import org.apache.maven.plugins.annotations.Execute;
+import org.apache.maven.plugins.annotations.LifecyclePhase;
+import org.apache.maven.plugins.annotations.Mojo;
+import org.apache.maven.plugins.annotations.Parameter;
+import org.apache.maven.project.MavenProject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.util.List;
+
+@Mojo( name = "elasticsearch",
+ defaultPhase = LifecyclePhase.GENERATE_RESOURCES
+)
+@Execute( goal = "elasticsearch",
+ phase = LifecyclePhase.GENERATE_RESOURCES
+)
+public class StreamsElasticsearchResourceGeneratorMojo extends AbstractMojo {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(StreamsElasticsearchResourceGeneratorMojo.class);
+
+ private volatile MojoFailureException mojoFailureException;
+
+ @Component
+ private MavenProject project;
+
+// @Component
+// private Settings settings;
+//
+// @Parameter( defaultValue = "${localRepository}", readonly = true, required = true )
+// protected ArtifactRepository localRepository;
+//
+// @Parameter( defaultValue = "${plugin}", readonly = true ) // Maven 3 only
+// private PluginDescriptor plugin;
+//
+ @Parameter( defaultValue = "${project.basedir}", readonly = true )
+ private File basedir;
+
+ @Parameter( defaultValue = "src/main/jsonschema", readonly = true ) // Maven 3 only
+ public String sourceDirectory;
+
+ @Parameter( readonly = true ) // Maven 3 only
+ public List<String> sourcePaths;
+
+ @Parameter(defaultValue = "./target/generated-resources/streams-plugin-elasticsearch", readonly = true)
+ public String targetDirectory;
+
+ public void execute() throws MojoExecutionException, MojoFailureException {
+
+ //addProjectDependenciesToClasspath();
+
+ StreamsElasticsearchGenerationConfig config = new StreamsElasticsearchGenerationConfig();
+
+ if( sourcePaths != null && sourcePaths.size() > 0)
+ config.setSourcePaths(sourcePaths);
+ else
+ config.setSourceDirectory(sourceDirectory);
+ config.setTargetDirectory(targetDirectory);
+
+ StreamsElasticsearchResourceGenerator streamsElasticsearchResourceGenerator = new StreamsElasticsearchResourceGenerator(config);
+
+ Thread.UncaughtExceptionHandler h = new Thread.UncaughtExceptionHandler() {
+ public void uncaughtException(Thread th, Throwable ex) {
+ LOGGER.error("Exception", ex);
+ mojoFailureException = new MojoFailureException("Exception", ex);
+ }
+ };
+ Thread.setDefaultUncaughtExceptionHandler(h);
+ Thread thread = new Thread(streamsElasticsearchResourceGenerator);
+ thread.setUncaughtExceptionHandler(h);
+ try {
+ thread.start();
+ thread.join();
+ } catch (InterruptedException e) {
+ LOGGER.error("InterruptedException", e);
+ } catch (Exception e) {
+ LOGGER.error("Exception", e);
+ mojoFailureException = new MojoFailureException("Exception", e);
+ }
+
+ if( mojoFailureException != null )
+ throw mojoFailureException;
+
+ return;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1bc84dbc/streams-plugins/streams-plugin-elasticsearch/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-elasticsearch/src/site/markdown/index.md b/streams-plugins/streams-plugin-elasticsearch/src/site/markdown/index.md
new file mode 100644
index 0000000..9e80a3e
--- /dev/null
+++ b/streams-plugins/streams-plugin-elasticsearch/src/site/markdown/index.md
@@ -0,0 +1,22 @@
+org.apache.streams.plugins:streams-plugin-elasticsearch
+=======================================================
+
+streams-plugin-elasticsearch generates resources from json schemas to assist with indexing of json data using Elasticsearch.
+
+#### Usage
+
+Run within a module containing a src/main/jsonschema directory
+
+ mvn org.apache.streams.plugins:streams-plugin-elasticsearch:0.3-incubating-SNAPSHOT:elasticsearch
+
+Output will be placed in target/generated-resources/elasticsearch by default
+
+#### Example
+
+[streams-plugin-elasticsearch/pom.xml](streams-plugin-elasticsearch/pom.xml "streams-plugin-elasticsearch/pom.xml")
+
+#### Documentation
+
+[JavaDocs](apidocs/index.html "JavaDocs")
+
+###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1bc84dbc/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorMojoTest.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorMojoTest.java b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorMojoTest.java
new file mode 100644
index 0000000..72f4675
--- /dev/null
+++ b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorMojoTest.java
@@ -0,0 +1,50 @@
+package org.apache.streams.plugins.elasticsearch.test;
+
+import com.google.common.collect.Lists;
+import junit.framework.TestCase;
+import org.apache.maven.it.Verifier;
+import org.apache.maven.it.util.ResourceExtractor;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Tests that streams-plugin-hive running via maven generates hql resources
+ */
+public class StreamsElasticsearchResourceGeneratorMojoTest extends TestCase {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(StreamsElasticsearchResourceGeneratorMojoTest.class);
+
+ protected void setUp() throws Exception
+ {
+ // required for mojo lookups to work
+ super.setUp();
+ }
+
+
+ @Test
+ public void testStreamsElasticsearchResourceGeneratorMojo() throws Exception {
+
+ File testDir = ResourceExtractor.simpleExtractResources( getClass(), "/streams-plugin-elasticsearch" );
+
+ Verifier verifier;
+
+ verifier = new Verifier( testDir.getAbsolutePath() );
+
+ List cliOptions = new ArrayList();
+ cliOptions.add( "-N" );
+ verifier.executeGoals( Lists.<String>newArrayList(
+ "clean",
+ "dependency:unpack-dependencies",
+ "generate-resources"));
+
+ verifier.verifyErrorFreeLog();
+
+ verifier.resetStreams();
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1bc84dbc/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java
new file mode 100644
index 0000000..20bb561
--- /dev/null
+++ b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java
@@ -0,0 +1,139 @@
+package org.apache.streams.plugins.elasticsearch.test;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.common.io.Files;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.streams.plugins.elasticsearch.StreamsElasticsearchGenerationConfig;
+import org.apache.streams.plugins.elasticsearch.StreamsElasticsearchResourceGenerator;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+import java.io.File;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Collection;
+import java.util.Iterator;
+
+import static org.apache.streams.schema.FileUtil.dropSourcePathPrefix;
+
+/**
+ * Test that Elasticsearch resources are generated.
+ */
+public class StreamsElasticsearchResourceGeneratorTest {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(StreamsElasticsearchResourceGeneratorTest.class);
+
+ /**
+ * Test that Elasticsearch resources are generated
+ *
+ * @throws Exception
+ */
+ @Test
+ public void StreamsElasticsearchResourceGenerator() throws Exception {
+
+ StreamsElasticsearchGenerationConfig config = new StreamsElasticsearchGenerationConfig();
+
+ String sourceDirectory = "target/test-classes/streams-schemas";
+
+ config.setSourceDirectory(sourceDirectory);
+
+ config.setTargetDirectory("target/generated-sources/test");
+
+ config.setExclusions(Sets.newHashSet("attachments"));
+
+ config.setMaxDepth(2);
+
+ StreamsElasticsearchResourceGenerator streamsElasticsearchResourceGenerator = new StreamsElasticsearchResourceGenerator(config);
+ Thread thread = new Thread(streamsElasticsearchResourceGenerator);
+ thread.start();
+ try {
+ thread.join();
+ } catch (InterruptedException e) {
+ LOGGER.error("InterruptedException", e);
+ } catch (Exception e) {
+ LOGGER.error("Exception", e);
+ }
+
+ File testOutput = new File( "./target/generated-sources/test");
+ Predicate<File> jsonFilter = new Predicate<File>() {
+ @Override
+ public boolean apply(@Nullable File file) {
+ if( file.getName().endsWith(".json") )
+ return true;
+ else return false;
+ }
+ };
+
+ assert( testOutput != null );
+ assert( testOutput.exists() == true );
+ assert( testOutput.isDirectory() == true );
+
+ Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput)
+ .filter(jsonFilter);
+ Collection<File> outputCollection = Lists.newArrayList(outputIterator);
+ assert( outputCollection.size() == 133 );
+
+ String expectedDirectory = "target/test-classes/expected";
+ File testExpected = new File( expectedDirectory );
+
+ Iterable<File> expectedIterator = Files.fileTreeTraverser().breadthFirstTraversal(testExpected)
+ .filter(jsonFilter);
+ Collection<File> expectedCollection = Lists.newArrayList(expectedIterator);
+
+ int fails = 0;
+
+ Iterator<File> iterator = expectedCollection.iterator();
+ while( iterator.hasNext() ) {
+ File objectExpected = iterator.next();
+ String expectedEnd = dropSourcePathPrefix(objectExpected.getAbsolutePath(), expectedDirectory);
+ File objectActual = new File(config.getTargetDirectory() + "/" + expectedEnd);
+ LOGGER.info("Comparing: {} and {}", objectExpected.getAbsolutePath(), objectActual.getAbsolutePath());
+ assert( objectActual.exists());
+ if( FileUtils.contentEquals(objectActual, objectExpected) == true ) {
+ LOGGER.info("Exact Match!");
+ } else {
+ LOGGER.info("No Match!");
+ fails++;
+ }
+ }
+ if( fails > 0 ) {
+ LOGGER.info("Fails: {}", fails);
+ Assert.fail();
+ }
+
+// String expectedDirectory = "target/test-classes/expected";
+// File testExpected = new File( expectedDirectory );
+//
+// Iterable<File> expectedIterator = Files.fileTreeTraverser().breadthFirstTraversal(testExpected)
+// .filter(cqlFilter);
+// Collection<File> expectedCollection = Lists.newArrayList(expectedIterator);
+//
+// int fails = 0;
+//
+// Iterator<File> iterator = expectedCollection.iterator();
+// while( iterator.hasNext() ) {
+// File objectExpected = iterator.next();
+// String expectedEnd = dropSourcePathPrefix(objectExpected.getAbsolutePath(), expectedDirectory);
+// File objectActual = new File(config.getTargetDirectory() + "/" + expectedEnd);
+// LOGGER.info("Comparing: {} and {}", objectExpected.getAbsolutePath(), objectActual.getAbsolutePath());
+// assert( objectActual.exists());
+// if( FileUtils.contentEquals(objectActual, objectExpected) == true ) {
+// LOGGER.info("Exact Match!");
+// } else {
+// LOGGER.info("No Match!");
+// fails++;
+// }
+// }
+// if( fails > 0 ) {
+// LOGGER.info("Fails: {}", fails);
+// Assert.fail();
+// }
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1bc84dbc/streams-plugins/streams-plugin-elasticsearch/src/test/resources/streams-plugin-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-elasticsearch/src/test/resources/streams-plugin-elasticsearch/pom.xml b/streams-plugins/streams-plugin-elasticsearch/src/test/resources/streams-plugin-elasticsearch/pom.xml
new file mode 100644
index 0000000..2aaa037
--- /dev/null
+++ b/streams-plugins/streams-plugin-elasticsearch/src/test/resources/streams-plugin-elasticsearch/pom.xml
@@ -0,0 +1,75 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>org.apache.streams.plugins</groupId>
+ <artifactId>streams-plugin-elasticsearch-test</artifactId>
+ <version>0.3-incubating-SNAPSHOT</version>
+ <packaging>jar</packaging>
+ <name>Test StreamsElasticsearchResourceGeneratorMojo</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.11</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-schemas</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.streams.plugins</groupId>
+ <artifactId>streams-plugin-elasticsearch</artifactId>
+ <version>0.3-incubating-SNAPSHOT</version>
+ <configuration>
+ <sourcePaths>
+ <sourcePath>target/test-classes/streams-schemas/activity.json</sourcePath>
+ <sourcePath>target/test-classes/streams-schemas/collection.json</sourcePath>
+ <sourcePath>target/test-classes/streams-schemas/media_link.json</sourcePath>
+ <sourcePath>target/test-classes/streams-schemas/object.json</sourcePath>
+ <sourcePath>target/test-classes/streams-schemas/objectTypes</sourcePath>
+ <sourcePath>target/test-classes/streams-schemas/verbs</sourcePath>
+ </sourcePaths>
+ <targetDirectory>target/generated-resources/test-mojo</targetDirectory>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>elasticsearch</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.10</version>
+ <configuration>
+ <includes>**/*.json</includes>
+ <outputDirectory>${project.build.directory}/test-classes/streams-schemas</outputDirectory>
+ <includeGroupIds>org.apache.streams</includeGroupIds>
+ <includeArtifactIds>streams-schemas</includeArtifactIds>
+ <includeTypes>jar,test-jar</includeTypes>
+ </configuration>
+ <executions>
+ <execution>
+ <id>unpack-schemas</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>unpack-dependencies</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1bc84dbc/streams-plugins/streams-plugin-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hbase/pom.xml b/streams-plugins/streams-plugin-hbase/pom.xml
new file mode 100644
index 0000000..94a0115
--- /dev/null
+++ b/streams-plugins/streams-plugin-hbase/pom.xml
@@ -0,0 +1,244 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.apache.streams.plugins</groupId>
+ <artifactId>streams-plugin-hbase</artifactId>
+ <version>0.3-incubating-SNAPSHOT</version>
+ <packaging>maven-plugin</packaging>
+
+ <parent>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-plugins</artifactId>
+ <version>0.3-incubating-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <properties>
+ <hbase.version>3.0</hbase.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-config</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>commons-logging</artifactId>
+ <groupId>commons-logging</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-schemas</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections</artifactId>
+ <version>0.9.9</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-aether-provider</artifactId>
+ <version>3.3.3</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-artifact</artifactId>
+ <version>3.3.3</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-compat</artifactId>
+ <version>3.3.3</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-core</artifactId>
+ <version>3.3.3</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.sonatype.aether</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-plugin-api</artifactId>
+ <version>3.3.3</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.shared</groupId>
+ <artifactId>maven-verifier</artifactId>
+ <version>1.6</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>2.7</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.plexus</groupId>
+ <artifactId>plexus-utils</artifactId>
+ <version>3.0.15</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-plugin-plugin</artifactId>
+ <version>3.4</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>commons-logging</artifactId>
+ <groupId>commons-logging</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.plugin-tools</groupId>
+ <artifactId>maven-plugin-tools-api</artifactId>
+ <version>3.3</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.plugin-tools</groupId>
+ <artifactId>maven-plugin-annotations</artifactId>
+ <version>3.3</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.plugin-tools</groupId>
+ <artifactId>maven-plugin-tools-generators</artifactId>
+ <version>3.3</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.plugin-testing</groupId>
+ <artifactId>maven-plugin-testing-harness</artifactId>
+ <version>3.3.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>2.6</version>
+ </dependency>
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ <version>2.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.joda</groupId>
+ <artifactId>joda-convert</artifactId>
+ <version>1.8.1</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <sourceDirectory>src/main/java</sourceDirectory>
+ <testSourceDirectory>src/test/java</testSourceDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>src/test/resources</directory>
+ </testResource>
+ </testResources>
+ <plugins>
+ <plugin>
+ <artifactId>maven-plugin-plugin</artifactId>
+ <version>3.4</version>
+ <configuration>
+ <skipErrorNoDescriptorsFound>true</skipErrorNoDescriptorsFound>
+ </configuration>
+ <executions>
+ <execution>
+ <id>mojo-descriptor</id>
+ <phase>process-classes</phase>
+ <goals>
+ <goal>descriptor</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections-maven</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>reflections</goal>
+ </goals>
+ <phase>process-classes</phase>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.4</version>
+ <executions>
+ <execution>
+ <id>resource-dependencies</id>
+ <phase>process-test-resources</phase>
+ <goals>
+ <goal>unpack-dependencies</goal>
+ </goals>
+ <configuration>
+ <includeGroupIds>org.apache.streams</includeGroupIds>
+ <includeArtifactIds>streams-schemas</includeArtifactIds>
+ <includes>**/*.json</includes>
+ <outputDirectory>${project.build.directory}/test-classes/streams-schemas</outputDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1bc84dbc/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java
new file mode 100644
index 0000000..2612ec2
--- /dev/null
+++ b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java
@@ -0,0 +1,90 @@
+package org.apache.streams.plugins.hbase;
+
+import org.apache.streams.schema.GenerationConfig;
+import org.jsonschema2pojo.DefaultGenerationConfig;
+import org.jsonschema2pojo.util.URLUtil;
+
+import java.io.File;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Configures StreamsHiveResourceGenerator
+ *
+ *
+ */
+public class StreamsHbaseGenerationConfig extends DefaultGenerationConfig implements GenerationConfig {
+
+ public String getSourceDirectory() {
+ return sourceDirectory;
+ }
+
+ public List<String> getSourcePaths() {
+ return sourcePaths;
+ }
+
+ private String columnFamily;
+ private String sourceDirectory;
+ private List<String> sourcePaths = new ArrayList<String>();
+ private String targetDirectory;
+ private int maxDepth = 1;
+
+ public Set<String> getExclusions() {
+ return exclusions;
+ }
+
+ public void setExclusions(Set<String> exclusions) {
+ this.exclusions = exclusions;
+ }
+
+ private Set<String> exclusions = new HashSet<String>();
+
+ public int getMaxDepth() {
+ return maxDepth;
+ }
+
+ public void setSourceDirectory(String sourceDirectory) {
+ this.sourceDirectory = sourceDirectory;
+ }
+
+ public void setSourcePaths(List<String> sourcePaths) {
+ this.sourcePaths = sourcePaths;
+ }
+
+ public void setTargetDirectory(String targetDirectory) {
+ this.targetDirectory = targetDirectory;
+ }
+
+ public File getTargetDirectory() {
+ return new File(targetDirectory);
+ }
+
+ public Iterator<URL> getSource() {
+ if (null != sourceDirectory) {
+ return Collections.singleton(URLUtil.parseURL(sourceDirectory)).iterator();
+ }
+ List<URL> sourceURLs = new ArrayList<URL>();
+ if( sourcePaths != null && sourcePaths.size() > 0)
+ for (String source : sourcePaths) {
+ sourceURLs.add(URLUtil.parseURL(source));
+ }
+ return sourceURLs.iterator();
+ }
+
+ public void setMaxDepth(int maxDepth) {
+ this.maxDepth = maxDepth;
+ }
+
+ public String getColumnFamily() {
+ return columnFamily;
+ }
+
+ public void setColumnFamily(String columnFamily) {
+ this.columnFamily = columnFamily;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1bc84dbc/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java
new file mode 100644
index 0000000..649bdd9
--- /dev/null
+++ b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java
@@ -0,0 +1,204 @@
+package org.apache.streams.plugins.hbase;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Joiner;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+import org.apache.streams.schema.FieldType;
+import org.apache.streams.schema.FieldUtil;
+import org.apache.streams.schema.GenerationConfig;
+import org.apache.streams.schema.Schema;
+import org.apache.streams.schema.SchemaStore;
+import org.jsonschema2pojo.util.URLUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.net.URI;
+import java.net.URL;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.streams.schema.FileUtil.dropExtension;
+import static org.apache.streams.schema.FileUtil.dropSourcePathPrefix;
+import static org.apache.streams.schema.FileUtil.resolveRecursive;
+import static org.apache.streams.schema.FileUtil.swapExtension;
+import static org.apache.streams.schema.FileUtil.writeFile;
+
+/**
+ * Created by sblackmon on 5/3/16.
+ */
+public class StreamsHbaseResourceGenerator implements Runnable {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(StreamsHbaseResourceGenerator.class);
+
+ private final static String LS = System.getProperty("line.separator");
+
+ private StreamsHbaseGenerationConfig config;
+
+ private SchemaStore schemaStore = new SchemaStore();
+
+ private int currentDepth = 0;
+
+ public static void main(String[] args) {
+ StreamsHbaseGenerationConfig config = new StreamsHbaseGenerationConfig();
+
+ String sourceDirectory = "./target/test-classes/activities";
+ String targetDirectory = "./target/generated-resources/hbase";
+
+ if( args.length > 0 )
+ sourceDirectory = args[0];
+ if( args.length > 1 )
+ targetDirectory = args[1];
+
+ config.setSourceDirectory(sourceDirectory);
+ config.setTargetDirectory(targetDirectory);
+
+ StreamsHbaseResourceGenerator StreamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config);
+ Thread thread = new Thread(StreamsHbaseResourceGenerator);
+ thread.start();
+ try {
+ thread.join();
+ } catch (InterruptedException e) {
+ LOGGER.error("InterruptedException", e);
+ } catch (Exception e) {
+ LOGGER.error("Exception", e);
+ }
+ return;
+ }
+
+ public StreamsHbaseResourceGenerator(StreamsHbaseGenerationConfig config) {
+ this.config = config;
+ }
+
+ public void run() {
+
+ checkNotNull(config);
+
+ generate(config);
+
+ }
+
+ public void generate(StreamsHbaseGenerationConfig config) {
+
+ LinkedList<File> sourceFiles = new LinkedList<File>();
+
+ for (Iterator<URL> sources = config.getSource(); sources.hasNext();) {
+ URL source = sources.next();
+ sourceFiles.add(URLUtil.getFileFromURL(source));
+ }
+
+ LOGGER.info("Seeded with {} source paths:", sourceFiles.size());
+
+ resolveRecursive((GenerationConfig)config, sourceFiles);
+
+ LOGGER.info("Resolved {} schema files:", sourceFiles.size());
+
+ for (Iterator<File> iterator = sourceFiles.iterator(); iterator.hasNext();) {
+ File item = iterator.next();
+ schemaStore.create(item.toURI());
+ }
+
+ LOGGER.info("Identified {} objects:", schemaStore.getSize());
+
+ for (Iterator<Schema> schemaIterator = schemaStore.getSchemaIterator(); schemaIterator.hasNext(); ) {
+ Schema schema = schemaIterator.next();
+ currentDepth = 0;
+ if( schema.getURI().getScheme().equals("file")) {
+ String inputFile = schema.getURI().getPath();
+ String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory());
+ for (String sourcePath : config.getSourcePaths()) {
+ resourcePath = dropSourcePathPrefix(resourcePath, sourcePath);
+ }
+ String outputFile = config.getTargetDirectory() + "/" + swapExtension(resourcePath, "json", "txt");
+
+ LOGGER.info("Processing {}:", resourcePath);
+
+ String resourceId = dropExtension(resourcePath).replace("/", "_");
+
+ String resourceContent = generateResource(schema, resourceId);
+
+ writeFile(outputFile, resourceContent);
+
+ LOGGER.info("Wrote {}:", outputFile);
+ }
+ }
+
+ }
+
+ public String generateResource(Schema schema, String resourceId) {
+ StringBuilder resourceBuilder = new StringBuilder();
+ resourceBuilder.append("CREATE ");
+ resourceBuilder = appendRootObject(resourceBuilder, schema, resourceId);
+ return resourceBuilder.toString();
+ }
+
+ public StringBuilder appendRootObject(StringBuilder builder, Schema schema, String resourceId) {
+ checkNotNull(builder);
+ ObjectNode propertiesNode = schemaStore.resolveProperties(schema, null, resourceId);
+ if( propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0) {
+
+ List<String> fieldStrings = Lists.newArrayList();
+
+ // table
+ fieldStrings.add(hbaseEscape(schemaSymbol(schema)));
+
+ // column family
+ fieldStrings.add(hbaseEscape(schemaSymbol(schema)));
+
+ // parent column family
+ if( schema.getParent() != null )
+ fieldStrings.add(hbaseEscape(schemaSymbol(schema.getParent())));
+
+ // sub-object column families
+ if( propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0 ) {
+
+ Iterator<Map.Entry<String, JsonNode>> fields = propertiesNode.fields();
+ Joiner joiner = Joiner.on(", ").skipNulls();
+ for( ; fields.hasNext(); ) {
+ Map.Entry<String, JsonNode> field = fields.next();
+ String fieldId = field.getKey();
+ if( !config.getExclusions().contains(fieldId) && field.getValue().isObject()) {
+ ObjectNode fieldNode = (ObjectNode) field.getValue();
+ FieldType fieldType = FieldUtil.determineFieldType(fieldNode);
+ if (fieldType != null ) {
+ switch (fieldType) {
+ case OBJECT:
+ fieldStrings.add(hbaseEscape(fieldId));
+ }
+ }
+ }
+ }
+ builder.append(joiner.join(fieldStrings));
+
+ }
+ }
+ checkNotNull(builder);
+ return builder;
+ }
+
+ private static String hbaseEscape( String fieldId ) {
+ return "'"+fieldId+"'";
+ }
+
+ private String schemaSymbol( Schema schema ) {
+ if (schema == null) return null;
+ if (schema.getURI().getScheme().equals("file")) {
+ String inputFile = schema.getURI().getPath();
+ String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory());
+ for (String sourcePath : config.getSourcePaths()) {
+ resourcePath = dropSourcePathPrefix(resourcePath, sourcePath);
+ }
+ return dropExtension(resourcePath).replace("/", "_");
+ } else {
+ return "IDK";
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1bc84dbc/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java
new file mode 100644
index 0000000..0db68f0
--- /dev/null
+++ b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java
@@ -0,0 +1,93 @@
+package org.apache.streams.plugins.hbase;
+
+import org.apache.maven.plugin.AbstractMojo;
+import org.apache.maven.plugin.MojoExecutionException;
+import org.apache.maven.plugin.MojoFailureException;
+import org.apache.maven.plugins.annotations.Component;
+import org.apache.maven.plugins.annotations.Execute;
+import org.apache.maven.plugins.annotations.LifecyclePhase;
+import org.apache.maven.plugins.annotations.Mojo;
+import org.apache.maven.plugins.annotations.Parameter;
+import org.apache.maven.project.MavenProject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.util.List;
+
+@Mojo( name = "hbase",
+ defaultPhase = LifecyclePhase.GENERATE_RESOURCES
+)
+@Execute( goal = "hbase",
+ phase = LifecyclePhase.GENERATE_RESOURCES
+)
+public class StreamsHbaseResourceGeneratorMojo extends AbstractMojo {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(StreamsHbaseResourceGeneratorMojo.class);
+
+ private volatile MojoFailureException mojoFailureException;
+
+ @Component
+ private MavenProject project;
+
+// @Component
+// private Settings settings;
+//
+// @Parameter( defaultValue = "${localRepository}", readonly = true, required = true )
+// protected ArtifactRepository localRepository;
+//
+// @Parameter( defaultValue = "${plugin}", readonly = true ) // Maven 3 only
+// private PluginDescriptor plugin;
+//
+ @Parameter( defaultValue = "${project.basedir}", readonly = true )
+ private File basedir;
+
+ @Parameter( defaultValue = "src/main/jsonschema", readonly = true ) // Maven 3 only
+ public String sourceDirectory;
+
+ @Parameter( readonly = true ) // Maven 3 only
+ public List<String> sourcePaths;
+
+ @Parameter(defaultValue = "./target/generated-resources/hbase", readonly = true)
+ public String targetDirectory;
+
+ public void execute() throws MojoExecutionException, MojoFailureException {
+
+ //addProjectDependenciesToClasspath();
+
+ StreamsHbaseGenerationConfig config = new StreamsHbaseGenerationConfig();
+
+ if( sourcePaths != null && sourcePaths.size() > 0)
+ config.setSourcePaths(sourcePaths);
+ else
+ config.setSourceDirectory(sourceDirectory);
+ config.setTargetDirectory(targetDirectory);
+
+ StreamsHbaseResourceGenerator streamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config);
+
+ Thread.UncaughtExceptionHandler h = new Thread.UncaughtExceptionHandler() {
+ public void uncaughtException(Thread th, Throwable ex) {
+ LOGGER.error("Exception", ex);
+ mojoFailureException = new MojoFailureException("Exception", ex);
+ }
+ };
+ Thread.setDefaultUncaughtExceptionHandler(h);
+ Thread thread = new Thread(streamsHbaseResourceGenerator);
+ thread.setUncaughtExceptionHandler(h);
+ try {
+ thread.start();
+ thread.join();
+ } catch (InterruptedException e) {
+ LOGGER.error("InterruptedException", e);
+ } catch (Exception e) {
+ LOGGER.error("Exception", e);
+ mojoFailureException = new MojoFailureException("Exception", e);
+ }
+
+ if( mojoFailureException != null )
+ throw mojoFailureException;
+
+ return;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1bc84dbc/streams-plugins/streams-plugin-hbase/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hbase/src/site/markdown/index.md b/streams-plugins/streams-plugin-hbase/src/site/markdown/index.md
new file mode 100644
index 0000000..858cb85
--- /dev/null
+++ b/streams-plugins/streams-plugin-hbase/src/site/markdown/index.md
@@ -0,0 +1,22 @@
+org.apache.streams.plugins:streams-plugin-hbase
+===============================================
+
+streams-plugin-hbase generates resources from json schemas to assist with indexing of json data using Apache HBase.
+
+#### Usage
+
+Run within a module containing a src/main/jsonschema directory
+
+ mvn org.apache.streams.plugins:streams-plugin-hbase:0.3-incubating-SNAPSHOT:hbase
+
+Output will be placed in target/generated-resources/hive by default
+
+#### Example
+
+[streams-plugin-hbase/pom.xml](streams-plugin-hbase/pom.xml "streams-plugin-hbase/pom.xml")
+
+#### Documentation
+
+[JavaDocs](apidocs/index.html "JavaDocs")
+
+###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1bc84dbc/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java
new file mode 100644
index 0000000..290d601
--- /dev/null
+++ b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java
@@ -0,0 +1,41 @@
+package org.apache.streams.plugins.test;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.Lists;
+import com.google.common.io.Files;
+import org.apache.streams.plugins.hbase.StreamsHbaseResourceGenerator;
+import org.junit.Test;
+
+import javax.annotation.Nullable;
+import java.io.File;
+import java.util.Collection;
+import java.util.List;
+
+import static org.apache.streams.plugins.test.StreamsHbaseResourceGeneratorTest.txtFilter;
+
+/**
+ * Created by sblackmon on 5/5/16.
+ */
+public class StreamsHbaseResourceGeneratorCLITest {
+
+ @Test
+ public void testStreamsHiveResourceGeneratorCLI() throws Exception {
+
+ String sourceDirectory = "target/test-classes/streams-schemas";
+ String targetDirectory = "target/generated-resources/hbase-cli";
+
+ List<String> argsList = Lists.newArrayList(sourceDirectory, targetDirectory);
+ StreamsHbaseResourceGenerator.main(argsList.toArray(new String[0]));
+
+ File testOutput = new File(targetDirectory);
+
+ assert( testOutput != null );
+ assert( testOutput.exists() == true );
+ assert( testOutput.isDirectory() == true );
+
+ Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput)
+ .filter(txtFilter);
+ Collection<File> outputCollection = Lists.newArrayList(outputIterator);
+ assert( outputCollection.size() == 133 );
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1bc84dbc/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoTest.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoTest.java b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoTest.java
new file mode 100644
index 0000000..6293021
--- /dev/null
+++ b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoTest.java
@@ -0,0 +1,65 @@
+package org.apache.streams.plugins.test;
+
+import com.google.common.collect.Lists;
+import com.google.common.io.Files;
+import junit.framework.TestCase;
+import org.apache.maven.it.Verifier;
+import org.apache.maven.it.util.ResourceExtractor;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import static org.apache.streams.plugins.test.StreamsHbaseResourceGeneratorTest.txtFilter;
+
+/**
+ * Tests that streams-plugin-hive running via maven generates hql resources
+ */
+public class StreamsHbaseResourceGeneratorMojoTest extends TestCase {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(StreamsHbaseResourceGeneratorMojoTest.class);
+
+ protected void setUp() throws Exception
+ {
+ // required for mojo lookups to work
+ super.setUp();
+ }
+
+
+ @Test
+ public void testStreamsHbaseResourceGeneratorMojo() throws Exception {
+
+ File testDir = ResourceExtractor.simpleExtractResources( getClass(), "/streams-plugin-hbase" );
+
+ Verifier verifier;
+
+ verifier = new Verifier( testDir.getAbsolutePath() );
+
+ List cliOptions = new ArrayList();
+ cliOptions.add( "-N" );
+ verifier.executeGoals( Lists.<String>newArrayList(
+ "clean",
+ "dependency:unpack-dependencies",
+ "generate-resources"));
+
+ verifier.verifyErrorFreeLog();
+
+ verifier.resetStreams();
+
+ File testOutput = new File(testDir.getAbsolutePath() + "/target/generated-resources/test-mojo");
+
+ assert( testOutput != null );
+ assert( testOutput.exists() == true );
+ assert( testOutput.isDirectory() == true );
+
+ Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput)
+ .filter(txtFilter);
+ Collection<File> outputCollection = Lists.newArrayList(outputIterator);
+ assert( outputCollection.size() == 133 );
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1bc84dbc/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java
new file mode 100644
index 0000000..02b0214
--- /dev/null
+++ b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java
@@ -0,0 +1,125 @@
+package org.apache.streams.plugins.test;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.common.io.Files;
+import org.apache.commons.io.FileUtils;
+import org.apache.streams.plugins.hbase.StreamsHbaseGenerationConfig;
+import org.apache.streams.plugins.hbase.StreamsHbaseResourceGenerator;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+import java.io.File;
+import java.util.Collection;
+import java.util.Iterator;
+
+import static org.apache.streams.schema.FileUtil.dropSourcePathPrefix;
+
+/**
+ * Test that Activity beans are compatible with the example activities in the spec.
+ */
+public class StreamsHbaseResourceGeneratorTest {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(StreamsHbaseResourceGeneratorTest.class);
+
+ public static final Predicate<File> txtFilter = new Predicate<File>() {
+ @Override
+ public boolean apply(@Nullable File file) {
+ if( file.getName().endsWith(".txt") )
+ return true;
+ else return false;
+ }
+ };
+
+ /**
+ * Tests that all example activities can be loaded into Activity beans
+ *
+ * @throws Exception
+ */
+ @Test
+ public void StreamsHbaseResourceGenerator() throws Exception {
+
+ StreamsHbaseGenerationConfig config = new StreamsHbaseGenerationConfig();
+
+ String sourceDirectory = "target/test-classes/streams-schemas";
+
+ config.setSourceDirectory(sourceDirectory);
+
+ config.setTargetDirectory("target/generated-resources/test");
+
+ config.setExclusions(Sets.newHashSet("attachments"));
+
+ config.setColumnFamily("cf");
+ config.setMaxDepth(2);
+
+ StreamsHbaseResourceGenerator streamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config);
+ Thread thread = new Thread(streamsHbaseResourceGenerator);
+ thread.start();
+ try {
+ thread.join();
+ } catch (InterruptedException e) {
+ LOGGER.error("InterruptedException", e);
+ } catch (Exception e) {
+ LOGGER.error("Exception", e);
+ }
+
+ File testOutput = config.getTargetDirectory();
+
+
+ assert( testOutput != null );
+ assert( testOutput.exists() == true );
+ assert( testOutput.isDirectory() == true );
+
+ Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput)
+ .filter(txtFilter);
+ Collection<File> outputCollection = Lists.newArrayList(outputIterator);
+ assert( outputCollection.size() == 133 );
+
+ String expectedDirectory = "target/test-classes/expected";
+ File testExpected = new File( expectedDirectory );
+
+ Iterable<File> expectedIterator = Files.fileTreeTraverser().breadthFirstTraversal(testExpected)
+ .filter(txtFilter);
+ Collection<File> expectedCollection = Lists.newArrayList(expectedIterator);
+
+ int fails = 0;
+
+ Iterator<File> iterator = expectedCollection.iterator();
+ while( iterator.hasNext() ) {
+ File objectExpected = iterator.next();
+ String expectedEnd = dropSourcePathPrefix(objectExpected.getAbsolutePath(), expectedDirectory);
+ File objectActual = new File(config.getTargetDirectory() + "/" + expectedEnd);
+ LOGGER.info("Comparing: {} and {}", objectExpected.getAbsolutePath(), objectActual.getAbsolutePath());
+ assert( objectActual.exists());
+ if( FileUtils.contentEquals(objectActual, objectExpected) == true ) {
+ LOGGER.info("Exact Match!");
+ } else {
+ LOGGER.info("No Match!");
+ fails++;
+ }
+ }
+ if( fails > 0 ) {
+ LOGGER.info("Fails: {}", fails);
+ Assert.fail();
+ }
+
+
+// assert( new File(testOutput + "/traits").exists() == true );
+// assert( new File(testOutput + "/traits").isDirectory() == true );
+// assert( new File(testOutput + "/traits").listFiles(scalaFilter) != null );
+// assert( new File(testOutput + "/traits").listFiles(scalaFilter).length == 4 );
+// assert( new File(testOutput + "/objectTypes").exists() == true );
+// assert( new File(testOutput + "/objectTypes").isDirectory() == true );
+// assert( new File(testOutput + "/objectTypes").listFiles(scalaFilter) != null );
+// assert( new File(testOutput + "/objectTypes").listFiles(scalaFilter).length == 43 );
+// assert( new File(testOutput + "/verbs").exists() == true );
+// assert( new File(testOutput + "/verbs").isDirectory() == true );
+// assert( new File(testOutput + "/verbs").listFiles(scalaFilter) != null );
+// assert( new File(testOutput + "/verbs").listFiles(scalaFilter).length == 89 );
+ }
+}
\ No newline at end of file