You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2016/06/01 17:41:20 UTC
[31/32] incubator-streams git commit: refactoring, testing,
documentation, CLI modes
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-cassandra/src/test/java/org/apache/streams/plugins/cassandra/test/StreamsCassandraResourceGeneratorTest.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-cassandra/src/test/java/org/apache/streams/plugins/cassandra/test/StreamsCassandraResourceGeneratorTest.java b/streams-plugins/streams-plugin-cassandra/src/test/java/org/apache/streams/plugins/cassandra/test/StreamsCassandraResourceGeneratorTest.java
index d46eaa6..daf85ae 100644
--- a/streams-plugins/streams-plugin-cassandra/src/test/java/org/apache/streams/plugins/cassandra/test/StreamsCassandraResourceGeneratorTest.java
+++ b/streams-plugins/streams-plugin-cassandra/src/test/java/org/apache/streams/plugins/cassandra/test/StreamsCassandraResourceGeneratorTest.java
@@ -1,15 +1,12 @@
package org.apache.streams.plugins.cassandra.test;
-import com.google.common.base.Charsets;
import com.google.common.base.Predicate;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.io.Files;
-import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.streams.plugins.cassandra.StreamsCassandraGenerationConfig;
import org.apache.streams.plugins.cassandra.StreamsCassandraResourceGenerator;
-import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -19,10 +16,6 @@ import java.io.File;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-
-import static org.apache.streams.schema.FileUtil.dropSourcePathPrefix;
/**
* Test that cassandra resources are generated.
@@ -50,28 +43,20 @@ public class StreamsCassandraResourceGeneratorTest {
StreamsCassandraGenerationConfig config = new StreamsCassandraGenerationConfig();
- String sourceDirectory = "target/test-classes/streams-schemas";
+ String sourceDirectory = "target/test-classes/streams-schema-activitystreams";
config.setSourceDirectory(sourceDirectory);
- config.setTargetDirectory("target/generated-sources/test");
+ config.setTargetDirectory("target/generated-resources/cassandra");
config.setExclusions(Sets.newHashSet("attachments"));
config.setMaxDepth(2);
StreamsCassandraResourceGenerator streamsCassandraResourceGenerator = new StreamsCassandraResourceGenerator(config);
- Thread thread = new Thread(streamsCassandraResourceGenerator);
- thread.start();
- try {
- thread.join();
- } catch (InterruptedException e) {
- LOGGER.error("InterruptedException", e);
- } catch (Exception e) {
- LOGGER.error("Exception", e);
- }
+ streamsCassandraResourceGenerator.run();
- File testOutput = new File( "./target/generated-sources/test");
+ File testOutput = config.getTargetDirectory();
assert( testOutput != null );
assert( testOutput.exists() == true );
@@ -82,43 +67,14 @@ public class StreamsCassandraResourceGeneratorTest {
Collection<File> outputCollection = Lists.newArrayList(outputIterator);
assert( outputCollection.size() == 1 );
- Path path = Paths.get("./target/generated-sources/test/types.cql");
+ Path path = Paths.get(testOutput.getAbsolutePath()).resolve("types.cql");
+
+ assert( path.toFile().exists() );
String typesCqlBytes = new String(
java.nio.file.Files.readAllBytes(path));
assert( StringUtils.countMatches(typesCqlBytes, "CREATE TYPE") == 133 );
- assert( !typesCqlBytes.contains("IDK"));
-
-
-// String expectedDirectory = "target/test-classes/expected";
-// File testExpected = new File( expectedDirectory );
-//
-// Iterable<File> expectedIterator = Files.fileTreeTraverser().breadthFirstTraversal(testExpected)
-// .filter(cqlFilter);
-// Collection<File> expectedCollection = Lists.newArrayList(expectedIterator);
-//
-// int fails = 0;
-//
-// Iterator<File> iterator = expectedCollection.iterator();
-// while( iterator.hasNext() ) {
-// File objectExpected = iterator.next();
-// String expectedEnd = dropSourcePathPrefix(objectExpected.getAbsolutePath(), expectedDirectory);
-// File objectActual = new File(config.getTargetDirectory() + "/" + expectedEnd);
-// LOGGER.info("Comparing: {} and {}", objectExpected.getAbsolutePath(), objectActual.getAbsolutePath());
-// assert( objectActual.exists());
-// if( FileUtils.contentEquals(objectActual, objectExpected) == true ) {
-// LOGGER.info("Exact Match!");
-// } else {
-// LOGGER.info("No Match!");
-// fails++;
-// }
-// }
-// if( fails > 0 ) {
-// LOGGER.info("Fails: {}", fails);
-// Assert.fail();
-// }
-
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-cassandra/src/test/resources/streams-plugin-cassandra/pom.xml
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-cassandra/src/test/resources/streams-plugin-cassandra/pom.xml b/streams-plugins/streams-plugin-cassandra/src/test/resources/streams-plugin-cassandra/pom.xml
index 064ea52..2a69ec7 100644
--- a/streams-plugins/streams-plugin-cassandra/src/test/resources/streams-plugin-cassandra/pom.xml
+++ b/streams-plugins/streams-plugin-cassandra/src/test/resources/streams-plugin-cassandra/pom.xml
@@ -18,8 +18,9 @@
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
- <artifactId>streams-schemas</artifactId>
+ <artifactId>streams-schema-activitystreams</artifactId>
<version>${project.version}</version>
+ <type>test-jar</type>
<scope>test</scope>
</dependency>
</dependencies>
@@ -32,19 +33,19 @@
<version>0.3-incubating-SNAPSHOT</version>
<configuration>
<sourcePaths>
- <sourcePath>target/test-classes/streams-schemas/activity.json</sourcePath>
- <sourcePath>target/test-classes/streams-schemas/collection.json</sourcePath>
- <sourcePath>target/test-classes/streams-schemas/media_link.json</sourcePath>
- <sourcePath>target/test-classes/streams-schemas/object.json</sourcePath>
- <sourcePath>target/test-classes/streams-schemas/objectTypes</sourcePath>
- <sourcePath>target/test-classes/streams-schemas/verbs</sourcePath>
+ <sourcePath>target/test-classes/streams-schema-activitystreams/activity.json</sourcePath>
+ <sourcePath>target/test-classes/streams-schema-activitystreams/collection.json</sourcePath>
+ <sourcePath>target/test-classes/streams-schema-activitystreams/media_link.json</sourcePath>
+ <sourcePath>target/test-classes/streams-schema-activitystreams/object.json</sourcePath>
+ <sourcePath>target/test-classes/streams-schema-activitystreams/objectTypes</sourcePath>
+ <sourcePath>target/test-classes/streams-schema-activitystreams/verbs</sourcePath>
</sourcePaths>
<targetDirectory>target/generated-resources/test-mojo</targetDirectory>
</configuration>
<executions>
<execution>
<goals>
- <goal>cassandra</goal>
+ <goal>generate-resources</goal>
</goals>
</execution>
</executions>
@@ -55,9 +56,9 @@
<version>2.10</version>
<configuration>
<includes>**/*.json</includes>
- <outputDirectory>${project.build.directory}/test-classes/streams-schemas</outputDirectory>
+ <outputDirectory>${project.build.directory}/test-classes/streams-schema-activitystreams</outputDirectory>
<includeGroupIds>org.apache.streams</includeGroupIds>
- <includeArtifactIds>streams-schemas</includeArtifactIds>
+ <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
<includeTypes>jar,test-jar</includeTypes>
</configuration>
<executions>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-elasticsearch/pom.xml b/streams-plugins/streams-plugin-elasticsearch/pom.xml
index 4f0ed4a..07d7fc1 100644
--- a/streams-plugins/streams-plugin-elasticsearch/pom.xml
+++ b/streams-plugins/streams-plugin-elasticsearch/pom.xml
@@ -55,10 +55,17 @@
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
- <artifactId>streams-schemas</artifactId>
+ <artifactId>streams-util</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-schema-activitystreams</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
@@ -231,9 +238,9 @@
</goals>
<configuration>
<includeGroupIds>org.apache.streams</includeGroupIds>
- <includeArtifactIds>streams-schemas</includeArtifactIds>
+ <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
<includes>**/*.json</includes>
- <outputDirectory>${project.build.directory}/test-classes/streams-schemas</outputDirectory>
+ <outputDirectory>${project.build.directory}/test-classes/streams-schema-activitystreams</outputDirectory>
</configuration>
</execution>
</executions>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchGenerationConfig.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchGenerationConfig.java b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchGenerationConfig.java
index ef5b9ef..71bbdf8 100644
--- a/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchGenerationConfig.java
+++ b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchGenerationConfig.java
@@ -1,6 +1,6 @@
package org.apache.streams.plugins.elasticsearch;
-import org.apache.streams.schema.GenerationConfig;
+import org.apache.streams.util.schema.GenerationConfig;
import org.jsonschema2pojo.DefaultGenerationConfig;
import org.jsonschema2pojo.util.URLUtil;
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGenerator.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGenerator.java b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGenerator.java
index 0f506ca..4d16f7e 100644
--- a/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGenerator.java
+++ b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGenerator.java
@@ -1,17 +1,21 @@
package org.apache.streams.plugins.elasticsearch;
+import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.base.Joiner;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
-import org.apache.streams.schema.FieldType;
-import org.apache.streams.schema.FieldUtil;
-import org.apache.streams.schema.GenerationConfig;
-import org.apache.streams.schema.Schema;
-import org.apache.streams.schema.SchemaStore;
+import org.apache.streams.jackson.StreamsJacksonMapper;
+import org.apache.streams.util.schema.FieldType;
+import org.apache.streams.util.schema.FieldUtil;
+import org.apache.streams.util.schema.GenerationConfig;
+import org.apache.streams.util.schema.Schema;
+import org.apache.streams.util.schema.SchemaStore;
+import org.apache.streams.util.schema.SchemaStoreImpl;
import org.jsonschema2pojo.util.URLUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -25,10 +29,11 @@ import java.util.List;
import java.util.Map;
import static com.google.common.base.Preconditions.checkNotNull;
-import static org.apache.streams.schema.FileUtil.dropExtension;
-import static org.apache.streams.schema.FileUtil.dropSourcePathPrefix;
-import static org.apache.streams.schema.FileUtil.resolveRecursive;
-import static org.apache.streams.schema.FileUtil.writeFile;
+import static org.apache.streams.util.schema.FileUtil.dropExtension;
+import static org.apache.streams.util.schema.FileUtil.dropSourcePathPrefix;
+import static org.apache.streams.util.schema.FileUtil.resolveRecursive;
+import static org.apache.streams.util.schema.FileUtil.swapExtension;
+import static org.apache.streams.util.schema.FileUtil.writeFile;
/**
* Created by sblackmon on 5/3/16.
@@ -37,15 +42,17 @@ public class StreamsElasticsearchResourceGenerator implements Runnable {
private final static Logger LOGGER = LoggerFactory.getLogger(StreamsElasticsearchResourceGenerator.class);
+ ObjectMapper MAPPER = StreamsJacksonMapper.getInstance();
+
private final static String LS = System.getProperty("line.separator");
private StreamsElasticsearchGenerationConfig config;
- private SchemaStore schemaStore = new SchemaStore();
+ private SchemaStore schemaStore = new SchemaStoreImpl();
private int currentDepth = 0;
- public void main(String[] args) {
+ public static void main(String[] args) {
StreamsElasticsearchGenerationConfig config = new StreamsElasticsearchGenerationConfig();
String sourceDirectory = "target/test-classes/activities";
@@ -59,17 +66,9 @@ public class StreamsElasticsearchResourceGenerator implements Runnable {
config.setSourceDirectory(sourceDirectory);
config.setTargetDirectory(targetDirectory);
- StreamsElasticsearchResourceGenerator StreamsElasticsearchResourceGenerator = new StreamsElasticsearchResourceGenerator(config);
- Thread thread = new Thread(StreamsElasticsearchResourceGenerator);
- thread.start();
- try {
- thread.join();
- } catch (InterruptedException e) {
- LOGGER.error("InterruptedException", e);
- } catch (Exception e) {
- LOGGER.error("Exception", e);
- }
- return;
+ StreamsElasticsearchResourceGenerator streamsElasticsearchResourceGenerator = new StreamsElasticsearchResourceGenerator(config);
+ streamsElasticsearchResourceGenerator.run();
+
}
public StreamsElasticsearchResourceGenerator(StreamsElasticsearchGenerationConfig config) {
@@ -106,7 +105,6 @@ public class StreamsElasticsearchResourceGenerator implements Runnable {
LOGGER.info("Identified {} objects:", schemaStore.getSize());
- String outputFile = config.getTargetDirectory() + "/" + "types.cql";
StringBuilder typesContent = new StringBuilder();
for (Iterator<Schema> schemaIterator = schemaStore.getSchemaIterator(); schemaIterator.hasNext(); ) {
@@ -118,6 +116,7 @@ public class StreamsElasticsearchResourceGenerator implements Runnable {
for (String sourcePath : config.getSourcePaths()) {
resourcePath = dropSourcePathPrefix(resourcePath, sourcePath);
}
+ String outputFile = config.getTargetDirectory() + "/" + resourcePath;
LOGGER.info("Processing {}:", resourcePath);
@@ -125,25 +124,33 @@ public class StreamsElasticsearchResourceGenerator implements Runnable {
String resourceContent = generateResource(schema, resourceId);
- typesContent.append(resourceContent);
+ if( !Strings.isNullOrEmpty(resourceContent))
+ writeFile(outputFile, resourceContent);
- LOGGER.info("Added {}:", resourceId);
+ LOGGER.info("Wrote {}:", outputFile);
}
}
- writeFile(outputFile, typesContent.toString());
-
}
public String generateResource(Schema schema, String resourceId) {
StringBuilder resourceBuilder = new StringBuilder();
- resourceBuilder.append("CREATE TYPE ");
- resourceBuilder.append(resourceId);
- resourceBuilder.append(" IF NOT EXISTS (");
- resourceBuilder.append(LS);
- resourceBuilder = appendRootObject(resourceBuilder, schema, resourceId, ' ');
- resourceBuilder.append(");");
- resourceBuilder.append(LS);
+
+ ObjectNode rootNode = (ObjectNode) schema.getContent();
+
+ // remove java*
+ // remove description
+ // resolve all $ref
+ // replace format: date with type: date
+ // replace format: date-time with type: date
+ // replace array of primitive with just primitive
+
+ try {
+ String objectString = MAPPER.writeValueAsString(rootNode);
+ resourceBuilder.append(objectString);
+ } catch (JsonProcessingException e) {
+ LOGGER.error("{}: {}", e.getClass().getName(), e);
+ }
return resourceBuilder.toString();
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java
index 6969051..ed280ff 100644
--- a/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java
+++ b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java
@@ -15,10 +15,10 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.List;
-@Mojo( name = "elasticsearch",
+@Mojo( name = "generate-resources",
defaultPhase = LifecyclePhase.GENERATE_RESOURCES
)
-@Execute( goal = "elasticsearch",
+@Execute( goal = "generate-resources",
phase = LifecyclePhase.GENERATE_RESOURCES
)
public class StreamsElasticsearchResourceGeneratorMojo extends AbstractMojo {
@@ -65,29 +65,8 @@ public class StreamsElasticsearchResourceGeneratorMojo extends AbstractMojo {
StreamsElasticsearchResourceGenerator streamsElasticsearchResourceGenerator = new StreamsElasticsearchResourceGenerator(config);
- Thread.UncaughtExceptionHandler h = new Thread.UncaughtExceptionHandler() {
- public void uncaughtException(Thread th, Throwable ex) {
- LOGGER.error("Exception", ex);
- mojoFailureException = new MojoFailureException("Exception", ex);
- }
- };
- Thread.setDefaultUncaughtExceptionHandler(h);
- Thread thread = new Thread(streamsElasticsearchResourceGenerator);
- thread.setUncaughtExceptionHandler(h);
- try {
- thread.start();
- thread.join();
- } catch (InterruptedException e) {
- LOGGER.error("InterruptedException", e);
- } catch (Exception e) {
- LOGGER.error("Exception", e);
- mojoFailureException = new MojoFailureException("Exception", e);
- }
-
- if( mojoFailureException != null )
- throw mojoFailureException;
-
- return;
+ streamsElasticsearchResourceGenerator.run();
+
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-elasticsearch/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-elasticsearch/src/site/markdown/index.md b/streams-plugins/streams-plugin-elasticsearch/src/site/markdown/index.md
index 9e80a3e..f65433b 100644
--- a/streams-plugins/streams-plugin-elasticsearch/src/site/markdown/index.md
+++ b/streams-plugins/streams-plugin-elasticsearch/src/site/markdown/index.md
@@ -3,17 +3,33 @@ org.apache.streams.plugins:streams-plugin-elasticsearch
streams-plugin-elasticsearch generates resources from json schemas to assist with indexing of json data using Elasticsearch.
-#### Usage
+### Usage
+
+Output will be placed in target/generated-resources/elasticsearch by default
+
+##### Maven
Run within a module containing a src/main/jsonschema directory
mvn org.apache.streams.plugins:streams-plugin-elasticsearch:0.3-incubating-SNAPSHOT:elasticsearch
-Output will be placed in target/generated-resources/elasticsearch by default
+[streams-plugin-elasticsearch/pom.xml](streams-plugin-elasticsearch/pom.xml "streams-plugin-elasticsearch/pom.xml")
-#### Example
+##### SDK
-[streams-plugin-elasticsearch/pom.xml](streams-plugin-elasticsearch/pom.xml "streams-plugin-elasticsearch/pom.xml")
+Embed within your own java code
+
+ StreamsElasticsearchGenerationConfig config = new StreamsElasticsearchGenerationConfig();
+ config.setSourceDirectory("src/main/jsonschema");
+ config.setTargetDirectory("target/generated-resources");
+ StreamsElasticsearchResourceGenerator generator = new StreamsElasticsearchResourceGenerator(config);
+ generator.run();
+
+##### CLI
+
+Run from CLI without Maven
+
+ java -jar streams-plugin-elasticsearch-jar-with-dependencies.jar StreamsElasticsearchResourceGenerator src/main/jsonschema target/generated-resources
#### Documentation
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorCLITest.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorCLITest.java b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorCLITest.java
new file mode 100644
index 0000000..3805894
--- /dev/null
+++ b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorCLITest.java
@@ -0,0 +1,39 @@
+package org.apache.streams.plugins.elasticsearch.test;
+
+import com.google.common.collect.Lists;
+import com.google.common.io.Files;
+import org.apache.streams.plugins.elasticsearch.StreamsElasticsearchResourceGenerator;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.Collection;
+import java.util.List;
+
+import static org.apache.streams.plugins.elasticsearch.test.StreamsElasticsearchResourceGeneratorTest.jsonFilter;
+
+/**
+ * Created by sblackmon on 5/5/16.
+ */
+public class StreamsElasticsearchResourceGeneratorCLITest {
+
+ @Test
+ public void testStreamsElasticsearchResourceGeneratorCLI() throws Exception {
+
+ String sourceDirectory = "target/test-classes/streams-schema-activitystreams";
+ String targetDirectory = "target/generated-resources/elasticsearch-cli";
+
+ List<String> argsList = Lists.newArrayList(sourceDirectory, targetDirectory);
+ StreamsElasticsearchResourceGenerator.main(argsList.toArray(new String[0]));
+
+ File testOutput = new File(targetDirectory);
+
+ assert( testOutput != null );
+ assert( testOutput.exists() == true );
+ assert( testOutput.isDirectory() == true );
+
+ Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput)
+ .filter(jsonFilter);
+ Collection<File> outputCollection = Lists.newArrayList(outputIterator);
+ assert( outputCollection.size() == 133 );
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java
index 20bb561..7844442 100644
--- a/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java
+++ b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java
@@ -5,7 +5,6 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.io.Files;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.StringUtils;
import org.apache.streams.plugins.elasticsearch.StreamsElasticsearchGenerationConfig;
import org.apache.streams.plugins.elasticsearch.StreamsElasticsearchResourceGenerator;
import org.junit.Assert;
@@ -15,12 +14,10 @@ import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
import java.io.File;
-import java.nio.file.Path;
-import java.nio.file.Paths;
import java.util.Collection;
import java.util.Iterator;
-import static org.apache.streams.schema.FileUtil.dropSourcePathPrefix;
+import static org.apache.streams.util.schema.FileUtil.dropSourcePathPrefix;
/**
* Test that Elasticsearch resources are generated.
@@ -29,6 +26,15 @@ public class StreamsElasticsearchResourceGeneratorTest {
private final static Logger LOGGER = LoggerFactory.getLogger(StreamsElasticsearchResourceGeneratorTest.class);
+ public static final Predicate<File> jsonFilter = new Predicate<File>() {
+ @Override
+ public boolean apply(@Nullable File file) {
+ if( file.getName().endsWith(".json") )
+ return true;
+ else return false;
+ }
+ };
+
/**
* Test that Elasticsearch resources are generated
*
@@ -39,28 +45,21 @@ public class StreamsElasticsearchResourceGeneratorTest {
StreamsElasticsearchGenerationConfig config = new StreamsElasticsearchGenerationConfig();
- String sourceDirectory = "target/test-classes/streams-schemas";
+ String sourceDirectory = "target/test-classes/streams-schema-activitystreams";
config.setSourceDirectory(sourceDirectory);
- config.setTargetDirectory("target/generated-sources/test");
+ config.setTargetDirectory("target/generated-resources/elasticsearch");
config.setExclusions(Sets.newHashSet("attachments"));
config.setMaxDepth(2);
StreamsElasticsearchResourceGenerator streamsElasticsearchResourceGenerator = new StreamsElasticsearchResourceGenerator(config);
- Thread thread = new Thread(streamsElasticsearchResourceGenerator);
- thread.start();
- try {
- thread.join();
- } catch (InterruptedException e) {
- LOGGER.error("InterruptedException", e);
- } catch (Exception e) {
- LOGGER.error("Exception", e);
- }
+ streamsElasticsearchResourceGenerator.run();
+
+ File testOutput = config.getTargetDirectory();
- File testOutput = new File( "./target/generated-sources/test");
Predicate<File> jsonFilter = new Predicate<File>() {
@Override
public boolean apply(@Nullable File file) {
@@ -107,33 +106,5 @@ public class StreamsElasticsearchResourceGeneratorTest {
Assert.fail();
}
-// String expectedDirectory = "target/test-classes/expected";
-// File testExpected = new File( expectedDirectory );
-//
-// Iterable<File> expectedIterator = Files.fileTreeTraverser().breadthFirstTraversal(testExpected)
-// .filter(cqlFilter);
-// Collection<File> expectedCollection = Lists.newArrayList(expectedIterator);
-//
-// int fails = 0;
-//
-// Iterator<File> iterator = expectedCollection.iterator();
-// while( iterator.hasNext() ) {
-// File objectExpected = iterator.next();
-// String expectedEnd = dropSourcePathPrefix(objectExpected.getAbsolutePath(), expectedDirectory);
-// File objectActual = new File(config.getTargetDirectory() + "/" + expectedEnd);
-// LOGGER.info("Comparing: {} and {}", objectExpected.getAbsolutePath(), objectActual.getAbsolutePath());
-// assert( objectActual.exists());
-// if( FileUtils.contentEquals(objectActual, objectExpected) == true ) {
-// LOGGER.info("Exact Match!");
-// } else {
-// LOGGER.info("No Match!");
-// fails++;
-// }
-// }
-// if( fails > 0 ) {
-// LOGGER.info("Fails: {}", fails);
-// Assert.fail();
-// }
-
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-elasticsearch/src/test/resources/streams-plugin-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-elasticsearch/src/test/resources/streams-plugin-elasticsearch/pom.xml b/streams-plugins/streams-plugin-elasticsearch/src/test/resources/streams-plugin-elasticsearch/pom.xml
index 2aaa037..de74014 100644
--- a/streams-plugins/streams-plugin-elasticsearch/src/test/resources/streams-plugin-elasticsearch/pom.xml
+++ b/streams-plugins/streams-plugin-elasticsearch/src/test/resources/streams-plugin-elasticsearch/pom.xml
@@ -18,9 +18,10 @@
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
- <artifactId>streams-schemas</artifactId>
+ <artifactId>streams-schema-activitystreams</artifactId>
<version>${project.version}</version>
<scope>test</scope>
+ <type>test-jar</type>
</dependency>
</dependencies>
@@ -32,19 +33,19 @@
<version>0.3-incubating-SNAPSHOT</version>
<configuration>
<sourcePaths>
- <sourcePath>target/test-classes/streams-schemas/activity.json</sourcePath>
- <sourcePath>target/test-classes/streams-schemas/collection.json</sourcePath>
- <sourcePath>target/test-classes/streams-schemas/media_link.json</sourcePath>
- <sourcePath>target/test-classes/streams-schemas/object.json</sourcePath>
- <sourcePath>target/test-classes/streams-schemas/objectTypes</sourcePath>
- <sourcePath>target/test-classes/streams-schemas/verbs</sourcePath>
+ <sourcePath>target/test-classes/streams-schema-activitystreams/activity.json</sourcePath>
+ <sourcePath>target/test-classes/streams-schema-activitystreams/collection.json</sourcePath>
+ <sourcePath>target/test-classes/streams-schema-activitystreams/media_link.json</sourcePath>
+ <sourcePath>target/test-classes/streams-schema-activitystreams/object.json</sourcePath>
+ <sourcePath>target/test-classes/streams-schema-activitystreams/objectTypes</sourcePath>
+ <sourcePath>target/test-classes/streams-schema-activitystreams/verbs</sourcePath>
</sourcePaths>
- <targetDirectory>target/generated-resources/test-mojo</targetDirectory>
+ <targetDirectory>target/generated-resources/elasticsearch-mojo</targetDirectory>
</configuration>
<executions>
<execution>
<goals>
- <goal>elasticsearch</goal>
+ <goal>generate-resources</goal>
</goals>
</execution>
</executions>
@@ -55,9 +56,9 @@
<version>2.10</version>
<configuration>
<includes>**/*.json</includes>
- <outputDirectory>${project.build.directory}/test-classes/streams-schemas</outputDirectory>
+ <outputDirectory>${project.build.directory}/test-classes/streams-schema-activitystreams</outputDirectory>
<includeGroupIds>org.apache.streams</includeGroupIds>
- <includeArtifactIds>streams-schemas</includeArtifactIds>
+ <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
<includeTypes>jar,test-jar</includeTypes>
</configuration>
<executions>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hbase/pom.xml b/streams-plugins/streams-plugin-hbase/pom.xml
index 94a0115..b29bd38 100644
--- a/streams-plugins/streams-plugin-hbase/pom.xml
+++ b/streams-plugins/streams-plugin-hbase/pom.xml
@@ -54,10 +54,17 @@
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
- <artifactId>streams-schemas</artifactId>
+ <artifactId>streams-util</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-schema-activitystreams</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
@@ -230,9 +237,9 @@
</goals>
<configuration>
<includeGroupIds>org.apache.streams</includeGroupIds>
- <includeArtifactIds>streams-schemas</includeArtifactIds>
+ <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
<includes>**/*.json</includes>
- <outputDirectory>${project.build.directory}/test-classes/streams-schemas</outputDirectory>
+ <outputDirectory>${project.build.directory}/test-classes/streams-schema-activitystreams</outputDirectory>
</configuration>
</execution>
</executions>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java
index 2612ec2..3972a54 100644
--- a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java
+++ b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java
@@ -1,6 +1,6 @@
package org.apache.streams.plugins.hbase;
-import org.apache.streams.schema.GenerationConfig;
+import org.apache.streams.util.schema.GenerationConfig;
import org.jsonschema2pojo.DefaultGenerationConfig;
import org.jsonschema2pojo.util.URLUtil;
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java
index 649bdd9..71a8705 100644
--- a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java
+++ b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java
@@ -3,21 +3,18 @@ package org.apache.streams.plugins.hbase;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.base.Joiner;
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
import com.google.common.collect.Lists;
-import org.apache.streams.schema.FieldType;
-import org.apache.streams.schema.FieldUtil;
-import org.apache.streams.schema.GenerationConfig;
-import org.apache.streams.schema.Schema;
-import org.apache.streams.schema.SchemaStore;
+import org.apache.streams.util.schema.FieldType;
+import org.apache.streams.util.schema.FieldUtil;
+import org.apache.streams.util.schema.GenerationConfig;
+import org.apache.streams.util.schema.Schema;
+import org.apache.streams.util.schema.SchemaStore;
+import org.apache.streams.util.schema.SchemaStoreImpl;
import org.jsonschema2pojo.util.URLUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
-import java.net.URI;
import java.net.URL;
import java.util.Iterator;
import java.util.LinkedList;
@@ -25,11 +22,11 @@ import java.util.List;
import java.util.Map;
import static com.google.common.base.Preconditions.checkNotNull;
-import static org.apache.streams.schema.FileUtil.dropExtension;
-import static org.apache.streams.schema.FileUtil.dropSourcePathPrefix;
-import static org.apache.streams.schema.FileUtil.resolveRecursive;
-import static org.apache.streams.schema.FileUtil.swapExtension;
-import static org.apache.streams.schema.FileUtil.writeFile;
+import static org.apache.streams.util.schema.FileUtil.dropExtension;
+import static org.apache.streams.util.schema.FileUtil.dropSourcePathPrefix;
+import static org.apache.streams.util.schema.FileUtil.resolveRecursive;
+import static org.apache.streams.util.schema.FileUtil.swapExtension;
+import static org.apache.streams.util.schema.FileUtil.writeFile;
/**
* Created by sblackmon on 5/3/16.
@@ -42,7 +39,7 @@ public class StreamsHbaseResourceGenerator implements Runnable {
private StreamsHbaseGenerationConfig config;
- private SchemaStore schemaStore = new SchemaStore();
+ private SchemaStore schemaStore = new SchemaStoreImpl();
private int currentDepth = 0;
@@ -60,17 +57,9 @@ public class StreamsHbaseResourceGenerator implements Runnable {
config.setSourceDirectory(sourceDirectory);
config.setTargetDirectory(targetDirectory);
- StreamsHbaseResourceGenerator StreamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config);
- Thread thread = new Thread(StreamsHbaseResourceGenerator);
- thread.start();
- try {
- thread.join();
- } catch (InterruptedException e) {
- LOGGER.error("InterruptedException", e);
- } catch (Exception e) {
- LOGGER.error("Exception", e);
- }
- return;
+ StreamsHbaseResourceGenerator streamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config);
+ streamsHbaseResourceGenerator.run();
+
}
public StreamsHbaseResourceGenerator(StreamsHbaseGenerationConfig config) {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java
index 0db68f0..6bad40b 100644
--- a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java
+++ b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java
@@ -15,10 +15,10 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.List;
-@Mojo( name = "hbase",
+@Mojo( name = "generate-resources",
defaultPhase = LifecyclePhase.GENERATE_RESOURCES
)
-@Execute( goal = "hbase",
+@Execute( goal = "generate-resources",
phase = LifecyclePhase.GENERATE_RESOURCES
)
public class StreamsHbaseResourceGeneratorMojo extends AbstractMojo {
@@ -64,30 +64,8 @@ public class StreamsHbaseResourceGeneratorMojo extends AbstractMojo {
config.setTargetDirectory(targetDirectory);
StreamsHbaseResourceGenerator streamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config);
+ streamsHbaseResourceGenerator.run();
- Thread.UncaughtExceptionHandler h = new Thread.UncaughtExceptionHandler() {
- public void uncaughtException(Thread th, Throwable ex) {
- LOGGER.error("Exception", ex);
- mojoFailureException = new MojoFailureException("Exception", ex);
- }
- };
- Thread.setDefaultUncaughtExceptionHandler(h);
- Thread thread = new Thread(streamsHbaseResourceGenerator);
- thread.setUncaughtExceptionHandler(h);
- try {
- thread.start();
- thread.join();
- } catch (InterruptedException e) {
- LOGGER.error("InterruptedException", e);
- } catch (Exception e) {
- LOGGER.error("Exception", e);
- mojoFailureException = new MojoFailureException("Exception", e);
- }
-
- if( mojoFailureException != null )
- throw mojoFailureException;
-
- return;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hbase/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hbase/src/site/markdown/index.md b/streams-plugins/streams-plugin-hbase/src/site/markdown/index.md
index 858cb85..0a0c308 100644
--- a/streams-plugins/streams-plugin-hbase/src/site/markdown/index.md
+++ b/streams-plugins/streams-plugin-hbase/src/site/markdown/index.md
@@ -3,17 +3,35 @@ org.apache.streams.plugins:streams-plugin-hbase
streams-plugin-hbase generates resources from json schemas to assist with indexing of json data using Apache HBase.
-#### Usage
+### Usage
+
+Output will be placed in target/generated-resources/hbase by default
+
+[streams-plugin-hbase/pom.xml](streams-plugin-hbase/pom.xml "streams-plugin-hbase/pom.xml")
+
+##### Maven
Run within a module containing a src/main/jsonschema directory
mvn org.apache.streams.plugins:streams-plugin-hbase:0.3-incubating-SNAPSHOT:hbase
-Output will be placed in target/generated-resources/hive by default
+##### SDK
-#### Example
+Embed within your own java code
-[streams-plugin-hbase/pom.xml](streams-plugin-hbase/pom.xml "streams-plugin-hbase/pom.xml")
+ StreamsHbaseGenerationConfig config = new StreamsHbaseGenerationConfig();
+ config.setSourceDirectory("src/main/jsonschema");
+ config.setTargetDirectory("target/generated-resources");
+ StreamsHbaseResourceGenerator generator = new StreamsHbaseResourceGenerator(config);
+ generator.run();
+
+##### CLI
+
+Run from CLI without Maven
+
+ java -jar streams-plugin-hbase-jar-with-dependencies.jar StreamsHbaseResourceGenerator src/main/jsonschema target/generated-resources
+
+#### Example
#### Documentation
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java
index 290d601..24eba24 100644
--- a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java
+++ b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java
@@ -21,7 +21,7 @@ public class StreamsHbaseResourceGeneratorCLITest {
@Test
public void testStreamsHiveResourceGeneratorCLI() throws Exception {
- String sourceDirectory = "target/test-classes/streams-schemas";
+ String sourceDirectory = "target/test-classes/streams-schema-activitystreams";
String targetDirectory = "target/generated-resources/hbase-cli";
List<String> argsList = Lists.newArrayList(sourceDirectory, targetDirectory);
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoTest.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoTest.java b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoTest.java
index 6293021..fc8c77a 100644
--- a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoTest.java
+++ b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoTest.java
@@ -51,7 +51,7 @@ public class StreamsHbaseResourceGeneratorMojoTest extends TestCase {
verifier.resetStreams();
- File testOutput = new File(testDir.getAbsolutePath() + "/target/generated-resources/test-mojo");
+ File testOutput = new File(testDir.getAbsolutePath() + "/target/generated-resources/hbase-mojo");
assert( testOutput != null );
assert( testOutput.exists() == true );
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java
index 02b0214..91a1a05 100644
--- a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java
+++ b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java
@@ -8,7 +8,6 @@ import org.apache.commons.io.FileUtils;
import org.apache.streams.plugins.hbase.StreamsHbaseGenerationConfig;
import org.apache.streams.plugins.hbase.StreamsHbaseResourceGenerator;
import org.junit.Assert;
-import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -18,7 +17,7 @@ import java.io.File;
import java.util.Collection;
import java.util.Iterator;
-import static org.apache.streams.schema.FileUtil.dropSourcePathPrefix;
+import static org.apache.streams.util.schema.FileUtil.dropSourcePathPrefix;
/**
* Test that Activity beans are compatible with the example activities in the spec.
@@ -46,11 +45,11 @@ public class StreamsHbaseResourceGeneratorTest {
StreamsHbaseGenerationConfig config = new StreamsHbaseGenerationConfig();
- String sourceDirectory = "target/test-classes/streams-schemas";
+ String sourceDirectory = "target/test-classes/streams-schema-activitystreams";
config.setSourceDirectory(sourceDirectory);
- config.setTargetDirectory("target/generated-resources/test");
+ config.setTargetDirectory("target/generated-resources/hbase");
config.setExclusions(Sets.newHashSet("attachments"));
@@ -58,19 +57,10 @@ public class StreamsHbaseResourceGeneratorTest {
config.setMaxDepth(2);
StreamsHbaseResourceGenerator streamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config);
- Thread thread = new Thread(streamsHbaseResourceGenerator);
- thread.start();
- try {
- thread.join();
- } catch (InterruptedException e) {
- LOGGER.error("InterruptedException", e);
- } catch (Exception e) {
- LOGGER.error("Exception", e);
- }
+ streamsHbaseResourceGenerator.run();
File testOutput = config.getTargetDirectory();
-
assert( testOutput != null );
assert( testOutput.exists() == true );
assert( testOutput.isDirectory() == true );
@@ -108,18 +98,5 @@ public class StreamsHbaseResourceGeneratorTest {
Assert.fail();
}
-
-// assert( new File(testOutput + "/traits").exists() == true );
-// assert( new File(testOutput + "/traits").isDirectory() == true );
-// assert( new File(testOutput + "/traits").listFiles(scalaFilter) != null );
-// assert( new File(testOutput + "/traits").listFiles(scalaFilter).length == 4 );
-// assert( new File(testOutput + "/objectTypes").exists() == true );
-// assert( new File(testOutput + "/objectTypes").isDirectory() == true );
-// assert( new File(testOutput + "/objectTypes").listFiles(scalaFilter) != null );
-// assert( new File(testOutput + "/objectTypes").listFiles(scalaFilter).length == 43 );
-// assert( new File(testOutput + "/verbs").exists() == true );
-// assert( new File(testOutput + "/verbs").isDirectory() == true );
-// assert( new File(testOutput + "/verbs").listFiles(scalaFilter) != null );
-// assert( new File(testOutput + "/verbs").listFiles(scalaFilter).length == 89 );
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hbase/src/test/resources/streams-plugin-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hbase/src/test/resources/streams-plugin-hbase/pom.xml b/streams-plugins/streams-plugin-hbase/src/test/resources/streams-plugin-hbase/pom.xml
index 9aa7d8d..a3fbeeb 100644
--- a/streams-plugins/streams-plugin-hbase/src/test/resources/streams-plugin-hbase/pom.xml
+++ b/streams-plugins/streams-plugin-hbase/src/test/resources/streams-plugin-hbase/pom.xml
@@ -18,9 +18,10 @@
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
- <artifactId>streams-schemas</artifactId>
+ <artifactId>streams-schema-activitystreams</artifactId>
<version>${project.version}</version>
<scope>test</scope>
+ <type>test-jar</type>
</dependency>
</dependencies>
@@ -32,20 +33,20 @@
<version>0.3-incubating-SNAPSHOT</version>
<configuration>
<sourcePaths>
- <sourcePath>target/test-classes/streams-schemas/activity.json</sourcePath>
- <sourcePath>target/test-classes/streams-schemas/collection.json</sourcePath>
- <sourcePath>target/test-classes/streams-schemas/media_link.json</sourcePath>
- <sourcePath>target/test-classes/streams-schemas/object.json</sourcePath>
- <sourcePath>target/test-classes/streams-schemas/objectTypes</sourcePath>
- <sourcePath>target/test-classes/streams-schemas/verbs</sourcePath>
+ <sourcePath>target/test-classes/streams-schema-activitystreams/activity.json</sourcePath>
+ <sourcePath>target/test-classes/streams-schema-activitystreams/collection.json</sourcePath>
+ <sourcePath>target/test-classes/streams-schema-activitystreams/media_link.json</sourcePath>
+ <sourcePath>target/test-classes/streams-schema-activitystreams/object.json</sourcePath>
+ <sourcePath>target/test-classes/streams-schema-activitystreams/objectTypes</sourcePath>
+ <sourcePath>target/test-classes/streams-schema-activitystreams/verbs</sourcePath>
</sourcePaths>
- <targetDirectory>target/generated-resources/test-mojo</targetDirectory>
+ <targetDirectory>target/generated-resources/hbase-mojo</targetDirectory>
<columnFamily>cf</columnFamily>
</configuration>
<executions>
<execution>
<goals>
- <goal>hbase</goal>
+ <goal>generate-resources</goal>
</goals>
</execution>
</executions>
@@ -56,9 +57,9 @@
<version>2.10</version>
<configuration>
<includes>**/*.json</includes>
- <outputDirectory>${project.build.directory}/test-classes/streams-schemas</outputDirectory>
+ <outputDirectory>${project.build.directory}/test-classes/streams-schema-activitystreams</outputDirectory>
<includeGroupIds>org.apache.streams</includeGroupIds>
- <includeArtifactIds>streams-schemas</includeArtifactIds>
+ <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
<includeTypes>jar,test-jar</includeTypes>
</configuration>
<executions>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hive/pom.xml
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hive/pom.xml b/streams-plugins/streams-plugin-hive/pom.xml
index 515314a..38c69c1 100644
--- a/streams-plugins/streams-plugin-hive/pom.xml
+++ b/streams-plugins/streams-plugin-hive/pom.xml
@@ -54,10 +54,17 @@
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
- <artifactId>streams-schemas</artifactId>
+ <artifactId>streams-util</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-schema-activitystreams</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
@@ -230,9 +237,9 @@
</goals>
<configuration>
<includeGroupIds>org.apache.streams</includeGroupIds>
- <includeArtifactIds>streams-schemas</includeArtifactIds>
+ <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
<includes>**/*.json</includes>
- <outputDirectory>${project.build.directory}/test-classes/streams-schemas</outputDirectory>
+ <outputDirectory>${project.build.directory}/test-classes/streams-schema-activitystreams</outputDirectory>
</configuration>
</execution>
</executions>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveGenerationConfig.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveGenerationConfig.java b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveGenerationConfig.java
index 7e3bf35..f957757 100644
--- a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveGenerationConfig.java
+++ b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveGenerationConfig.java
@@ -1,11 +1,10 @@
package org.apache.streams.plugins.hive;
-import org.apache.streams.schema.GenerationConfig;
+import org.apache.streams.util.schema.GenerationConfig;
import org.jsonschema2pojo.DefaultGenerationConfig;
import org.jsonschema2pojo.util.URLUtil;
import java.io.File;
-import java.io.FileFilter;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGenerator.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGenerator.java b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGenerator.java
index c09f7dd..60c59ca 100644
--- a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGenerator.java
+++ b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGenerator.java
@@ -1,33 +1,29 @@
package org.apache.streams.plugins.hive;
-import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
-import org.apache.streams.schema.FieldType;
-import org.apache.streams.schema.FieldUtil;
-import org.apache.streams.schema.FileUtil;
-import org.apache.streams.schema.GenerationConfig;
-import org.apache.streams.schema.Schema;
-import org.apache.streams.schema.SchemaStore;
-import org.apache.streams.schema.SchemaUtil;
-import org.apache.streams.schema.URIUtil;
+import org.apache.streams.util.schema.FieldType;
+import org.apache.streams.util.schema.FieldUtil;
+import org.apache.streams.util.schema.FileUtil;
+import org.apache.streams.util.schema.GenerationConfig;
+import org.apache.streams.util.schema.Schema;
+import org.apache.streams.util.schema.SchemaStore;
+import org.apache.streams.util.schema.SchemaStoreImpl;
import org.jsonschema2pojo.util.URLUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
-import java.net.URI;
import java.net.URL;
import java.util.*;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.apache.commons.lang3.StringUtils.defaultString;
-import static org.apache.streams.schema.FileUtil.*;
+import static org.apache.streams.util.schema.FileUtil.*;
/**
* Generates hive table definitions for using org.openx.data.jsonserde.JsonSerDe on new-line delimited json documents.
@@ -42,7 +38,7 @@ public class StreamsHiveResourceGenerator implements Runnable {
private StreamsHiveGenerationConfig config;
- private SchemaStore schemaStore = new SchemaStore();
+ private SchemaStore schemaStore = new SchemaStoreImpl();
private int currentDepth = 0;
@@ -60,17 +56,8 @@ public class StreamsHiveResourceGenerator implements Runnable {
config.setSourceDirectory(sourceDirectory);
config.setTargetDirectory(targetDirectory);
- StreamsHiveResourceGenerator streamsPojoSourceGenerator = new StreamsHiveResourceGenerator(config);
- Thread thread = new Thread(streamsPojoSourceGenerator);
- thread.start();
- try {
- thread.join();
- } catch (InterruptedException e) {
- LOGGER.error("InterruptedException", e);
- } catch (Exception e) {
- LOGGER.error("Exception", e);
- }
- return;
+ StreamsHiveResourceGenerator streamsHiveResourceGenerator = new StreamsHiveResourceGenerator(config);
+ streamsHiveResourceGenerator.run();
}
public StreamsHiveResourceGenerator(StreamsHiveGenerationConfig config) {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGeneratorMojo.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGeneratorMojo.java b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGeneratorMojo.java
index 16466cf..76e3a18 100644
--- a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGeneratorMojo.java
+++ b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGeneratorMojo.java
@@ -15,10 +15,10 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.List;
-@Mojo( name = "hive",
+@Mojo( name = "generate-resources",
defaultPhase = LifecyclePhase.GENERATE_RESOURCES
)
-@Execute( goal = "hive",
+@Execute( goal = "generate-resources",
phase = LifecyclePhase.GENERATE_RESOURCES
)
public class StreamsHiveResourceGeneratorMojo extends AbstractMojo {
@@ -30,15 +30,6 @@ public class StreamsHiveResourceGeneratorMojo extends AbstractMojo {
@Component
private MavenProject project;
-// @Component
-// private Settings settings;
-//
-// @Parameter( defaultValue = "${localRepository}", readonly = true, required = true )
-// protected ArtifactRepository localRepository;
-//
-// @Parameter( defaultValue = "${plugin}", readonly = true ) // Maven 3 only
-// private PluginDescriptor plugin;
-//
@Parameter( defaultValue = "${project.basedir}", readonly = true )
private File basedir;
@@ -65,29 +56,8 @@ public class StreamsHiveResourceGeneratorMojo extends AbstractMojo {
StreamsHiveResourceGenerator streamsHiveResourceGenerator = new StreamsHiveResourceGenerator(config);
- Thread.UncaughtExceptionHandler h = new Thread.UncaughtExceptionHandler() {
- public void uncaughtException(Thread th, Throwable ex) {
- LOGGER.error("Exception", ex);
- mojoFailureException = new MojoFailureException("Exception", ex);
- }
- };
- Thread.setDefaultUncaughtExceptionHandler(h);
- Thread thread = new Thread(streamsHiveResourceGenerator);
- thread.setUncaughtExceptionHandler(h);
- try {
- thread.start();
- thread.join();
- } catch (InterruptedException e) {
- LOGGER.error("InterruptedException", e);
- } catch (Exception e) {
- LOGGER.error("Exception", e);
- mojoFailureException = new MojoFailureException("Exception", e);
- }
-
- if( mojoFailureException != null )
- throw mojoFailureException;
-
- return;
+ streamsHiveResourceGenerator.run();
+
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hive/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hive/src/site/markdown/index.md b/streams-plugins/streams-plugin-hive/src/site/markdown/index.md
index 4df9103..12ce831 100644
--- a/streams-plugins/streams-plugin-hive/src/site/markdown/index.md
+++ b/streams-plugins/streams-plugin-hive/src/site/markdown/index.md
@@ -3,18 +3,36 @@ org.apache.streams.plugins:streams-plugin-hive
streams-plugin-hive generates resources from json schemas to assist with analysis of json data using Apache Hive.
-#### Usage
+### Usage
-Run within a module containing a src/main/jsonschema directory
+Output will be placed in target/generated-resources/hive by default
- mvn org.apache.streams.plugins:streams-plugin-hive:0.3-incubating-SNAPSHOT:hive
+##### Maven
-Output will be placed in target/generated-resources/hive by default
+Run within a module containing a src/main/jsonschema directory
-#### Example
+ mvn org.apache.streams.plugins:streams-plugin-hive:0.3-incubating-SNAPSHOT:generate-resources
[streams-plugin-hive/pom.xml](streams-plugin-hive/pom.xml "streams-plugin-hive/pom.xml")
+##### SDK
+
+Embed within your own java code
+
+ StreamsHiveGenerationConfig config = new StreamsHiveGenerationConfig();
+ config.setSourceDirectory("src/main/jsonschema");
+ config.setTargetDirectory("target/generated-resources");
+ StreamsHiveGenerationConfig generator = new StreamsHiveGenerationConfig(config);
+ generator.run();
+
+##### CLI
+
+Run from CLI without Maven
+
+ java -jar streams-plugin-hive-jar-with-dependencies.jar StreamsHiveResourceGenerator src/main/jsonschema target/generated-resources
+
+#### Example
+
#### Documentation
[JavaDocs](apidocs/index.html "JavaDocs")
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorCLITest.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorCLITest.java b/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorCLITest.java
index 21c82f9..ed08c31 100644
--- a/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorCLITest.java
+++ b/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorCLITest.java
@@ -21,7 +21,7 @@ public class StreamsHiveResourceGeneratorCLITest {
@Test
public void testStreamsHiveResourceGeneratorCLI() throws Exception {
- String sourceDirectory = "target/test-classes/streams-schemas";
+ String sourceDirectory = "target/test-classes/streams-schema-activitystreams";
String targetDirectory = "target/generated-resources/hive-cli";
List<String> argsList = Lists.newArrayList(sourceDirectory, targetDirectory);
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorMojoTest.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorMojoTest.java b/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorMojoTest.java
index b2f3e14..e2fbecc 100644
--- a/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorMojoTest.java
+++ b/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorMojoTest.java
@@ -50,7 +50,7 @@ public class StreamsHiveResourceGeneratorMojoTest extends TestCase {
verifier.resetStreams();
- File testOutput = new File(testDir.getAbsolutePath() + "/target/generated-resources/test-mojo");
+ File testOutput = new File(testDir.getAbsolutePath() + "/target/generated-resources/hive-mojo");
assert( testOutput != null );
assert( testOutput.exists() == true );
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorTest.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorTest.java b/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorTest.java
index 3944ddf..9778bb6 100644
--- a/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorTest.java
+++ b/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorTest.java
@@ -18,7 +18,7 @@ import java.io.File;
import java.util.Collection;
import java.util.Iterator;
-import static org.apache.streams.schema.FileUtil.dropSourcePathPrefix;
+import static org.apache.streams.util.schema.FileUtil.dropSourcePathPrefix;
/**
* Test that Activity beans are compatible with the example activities in the spec.
@@ -47,26 +47,18 @@ public class StreamsHiveResourceGeneratorTest {
StreamsHiveGenerationConfig config = new StreamsHiveGenerationConfig();
- String sourceDirectory = "target/test-classes/streams-schemas";
+ String sourceDirectory = "target/test-classes/streams-schema-activitystreams";
config.setSourceDirectory(sourceDirectory);
- config.setTargetDirectory("target/generated-sources/test");
+ config.setTargetDirectory("target/generated-resources/test");
config.setExclusions(Sets.newHashSet("attachments"));
config.setMaxDepth(2);
StreamsHiveResourceGenerator streamsHiveResourceGenerator = new StreamsHiveResourceGenerator(config);
- Thread thread = new Thread(streamsHiveResourceGenerator);
- thread.start();
- try {
- thread.join();
- } catch (InterruptedException e) {
- LOGGER.error("InterruptedException", e);
- } catch (Exception e) {
- LOGGER.error("Exception", e);
- }
+ streamsHiveResourceGenerator.run();
File testOutput = config.getTargetDirectory();
@@ -106,19 +98,5 @@ public class StreamsHiveResourceGeneratorTest {
LOGGER.info("Fails: {}", fails);
Assert.fail();
}
-
-
-// assert( new File(testOutput + "/traits").exists() == true );
-// assert( new File(testOutput + "/traits").isDirectory() == true );
-// assert( new File(testOutput + "/traits").listFiles(scalaFilter) != null );
-// assert( new File(testOutput + "/traits").listFiles(scalaFilter).length == 4 );
-// assert( new File(testOutput + "/objectTypes").exists() == true );
-// assert( new File(testOutput + "/objectTypes").isDirectory() == true );
-// assert( new File(testOutput + "/objectTypes").listFiles(scalaFilter) != null );
-// assert( new File(testOutput + "/objectTypes").listFiles(scalaFilter).length == 43 );
-// assert( new File(testOutput + "/verbs").exists() == true );
-// assert( new File(testOutput + "/verbs").isDirectory() == true );
-// assert( new File(testOutput + "/verbs").listFiles(scalaFilter) != null );
-// assert( new File(testOutput + "/verbs").listFiles(scalaFilter).length == 89 );
- }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hive/src/test/resources/streams-plugin-hive/pom.xml
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hive/src/test/resources/streams-plugin-hive/pom.xml b/streams-plugins/streams-plugin-hive/src/test/resources/streams-plugin-hive/pom.xml
index ae4bb66..40931ae 100644
--- a/streams-plugins/streams-plugin-hive/src/test/resources/streams-plugin-hive/pom.xml
+++ b/streams-plugins/streams-plugin-hive/src/test/resources/streams-plugin-hive/pom.xml
@@ -18,9 +18,10 @@
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
- <artifactId>streams-schemas</artifactId>
+ <artifactId>streams-schema-activitystreams</artifactId>
<version>${project.version}</version>
<scope>test</scope>
+ <type>test-jar</type>
</dependency>
</dependencies>
@@ -32,19 +33,19 @@
<version>0.3-incubating-SNAPSHOT</version>
<configuration>
<sourcePaths>
- <sourcePath>target/test-classes/streams-schemas/activity.json</sourcePath>
- <sourcePath>target/test-classes/streams-schemas/collection.json</sourcePath>
- <sourcePath>target/test-classes/streams-schemas/media_link.json</sourcePath>
- <sourcePath>target/test-classes/streams-schemas/object.json</sourcePath>
- <sourcePath>target/test-classes/streams-schemas/objectTypes</sourcePath>
- <sourcePath>target/test-classes/streams-schemas/verbs</sourcePath>
+ <sourcePath>target/test-classes/streams-schema-activitystreams/activity.json</sourcePath>
+ <sourcePath>target/test-classes/streams-schema-activitystreams/collection.json</sourcePath>
+ <sourcePath>target/test-classes/streams-schema-activitystreams/media_link.json</sourcePath>
+ <sourcePath>target/test-classes/streams-schema-activitystreams/object.json</sourcePath>
+ <sourcePath>target/test-classes/streams-schema-activitystreams/objectTypes</sourcePath>
+ <sourcePath>target/test-classes/streams-schema-activitystreams/verbs</sourcePath>
</sourcePaths>
- <targetDirectory>target/generated-resources/test-mojo</targetDirectory>
+ <targetDirectory>target/generated-resources/hive-mojo</targetDirectory>
</configuration>
<executions>
<execution>
<goals>
- <goal>hive</goal>
+ <goal>generate-resources</goal>
</goals>
</execution>
</executions>
@@ -55,9 +56,9 @@
<version>2.10</version>
<configuration>
<includes>**/*.json</includes>
- <outputDirectory>${project.build.directory}/test-classes/streams-schemas</outputDirectory>
+ <outputDirectory>${project.build.directory}/test-classes/streams-schema-activitystreams</outputDirectory>
<includeGroupIds>org.apache.streams</includeGroupIds>
- <includeArtifactIds>streams-schemas</includeArtifactIds>
+ <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
<includeTypes>jar,test-jar</includeTypes>
</configuration>
<executions>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-pig/pom.xml
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-pig/pom.xml b/streams-plugins/streams-plugin-pig/pom.xml
index ef59587..0c7bf1a 100644
--- a/streams-plugins/streams-plugin-pig/pom.xml
+++ b/streams-plugins/streams-plugin-pig/pom.xml
@@ -54,10 +54,17 @@
</dependency>
<dependency>
<groupId>org.apache.streams</groupId>
- <artifactId>streams-schemas</artifactId>
+ <artifactId>streams-util</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-schema-activitystreams</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
@@ -230,9 +237,9 @@
</goals>
<configuration>
<includeGroupIds>org.apache.streams</includeGroupIds>
- <includeArtifactIds>streams-schemas</includeArtifactIds>
+ <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds>
<includes>**/*.json</includes>
- <outputDirectory>${project.build.directory}/test-classes/streams-schemas</outputDirectory>
+ <outputDirectory>${project.build.directory}/test-classes/streams-schema-activitystreams</outputDirectory>
</configuration>
</execution>
</executions>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigGenerationConfig.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigGenerationConfig.java b/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigGenerationConfig.java
index 5aa55ed..1b346f4 100644
--- a/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigGenerationConfig.java
+++ b/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigGenerationConfig.java
@@ -1,11 +1,10 @@
package org.apache.streams.plugins.pig;
-import org.apache.streams.schema.GenerationConfig;
+import org.apache.streams.util.schema.GenerationConfig;
import org.jsonschema2pojo.DefaultGenerationConfig;
import org.jsonschema2pojo.util.URLUtil;
import java.io.File;
-import java.io.FileFilter;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigResourceGenerator.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigResourceGenerator.java b/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigResourceGenerator.java
index 072e1b6..f1f94a6 100644
--- a/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigResourceGenerator.java
+++ b/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigResourceGenerator.java
@@ -6,13 +6,13 @@ import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
-import org.apache.streams.plugins.pig.StreamsPigGenerationConfig;
-import org.apache.streams.schema.FieldType;
-import org.apache.streams.schema.FieldUtil;
-import org.apache.streams.schema.FileUtil;
-import org.apache.streams.schema.GenerationConfig;
-import org.apache.streams.schema.Schema;
-import org.apache.streams.schema.SchemaStore;
+import org.apache.streams.util.schema.FieldType;
+import org.apache.streams.util.schema.FieldUtil;
+import org.apache.streams.util.schema.FileUtil;
+import org.apache.streams.util.schema.GenerationConfig;
+import org.apache.streams.util.schema.Schema;
+import org.apache.streams.util.schema.SchemaStore;
+import org.apache.streams.util.schema.SchemaStoreImpl;
import org.jsonschema2pojo.util.URLUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -25,10 +25,10 @@ import java.util.List;
import java.util.Map;
import static com.google.common.base.Preconditions.checkNotNull;
-import static org.apache.streams.schema.FileUtil.dropExtension;
-import static org.apache.streams.schema.FileUtil.dropSourcePathPrefix;
-import static org.apache.streams.schema.FileUtil.swapExtension;
-import static org.apache.streams.schema.FileUtil.writeFile;
+import static org.apache.streams.util.schema.FileUtil.dropExtension;
+import static org.apache.streams.util.schema.FileUtil.dropSourcePathPrefix;
+import static org.apache.streams.util.schema.FileUtil.swapExtension;
+import static org.apache.streams.util.schema.FileUtil.writeFile;
public class StreamsPigResourceGenerator implements Runnable {
@@ -38,7 +38,7 @@ public class StreamsPigResourceGenerator implements Runnable {
private StreamsPigGenerationConfig config;
- private SchemaStore schemaStore = new SchemaStore();
+ private SchemaStore schemaStore = new SchemaStoreImpl();
private int currentDepth = 0;
@@ -57,16 +57,8 @@ public class StreamsPigResourceGenerator implements Runnable {
config.setTargetDirectory(targetDirectory);
StreamsPigResourceGenerator streamsPigResourceGenerator = new StreamsPigResourceGenerator(config);
- Thread thread = new Thread(streamsPigResourceGenerator);
- thread.start();
- try {
- thread.join();
- } catch (InterruptedException e) {
- LOGGER.error("InterruptedException", e);
- } catch (Exception e) {
- LOGGER.error("Exception", e);
- }
- return;
+ streamsPigResourceGenerator.run();
+
}
public StreamsPigResourceGenerator(StreamsPigGenerationConfig config) {
@@ -160,13 +152,9 @@ public class StreamsPigResourceGenerator implements Runnable {
if (fieldType != null ) {
switch (fieldType) {
case ARRAY:
- ObjectNode itemsNode = FieldUtil.resolveItemsNode(fieldNode);
- if( itemsNode == null ) {
- ObjectNode resolvedItems = schemaStore.resolveProperties(schema, fieldNode, fieldId);
- itemsNode = FieldUtil.resolveItemsNode();
- }
- if( itemsNode != null && currentDepth <= config.getMaxDepth()) {
- StringBuilder arrayItemsBuilder = appendArrayItems(new StringBuilder(), schema, fieldId, itemsNode, seperator);
+ ObjectNode resolvedItems = schemaStore.resolveItems(schema, fieldNode, fieldId);
+ if( resolvedItems != null && currentDepth <= config.getMaxDepth()) {
+ StringBuilder arrayItemsBuilder = appendArrayItems(new StringBuilder(), schema, fieldId, resolvedItems, seperator);
if( !Strings.isNullOrEmpty(arrayItemsBuilder.toString())) {
fieldStrings.add(arrayItemsBuilder.toString());
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigResourceGeneratorMojo.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigResourceGeneratorMojo.java b/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigResourceGeneratorMojo.java
index 4763b60..571c75a 100644
--- a/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigResourceGeneratorMojo.java
+++ b/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigResourceGeneratorMojo.java
@@ -15,10 +15,10 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.List;
-@Mojo( name = "pig",
+@Mojo( name = "generate-resources",
defaultPhase = LifecyclePhase.GENERATE_RESOURCES
)
-@Execute( goal = "pig",
+@Execute( goal = "generate-resources",
phase = LifecyclePhase.GENERATE_RESOURCES
)
public class StreamsPigResourceGeneratorMojo extends AbstractMojo {
@@ -30,15 +30,6 @@ public class StreamsPigResourceGeneratorMojo extends AbstractMojo {
@Component
private MavenProject project;
-// @Component
-// private Settings settings;
-//
-// @Parameter( defaultValue = "${localRepository}", readonly = true, required = true )
-// protected ArtifactRepository localRepository;
-//
-// @Parameter( defaultValue = "${plugin}", readonly = true ) // Maven 3 only
-// private PluginDescriptor plugin;
-//
@Parameter( defaultValue = "${project.basedir}", readonly = true )
private File basedir;
@@ -64,30 +55,7 @@ public class StreamsPigResourceGeneratorMojo extends AbstractMojo {
config.setTargetDirectory(targetDirectory);
StreamsPigResourceGenerator streamsPigResourceGenerator = new StreamsPigResourceGenerator(config);
-
- Thread.UncaughtExceptionHandler h = new Thread.UncaughtExceptionHandler() {
- public void uncaughtException(Thread th, Throwable ex) {
- LOGGER.error("Exception", ex);
- mojoFailureException = new MojoFailureException("Exception", ex);
- }
- };
- Thread.setDefaultUncaughtExceptionHandler(h);
- Thread thread = new Thread(streamsPigResourceGenerator);
- thread.setUncaughtExceptionHandler(h);
- try {
- thread.start();
- thread.join();
- } catch (InterruptedException e) {
- LOGGER.error("InterruptedException", e);
- } catch (Exception e) {
- LOGGER.error("Exception", e);
- mojoFailureException = new MojoFailureException("Exception", e);
- }
-
- if( mojoFailureException != null )
- throw mojoFailureException;
-
- return;
+ streamsPigResourceGenerator.run();
}
}
\ No newline at end of file