You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2016/10/12 17:41:46 UTC
[13/38] incubator-streams git commit: start of implementation for
STREAMS-398
start of implementation for STREAMS-398
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/50f67bf8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/50f67bf8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/50f67bf8
Branch: refs/heads/master
Commit: 50f67bf8d97a61bb325f8524472f5c20e86287d0
Parents: 0e7fbcb
Author: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Authored: Tue Dec 29 17:19:15 2015 -0600
Committer: Steve Blackmon @steveblackmon <sb...@apache.org>
Committed: Wed Jun 1 12:49:27 2016 -0500
----------------------------------------------------------------------
streams-plugins/pom.xml | 56 ++++
streams-plugins/streams-plugin-hive/pom.xml | 232 +++++++++++++++
.../apache/streams/plugins/StreamsPojoHive.java | 242 +++++++++++++++
.../streams/plugins/StreamsPojoHiveMojo.java | 71 +++++
.../plugins/test/StreamsPojoHiveTest.java | 55 ++++
.../src/test/resources/Tweet.hql | 297 +++++++++++++++++++
.../test/resources/streams-plugin-hive/pom.xml | 42 +++
7 files changed, 995 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/50f67bf8/streams-plugins/pom.xml
----------------------------------------------------------------------
diff --git a/streams-plugins/pom.xml b/streams-plugins/pom.xml
new file mode 100644
index 0000000..2306aab
--- /dev/null
+++ b/streams-plugins/pom.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>streams-project</artifactId>
+ <groupId>org.apache.streams</groupId>
+ <version>0.3-incubating-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>streams-plugins</artifactId>
+
+ <packaging>pom</packaging>
+ <name>streams-plugins</name>
+
+ <properties>
+
+ </properties>
+
+ <modules>
+ <module>streams-plugin-scala</module>
+ </modules>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-config</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-pojo</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/50f67bf8/streams-plugins/streams-plugin-hive/pom.xml
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hive/pom.xml b/streams-plugins/streams-plugin-hive/pom.xml
new file mode 100644
index 0000000..cbc055b
--- /dev/null
+++ b/streams-plugins/streams-plugin-hive/pom.xml
@@ -0,0 +1,232 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.apache.streams.plugins</groupId>
+ <artifactId>streams-plugin-hive</artifactId>
+ <version>0.3-incubating-SNAPSHOT</version>
+ <packaging>maven-plugin</packaging>
+
+ <parent>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-plugins</artifactId>
+ <version>0.3-incubating-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <properties>
+ <hive.version>1.14</hive.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-config</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.jsonschema2pojo</groupId>
+ <artifactId>jsonschema2pojo-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-pojo</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections</artifactId>
+ <version>0.9.9</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-aether-provider</artifactId>
+ <version>3.3.3</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-artifact</artifactId>
+ <version>3.3.3</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-compat</artifactId>
+ <version>3.3.3</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-core</artifactId>
+ <version>3.3.3</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.sonatype.aether</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-plugin-api</artifactId>
+ <version>3.3.3</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>2.7</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.plexus</groupId>
+ <artifactId>plexus-utils</artifactId>
+ <version>3.0.15</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-plugin-plugin</artifactId>
+ <version>3.4</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>commons-logging</artifactId>
+ <groupId>commons-logging</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.plugin-tools</groupId>
+ <artifactId>maven-plugin-tools-api</artifactId>
+ <version>3.3</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.plugin-tools</groupId>
+ <artifactId>maven-plugin-annotations</artifactId>
+ <version>3.3</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.plugin-tools</groupId>
+ <artifactId>maven-plugin-tools-generators</artifactId>
+ <version>3.3</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.plugin-testing</groupId>
+ <artifactId>maven-plugin-testing-harness</artifactId>
+ <version>3.3.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>2.6</version>
+ </dependency>
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ <version>2.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.joda</groupId>
+ <artifactId>joda-convert</artifactId>
+ <version>1.8.1</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <sourceDirectory>src/main/java</sourceDirectory>
+ <testSourceDirectory>src/test/java</testSourceDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>src/test/resources</directory>
+ </testResource>
+ </testResources>
+ <plugins>
+ <plugin>
+ <artifactId>maven-plugin-plugin</artifactId>
+ <version>3.4</version>
+ <configuration>
+ <skipErrorNoDescriptorsFound>true</skipErrorNoDescriptorsFound>
+ </configuration>
+ <executions>
+ <execution>
+ <id>mojo-descriptor</id>
+ <phase>process-classes</phase>
+ <goals>
+ <goal>descriptor</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections-maven</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>reflections</goal>
+ </goals>
+ <phase>process-classes</phase>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.4</version>
+ <executions>
+ <execution>
+ <id>streams-pojo-resource-dependencies</id>
+ <phase>process-test-resources</phase>
+ <goals>
+ <goal>unpack-dependencies</goal>
+ </goals>
+ <configuration>
+ <includeArtifactIds>streams-pojo</includeArtifactIds>
+ <includes>org/apache/streams/pojo/json/**</includes>
+ <outputDirectory>${project.build.directory}/test-classes</outputDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/50f67bf8/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/StreamsPojoHive.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/StreamsPojoHive.java b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/StreamsPojoHive.java
new file mode 100644
index 0000000..f8a00f7
--- /dev/null
+++ b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/StreamsPojoHive.java
@@ -0,0 +1,242 @@
+package org.apache.streams.plugins;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import org.apache.streams.data.DocumentClassifier;
+import org.reflections.ReflectionUtils;
+import org.reflections.Reflections;
+import org.reflections.scanners.SubTypesScanner;
+import org.reflections.scanners.TypeAnnotationsScanner;
+import org.reflections.util.ClasspathHelper;
+import org.reflections.util.ConfigurationBuilder;
+import org.reflections.ReflectionUtils.*;
+import org.reflections.util.FilterBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Generated;
+import java.io.File;
+import java.io.IOException;
+import java.io.Serializable;
+import java.lang.reflect.Field;
+import java.nio.file.Files;
+import java.nio.file.OpenOption;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Created by sblackmon on 11/18/15.
+ */
+public class StreamsPojoHive implements Runnable {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(StreamsPojoHive.class);
+
+ private final static String LS = System.getProperty("line.separator");
+
+ private StreamsPojoHiveMojo mojo;
+
+ String outDir = "./target/generated-sources/hive";
+ String packages[] = {"org.apache.streams.pojo.json"};
+
+ private final Reflections reflections = new Reflections(
+ new ConfigurationBuilder()
+ .forPackages(packages)
+ .filterInputsBy(new FilterBuilder().includePackage(packages))
+ .setScanners(
+ new SubTypesScanner(),
+ new TypeAnnotationsScanner()));
+
+ public void main(String[] args) {
+ StreamsPojoHive streamsPojoScala = new StreamsPojoHive();
+ Thread thread = new Thread(streamsPojoScala);
+ thread.start();
+ try {
+ thread.join();
+ } catch (InterruptedException e) {
+ LOGGER.error("InterruptedException", e);
+ } catch (Exception e) {
+ LOGGER.error("Exception", e);
+ }
+ return;
+ }
+
+ public StreamsPojoHive(StreamsPojoHiveMojo mojo) {
+ this.mojo = mojo;
+ if ( mojo != null &&
+ mojo.getTarget() != null &&
+ !Strings.isNullOrEmpty(mojo.getTarget().getAbsolutePath())
+ )
+ outDir = mojo.getTarget().getAbsolutePath();
+
+ if ( mojo != null &&
+ mojo.getPackages() != null &&
+ mojo.getPackages().length > 0
+ )
+ packages = mojo.getPackages();
+ }
+
+ public StreamsPojoHive() {
+ }
+
+ public void run() {
+
+ List<Class<?>> serializableClasses = detectSerializableClasses();
+
+ LOGGER.info("Detected {} serialiables:", serializableClasses.size());
+ for( Class clazz : serializableClasses )
+ LOGGER.debug(clazz.toString());
+
+ List<Class<?>> pojoClasses = detectPojoClasses(serializableClasses);
+
+ LOGGER.info("Detected {} pojos:", pojoClasses.size());
+ for( Class clazz : pojoClasses ) {
+ LOGGER.debug(clazz.toString());
+
+ }
+
+
+ for( Class clazz : pojoClasses ) {
+ String pojoPath = clazz.getPackage().getName().replace(".pojo.json", ".hive").replace(".","/")+"/";
+ String pojoName = clazz.getSimpleName()+".hql";
+ String pojoHive = renderPojo(clazz);
+ writeFile(outDir+"/"+pojoPath+pojoName, pojoHive);
+ }
+
+ }
+
+ private void writeFile(String pojoFile, String pojoHive) {
+ try {
+ File path = new File(pojoFile);
+ File dir = path.getParentFile();
+ if( !dir.exists() )
+ dir.mkdirs();
+ Files.write(Paths.get(pojoFile), pojoHive.getBytes(), StandardOpenOption.CREATE_NEW);
+ } catch (Exception e) {
+ LOGGER.error("Write Exception: {}", e);
+ }
+ }
+
+ public List<Class<?>> detectSerializableClasses() {
+
+ Set<Class<? extends Serializable>> classes =
+ reflections.getSubTypesOf(java.io.Serializable.class);
+
+ List<Class<?>> result = Lists.newArrayList();
+
+ for( Class clazz : classes ) {
+ result.add(clazz);
+ }
+
+ return result;
+ }
+
+ public List<Class<?>> detectPojoClasses(List<Class<?>> classes) {
+
+ List<Class<?>> result = Lists.newArrayList();
+
+ for( Class clazz : classes ) {
+ try {
+ clazz.newInstance().toString();
+ } catch( Exception e) {}
+ // super-halfass way to know if this is a jsonschema2pojo
+ if( clazz.getAnnotations().length >= 1 )
+ result.add(clazz);
+ }
+
+ return result;
+ }
+
+ public String renderPojo(Class<?> pojoClass) {
+ StringBuffer stringBuffer = new StringBuffer();
+ stringBuffer.append("CREATE TABLE ");
+ stringBuffer.append(pojoClass.getPackage().getName().replace(".pojo.json", ".hive"));
+ stringBuffer.append(LS);
+ stringBuffer.append("(");
+ stringBuffer.append(LS);
+
+ Set<Field> fields = ReflectionUtils.getAllFields(pojoClass);
+ appendFields(stringBuffer, fields, "", ",");
+
+ stringBuffer.append(")");
+
+ return stringBuffer.toString();
+ }
+
+ private void appendFields(StringBuffer stringBuffer, Set<Field> fields, String varDef, String fieldDelimiter) {
+ if( fields.size() > 0 ) {
+ stringBuffer.append(LS);
+ Map<String,Field> fieldsToAppend = uniqueFields(fields);
+ for( Iterator<Field> iter = fieldsToAppend.values().iterator(); iter.hasNext(); ) {
+ Field field = iter.next();
+ stringBuffer.append(name(field));
+ stringBuffer.append(": ");
+ stringBuffer.append(type(field));
+ if( iter.hasNext()) stringBuffer.append(fieldDelimiter);
+ stringBuffer.append(LS);
+ }
+ } else {
+ stringBuffer.append(LS);
+ }
+ }
+
+ private String value(Field field) {
+ if( field.getName().equals("verb")) {
+ return "\"post\"";
+ } else if( field.getName().equals("objectType")) {
+ return "\"application\"";
+ } else return null;
+ }
+
+ private String type(Field field) {
+ if( field.getType().equals(java.lang.String.class)) {
+ return "STRING";
+ } else if( field.getType().equals(java.lang.Integer.class)) {
+ return "INT";
+ } else if( field.getType().equals(org.joda.time.DateTime.class)) {
+ return "DATE";
+ }else if( field.getType().equals(java.util.Map.class)) {
+ return "MAP";
+ } else if( field.getType().equals(java.util.List.class)) {
+ return "ARRAY";
+ }
+ return field.getType().getCanonicalName().replace(".pojo.json", ".scala");
+ }
+
+ private Map<String,Field> uniqueFields(Set<Field> fieldset) {
+ Map<String,Field> fields = Maps.newTreeMap();
+ Field item = null;
+ for( Iterator<Field> it = fieldset.iterator(); it.hasNext(); item = it.next() ) {
+ if( item != null && item.getName() != null ) {
+ Field added = fields.put(item.getName(), item);
+ }
+ // ensure right class will get used
+ }
+ return fields;
+ }
+
+ private String name(Field field) {
+ if( field.getName().equals("object"))
+ return "obj";
+ else return field.getName();
+ }
+
+ private boolean override(Field field) {
+ try {
+ if( field.getDeclaringClass().getSuperclass().getField(field.getName()) != null )
+ return true;
+ else return false;
+ } catch( Exception e ) {
+ return false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/50f67bf8/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/StreamsPojoHiveMojo.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/StreamsPojoHiveMojo.java b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/StreamsPojoHiveMojo.java
new file mode 100644
index 0000000..a0af5ac
--- /dev/null
+++ b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/StreamsPojoHiveMojo.java
@@ -0,0 +1,71 @@
+package org.apache.streams.plugins;
+
+import org.apache.maven.artifact.repository.ArtifactRepository;
+import org.apache.maven.plugin.AbstractMojo;
+import org.apache.maven.plugin.MojoExecutionException;
+import org.apache.maven.plugin.descriptor.PluginDescriptor;
+import org.apache.maven.plugins.annotations.Component;
+import org.apache.maven.plugins.annotations.Execute;
+import org.apache.maven.plugins.annotations.LifecyclePhase;
+import org.apache.maven.plugins.annotations.Mojo;
+import org.apache.maven.plugins.annotations.Parameter;
+import org.apache.maven.project.MavenProject;
+import org.apache.maven.settings.Settings;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+
+@Mojo( name = "hive",
+ defaultPhase = LifecyclePhase.GENERATE_SOURCES
+)
+@Execute( goal = "hive",
+ phase = LifecyclePhase.GENERATE_SOURCES
+)
+public class StreamsPojoHiveMojo extends AbstractMojo {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(StreamsPojoHiveMojo.class);
+
+ @Component
+ private MavenProject project;
+
+// @Component
+// private Settings settings;
+//
+// @Parameter( defaultValue = "${localRepository}", readonly = true, required = true )
+// protected ArtifactRepository localRepository;
+//
+// @Parameter( defaultValue = "${plugin}", readonly = true ) // Maven 3 only
+// private PluginDescriptor plugin;
+//
+ @Parameter( defaultValue = "${project.basedir}", readonly = true )
+ private File basedir;
+
+ @Parameter(defaultValue = "${project.build.directory}", readonly = true)
+ private File target;
+
+ @Parameter(defaultValue = "org.apache.streams.pojo.json", readonly = true)
+ private String[] packages;
+
+ public void execute() throws MojoExecutionException {
+ StreamsPojoHive streamsPojoScala = new StreamsPojoHive(this);
+ Thread thread = new Thread(streamsPojoScala);
+ thread.start();
+ try {
+ thread.join();
+ } catch (InterruptedException e) {
+ LOGGER.error("InterruptedException", e);
+ } catch (Exception e) {
+ LOGGER.error("Exception", e);
+ }
+ return;
+ }
+
+ public File getTarget() {
+ return target;
+ }
+
+ public String[] getPackages() {
+ return packages;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/50f67bf8/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsPojoHiveTest.java
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsPojoHiveTest.java b/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsPojoHiveTest.java
new file mode 100644
index 0000000..e5a7abd
--- /dev/null
+++ b/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsPojoHiveTest.java
@@ -0,0 +1,55 @@
+package org.apache.streams.plugins.test;
+
+import org.apache.streams.plugins.StreamsPojoHive;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileFilter;
+
+/**
+ * Test that Activity beans are compatible with the example activities in the spec.
+ */
+public class StreamsPojoHiveTest {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(StreamsPojoHiveTest.class);
+
+ /**
+ * Tests that all example activities can be loaded into Activity beans
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testStreamsPojoHive() throws Exception {
+ StreamsPojoHive streamsPojoHive = new StreamsPojoHive();
+ streamsPojoHive.main(new String[0]);
+
+ File testOutput = new File( "./target/generated-sources/hive/org/apache/streams/hive");
+ FileFilter hqlFilter = new FileFilter() {
+ @Override
+ public boolean accept(File pathname) {
+ if( pathname.getName().endsWith(".hql") )
+ return true;
+ return false;
+ }
+ };
+
+ assert( testOutput != null );
+ assert( testOutput.exists() == true );
+ assert( testOutput.isDirectory() == true );
+ assert( testOutput.listFiles(hqlFilter).length == 11 );
+// assert( new File(testOutput + "/traits").exists() == true );
+// assert( new File(testOutput + "/traits").isDirectory() == true );
+// assert( new File(testOutput + "/traits").listFiles(scalaFilter) != null );
+// assert( new File(testOutput + "/traits").listFiles(scalaFilter).length == 4 );
+// assert( new File(testOutput + "/objectTypes").exists() == true );
+// assert( new File(testOutput + "/objectTypes").isDirectory() == true );
+// assert( new File(testOutput + "/objectTypes").listFiles(scalaFilter) != null );
+// assert( new File(testOutput + "/objectTypes").listFiles(scalaFilter).length == 43 );
+// assert( new File(testOutput + "/verbs").exists() == true );
+// assert( new File(testOutput + "/verbs").isDirectory() == true );
+// assert( new File(testOutput + "/verbs").listFiles(scalaFilter) != null );
+// assert( new File(testOutput + "/verbs").listFiles(scalaFilter).length == 89 );
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/50f67bf8/streams-plugins/streams-plugin-hive/src/test/resources/Tweet.hql
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hive/src/test/resources/Tweet.hql b/streams-plugins/streams-plugin-hive/src/test/resources/Tweet.hql
new file mode 100644
index 0000000..dfd3a72
--- /dev/null
+++ b/streams-plugins/streams-plugin-hive/src/test/resources/Tweet.hql
@@ -0,0 +1,297 @@
+/*
+root
+ |-- contributors: string (nullable = true)
+ |-- coordinates: string (nullable = true)
+ |-- created_at: string (nullable = true)
+ |-- entities: struct (nullable = true)
+ | |-- hashtags: array (nullable = true)
+ | | |-- element: struct (containsNull = true)
+ | | | |-- indices: array (nullable = true)
+ | | | | |-- element: long (containsNull = true)
+ | | | |-- text: string (nullable = true)
+ | |-- symbols: array (nullable = true)
+ | | |-- element: string (containsNull = true)
+ | |-- urls: array (nullable = true)
+ | | |-- element: struct (containsNull = true)
+ | | | |-- display_url: string (nullable = true)
+ | | | |-- expanded_url: string (nullable = true)
+ | | | |-- indices: array (nullable = true)
+ | | | | |-- element: long (containsNull = true)
+ | | | |-- url: string (nullable = true)
+ | |-- user_mentions: array (nullable = true)
+ | | |-- element: struct (containsNull = true)
+ | | | |-- id: long (nullable = true)
+ | | | |-- id_str: string (nullable = true)
+ | | | |-- indices: array (nullable = true)
+ | | | | |-- element: long (containsNull = true)
+ | | | |-- name: string (nullable = true)
+ | | | |-- screen_name: string (nullable = true)
+ |-- favorite_count: long (nullable = true)
+ |-- favorited: boolean (nullable = true)
+ |-- geo: string (nullable = true)
+ |-- id: long (nullable = true)
+ |-- id_str: string (nullable = true)
+ |-- in_reply_to_screen_name: string (nullable = true)
+ |-- in_reply_to_status_id: long (nullable = true)
+ |-- in_reply_to_status_id_str: string (nullable = true)
+ |-- in_reply_to_user_id: long (nullable = true)
+ |-- in_reply_to_user_id_str: string (nullable = true)
+ |-- lang: string (nullable = true)
+ |-- place: struct (nullable = true)
+ | |-- attributes: struct (nullable = true)
+ | |-- bounding_box: struct (nullable = true)
+ | | |-- coordinates: array (nullable = true)
+ | | | |-- element: array (containsNull = true)
+ | | | | |-- element: array (containsNull = true)
+ | | | | | |-- element: double (containsNull = true)
+ | | |-- type: string (nullable = true)
+ | |-- contained_within: array (nullable = true)
+ | | |-- element: string (containsNull = true)
+ | |-- country: string (nullable = true)
+ | |-- country_code: string (nullable = true)
+ | |-- full_name: string (nullable = true)
+ | |-- id: string (nullable = true)
+ | |-- name: string (nullable = true)
+ | |-- place_type: string (nullable = true)
+ | |-- url: string (nullable = true)
+ |-- possibly_sensitive: boolean (nullable = true)
+ |-- retweet_count: long (nullable = true)
+ |-- retweeted: boolean (nullable = true)
+ |-- retweeted_status: struct (nullable = true)
+ | |-- contributors: string (nullable = true)
+ | |-- coordinates: string (nullable = true)
+ | |-- created_at: string (nullable = true)
+ | |-- entities: struct (nullable = true)
+ | | |-- hashtags: array (nullable = true)
+ | | | |-- element: struct (containsNull = true)
+ | | | | |-- indices: array (nullable = true)
+ | | | | | |-- element: long (containsNull = true)
+ | | | | |-- text: string (nullable = true)
+ | | |-- symbols: array (nullable = true)
+ | | | |-- element: string (containsNull = true)
+ | | |-- urls: array (nullable = true)
+ | | | |-- element: struct (containsNull = true)
+ | | | | |-- display_url: string (nullable = true)
+ | | | | |-- expanded_url: string (nullable = true)
+ | | | | |-- indices: array (nullable = true)
+ | | | | | |-- element: long (containsNull = true)
+ | | | | |-- url: string (nullable = true)
+ | | |-- user_mentions: array (nullable = true)
+ | | | |-- element: struct (containsNull = true)
+ | | | | |-- id: long (nullable = true)
+ | | | | |-- id_str: string (nullable = true)
+ | | | | |-- indices: array (nullable = true)
+ | | | | | |-- element: long (containsNull = true)
+ | | | | |-- name: string (nullable = true)
+ | | | | |-- screen_name: string (nullable = true)
+ | |-- favorite_count: long (nullable = true)
+ | |-- favorited: boolean (nullable = true)
+ | |-- geo: string (nullable = true)
+ | |-- id: long (nullable = true)
+ | |-- id_str: string (nullable = true)
+ | |-- in_reply_to_screen_name: string (nullable = true)
+ | |-- in_reply_to_status_id: string (nullable = true)
+ | |-- in_reply_to_status_id_str: string (nullable = true)
+ | |-- in_reply_to_user_id: string (nullable = true)
+ | |-- in_reply_to_user_id_str: string (nullable = true)
+ | |-- lang: string (nullable = true)
+ | |-- place: struct (nullable = true)
+ | | |-- attributes: struct (nullable = true)
+ | | |-- bounding_box: struct (nullable = true)
+ | | | |-- coordinates: array (nullable = true)
+ | | | | |-- element: array (containsNull = true)
+ | | | | | |-- element: array (containsNull = true)
+ | | | | | | |-- element: double (containsNull = true)
+ | | | |-- type: string (nullable = true)
+ | | |-- contained_within: array (nullable = true)
+ | | | |-- element: string (containsNull = true)
+ | | |-- country: string (nullable = true)
+ | | |-- country_code: string (nullable = true)
+ | | |-- full_name: string (nullable = true)
+ | | |-- id: string (nullable = true)
+ | | |-- name: string (nullable = true)
+ | | |-- place_type: string (nullable = true)
+ | | |-- url: string (nullable = true)
+ | |-- possibly_sensitive: boolean (nullable = true)
+ | |-- retweet_count: long (nullable = true)
+ | |-- retweeted: boolean (nullable = true)
+ | |-- source: string (nullable = true)
+ | |-- text: string (nullable = true)
+ | |-- truncated: boolean (nullable = true)
+ | |-- user: struct (nullable = true)
+ | | |-- contributors_enabled: boolean (nullable = true)
+ | | |-- created_at: string (nullable = true)
+ | | |-- default_profile: boolean (nullable = true)
+ | | |-- default_profile_image: boolean (nullable = true)
+ | | |-- description: string (nullable = true)
+ | | |-- entities: struct (nullable = true)
+ | | | |-- description: struct (nullable = true)
+ | | | | |-- urls: array (nullable = true)
+ | | | | | |-- element: struct (containsNull = true)
+ | | | | | | |-- display_url: string (nullable = true)
+ | | | | | | |-- expanded_url: string (nullable = true)
+ | | | | | | |-- indices: array (nullable = true)
+ | | | | | | | |-- element: long (containsNull = true)
+ | | | | | | |-- url: string (nullable = true)
+ | | | |-- url: struct (nullable = true)
+ | | | | |-- urls: array (nullable = true)
+ | | | | | |-- element: struct (containsNull = true)
+ | | | | | | |-- display_url: string (nullable = true)
+ | | | | | | |-- expanded_url: string (nullable = true)
+ | | | | | | |-- indices: array (nullable = true)
+ | | | | | | | |-- element: long (containsNull = true)
+ | | | | | | |-- url: string (nullable = true)
+ | | |-- favourites_count: long (nullable = true)
+ | | |-- follow_request_sent: boolean (nullable = true)
+ | | |-- followers_count: long (nullable = true)
+ | | |-- following: boolean (nullable = true)
+ | | |-- friends_count: long (nullable = true)
+ | | |-- geo_enabled: boolean (nullable = true)
+ | | |-- id: long (nullable = true)
+ | | |-- id_str: string (nullable = true)
+ | | |-- is_translation_enabled: boolean (nullable = true)
+ | | |-- is_translator: boolean (nullable = true)
+ | | |-- lang: string (nullable = true)
+ | | |-- listed_count: long (nullable = true)
+ | | |-- location: string (nullable = true)
+ | | |-- name: string (nullable = true)
+ | | |-- notifications: boolean (nullable = true)
+ | | |-- profile_background_color: string (nullable = true)
+ | | |-- profile_background_image_url: string (nullable = true)
+ | | |-- profile_background_image_url_https: string (nullable = true)
+ | | |-- profile_background_tile: boolean (nullable = true)
+ | | |-- profile_banner_url: string (nullable = true)
+ | | |-- profile_image_url: string (nullable = true)
+ | | |-- profile_image_url_https: string (nullable = true)
+ | | |-- profile_link_color: string (nullable = true)
+ | | |-- profile_sidebar_border_color: string (nullable = true)
+ | | |-- profile_sidebar_fill_color: string (nullable = true)
+ | | |-- profile_text_color: string (nullable = true)
+ | | |-- profile_use_background_image: boolean (nullable = true)
+ | | |-- protected: boolean (nullable = true)
+ | | |-- screen_name: string (nullable = true)
+ | | |-- statuses_count: long (nullable = true)
+ | | |-- time_zone: string (nullable = true)
+ | | |-- url: string (nullable = true)
+ | | |-- utc_offset: long (nullable = true)
+ | | |-- verified: boolean (nullable = true)
+ |-- source: string (nullable = true)
+ |-- text: string (nullable = true)
+ |-- truncated: boolean (nullable = true)
+ |-- user: struct (nullable = true)
+ | |-- contributors_enabled: boolean (nullable = true)
+ | |-- created_at: string (nullable = true)
+ | |-- default_profile: boolean (nullable = true)
+ | |-- default_profile_image: boolean (nullable = true)
+ | |-- description: string (nullable = true)
+ | |-- entities: struct (nullable = true)
+ | | |-- description: struct (nullable = true)
+ | | | |-- urls: array (nullable = true)
+ | | | | |-- element: string (containsNull = true)
+ | | |-- url: struct (nullable = true)
+ | | | |-- urls: array (nullable = true)
+ | | | | |-- element: struct (containsNull = true)
+ | | | | | |-- display_url: string (nullable = true)
+ | | | | | |-- expanded_url: string (nullable = true)
+ | | | | | |-- indices: array (nullable = true)
+ | | | | | | |-- element: long (containsNull = true)
+ | | | | | |-- url: string (nullable = true)
+ | |-- favourites_count: long (nullable = true)
+ | |-- follow_request_sent: boolean (nullable = true)
+ | |-- followers_count: long (nullable = true)
+ | |-- following: boolean (nullable = true)
+ | |-- friends_count: long (nullable = true)
+ | |-- geo_enabled: boolean (nullable = true)
+ | |-- id: long (nullable = true)
+ | |-- id_str: string (nullable = true)
+ | |-- is_translation_enabled: boolean (nullable = true)
+ | |-- is_translator: boolean (nullable = true)
+ | |-- lang: string (nullable = true)
+ | |-- listed_count: long (nullable = true)
+ | |-- location: string (nullable = true)
+ | |-- name: string (nullable = true)
+ | |-- notifications: boolean (nullable = true)
+ | |-- profile_background_color: string (nullable = true)
+ | |-- profile_background_image_url: string (nullable = true)
+ | |-- profile_background_image_url_https: string (nullable = true)
+ | |-- profile_background_tile: boolean (nullable = true)
+ | |-- profile_banner_url: string (nullable = true)
+ | |-- profile_image_url: string (nullable = true)
+ | |-- profile_image_url_https: string (nullable = true)
+ | |-- profile_link_color: string (nullable = true)
+ | |-- profile_sidebar_border_color: string (nullable = true)
+ | |-- profile_sidebar_fill_color: string (nullable = true)
+ | |-- profile_text_color: string (nullable = true)
+ | |-- profile_use_background_image: boolean (nullable = true)
+ | |-- protected: boolean (nullable = true)
+ | |-- screen_name: string (nullable = true)
+ | |-- statuses_count: long (nullable = true)
+ | |-- time_zone: string (nullable = true)
+ | |-- url: string (nullable = true)
+ | |-- utc_offset: long (nullable = true)
+ | |-- verified: boolean (nullable = true)
+*/
+create table tweet (
+ created_at string,
+ entities struct
+ <
+ hashtags: array
+ <
+ struct
+ <
+ text: string
+ >
+ >,
+ urls: array
+ <
+ struct
+ <
+ expanded_url: string
+ >
+ >,
+ user_mentions: array
+ <
+ struct
+ <
+ id_str: string
+ >
+ >
+ >,
+ in_reply_to_user_id_str: string,
+ user: struct
+ <
+ id_str: string
+ >,
+ retweeted_status struct
+ <
+ entities struct
+ <
+ hashtags: array
+ <
+ struct
+ <
+ text: string
+ >
+ >,
+ urls: array
+ <
+ struct
+ <
+ expanded_url: string
+ >
+ >,
+ user_mentions: array
+ <
+ struct
+ <
+ id_str: string
+ >
+ >
+ >,
+ in_reply_to_user_id_str: string,
+ user: struct
+ <
+ id_str: string
+ >
+ >
+)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/50f67bf8/streams-plugins/streams-plugin-hive/src/test/resources/streams-plugin-hive/pom.xml
----------------------------------------------------------------------
diff --git a/streams-plugins/streams-plugin-hive/src/test/resources/streams-plugin-hive/pom.xml b/streams-plugins/streams-plugin-hive/src/test/resources/streams-plugin-hive/pom.xml
new file mode 100644
index 0000000..b1de7b8
--- /dev/null
+++ b/streams-plugins/streams-plugin-hive/src/test/resources/streams-plugin-hive/pom.xml
@@ -0,0 +1,42 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>org.apache.streams.plugins</groupId>
+ <artifactId>streams-plugin-hive-test</artifactId>
+ <version>0.3-incubating-SNAPSHOT</version>
+ <packaging>jar</packaging>
+ <name>Test StreamsPojoHiveMojo</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.streams.plugins</groupId>
+ <artifactId>streams-plugin-hive</artifactId>
+ <version>0.3-incubating-SNAPSHOT</version>
+ <configuration>
+ <packages>
+ <package>org.apache.streams.pojo.json</package>
+ </packages>
+ <target>target/test-classes/streams-hive-plugin/</target>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>hive</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file