You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by sh...@apache.org on 2014/08/13 10:06:04 UTC
git commit: FALCON-588 Baselining designer code. Contributed by samar
Repository: incubator-falcon
Updated Branches:
refs/heads/master 839d44d60 -> aee6d3bf8
FALCON-588 Baselining designer code. Contributed by samar
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/aee6d3bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/aee6d3bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/aee6d3bf
Branch: refs/heads/master
Commit: aee6d3bf8e63763ac7924d1f4320cf23e4e6ef3a
Parents: 839d44d
Author: Shwetha GS <sh...@inmobi.com>
Authored: Wed Aug 13 13:34:47 2014 +0530
Committer: Shwetha GS <sh...@inmobi.com>
Committed: Wed Aug 13 13:35:15 2014 +0530
----------------------------------------------------------------------
CHANGES.txt | 2 +
addons/designer/actions/pom.xml | 14 ++-
.../configuration/EmailActionConfiguration.java | 74 ++++++++++++
.../designer/primitive/action/EmailAction.java | 92 +++++++++++++++
addons/designer/core/pom.xml | 66 +++++++----
.../configuration/ActionConfiguration.java | 32 ++++++
.../designer/configuration/Configuration.java | 63 ++++++++++-
.../designer/configuration/FlowConfig.java | 69 +++++++++++
.../designer/configuration/SerdeException.java | 61 ++++++++++
.../configuration/TransformConfiguration.java | 33 ++++++
.../falcon/designer/primitive/Action.java | 12 +-
.../falcon/designer/primitive/Primitive.java | 113 +++++++++++++------
.../falcon/designer/primitive/Transform.java | 4 +-
.../designer/storage/impl/HDFSStorage.java | 98 ++++++++++++++++
.../designer/storage/impl/HDFSStorageTest.java | 78 +++++++++++++
addons/designer/flows/pom.xml | 4 +
.../apache/falcon/designer/primitive/Flow.java | 83 ++++++++++++++
addons/designer/pom.xml | 47 +++++++-
18 files changed, 867 insertions(+), 78 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 8ee08e8..63552b0 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -5,6 +5,8 @@ Trunk (Unreleased)
INCOMPATIBLE CHANGES
NEW FEATURES
+ FALCON-588 Baselining designer code. (samar via Shwetha GS)
+
FALCON-400 Add Authorization for Entities (Venkatesh Seetharam)
FALCON-279 Add ACL for Cluster Entity
FALCON-462 Add ACL for process entity
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/actions/pom.xml
----------------------------------------------------------------------
diff --git a/addons/designer/actions/pom.xml b/addons/designer/actions/pom.xml
index 7d5afbb..bba43fd 100644
--- a/addons/designer/actions/pom.xml
+++ b/addons/designer/actions/pom.xml
@@ -33,10 +33,14 @@
<packaging>jar</packaging>
<dependencies>
- <dependency>
- <groupId>org.testng</groupId>
- <artifactId>testng</artifactId>
- </dependency>
- </dependencies>
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.falcon.designer</groupId>
+ <artifactId>designer-core</artifactId>
+ </dependency>
+</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/actions/src/main/java/org/apache/falcon/designer/action/configuration/EmailActionConfiguration.java
----------------------------------------------------------------------
diff --git a/addons/designer/actions/src/main/java/org/apache/falcon/designer/action/configuration/EmailActionConfiguration.java b/addons/designer/actions/src/main/java/org/apache/falcon/designer/action/configuration/EmailActionConfiguration.java
new file mode 100644
index 0000000..fd37a49
--- /dev/null
+++ b/addons/designer/actions/src/main/java/org/apache/falcon/designer/action/configuration/EmailActionConfiguration.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.falcon.designer.action.configuration;
+
+import org.apache.falcon.designer.configuration.ActionConfiguration;
+/**
+ * Action object holding Email data.
+ */
+public class EmailActionConfiguration extends ActionConfiguration<EmailActionConfiguration> {
+
+ private String to;
+ private String cc;
+ private String subject;
+ private String body;
+
+ private static final String NAME = "Email";
+
+ public String getTo() {
+ return to;
+ }
+
+ public void setTo(String to) {
+ this.to = to;
+ }
+
+ public String getCc() {
+ return cc;
+ }
+
+ public void setCc(String cc) {
+ this.cc = cc;
+ }
+
+ public String getSubject() {
+ return subject;
+ }
+
+ public void setSubject(String subject) {
+ this.subject = subject;
+ }
+
+ public String getBody() {
+ return body;
+ }
+
+ public void setBody(String body) {
+ this.body = body;
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public Class<EmailActionConfiguration> getConfigClass() {
+ return EmailActionConfiguration.class;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/actions/src/main/java/org/apache/falcon/designer/primitive/action/EmailAction.java
----------------------------------------------------------------------
diff --git a/addons/designer/actions/src/main/java/org/apache/falcon/designer/primitive/action/EmailAction.java b/addons/designer/actions/src/main/java/org/apache/falcon/designer/primitive/action/EmailAction.java
new file mode 100644
index 0000000..1bd1197
--- /dev/null
+++ b/addons/designer/actions/src/main/java/org/apache/falcon/designer/primitive/action/EmailAction.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.falcon.designer.primitive.action;
+
+import org.apache.falcon.designer.action.configuration.EmailActionConfiguration;
+import org.apache.falcon.designer.primitive.Action;
+import org.apache.falcon.designer.primitive.Code;
+import org.apache.falcon.designer.primitive.Message;
+
+/**
+ * EmailAction Primitive containing implementation to compile.
+ */
+public class EmailAction extends Action<EmailAction, EmailActionConfiguration> {
+
+ private String nameSpace;
+ private String entity;
+
+ private EmailActionConfiguration emailConfig;
+
+ public EmailAction(EmailActionConfiguration config , String nameSpace, String entity) {
+ this.emailConfig = config;
+ this.nameSpace = nameSpace;
+ this.entity = entity;
+ }
+
+ @Override
+ public EmailActionConfiguration getConfiguration() {
+ return emailConfig;
+ }
+
+ @Override
+ public void setConfiguration(EmailActionConfiguration config) {
+ this.emailConfig = config;
+
+ }
+
+ @Override
+ public boolean hasOutput() {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ protected EmailAction copy() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Iterable<Message> validate() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ protected Code doCompile() {
+ getConfiguration().getBody();
+ return null;
+ }
+
+ @Override
+ protected EmailAction doOptimize() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public String getNamespace() {
+ return nameSpace;
+ }
+
+ @Override
+ public String getEntity() {
+ return entity;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/core/pom.xml
----------------------------------------------------------------------
diff --git a/addons/designer/core/pom.xml b/addons/designer/core/pom.xml
index ddd8814..8ebc79a 100644
--- a/addons/designer/core/pom.xml
+++ b/addons/designer/core/pom.xml
@@ -32,26 +32,50 @@
<name>Apache Falcon Designer Core</name>
<packaging>jar</packaging>
- <dependencies>
- <dependency>
- <groupId>org.apache.falcon</groupId>
- <artifactId>falcon-client</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.testng</groupId>
- <artifactId>testng</artifactId>
- </dependency>
-
- <dependency>
- <groupId>net.sourceforge.findbugs</groupId>
- <artifactId>jsr305</artifactId>
- </dependency>
-
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </dependency>
- </dependencies>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.falcon</groupId>
+ <artifactId>falcon-client</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>net.sourceforge.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
+ </dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/ActionConfiguration.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/ActionConfiguration.java b/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/ActionConfiguration.java
new file mode 100644
index 0000000..884f459
--- /dev/null
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/ActionConfiguration.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.falcon.designer.configuration;
+
+/**
+ * Abstract ActionConfiguration extending Configuration.
+ */
+public abstract class ActionConfiguration<A extends ActionConfiguration> extends Configuration<A> {
+
+ private static final String CATEGORY = "ACTION";
+
+ @Override
+ public String getCategory() {
+ return CATEGORY;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/Configuration.java b/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/Configuration.java
index dba908a..30c0314 100644
--- a/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/Configuration.java
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/Configuration.java
@@ -15,12 +15,67 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.falcon.designer.configuration;
+import java.io.IOException;
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.JsonParseException;
+import org.codehaus.jackson.map.JsonMappingException;
+import org.codehaus.jackson.map.ObjectMapper;
+
/**
- * This is the configuration that primitives will use
- * to configure their instance.
+ * This is the configuration that primitives will use to configure their
+ * instance.
*/
-public interface Configuration {
+public abstract class Configuration<T extends Configuration> {
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ public abstract String getName();
+
+ public abstract String getCategory();
+
+ /**
+ * Serialize it from a action type to string type.
+ * @param act
+ * actual Action Configuration data
+ * @return
+ */
+ public String serialize() throws SerdeException {
+ String returnJsonString;
+ try {
+ returnJsonString = MAPPER.writeValueAsString(this);
+ } catch (JsonGenerationException e) {
+ throw new SerdeException("Failed serializing object ", e);
+ } catch (JsonMappingException e) {
+ throw new SerdeException("Failed serializing object ", e);
+ } catch (IOException e) {
+ throw new SerdeException("Failed serializing object ", e);
+ }
+ return returnJsonString;
+ }
+
+ /**
+ * Deserialize from string to object of Action type.
+ * @param actString
+ * actual Data
+ * @return
+ */
+ public T deserialize(String actString) throws SerdeException {
+ T returnEmailAction;
+ try {
+ returnEmailAction = MAPPER.readValue(actString, getConfigClass());
+ } catch (JsonParseException e) {
+ throw new SerdeException("Failed deserialize string " + actString,
+ e);
+ } catch (JsonMappingException e) {
+ throw new SerdeException("Failed deserialize string " + actString,
+ e);
+ } catch (IOException e) {
+ throw new SerdeException("Failed deserialize string " + actString,
+ e);
+ }
+ return returnEmailAction;
+ }
+
+ public abstract Class<T> getConfigClass();
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/FlowConfig.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/FlowConfig.java b/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/FlowConfig.java
new file mode 100644
index 0000000..ba7d074
--- /dev/null
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/FlowConfig.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.designer.configuration;
+
+/**
+ * Flow data. More parameters would need to be added.
+ */
+public class FlowConfig extends Configuration<FlowConfig> {
+
+ private String namespace;
+ private String entity;
+ private String name;
+ private static final String CATEGORY = "FLOW";
+
+ public FlowConfig(String namespace, String entity, String name) {
+ this.namespace = namespace;
+ this.entity = entity;
+ this.name = name;
+
+ }
+
+ @Override
+ public String getCategory() {
+ return CATEGORY;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ public String getNamespace() {
+ return namespace;
+ }
+
+ public void setNamespace(String namespace) {
+ this.namespace = namespace;
+ }
+
+ public String getEntity() {
+ return entity;
+ }
+
+ public void setEntity(String entity) {
+ this.entity = entity;
+ }
+
+ @Override
+ public Class<FlowConfig> getConfigClass() {
+ return FlowConfig.class;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/SerdeException.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/SerdeException.java b/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/SerdeException.java
new file mode 100644
index 0000000..107180c
--- /dev/null
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/SerdeException.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.falcon.designer.configuration;
+
+/**
+ * Checked Exception that is thrown on serialization/deserialization of a config
+ * object failure.
+ */
+public class SerdeException extends Exception {
+ /**
+ * Constructs a default exception with no cause or message.
+ */
+ public SerdeException() {
+ super();
+ }
+
+ /**
+ * Constructs an exception with a specific message.
+ * @param message
+ * - Message on the exception
+ */
+ public SerdeException(String message) {
+ super(message);
+ }
+
+ /**
+ * Constructs an exception with a specific message and cause.
+ * @param message
+ * - Message on the exception
+ * @param cause
+ * - Underlying exception that resulted in this being thrown
+ */
+ public SerdeException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ /**
+ * Constructs an exception with a cause and message is initialized to be
+ * same as that of the cause.
+ * @param cause
+ * - Underlying exception that resulted in this being thrown
+ */
+ public SerdeException(Throwable cause) {
+ super(cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/TransformConfiguration.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/TransformConfiguration.java b/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/TransformConfiguration.java
new file mode 100644
index 0000000..6e0fedb
--- /dev/null
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/configuration/TransformConfiguration.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.falcon.designer.configuration;
+
+/**
+ * Concrete implementation of Transform Configuration.
+ */
+public abstract class TransformConfiguration<A extends TransformConfiguration> extends Configuration<A> {
+
+
+ private static final String CATEGORY = "TRANSFORM";
+
+ @Override
+ public String getCategory() {
+ return CATEGORY;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Action.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Action.java b/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Action.java
index c40e462..e16c013 100644
--- a/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Action.java
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Action.java
@@ -19,9 +19,11 @@
package org.apache.falcon.designer.primitive;
import org.apache.commons.lang.NotImplementedException;
+import org.apache.falcon.designer.configuration.ActionConfiguration;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
+
import java.net.URI;
import java.util.Map;
@@ -39,17 +41,13 @@ import java.util.Map;
* There are no restrictions imposed on an Action with respect to the input
* or the output such as in a Transformation.
*/
-public abstract class Action extends Primitive {
-
- protected final String type;
+public abstract class Action<A extends Action, B extends ActionConfiguration> extends Primitive<A, B> {
- protected Action(@Nonnull String type) {
- this.type = type;
- }
+ protected static final String TYPE="ACTION";
@Nonnull
public String getType() {
- return type;
+ return TYPE;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Primitive.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Primitive.java b/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Primitive.java
index aa2b988..aa825a9 100644
--- a/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Primitive.java
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Primitive.java
@@ -18,53 +18,53 @@
package org.apache.falcon.designer.primitive;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
import org.apache.falcon.designer.configuration.Configuration;
+import org.apache.falcon.designer.configuration.SerdeException;
+import org.apache.falcon.designer.storage.Storage;
+import org.apache.falcon.designer.storage.StorageException;
import org.apache.falcon.designer.storage.Storeable;
-
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
/**
- * All elements of the pipeline are essentially a primitive.
- * These primitives only have life during the pipeline design time.
- *
+ * All elements of the pipeline are essentially a primitive. These primitives
+ * only have life during the pipeline design time.
*/
-public abstract class Primitive<T extends Primitive> implements Storeable {
+public abstract class Primitive<T extends Primitive, V extends Configuration>
+ implements Storeable {
- protected Configuration configuration;
-
- public void setConfiguration(@Nonnull Configuration config) {
- configuration = config;
- }
+ public abstract void setConfiguration(V config);
protected abstract T copy();
@Nonnull
- public Configuration getConfiguration() {
- return configuration;
- }
+ public abstract V getConfiguration();
/**
- * Perform a validation to see if the primitive configuration
- * is consistent with this primitive.
- *
+ * Perform a validation to see if the primitive configuration is consistent
+ * with this primitive.
* @return - Messages that need to be sent as feedback from the validation.
- * As long as one of the messages returned has a message type.
- * Returns null if there are no messages to return from the validation.
- * {@link Message.Type} is {@link Message.Type#ERROR}, the primitive cannot
- * be compiled successfully.
+ * As long as one of the messages returned has a message type.
+ * Returns null if there are no messages to return from the
+ * validation. {@link Message.Type} is {@link Message.Type#ERROR},
+ * the primitive cannot be compiled successfully.
*/
@Nullable
public abstract Iterable<Message> validate();
/**
- * Compile the primitive and generate corresponding binary/source code
- * The compile method fails if there are one or more
- * {@link Message.Type#ERROR} messages from the validation phase.
- *
+ * Compile the primitive and generate corresponding binary/source code The
+ * compile method fails if there are one or more {@link Message.Type#ERROR}
+ * messages from the validation phase.
* @return - Code object generated for the primitive.
- * @throws CompilationException - Compilation issues as returned by validate
- * (if any of the {@link Message} is of type {@link Message.Type#ERROR}
+ * @throws CompilationException
+ * - Compilation issues as returned by validate (if any of the
+ * {@link Message} is of type {@link Message.Type#ERROR}
*/
@Nonnull
public Code compile() throws CompilationException {
@@ -87,20 +87,18 @@ public abstract class Primitive<T extends Primitive> implements Storeable {
}
/**
- * This method is invoked only when the primitive is confirmed to be
- * valid. This would generate binary or source code for this primitive
- * and its configuration.
- *
+ * This method is invoked only when the primitive is confirmed to be valid.
+ * This would generate binary or source code for this primitive and its
+ * configuration.
* @return - Code object generated by the primitive.
*/
@Nonnull
protected abstract Code doCompile();
/**
- * This method is invoked only when the primitive is confirmed to be
- * valid. This would operate on the current instance and return back
- * an optimized version of the same type.
- *
+ * This method is invoked only when the primitive is confirmed to be valid.
+ * This would operate on the current instance and return back an optimized
+ * version of the same type.
* @return - Optimized object of the same type.
*/
@Nonnull
@@ -111,4 +109,51 @@ public abstract class Primitive<T extends Primitive> implements Storeable {
}
protected abstract T doOptimize();
+
+ public abstract String getNamespace();
+
+ public abstract String getEntity();
+
+ @Override
+ public void store(Storage storage) throws StorageException {
+ try {
+
+ BufferedWriter bufferedWriterInst =
+ new BufferedWriter(new OutputStreamWriter(storage.create(
+ getNamespace(), getEntity())));
+ String serializedResource =
+ getConfiguration().serialize();
+ bufferedWriterInst.write(serializedResource);
+ bufferedWriterInst.close();
+ } catch (IOException e) {
+ throw new StorageException(e.getMessage());
+ } catch (SerdeException e) {
+ throw new StorageException(e.getMessage());
+ }
+
+ }
+
+ @Override
+ public void restore(Storage storage) throws StorageException {
+ try {
+ BufferedReader bufferedReaderInst =
+ new BufferedReader(new InputStreamReader(storage.open(
+ getNamespace(), getEntity())));
+ String configInString = bufferedReaderInst.readLine();
+ setConfiguration((V)getConfiguration().deserialize(configInString));
+ bufferedReaderInst.close();
+ } catch (IOException e) {
+ throw new StorageException(e.getMessage());
+ } catch (SerdeException e) {
+ throw new StorageException(e.getMessage());
+ }
+
+ }
+
+ @Override
+ public void delete(Storage storage) throws StorageException {
+ storage.delete(getNamespace(), getEntity());
+ setConfiguration(null);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Transform.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Transform.java b/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Transform.java
index 72cf988..b2d4396 100644
--- a/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Transform.java
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/primitive/Transform.java
@@ -18,9 +18,11 @@
package org.apache.falcon.designer.primitive;
+import org.apache.falcon.designer.configuration.TransformConfiguration;
import org.apache.falcon.designer.schema.RelationalData;
import javax.annotation.Nonnull;
+
import java.util.Collections;
import java.util.List;
@@ -32,7 +34,7 @@ import java.util.List;
* to a schema and would produce one or more output typically with an uniform schema.
*
*/
-public abstract class Transform extends Primitive<Transform> {
+public abstract class Transform extends Primitive<Transform , TransformConfiguration> {
protected List<RelationalData> inputData;
protected List<RelationalData> outputData;
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/core/src/main/java/org/apache/falcon/designer/storage/impl/HDFSStorage.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/main/java/org/apache/falcon/designer/storage/impl/HDFSStorage.java b/addons/designer/core/src/main/java/org/apache/falcon/designer/storage/impl/HDFSStorage.java
new file mode 100644
index 0000000..4d0ff64
--- /dev/null
+++ b/addons/designer/core/src/main/java/org/apache/falcon/designer/storage/impl/HDFSStorage.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.falcon.designer.storage.impl;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.falcon.designer.storage.Storage;
+import org.apache.falcon.designer.storage.StorageException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Storage implementation to store in HDFS.
+ *
+ */
+public class HDFSStorage implements Storage {
+
+ private FileSystem fs;
+ private String basePath;
+ private static final String SEPERATOR = "/";
+ private static final String BASEPATH_CONFIG_NAME =
+ "falcon.designer.hdfsstorage.defaultpath";
+
+ HDFSStorage(Configuration conf) throws StorageException {
+ try {
+ this.fs = FileSystem.get(conf);
+ } catch (IOException e) {
+ throw new StorageException(e);
+ }
+ this.basePath = conf.get(BASEPATH_CONFIG_NAME);
+ if (this.basePath == null || this.basePath.isEmpty()) {
+ throw new StorageException(BASEPATH_CONFIG_NAME
+ + " cannot be empty");
+ }
+
+ }
+
+ @Override
+ public InputStream open(String namespace, String entity)
+ throws StorageException {
+ try {
+ return fs.open(new Path(basePath + SEPERATOR + namespace
+ + SEPERATOR + entity));
+
+ } catch (IllegalArgumentException e) {
+ throw new StorageException(e);
+ } catch (IOException e) {
+ throw new StorageException(e);
+ }
+
+ }
+
+ @Override
+ public OutputStream create(String namespace, String entity)
+ throws StorageException {
+ try {
+ return fs.create(new Path(basePath + SEPERATOR + namespace
+ + SEPERATOR + entity));
+ } catch (IllegalArgumentException e) {
+ throw new StorageException(e);
+ } catch (IOException e) {
+ throw new StorageException(e);
+ }
+
+ }
+
+ @Override
+ public void delete(String namespace, String entity) throws StorageException {
+ try {
+ fs.delete(new Path(basePath + SEPERATOR + namespace + SEPERATOR
+ + entity), true);
+ } catch (IllegalArgumentException e) {
+ throw new StorageException(e);
+ } catch (IOException e) {
+ throw new StorageException(e);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/core/src/test/java/org/apache/falcon/designer/storage/impl/HDFSStorageTest.java
----------------------------------------------------------------------
diff --git a/addons/designer/core/src/test/java/org/apache/falcon/designer/storage/impl/HDFSStorageTest.java b/addons/designer/core/src/test/java/org/apache/falcon/designer/storage/impl/HDFSStorageTest.java
new file mode 100644
index 0000000..187a7d2
--- /dev/null
+++ b/addons/designer/core/src/test/java/org/apache/falcon/designer/storage/impl/HDFSStorageTest.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.falcon.designer.storage.impl;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.falcon.designer.storage.StorageException;
+import org.apache.hadoop.conf.Configuration;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+/**
+ * Will test HDFSStorage for create open remove.
+ */
+public class HDFSStorageTest {
+ private HDFSStorage hdfsStorageInst;
+
+ @BeforeClass
+ public void setUpDFS() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set("falcon.designer.hdfsstorage.defaultpath", "/tmp/");
+ conf.set("fs.default.name", "file:///");
+ hdfsStorageInst = new HDFSStorage(conf);
+
+ }
+
+ @Test
+ public void testCreateOpenDelete() {
+ try {
+ final String testNameSpace = "testNS";
+ final String testEntity = "testEntity";
+ OutputStream opStream =
+ hdfsStorageInst.create(testNameSpace, testEntity);
+ String testMessage = "testing HDFSStorage";
+ byte[] outputByte = new byte[testMessage.length()];
+ opStream.write(testMessage.getBytes());
+ opStream.close();
+ InputStream ipStream =
+ hdfsStorageInst.open(testNameSpace, testEntity);
+ ipStream.read(outputByte, 0, testMessage.length());
+ ipStream.close();
+ hdfsStorageInst.delete(testNameSpace, testEntity);
+ try {
+ hdfsStorageInst.open(testNameSpace, testEntity);
+ Assert
+ .fail("file should be present and should have thrown an exception");
+ } catch (StorageException ex) {
+ Assert.assertEquals(ex.getCause().getClass(),
+ FileNotFoundException.class);
+ }
+ Assert.assertEquals(new String(outputByte), testMessage);
+ } catch (StorageException ex) {
+ Assert.fail(ex.getMessage());
+ } catch (IOException ex) {
+ Assert.fail(ex.getMessage());
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/flows/pom.xml
----------------------------------------------------------------------
diff --git a/addons/designer/flows/pom.xml b/addons/designer/flows/pom.xml
index ce706a3..e47154f 100644
--- a/addons/designer/flows/pom.xml
+++ b/addons/designer/flows/pom.xml
@@ -37,6 +37,10 @@
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.falcon.designer</groupId>
+ <artifactId>designer-core</artifactId>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/flows/src/main/java/org/apache/falcon/designer/primitive/Flow.java
----------------------------------------------------------------------
diff --git a/addons/designer/flows/src/main/java/org/apache/falcon/designer/primitive/Flow.java b/addons/designer/flows/src/main/java/org/apache/falcon/designer/primitive/Flow.java
new file mode 100644
index 0000000..f7d227c
--- /dev/null
+++ b/addons/designer/flows/src/main/java/org/apache/falcon/designer/primitive/Flow.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.falcon.designer.primitive;
+
+import org.apache.falcon.designer.configuration.FlowConfig;
+
+/**
+ * Concrete implementation for a Flow.
+ */
+public class Flow extends Primitive<Flow, FlowConfig> {
+
+ private FlowConfig process;
+ private String nameSpace;
+ private String entity;
+
+ public Flow(FlowConfig process, String nameSpace, String entity) {
+ this.process = process;
+ this.nameSpace = nameSpace;
+ this.entity = entity;
+ }
+
+ @Override
+ protected Flow copy() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Iterable<Message> validate() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ protected Code doCompile() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ protected Flow doOptimize() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public String getNamespace() {
+ return nameSpace;
+ }
+
+ @Override
+ public String getEntity() {
+ return entity;
+ }
+
+
+ @Override
+ public void setConfiguration(FlowConfig config) {
+ this.process = config;
+ }
+
+ @Override
+ public FlowConfig getConfiguration() {
+ return process;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/aee6d3bf/addons/designer/pom.xml
----------------------------------------------------------------------
diff --git a/addons/designer/pom.xml b/addons/designer/pom.xml
index 3e1a98a..2330682 100644
--- a/addons/designer/pom.xml
+++ b/addons/designer/pom.xml
@@ -102,6 +102,7 @@
<jetty.version>6.1.26</jetty.version>
<internal.maven.repo>file:///tmp/falcontemprepo</internal.maven.repo>
<skipCheck>false</skipCheck>
+ <hadoop.version>2.2.0</hadoop.version>
</properties>
<modules>
@@ -368,11 +369,40 @@
<version>${hcatalog.version}</version>
</dependency>
- <dependency>
- <groupId>net.sourceforge.findbugs</groupId>
- <artifactId>jsr305</artifactId>
- <version>1.3.2</version>
- </dependency>
+ <dependency>
+ <groupId>net.sourceforge.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ <version>1.3.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.falcon.designer</groupId>
+ <artifactId>designer-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
</dependencies>
</dependencyManagement>
@@ -575,6 +605,8 @@
<exclude>**/.settings</exclude>
<exclude>**/maven-eclipse.xml</exclude>
<exclude>**/.externalToolBuilders</exclude>
+ <exclude>build/**</exclude>
+ <exclude>test-output/**</exclude>
</excludes>
</configuration>
<executions>
@@ -584,6 +616,9 @@
<goal>check</goal>
</goals>
<phase>verify</phase>
+ <configuration>
+ <failOnViolation>false</failOnViolation>
+ </configuration>
</execution>
</executions>
</plugin>
@@ -609,7 +644,7 @@
<consoleOutput>true</consoleOutput>
<includeTestSourceDirectory>true</includeTestSourceDirectory>
<configLocation>falcon/checkstyle.xml</configLocation>
- <failOnViolation>true</failOnViolation>
+ <failOnViolation>true</failOnViolation>
<skip>${skipCheck}</skip>
</configuration>
</execution>