You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by tb...@apache.org on 2014/05/02 18:24:54 UTC
[7/7] git commit: AMBARI-5616 - Ambari Views: Pig view (Roman Rader
via tbeerbower)
AMBARI-5616 - Ambari Views: Pig view (Roman Rader via tbeerbower)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5eb22214
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5eb22214
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5eb22214
Branch: refs/heads/trunk
Commit: 5eb22214397b55a153594faa75705517f55d945c
Parents: 75505c6
Author: tbeerbower <tb...@hortonworks.com>
Authored: Fri May 2 12:22:07 2014 -0400
Committer: tbeerbower <tb...@hortonworks.com>
Committed: Fri May 2 12:24:30 2014 -0400
----------------------------------------------------------------------
contrib/views/pig/pom.xml | 198 +++++++++
.../ambari/view/pig/PigServiceRouter.java | 48 +++
.../persistence/InstanceKeyValueStorage.java | 107 +++++
.../view/pig/persistence/KeyValueStorage.java | 154 +++++++
.../pig/persistence/LocalKeyValueStorage.java | 61 +++
.../persistence/PersistentConfiguration.java | 42 ++
.../ambari/view/pig/persistence/Storage.java | 39 ++
.../utils/ContextConfigurationAdapter.java | 250 +++++++++++
.../persistence/utils/FilteringStrategy.java | 23 ++
.../view/pig/persistence/utils/Indexed.java | 24 ++
.../pig/persistence/utils/ItemNotFound.java | 22 +
.../utils/OnlyOwnersFilteringStrategy.java | 33 ++
.../view/pig/persistence/utils/Owned.java | 24 ++
.../pig/persistence/utils/PersonalResource.java | 22 +
.../view/pig/persistence/utils/StorageUtil.java | 53 +++
.../view/pig/resources/CRUDResourceManager.java | 91 ++++
.../resources/PersonalCRUDResourceManager.java | 81 ++++
.../resources/SharedCRUDResourceManager.java | 45 ++
.../view/pig/resources/files/FileResource.java | 27 ++
.../view/pig/resources/files/FileService.java | 141 +++++++
.../pig/resources/jobs/JobResourceManager.java | 280 +++++++++++++
.../pig/resources/jobs/JobResourceProvider.java | 100 +++++
.../view/pig/resources/jobs/JobService.java | 243 +++++++++++
.../view/pig/resources/jobs/models/PigJob.java | 253 ++++++++++++
.../pig/resources/jobs/utils/JobPolling.java | 143 +++++++
.../scripts/ScriptResourceManager.java | 101 +++++
.../scripts/ScriptResourceProvider.java | 103 +++++
.../pig/resources/scripts/ScriptService.java | 154 +++++++
.../pig/resources/scripts/models/PigScript.java | 131 ++++++
.../pig/resources/udf/UDFResourceManager.java | 34 ++
.../pig/resources/udf/UDFResourceProvider.java | 105 +++++
.../view/pig/resources/udf/UDFService.java | 156 +++++++
.../view/pig/resources/udf/models/UDF.java | 79 ++++
.../ambari/view/pig/services/BaseService.java | 117 ++++++
.../ambari/view/pig/services/HelpService.java | 56 +++
.../view/pig/templeton/client/Request.java | 213 ++++++++++
.../view/pig/templeton/client/TempletonApi.java | 164 ++++++++
.../pig/templeton/client/TempletonRequest.java | 80 ++++
.../ambari/view/pig/utils/FilePaginator.java | 83 ++++
.../apache/ambari/view/pig/utils/HdfsApi.java | 189 +++++++++
.../src/main/resources/ui/pig-web/.gitignore | 34 ++
.../pig/src/main/resources/ui/pig-web/README.md | 15 +
.../src/main/resources/ui/pig-web/app/app.js | 23 ++
.../fonts/glyphicons-halflings-regular.svg | 229 +++++++++++
.../resources/ui/pig-web/app/assets/index.html | 29 ++
.../fonts/glyphicons-halflings-regular.svg | 229 +++++++++++
.../ui/pig-web/app/controllers/edit.js | 165 ++++++++
.../ui/pig-web/app/controllers/jobResults.js | 58 +++
.../pig-web/app/controllers/modal/pigModal.js | 22 +
.../resources/ui/pig-web/app/controllers/pig.js | 26 ++
.../ui/pig-web/app/controllers/pigHistory.js | 28 ++
.../ui/pig-web/app/controllers/pigJob.js | 31 ++
.../ui/pig-web/app/controllers/pigScriptEdit.js | 61 +++
.../app/controllers/pigScriptEditResults.js | 22 +
.../ui/pig-web/app/controllers/pigScriptList.js | 23 ++
.../ui/pig-web/app/controllers/pigUdfs.js | 23 ++
.../ui/pig-web/app/controllers/poll.js | 56 +++
.../app/controllers/util/pigUtilAlert.js | 28 ++
.../main/resources/ui/pig-web/app/initialize.js | 157 +++++++
.../resources/ui/pig-web/app/models/file.js | 26 ++
.../resources/ui/pig-web/app/models/pig_job.js | 90 ++++
.../ui/pig-web/app/models/pig_script.js | 43 ++
.../main/resources/ui/pig-web/app/models/udf.js | 25 ++
.../src/main/resources/ui/pig-web/app/router.js | 37 ++
.../ui/pig-web/app/routes/jobResults.js | 29 ++
.../main/resources/ui/pig-web/app/routes/pig.js | 69 ++++
.../ui/pig-web/app/routes/pigHistory.js | 28 ++
.../resources/ui/pig-web/app/routes/pigIndex.js | 25 ++
.../resources/ui/pig-web/app/routes/pigJob.js | 54 +++
.../ui/pig-web/app/routes/pigScriptEdit.js | 71 ++++
.../ui/pig-web/app/routes/pigScriptEditIndex.js | 27 ++
.../pig-web/app/routes/pigScriptEditResults.js | 36 ++
.../ui/pig-web/app/routes/pigScriptList.js | 88 ++++
.../resources/ui/pig-web/app/routes/pigUdfs.js | 55 +++
.../resources/ui/pig-web/app/styles/style.less | 295 +++++++++++++
.../ui/pig-web/app/templates/application.hbs | 21 +
.../ui/pig-web/app/templates/index.hbs | 18 +
.../ui/pig-web/app/templates/loading.hbs | 19 +
.../resources/ui/pig-web/app/templates/pig.hbs | 31 ++
.../ui/pig-web/app/templates/pig/history.hbs | 46 +++
.../ui/pig-web/app/templates/pig/index.hbs | 19 +
.../ui/pig-web/app/templates/pig/job.hbs | 39 ++
.../ui/pig-web/app/templates/pig/jobEdit.hbs | 18 +
.../ui/pig-web/app/templates/pig/jobResults.hbs | 19 +
.../app/templates/pig/jobResultsOutput.hbs | 40 ++
.../ui/pig-web/app/templates/pig/jobStatus.hbs | 18 +
.../ui/pig-web/app/templates/pig/loading.hbs | 20 +
.../app/templates/pig/modal/confirmdelete.hbs | 29 ++
.../app/templates/pig/modal/createScript.hbs | 37 ++
.../app/templates/pig/modal/createUdf.hbs | 36 ++
.../app/templates/pig/modal/modalLayout.hbs | 25 ++
.../ui/pig-web/app/templates/pig/scriptEdit.hbs | 120 ++++++
.../app/templates/pig/scriptEditIndex.hbs | 18 +
.../ui/pig-web/app/templates/pig/scriptList.hbs | 57 +++
.../pig-web/app/templates/pig/scriptResults.hbs | 18 +
.../app/templates/pig/scriptResultsNav.hbs | 19 +
.../ui/pig-web/app/templates/pig/udfs.hbs | 45 ++
.../app/templates/pig/util/alert-content.hbs | 22 +
.../ui/pig-web/app/templates/pig/util/alert.hbs | 21 +
.../app/templates/pig/util/pigHelper.hbs | 34 ++
.../app/templates/pig/util/script-nav.hbs | 30 ++
.../resources/ui/pig-web/app/translations.js | 110 +++++
.../main/resources/ui/pig-web/app/views/pig.js | 59 +++
.../ui/pig-web/app/views/pig/jobResults.js | 52 +++
.../app/views/pig/modal/confirmDelete.js | 29 ++
.../pig-web/app/views/pig/modal/createScript.js | 34 ++
.../ui/pig-web/app/views/pig/modal/createUdf.js | 38 ++
.../ui/pig-web/app/views/pig/modal/pigModal.js | 35 ++
.../ui/pig-web/app/views/pig/pigHistory.js | 36 ++
.../ui/pig-web/app/views/pig/pigJob.js | 41 ++
.../ui/pig-web/app/views/pig/pigUdfs.js | 23 ++
.../ui/pig-web/app/views/pig/scriptEdit.js | 257 ++++++++++++
.../ui/pig-web/app/views/pig/scriptList.js | 22 +
.../ui/pig-web/app/views/pig/scriptResults.js | 23 ++
.../pig-web/app/views/pig/scriptResultsNav.js | 49 +++
.../pig-web/app/views/pig/util/pigUtilAlert.js | 55 +++
.../views/pig/src/main/resources/ui/pig-web/bin | 1 +
.../src/main/resources/ui/pig-web/bower.json | 22 +
.../src/main/resources/ui/pig-web/config.coffee | 51 +++
.../generators/collection/collection.js.hbs | 23 ++
.../generators/collection/generator.json | 9 +
.../generators/controller/controller.js.hbs | 23 ++
.../generators/controller/generator.json | 9 +
.../ui/pig-web/generators/model/generator.json | 9 +
.../ui/pig-web/generators/model/model.js.hbs | 23 ++
.../ui/pig-web/generators/route/generator.json | 9 +
.../ui/pig-web/generators/route/route.js.hbs | 25 ++
.../pig-web/generators/template/generator.json | 9 +
.../generators/template/template.hbs.hbs | 18 +
.../ui/pig-web/generators/view/generator.json | 9 +
.../ui/pig-web/generators/view/view.js.hbs | 23 ++
.../src/main/resources/ui/pig-web/package.json | 22 +
.../main/resources/ui/pig-web/test/spec.coffee | 17 +
.../main/resources/ui/pig-web/vendor/emacs.js | 50 +++
contrib/views/pig/src/main/resources/view.xml | 140 +++++++
.../org/apache/ambari/view/pig/BasePigTest.java | 96 +++++
.../org/apache/ambari/view/pig/HDFSTest.java | 60 +++
.../apache/ambari/view/pig/test/FileTest.java | 203 +++++++++
.../apache/ambari/view/pig/test/HelpTest.java | 59 +++
.../apache/ambari/view/pig/test/JobTest.java | 410 +++++++++++++++++++
.../apache/ambari/view/pig/test/ScriptTest.java | 175 ++++++++
.../view/pig/test/ScriptTestHDFSUnmanaged.java | 109 +++++
.../view/pig/test/ScriptTestUnmanaged.java | 92 +++++
.../apache/ambari/view/pig/test/UDFTest.java | 116 ++++++
contrib/views/pom.xml | 133 ++++++
145 files changed, 10153 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/5eb22214/contrib/views/pig/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/views/pig/pom.xml b/contrib/views/pig/pom.xml
new file mode 100644
index 0000000..3dd71a7
--- /dev/null
+++ b/contrib/views/pig/pom.xml
@@ -0,0 +1,198 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.apache.ambari.view</groupId>
+ <artifactId>pig</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ <name>Pig</name>
+
+ <parent>
+ <groupId>org.apache.ambari.views</groupId>
+ <artifactId>ambari-views-poc</artifactId>
+ <version>0.1.0-SNAPSHOT</version>
+ </parent>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.google.inject</groupId>
+ <artifactId>guice</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey.contribs</groupId>
+ <artifactId>jersey-multipart</artifactId>
+ <version>1.18</version>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-client</artifactId>
+ <version>1.8</version>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ <version>1.18.1</version>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+ <version>1.9</version>
+ </dependency>
+ <dependency>
+ <groupId>com.googlecode.json-simple</groupId>
+ <artifactId>json-simple</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-configuration</groupId>
+ <artifactId>commons-configuration</artifactId>
+ <version>1.6</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ambari</groupId>
+ <artifactId>ambari-views</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>2.2.2</version>
+ </dependency>
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ <version>3.0.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.7.5</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop-version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop-version}</version>
+ </dependency>
+ <dependency>
+ <groupId>javax.ws.rs</groupId>
+ <artifactId>javax.ws.rs-api</artifactId>
+ <version>2.0</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.easymock</groupId>
+ <artifactId>easymock</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <version>${hadoop-version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <properties>
+ <hadoop-version>2.2.0</hadoop-version>
+ <ambari.version>1.3.0-SNAPSHOT</ambari.version>
+ </properties>
+ <build>
+ <plugins>
+
+ <!-- Building frontend -->
+ <plugin>
+ <groupId>com.github.eirslett</groupId>
+ <artifactId>frontend-maven-plugin</artifactId>
+ <version>0.0.14</version>
+ <configuration>
+ <nodeVersion>v0.10.26</nodeVersion>
+ <npmVersion>1.4.3</npmVersion>
+ <workingDirectory>src/main/resources/ui/pig-web/</workingDirectory>
+ </configuration>
+ <executions>
+ <execution>
+ <id>install node and npm</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>install-node-and-npm</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>npm install</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>npm</goal>
+ </goals>
+ <configuration>
+ <arguments>install --unsafe-perm</arguments>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>exec-maven-plugin</artifactId>
+ <groupId>org.codehaus.mojo</groupId>
+ <executions>
+ <execution>
+ <id>Brunch build</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <workingDirectory>${basedir}/src/main/resources/ui/pig-web</workingDirectory>
+ <executable>node/node</executable>
+ <arguments>
+ <argument>node_modules/.bin/brunch</argument>
+ <argument>build</argument>
+ <argument>--production</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ <filtering>false</filtering>
+ <includes>
+ <include>META-INF/**/*</include>
+ <include>view.xml</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>src/main/resources/ui/pig-web/public</directory>
+ <filtering>false</filtering>
+ </resource>
+ </resources>
+ </build>
+
+
+</project>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5eb22214/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/PigServiceRouter.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/PigServiceRouter.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/PigServiceRouter.java
new file mode 100644
index 0000000..e1098e7
--- /dev/null
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/PigServiceRouter.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.pig;
+
+
+import com.google.inject.Inject;
+import org.apache.ambari.view.ViewContext;
+import org.apache.ambari.view.ViewResourceHandler;
+import org.apache.ambari.view.pig.persistence.Storage;
+import org.apache.ambari.view.pig.services.HelpService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.Path;
+
+public class PigServiceRouter {
+ @Inject
+ ViewContext context;
+
+ @Inject
+ protected ViewResourceHandler handler;
+
+ protected final static Logger LOG =
+ LoggerFactory.getLogger(PigServiceRouter.class);
+
+ private Storage storage = null;
+
+ @Path("/help")
+ public HelpService help(){
+ return new HelpService(context, handler);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5eb22214/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/InstanceKeyValueStorage.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/InstanceKeyValueStorage.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/InstanceKeyValueStorage.java
new file mode 100644
index 0000000..101dcb9
--- /dev/null
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/InstanceKeyValueStorage.java
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.pig.persistence;
+
+import org.apache.ambari.view.ViewContext;
+import org.apache.ambari.view.pig.persistence.utils.*;
+import org.apache.commons.configuration.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Persistent storage engine for storing java beans to
+ * properties file
+ * Path to file should be in 'dataworker.storagePath' parameter
+ */
+public class InstanceKeyValueStorage extends KeyValueStorage {
+ private final static Logger LOG =
+ LoggerFactory.getLogger(InstanceKeyValueStorage.class);
+
+ private ContextConfigurationAdapter config = null;
+ private int VALUE_LENGTH_LIMIT = 254;
+
+ public InstanceKeyValueStorage(ViewContext context) {
+ super(context);
+ }
+
+ @Override
+ protected synchronized Configuration getConfig() {
+ if (config == null) {
+ config = new ContextConfigurationAdapter(context);
+ }
+ return config;
+ }
+
+ /**
+ * Value is limited to 256 symbols, this code splits value into chunks and saves them as <key>#<chunk_id>
+ * @param modelPropName key
+ * @param json value
+ */
+ protected void write(String modelPropName, String json) {
+ int saved = 0;
+ int page = 1;
+ while (saved < json.length()) {
+ int end = Math.min(saved + VALUE_LENGTH_LIMIT, json.length());
+ String substring = json.substring(saved, end);
+ getConfig().setProperty(modelPropName + "#" + page, substring);
+ saved += VALUE_LENGTH_LIMIT;
+ page += 1;
+ LOG.debug("Chunk saved: " + modelPropName + "#" + page + "=" + substring);
+ }
+ getConfig().setProperty(modelPropName, page - 1);
+ LOG.debug("Write finished: " + modelPropName + " pages:" + (page - 1));
+ }
+
+ /**
+ * Read chunked value (keys format <key>#<chunk_id>)
+ * @param modelPropName key
+ * @return value
+ */
+ protected String read(String modelPropName) {
+ StringBuilder result = new StringBuilder();
+ int pages = getConfig().getInt(modelPropName);
+ LOG.debug("Read started: " + modelPropName + " pages:" + pages);
+
+ for(int page = 1; page <= pages; page++) {
+ String substring = getConfig().getString(modelPropName + "#" + page);
+ LOG.debug("Chunk read: " + modelPropName + "#" + page + "=" + substring);
+ if (substring != null) {
+ result.append(substring);
+ }
+ }
+
+ return result.toString();
+ }
+
+ /**
+ * Remove chunked value (keys format <key>#<chunk_id>)
+ * @param modelPropName key
+ */
+ protected void clear(String modelPropName) {
+ int pages = getConfig().getInt(modelPropName);
+ LOG.debug("Clean started: " + modelPropName + " pages:" + pages);
+
+ for(int page = 1; page <= pages; page++) {
+ getConfig().clearProperty(modelPropName + "#" + page);
+ LOG.debug("Chunk clean: " + modelPropName + "#" + page);
+ }
+ getConfig().clearProperty(modelPropName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5eb22214/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/KeyValueStorage.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/KeyValueStorage.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/KeyValueStorage.java
new file mode 100644
index 0000000..db18680
--- /dev/null
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/KeyValueStorage.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.pig.persistence;
+
+import com.google.gson.Gson;
+import org.apache.ambari.view.ViewContext;
+import org.apache.ambari.view.pig.persistence.utils.FilteringStrategy;
+import org.apache.ambari.view.pig.persistence.utils.Indexed;
+import org.apache.ambari.view.pig.persistence.utils.ItemNotFound;
+import org.apache.ambari.view.pig.persistence.utils.OnlyOwnersFilteringStrategy;
+import org.apache.commons.configuration.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Engine storing objects to key-value storage
+ */
+public abstract class KeyValueStorage implements Storage {
+ private final static Logger LOG =
+ LoggerFactory.getLogger(KeyValueStorage.class);
+ protected final Gson gson = new Gson();
+ protected ViewContext context;
+
+ public KeyValueStorage(ViewContext context) {
+ this.context = context;
+ }
+
+ protected abstract Configuration getConfig();
+
+ @Override
+ public synchronized void store(Indexed obj) {
+ String modelIndexingPropName = getIndexPropertyName(obj.getClass());
+
+ if (obj.getId() == null) {
+ int lastIndex = getConfig().getInt(modelIndexingPropName, 0);
+ lastIndex ++;
+ getConfig().setProperty(modelIndexingPropName, lastIndex);
+ obj.setId(Integer.toString(lastIndex));
+ }
+
+ String modelPropName = getItemPropertyName(obj.getClass(), Integer.parseInt(obj.getId()));
+ String json = serialize(obj);
+ write(modelPropName, json);
+ }
+
+ @Override
+ public <T extends Indexed> T load(Class<T> model, int id) throws ItemNotFound {
+ String modelPropName = getItemPropertyName(model, id);
+ LOG.debug(String.format("Loading %s", modelPropName));
+ if (getConfig().containsKey(modelPropName)) {
+ String json = read(modelPropName);
+ LOG.debug(String.format("json: %s", json));
+ return deserialize(model, json);
+ } else {
+ throw new ItemNotFound();
+ }
+ }
+
+ /**
+ * Write json to storage
+ * @param modelPropName key
+ * @param json value
+ */
+ protected void write(String modelPropName, String json) {
+ getConfig().setProperty(modelPropName, json);
+ }
+
+ /**
+ * Read json from storage
+ * @param modelPropName key
+ * @return value
+ */
+ protected String read(String modelPropName) {
+ return getConfig().getString(modelPropName);
+ }
+
+ /**
+ * Remove line from storage
+ * @param modelPropName key
+ */
+ protected void clear(String modelPropName) {
+ getConfig().clearProperty(modelPropName);
+ }
+
+ protected String serialize(Indexed obj) {
+ return gson.toJson(obj);
+ }
+
+ protected <T extends Indexed> T deserialize(Class<T> model, String json) {
+ return gson.fromJson(json, model);
+ }
+
+ @Override
+ public synchronized <T extends Indexed> List<T> loadAll(Class<T> model, FilteringStrategy filter) {
+ ArrayList<T> list = new ArrayList<T>();
+ String modelIndexingPropName = getIndexPropertyName(model);
+ LOG.debug(String.format("Loading all %s-s", model.getName()));
+ int lastIndex = getConfig().getInt(modelIndexingPropName, 0);
+ for(int i=1; i<=lastIndex; i++) {
+ try {
+ T item = load(model, i);
+ if ((filter == null) || filter.is_conform(item)) {
+ list.add(item);
+ }
+ } catch (ItemNotFound ignored) {
+ }
+ }
+ return list;
+ }
+
+ @Override
+ public synchronized <T extends Indexed> List<T> loadAll(Class<T> model) {
+ return loadAll(model, new OnlyOwnersFilteringStrategy(this.context.getUsername()));
+ }
+
+ @Override
+ public synchronized void delete(Class model, int id) {
+ LOG.debug(String.format("Deleting %s:%d", model.getName(), id));
+ String modelPropName = getItemPropertyName(model, id);
+ clear(modelPropName);
+ }
+
+ @Override
+ public boolean exists(Class model, int id) {
+ return getConfig().containsKey(getItemPropertyName(model, id));
+ }
+
+ private String getIndexPropertyName(Class model) {
+ return String.format("%s:index", model.getName());
+ }
+
+ private String getItemPropertyName(Class model, int id) {
+ return String.format("%s.%d", model.getName(), id);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5eb22214/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/LocalKeyValueStorage.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/LocalKeyValueStorage.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/LocalKeyValueStorage.java
new file mode 100644
index 0000000..54dcb7f
--- /dev/null
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/LocalKeyValueStorage.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.pig.persistence;
+
+import org.apache.ambari.view.ViewContext;
+import org.apache.commons.configuration.ConfigurationException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.xml.ws.WebServiceException;
+
+/**
+ * Persistent storage engine for storing java beans to
+ * properties file
+ * Path to file should be in 'dataworker.storagePath' parameter
+ */
+public class LocalKeyValueStorage extends KeyValueStorage {
+ private final static Logger LOG =
+ LoggerFactory.getLogger(LocalKeyValueStorage.class);
+
+ private PersistentConfiguration config = null;
+
+ public LocalKeyValueStorage(ViewContext context) {
+ super(context);
+ }
+
+ @Override
+ protected synchronized PersistentConfiguration getConfig() {
+ if (config == null) {
+ String fileName = context.getProperties().get("dataworker.storagePath");
+ if (fileName == null) {
+ String msg = "dataworker.storagePath is not configured!";
+ LOG.error(msg);
+ throw new WebServiceException(msg);
+ }
+ try {
+ config = new PersistentConfiguration(fileName);
+ } catch (ConfigurationException e) {
+ e.printStackTrace();
+ }
+ }
+ return config;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5eb22214/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/PersistentConfiguration.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/PersistentConfiguration.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/PersistentConfiguration.java
new file mode 100644
index 0000000..7e191f2
--- /dev/null
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/PersistentConfiguration.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.pig.persistence;
+
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.configuration.reloading.FileChangedReloadingStrategy;
+
+import java.io.File;
+
+public class PersistentConfiguration extends PropertiesConfiguration {
+ public PersistentConfiguration(String fileName) throws ConfigurationException {
+ super();
+
+ File config = new File(fileName);
+ setFile(config);
+ this.setAutoSave(true);
+ this.setReloadingStrategy(new FileChangedReloadingStrategy());
+ this.setDelimiterParsingDisabled(true);
+ this.setListDelimiter((char) 0);
+
+ if (config.exists()) {
+ this.load();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5eb22214/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/Storage.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/Storage.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/Storage.java
new file mode 100644
index 0000000..1507918
--- /dev/null
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/Storage.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.pig.persistence;
+
+import org.apache.ambari.view.pig.persistence.utils.FilteringStrategy;
+import org.apache.ambari.view.pig.persistence.utils.Indexed;
+import org.apache.ambari.view.pig.persistence.utils.ItemNotFound;
+
+import java.util.List;
+
+public interface Storage {
+ void store(Indexed obj);
+
+ <T extends Indexed> T load(Class<T> model, int id) throws ItemNotFound;
+
+ <T extends Indexed> List<T> loadAll(Class<T> model, FilteringStrategy filter);
+
+ <T extends Indexed> List<T> loadAll(Class<T> model);
+
+ void delete(Class model, int id);
+
+ boolean exists(Class model, int id);
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5eb22214/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/ContextConfigurationAdapter.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/ContextConfigurationAdapter.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/ContextConfigurationAdapter.java
new file mode 100644
index 0000000..829905d
--- /dev/null
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/ContextConfigurationAdapter.java
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.pig.persistence.utils;
+
+import org.apache.ambari.view.ViewContext;
+import org.apache.commons.configuration.Configuration;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Properties;
+
+public class ContextConfigurationAdapter implements Configuration {
+ private ViewContext context;
+
+ public ContextConfigurationAdapter(ViewContext context) {
+ this.context = context;
+ }
+
+ @Override
+ public Configuration subset(String prefix) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return context.getInstanceData().isEmpty();
+ }
+
+ @Override
+ public boolean containsKey(String s) {
+ return context.getInstanceData().containsKey(s);
+ }
+
+ @Override
+ public void addProperty(String s, Object o) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void setProperty(String s, Object o) {
+ context.putInstanceData(s, o.toString());
+ }
+
+ @Override
+ public void clearProperty(String key) {
+ context.removeInstanceData(key);
+ }
+
+ @Override
+ public void clear() {
+ for (String key : context.getInstanceData().keySet())
+ context.removeInstanceData(key);
+ }
+
+ @Override
+ public Object getProperty(String key) {
+ return context.getInstanceData(key);
+ }
+
+ @Override
+ public Iterator getKeys(String s) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Iterator getKeys() {
+ return context.getInstanceData().keySet().iterator();
+ }
+
+ @Override
+ public Properties getProperties(String s) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean getBoolean(String s) {
+ return getBoolean(s, null);
+ }
+
+ @Override
+ public boolean getBoolean(String s, boolean b) {
+ return getBoolean(s, (Boolean)b);
+ }
+
+ @Override
+ public Boolean getBoolean(String s, Boolean aBoolean) {
+ String data = context.getInstanceData(s);
+ return (data != null)?Boolean.parseBoolean(data):aBoolean;
+ }
+
+ @Override
+ public byte getByte(String s) {
+ return getByte(s, null);
+ }
+
+ @Override
+ public byte getByte(String s, byte b) {
+ return getByte(s, (Byte)b);
+ }
+
+ @Override
+ public Byte getByte(String s, Byte aByte) {
+ String data = context.getInstanceData(s);
+ return (data != null)?Byte.parseByte(data):aByte;
+ }
+
+ @Override
+ public double getDouble(String s) {
+ return getDouble(s, null);
+ }
+
+ @Override
+ public double getDouble(String s, double v) {
+ return getDouble(s, (Double)v);
+ }
+
+ @Override
+ public Double getDouble(String s, Double aDouble) {
+ String data = context.getInstanceData(s);
+ return (data != null)?Double.parseDouble(data):aDouble;
+ }
+
+ @Override
+ public float getFloat(String s) {
+ return getFloat(s, null);
+ }
+
+ @Override
+ public float getFloat(String s, float v) {
+ return getFloat(s, (Float)v);
+ }
+
+ @Override
+ public Float getFloat(String s, Float aFloat) {
+ String data = context.getInstanceData(s);
+ return (data != null)?Float.parseFloat(data):aFloat;
+ }
+
+ @Override
+ public int getInt(String s) {
+ return getInteger(s, null);
+ }
+
+ @Override
+ public int getInt(String s, int i) {
+ return getInteger(s, i);
+ }
+
+ @Override
+ public Integer getInteger(String s, Integer integer) {
+ String data = context.getInstanceData(s);
+ return (data != null)?Integer.parseInt(data):integer;
+ }
+
+ @Override
+ public long getLong(String s) {
+ return getLong(s, null);
+ }
+
+ @Override
+ public long getLong(String s, long l) {
+ return getLong(s, (Long)l);
+ }
+
+ @Override
+ public Long getLong(String s, Long aLong) {
+ String data = context.getInstanceData(s);
+ return (data != null)?Long.parseLong(data):aLong;
+ }
+
+ @Override
+ public short getShort(String s) {
+ return getShort(s, null);
+ }
+
+ @Override
+ public short getShort(String s, short i) {
+ return getShort(s, (Short)i);
+ }
+
+ @Override
+ public Short getShort(String s, Short aShort) {
+ String data = context.getInstanceData(s);
+ return (data != null)?Short.parseShort(data):aShort;
+ }
+
+ @Override
+ public BigDecimal getBigDecimal(String s) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public BigDecimal getBigDecimal(String s, BigDecimal bigDecimal) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public BigInteger getBigInteger(String s) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public BigInteger getBigInteger(String s, BigInteger bigInteger) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public String getString(String s) {
+ return context.getInstanceData(s);
+ }
+
+ @Override
+ public String getString(String s, String s2) {
+ String data = getString(s);
+ return (data != null)?data:s2;
+ }
+
+ @Override
+ public String[] getStringArray(String s) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List getList(String s) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List getList(String s, List list) {
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5eb22214/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/FilteringStrategy.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/FilteringStrategy.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/FilteringStrategy.java
new file mode 100644
index 0000000..75a0953
--- /dev/null
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/FilteringStrategy.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.pig.persistence.utils;
+
+public interface FilteringStrategy {
+ boolean is_conform(Indexed item);
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5eb22214/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/Indexed.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/Indexed.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/Indexed.java
new file mode 100644
index 0000000..cbe2016
--- /dev/null
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/Indexed.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.pig.persistence.utils;
+
+public interface Indexed {
+ String getId();
+ void setId(String id);
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5eb22214/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/ItemNotFound.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/ItemNotFound.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/ItemNotFound.java
new file mode 100644
index 0000000..df56036
--- /dev/null
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/ItemNotFound.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.pig.persistence.utils;
+
+public class ItemNotFound extends Exception {
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5eb22214/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/OnlyOwnersFilteringStrategy.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/OnlyOwnersFilteringStrategy.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/OnlyOwnersFilteringStrategy.java
new file mode 100644
index 0000000..7964cf7
--- /dev/null
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/OnlyOwnersFilteringStrategy.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.pig.persistence.utils;
+
+public class OnlyOwnersFilteringStrategy implements FilteringStrategy {
+ private final String username;
+
+ public OnlyOwnersFilteringStrategy(String username) {
+ this.username = username;
+ }
+
+ @Override
+ public boolean is_conform(Indexed item) {
+ Owned object = (Owned) item;
+ return object.getOwner().compareTo(username) == 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5eb22214/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/Owned.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/Owned.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/Owned.java
new file mode 100644
index 0000000..30918a2
--- /dev/null
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/Owned.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.pig.persistence.utils;
+
+public interface Owned {
+ String getOwner();
+ void setOwner(String owner);
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5eb22214/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/PersonalResource.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/PersonalResource.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/PersonalResource.java
new file mode 100644
index 0000000..3096071
--- /dev/null
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/PersonalResource.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.pig.persistence.utils;
+
+public interface PersonalResource extends Indexed, Owned {
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5eb22214/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/StorageUtil.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/StorageUtil.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/StorageUtil.java
new file mode 100644
index 0000000..0c7b25b
--- /dev/null
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/StorageUtil.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.pig.persistence.utils;
+
+import org.apache.ambari.view.ViewContext;
+import org.apache.ambari.view.pig.persistence.InstanceKeyValueStorage;
+import org.apache.ambari.view.pig.persistence.LocalKeyValueStorage;
+import org.apache.ambari.view.pig.persistence.Storage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class StorageUtil {
+ private static Storage storageInstance = null;
+
+ protected final static Logger LOG =
+ LoggerFactory.getLogger(StorageUtil.class);
+
+ public synchronized static Storage getStorage(ViewContext context) {
+ if (storageInstance == null) {
+ String fileName = context.getProperties().get("dataworker.storagePath");
+ if (fileName != null) {
+ LOG.debug("Using local storage in " + fileName + " to store data");
+ // If specifed, use LocalKeyValueStorage - key-value file based storage
+ storageInstance = new LocalKeyValueStorage(context);
+ } else {
+ LOG.debug("Using Persistence API to store data");
+ // If not specifed, use ambari-views Persistence API
+ storageInstance = new InstanceKeyValueStorage(context);
+ }
+ }
+ return storageInstance;
+ }
+
+ public static void setStorage(Storage storage) {
+ storageInstance = storage;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5eb22214/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/CRUDResourceManager.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/CRUDResourceManager.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/CRUDResourceManager.java
new file mode 100644
index 0000000..9319d3b
--- /dev/null
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/CRUDResourceManager.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.pig.resources;
+
+import org.apache.ambari.view.ViewContext;
+import org.apache.ambari.view.pig.persistence.Storage;
+import org.apache.ambari.view.pig.persistence.utils.FilteringStrategy;
+import org.apache.ambari.view.pig.persistence.utils.Indexed;
+import org.apache.ambari.view.pig.persistence.utils.ItemNotFound;
+import org.apache.ambari.view.pig.persistence.utils.StorageUtil;
+
+import java.util.List;
+
+/**
+ * CRUD resource manager
+ * @param <T> Data type with ID
+ */
+abstract public class CRUDResourceManager<T extends Indexed> {
+ private Storage storage = null;
+
+ protected final Class<T> resourceClass;
+
+ public CRUDResourceManager(Class<T> responseClass) {
+ this.resourceClass = responseClass;
+ }
+ // CRUD operations
+
+ public T create(T object) {
+ object.setId(null);
+ return this.save(object);
+ }
+
+ public T read(String id) throws ItemNotFound {
+ T object = null;
+ object = getPigStorage().load(this.resourceClass, Integer.parseInt(id));
+ if (!checkPermissions(object))
+ throw new ItemNotFound();
+ return object;
+ }
+
+ public List<T> readAll(FilteringStrategy filteringStrategy) {
+ return getPigStorage().loadAll(this.resourceClass, filteringStrategy);
+ }
+
+ public T update(T newObject, String id) throws ItemNotFound {
+ newObject.setId(id);
+ this.save(newObject);
+ return newObject;
+ }
+
+ public void delete(String resourceId) throws ItemNotFound {
+ int id = Integer.parseInt(resourceId);
+ if (!getPigStorage().exists(this.resourceClass, id)) {
+ throw new ItemNotFound();
+ }
+ getPigStorage().delete(this.resourceClass, id);
+ }
+
+ // UTILS
+
+ protected T save(T object) {
+ getPigStorage().store(object);
+ return object;
+ }
+
+ protected Storage getPigStorage() {
+ if (storage == null) {
+ storage = StorageUtil.getStorage(getContext());
+ }
+ return storage;
+ }
+
+ protected abstract boolean checkPermissions(T object);
+ protected abstract ViewContext getContext();
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5eb22214/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/PersonalCRUDResourceManager.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/PersonalCRUDResourceManager.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/PersonalCRUDResourceManager.java
new file mode 100644
index 0000000..ffccfd9
--- /dev/null
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/PersonalCRUDResourceManager.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.pig.resources;
+
+import org.apache.ambari.view.ViewContext;
+import org.apache.ambari.view.pig.persistence.utils.ItemNotFound;
+import org.apache.ambari.view.pig.persistence.utils.PersonalResource;
+
+import java.util.concurrent.Callable;
+
+/**
+ * Resource manager that returns only user owned elements from DB
+ * @param <T> Data type with ID and Owner
+ */
+public class PersonalCRUDResourceManager<T extends PersonalResource> extends CRUDResourceManager<T> {
+ protected ViewContext context;
+ protected boolean ignorePermissions = false;
+
+ public PersonalCRUDResourceManager(Class<T> responseClass, ViewContext context) {
+ super(responseClass);
+ this.context = context;
+ }
+
+ public T update(T newObject, String id) throws ItemNotFound {
+ T object = getPigStorage().load(this.resourceClass, Integer.parseInt(id));
+ if (object.getOwner().compareTo(this.context.getUsername()) != 0) {
+ throw new ItemNotFound();
+ }
+
+ newObject.setOwner(this.context.getUsername());
+ return super.update(newObject, id);
+ }
+
+ public T save(T object) {
+ object.setOwner(this.context.getUsername());
+ return super.save(object);
+ }
+
+ @Override
+ protected boolean checkPermissions(T object) {
+ if (ignorePermissions)
+ return true;
+ return object.getOwner().compareTo(this.context.getUsername()) == 0;
+ }
+
+ @Override
+ protected ViewContext getContext() {
+ return context;
+ }
+
+ public <T> T ignorePermissions(String fakeUser, Callable<T> actions) throws Exception {
+ ignorePermissions = true;
+ T result;
+ try {
+ result = actions.call();
+ } finally {
+ ignorePermissions = false;
+ }
+ return result;
+ }
+
+ public <T> T ignorePermissions(Callable<T> actions) throws Exception {
+ return ignorePermissions("", actions);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5eb22214/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/SharedCRUDResourceManager.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/SharedCRUDResourceManager.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/SharedCRUDResourceManager.java
new file mode 100644
index 0000000..be89d36
--- /dev/null
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/SharedCRUDResourceManager.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.pig.resources;
+
+import org.apache.ambari.view.ViewContext;
+import org.apache.ambari.view.pig.persistence.utils.Indexed;
+
+/**
+ * Resource manager that doesn't restrict access (Allow all)
+ * @param <T> Data type with ID
+ */
+public class SharedCRUDResourceManager<T extends Indexed> extends CRUDResourceManager<T> {
+ protected ViewContext context;
+
+ public SharedCRUDResourceManager(Class<T> responseClass, ViewContext context) {
+ super(responseClass);
+ this.context = context;
+ }
+
+ @Override
+ protected boolean checkPermissions(T object) {
+ return true; //everyone has permission
+ }
+
+ @Override
+ protected ViewContext getContext() {
+ return context;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5eb22214/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/files/FileResource.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/files/FileResource.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/files/FileResource.java
new file mode 100644
index 0000000..9ae45c7
--- /dev/null
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/files/FileResource.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.pig.resources.files;
+
+public class FileResource {
+ public String filePath;
+ public String fileContent;
+ public boolean hasNext;
+ public long page;
+ public long pageCount;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/5eb22214/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/files/FileService.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/files/FileService.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/files/FileService.java
new file mode 100644
index 0000000..c36c582
--- /dev/null
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/files/FileService.java
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.pig.resources.files;
+
+import com.google.inject.Inject;
+import org.apache.ambari.view.ViewResourceHandler;
+import org.apache.ambari.view.pig.services.BaseService;
+import org.apache.ambari.view.pig.utils.FilePaginator;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.*;
+import javax.ws.rs.core.*;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+/**
+ * File access resource
+ * API:
+ * GET /:path
+ * read entire file
+ * POST /
+ * create new file
+ * Required: filePath
+ * file should not already exists
+ * PUT /:path
+ * update file content
+ */
+public class FileService extends BaseService {
+ @Inject
+ ViewResourceHandler handler;
+
+ protected final static Logger LOG =
+ LoggerFactory.getLogger(FileService.class);
+
+ /**
+ * Get single item
+ */
+ @GET
+ @Path("{filePath:.*}")
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getFile(@PathParam("filePath") String filePath, @QueryParam("page") Long page) throws IOException, InterruptedException {
+ LOG.debug("Reading file " + filePath);
+ try {
+ FilePaginator paginator = new FilePaginator(filePath, context);
+
+ if (page == null)
+ page = 0L;
+
+ FileResource file = new FileResource();
+ file.filePath = filePath;
+ file.fileContent = paginator.readPage(page);
+ file.hasNext = paginator.pageCount() > page + 1;
+ file.page = page;
+ file.pageCount = paginator.pageCount();
+
+ JSONObject object = new JSONObject();
+ object.put("file", file);
+ return Response.ok(object).status(200).build();
+ } catch (FileNotFoundException e) {
+ return notFoundResponse(e.toString());
+ } catch (IllegalArgumentException e) {
+ return badRequestResponse(e.toString());
+ }
+ }
+
+ /**
+ * Delete single item
+ */
+ @DELETE
+ @Path("{filePath:.*}")
+ public Response deleteFile(@PathParam("filePath") String filePath) throws IOException, InterruptedException {
+ LOG.debug("Deleting file " + filePath);
+ if (getHdfsApi().delete(filePath, false)) {
+ return Response.status(204).build();
+ }
+ return notFoundResponse("FileSystem.delete returned false");
+ }
+
+ /**
+ * Update item
+ */
+ @PUT
+ @Path("{filePath:.*}")
+ @Consumes(MediaType.APPLICATION_JSON)
+ public Response updateFile(FileResourceRequest request,
+ @PathParam("filePath") String filePath) throws IOException, InterruptedException {
+ LOG.debug("Rewriting file " + filePath);
+ FSDataOutputStream output = getHdfsApi().create(filePath, true);
+ output.writeBytes(request.file.fileContent);
+ output.close();
+ return Response.status(204).build();
+ }
+
+ /**
+ * Create script
+ */
+ @POST
+ @Consumes(MediaType.APPLICATION_JSON)
+ public Response createFile(FileResourceRequest request,
+ @Context HttpServletResponse response, @Context UriInfo ui)
+ throws IOException, InterruptedException {
+ LOG.debug("Creating file " + request.file.filePath);
+ try {
+ FSDataOutputStream output = getHdfsApi().create(request.file.filePath, false);
+ if (request.file.fileContent != null) {
+ output.writeBytes(request.file.fileContent);
+ }
+ output.close();
+ } catch (FileAlreadyExistsException e) {
+ return badRequestResponse(e.toString());
+ }
+ response.setHeader("Location",
+ String.format("%s/%s", ui.getAbsolutePath().toString(), request.file.filePath));
+ return Response.status(204).build();
+ }
+
+ public static class FileResourceRequest {
+ public FileResource file;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5eb22214/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobResourceManager.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobResourceManager.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobResourceManager.java
new file mode 100644
index 0000000..67a038b
--- /dev/null
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobResourceManager.java
@@ -0,0 +1,280 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.pig.resources.jobs;
+
+import org.apache.ambari.view.ViewContext;
+import org.apache.ambari.view.pig.persistence.utils.FilteringStrategy;
+import org.apache.ambari.view.pig.persistence.utils.Indexed;
+import org.apache.ambari.view.pig.resources.PersonalCRUDResourceManager;
+import org.apache.ambari.view.pig.resources.jobs.models.PigJob;
+import org.apache.ambari.view.pig.resources.jobs.utils.JobPolling;
+import org.apache.ambari.view.pig.services.BaseService;
+import org.apache.ambari.view.pig.templeton.client.TempletonApi;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.xml.ws.WebServiceException;
+import java.io.File;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Object that provides operations for templeton jobs
+ * CRUD overridden to support
+ */
+public class JobResourceManager extends PersonalCRUDResourceManager<PigJob> {
+ protected TempletonApi api;
+
+ private final static Logger LOG =
+ LoggerFactory.getLogger(JobResourceManager.class);
+
+ public JobResourceManager(ViewContext context) {
+ super(PigJob.class, context);
+ setupPolling();
+ }
+
+ public TempletonApi getTempletonApi() {
+ if (api == null) {
+ api = new TempletonApi(context.getProperties().get("dataworker.templeton_url"),
+ getTempletonUser(), getTempletonUser(), context);
+ }
+ return api;
+ }
+
+ public void setTempletonApi(TempletonApi api) {
+ this.api = api;
+ }
+
+ private void setupPolling() {
+ List<PigJob> notCompleted = this.readAll(new FilteringStrategy() {
+ @Override
+ public boolean is_conform(Indexed item) {
+ PigJob job = (PigJob) item;
+ return job.isInProgress();
+ }
+ });
+
+ for(PigJob job : notCompleted) {
+ JobPolling.pollJob(context, job);
+ }
+ }
+
+ @Override
+ public PigJob create(PigJob object) {
+ object.setStatus(PigJob.Status.SUBMITTING);
+ PigJob job = super.create(object);
+ LOG.debug("Submitting job...");
+
+ try {
+ submitJob(object);
+ } catch (RuntimeException e) {
+ object.setStatus(PigJob.Status.SUBMIT_FAILED);
+ save(object);
+ LOG.debug("Job submit FAILED");
+ throw e;
+ }
+ LOG.debug("Job submit OK");
+ object.setStatus(PigJob.Status.SUBMITTED);
+ save(object);
+ return job;
+ }
+
+ public void killJob(PigJob object) throws IOException {
+ LOG.debug("Killing job...");
+
+ try {
+ getTempletonApi().killJob(object.getJobId());
+ } catch (IOException e) {
+ LOG.debug("Job kill FAILED");
+ throw e;
+ }
+ LOG.debug("Job kill OK");
+ }
+
+ /**
+ * Running job
+ * @param job job bean
+ */
+ private void submitJob(PigJob job) {
+ String date = new SimpleDateFormat("dd-MM-yyyy-HH-mm-ss").format(new Date());
+ String statusdir = String.format(context.getProperties().get("dataworker.pigJobsPath") +
+ "/%s/%s_%s", getTempletonUser(),
+ job.getTitle().toLowerCase().replaceAll("[^a-zA-Z0-9 ]+", "").replace(" ", "_"),
+ date);
+
+ String newPigScriptPath = statusdir + "/script.pig";
+ String newSourceFilePath = statusdir + "/source.pig";
+ String newPythonScriptPath = statusdir + "/udf.py";
+ String templetonParamsFilePath = statusdir + "/params";
+ try {
+ // additional file can be passed to copy into work directory
+ if (job.getSourceFileContent() != null && !job.getSourceFileContent().isEmpty()) {
+ String sourceFileContent = job.getSourceFileContent();
+ job.setSourceFileContent(null); // we should not store content in DB
+ save(job);
+
+ FSDataOutputStream stream = BaseService.getHdfsApi(context).create(newSourceFilePath, true);
+ stream.writeBytes(sourceFileContent);
+ stream.close();
+ } else {
+ if (job.getSourceFile() != null && !job.getSourceFile().isEmpty()) {
+ // otherwise, just copy original file
+ if (!BaseService.getHdfsApi(context).copy(job.getSourceFile(), newSourceFilePath)) {
+ throw new WebServiceException("Can't copy source file from " + job.getSourceFile() +
+ " to " + newPigScriptPath);
+ }
+ }
+ }
+ } catch (IOException e) {
+ throw new WebServiceException("Can't create/copy source file: " + e.toString(), e);
+ } catch (InterruptedException e) {
+ throw new WebServiceException("Can't create/copy source file: " + e.toString(), e);
+ }
+
+ try {
+ // content can be passed from front-end with substituted arguments
+ if (job.getForcedContent() != null && !job.getForcedContent().isEmpty()) {
+ String forcedContent = job.getForcedContent();
+ // variable for sourceFile can be passed from front-ent
+ forcedContent = forcedContent.replace("${sourceFile}",
+ context.getProperties().get("dataworker.defaultFs") + newSourceFilePath);
+ job.setForcedContent(null); // we should not store content in DB
+ save(job);
+
+ FSDataOutputStream stream = BaseService.getHdfsApi(context).create(newPigScriptPath, true);
+ stream.writeBytes(forcedContent);
+ stream.close();
+ } else {
+ // otherwise, just copy original file
+ if (!BaseService.getHdfsApi(context).copy(job.getPigScript(), newPigScriptPath)) {
+ throw new WebServiceException("Can't copy pig script file from " + job.getPigScript() +
+ " to " + newPigScriptPath);
+ }
+ }
+ } catch (IOException e) {
+ throw new WebServiceException("Can't create/copy pig script file: " + e.toString(), e);
+ } catch (InterruptedException e) {
+ throw new WebServiceException("Can't create/copy pig script file: " + e.toString(), e);
+ }
+
+ if (job.getPythonScript() != null && !job.getPythonScript().isEmpty()) {
+ try {
+ if (!BaseService.getHdfsApi(context).copy(job.getPythonScript(), newPythonScriptPath)) {
+ throw new WebServiceException("Can't copy python udf script file from " + job.getPythonScript() +
+ " to " + newPythonScriptPath);
+ }
+ } catch (IOException e) {
+ throw new WebServiceException("Can't create/copy python udf file: " + e.toString(), e);
+ } catch (InterruptedException e) {
+ throw new WebServiceException("Can't create/copy python udf file: " + e.toString(), e);
+ }
+ }
+
+ try {
+ FSDataOutputStream stream = BaseService.getHdfsApi(context).create(templetonParamsFilePath, true);
+ if (job.getTempletonArguments() != null) {
+ stream.writeBytes(job.getTempletonArguments());
+ }
+ stream.close();
+ } catch (IOException e) {
+ throw new WebServiceException("Can't create params file: " + e.toString(), e);
+ } catch (InterruptedException e) {
+ throw new WebServiceException("Can't create params file: " + e.toString(), e);
+ }
+ job.setPigScript(newPigScriptPath);
+
+ job.setStatusDir(statusdir);
+ job.setDateStarted(System.currentTimeMillis() / 1000L);
+
+ TempletonApi.JobData data = null;
+ try {
+ data = getTempletonApi().runPigQuery(new File(job.getPigScript()), statusdir, job.getTempletonArguments());
+ } catch (IOException templetonBadResponse) {
+ String msg = String.format("Templeton bad response: %s", templetonBadResponse.toString());
+ LOG.debug(msg);
+ throw new WebServiceException(msg, templetonBadResponse);
+ }
+ job.setJobId(data.id);
+
+ JobPolling.pollJob(context, job);
+ }
+
+ public void retrieveJobStatus(PigJob job) {
+ TempletonApi.JobInfo info = null;
+ try {
+ info = getTempletonApi().checkJob(job.getJobId());
+ } catch (IOException e) {
+ LOG.warn(String.format("IO Exception: %s", e));
+ return;
+ }
+
+ if (info.status != null && (info.status.containsKey("runState"))) {
+ //TODO: retrieve from RM
+ int runState = ((Double) info.status.get("runState")).intValue();
+ switch (runState) {
+ case PigJob.RUN_STATE_KILLED:
+ LOG.debug(String.format("Job KILLED: %s", job.getJobId()));
+ job.setStatus(PigJob.Status.KILLED);
+ break;
+ case PigJob.RUN_STATE_FAILED:
+ LOG.debug(String.format("Job FAILED: %s", job.getJobId()));
+ job.setStatus(PigJob.Status.FAILED);
+ break;
+ case PigJob.RUN_STATE_PREP:
+ case PigJob.RUN_STATE_RUNNING:
+ job.setStatus(PigJob.Status.RUNNING);
+ break;
+ case PigJob.RUN_STATE_SUCCEEDED:
+ LOG.debug(String.format("Job COMPLETED: %s", job.getJobId()));
+ job.setStatus(PigJob.Status.COMPLETED);
+ break;
+ default:
+ LOG.debug(String.format("Job in unknown state: %s", job.getJobId()));
+ job.setStatus(PigJob.Status.UNKNOWN);
+ break;
+ }
+ }
+ Pattern pattern = Pattern.compile("\\d+");
+ Matcher matcher = null;
+ if (info.percentComplete != null) {
+ matcher = pattern.matcher(info.percentComplete);
+ }
+ if (matcher != null && matcher.find()) {
+ job.setPercentComplete(Integer.valueOf(matcher.group()));
+ } else {
+ job.setPercentComplete(null);
+ }
+ save(job);
+ }
+
+ /**
+ * Extension point to use different usernames in templeton
+ * requests instead of logged in user
+ * @return username in templeton
+ */
+ private String getTempletonUser() {
+ return context.getProperties().get("dataworker.templeton_user");
+// return context.getTempletonUser();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5eb22214/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobResourceProvider.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobResourceProvider.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobResourceProvider.java
new file mode 100644
index 0000000..9981952
--- /dev/null
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobResourceProvider.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.pig.resources.jobs;
+
+import com.google.inject.Inject;
+import org.apache.ambari.view.*;
+import org.apache.ambari.view.pig.persistence.utils.ItemNotFound;
+import org.apache.ambari.view.pig.persistence.utils.OnlyOwnersFilteringStrategy;
+import org.apache.ambari.view.pig.resources.jobs.models.PigJob;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+public class JobResourceProvider implements ResourceProvider<PigJob> {
+ @Inject
+ ViewContext context;
+
+ protected JobResourceManager resourceManager = null;
+
+ protected synchronized JobResourceManager getResourceManager() {
+ if (resourceManager == null) {
+ resourceManager = new JobResourceManager(context);
+ }
+ return resourceManager;
+ }
+
+ @Override
+ public PigJob getResource(String resourceId, Set<String> strings) throws SystemException, NoSuchResourceException, UnsupportedPropertyException {
+ try {
+ return getResourceManager().read(resourceId);
+ } catch (ItemNotFound itemNotFound) {
+ throw new NoSuchResourceException(resourceId);
+ }
+ }
+
+ @Override
+ public Set<PigJob> getResources(ReadRequest readRequest) throws SystemException, NoSuchResourceException, UnsupportedPropertyException {
+ return new HashSet<PigJob>(getResourceManager().readAll(
+ new OnlyOwnersFilteringStrategy(this.context.getUsername())));
+ }
+
+ @Override
+ public void createResource(String s, Map<String, Object> stringObjectMap) throws SystemException, ResourceAlreadyExistsException, NoSuchResourceException, UnsupportedPropertyException {
+ PigJob job = null;
+ try {
+ job = new PigJob(stringObjectMap);
+ } catch (InvocationTargetException e) {
+ throw new SystemException("error on creating resource", e);
+ } catch (IllegalAccessException e) {
+ throw new SystemException("error on creating resource", e);
+ }
+ getResourceManager().create(job);
+ }
+
+ @Override
+ public boolean updateResource(String resourceId, Map<String, Object> stringObjectMap) throws SystemException, NoSuchResourceException, UnsupportedPropertyException {
+ PigJob job = null;
+ try {
+ job = new PigJob(stringObjectMap);
+ } catch (InvocationTargetException e) {
+ throw new SystemException("error on updating resource", e);
+ } catch (IllegalAccessException e) {
+ throw new SystemException("error on updating resource", e);
+ }
+ try {
+ getResourceManager().update(job, resourceId);
+ } catch (ItemNotFound itemNotFound) {
+ throw new NoSuchResourceException(resourceId);
+ }
+ return true;
+ }
+
+ @Override
+ public boolean deleteResource(String resourceId) throws SystemException, NoSuchResourceException, UnsupportedPropertyException {
+ try {
+ getResourceManager().delete(resourceId);
+ } catch (ItemNotFound itemNotFound) {
+ throw new NoSuchResourceException(resourceId);
+ }
+ return true;
+ }
+}