You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by al...@apache.org on 2014/11/25 18:14:34 UTC
[10/10] ambari git commit: AMBARI-8402. View: Pig ui updates + flow
changes (alexantonenko)
AMBARI-8402. View: Pig ui updates + flow changes (alexantonenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/82e72e51
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/82e72e51
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/82e72e51
Branch: refs/heads/trunk
Commit: 82e72e517f3b86db25d2026b53b77c5c72fa8eab
Parents: bf19364
Author: Alex Antonenko <hi...@gmail.com>
Authored: Mon Nov 24 20:20:32 2014 +0200
Committer: Alex Antonenko <hi...@gmail.com>
Committed: Tue Nov 25 19:13:19 2014 +0200
----------------------------------------------------------------------
contrib/views/pig/pom.xml | 1 -
.../view/pig/persistence/DataStoreStorage.java | 174 +++
.../persistence/InstanceKeyValueStorage.java | 4 +-
.../pig/persistence/LocalKeyValueStorage.java | 1 +
.../persistence/PersistentConfiguration.java | 1 +
.../ambari/view/pig/persistence/Storage.java | 2 +-
.../utils/ContextConfigurationAdapter.java | 5 +-
.../view/pig/persistence/utils/StorageUtil.java | 35 +-
.../view/pig/resources/CRUDResourceManager.java | 2 +-
.../resources/PersonalCRUDResourceManager.java | 16 +-
.../view/pig/resources/files/FileService.java | 2 +-
.../pig/resources/jobs/JobResourceManager.java | 113 +-
.../view/pig/resources/jobs/JobService.java | 26 +-
.../view/pig/resources/jobs/models/PigJob.java | 48 +-
.../pig/resources/jobs/utils/JobPolling.java | 99 +-
.../scripts/ScriptResourceManager.java | 12 +-
.../pig/resources/scripts/ScriptService.java | 2 +-
.../ambari/view/pig/services/BaseService.java | 57 +-
.../ambari/view/pig/services/HelpService.java | 7 +-
.../view/pig/templeton/client/Request.java | 24 +-
.../view/pig/templeton/client/TempletonApi.java | 2 +-
.../ambari/view/pig/utils/FilePaginator.java | 2 +-
.../apache/ambari/view/pig/utils/HdfsApi.java | 60 +
.../ui/pig-web/app/assets/data/pigHelpers.json | 128 ++
.../app/assets/static/fonts/FontAwesome.otf | Bin 0 -> 85908 bytes
.../assets/static/fonts/fontawesome-webfont.eot | Bin 0 -> 56006 bytes
.../assets/static/fonts/fontawesome-webfont.svg | 520 ++++++++
.../assets/static/fonts/fontawesome-webfont.ttf | Bin 0 -> 112160 bytes
.../static/fonts/fontawesome-webfont.woff | Bin 0 -> 65452 bytes
.../ui/pig-web/app/components/codeMirror.js | 83 ++
.../ui/pig-web/app/components/helpers-data.js | 146 +++
.../ui/pig-web/app/components/jobProgress.js | 32 +
.../ui/pig-web/app/components/pigHelper.js | 54 +
.../ui/pig-web/app/components/scriptListRow.js | 39 +
.../ui/pig-web/app/components/tabControl.js | 44 +
.../ui/pig-web/app/controllers/edit.js | 195 ---
.../ui/pig-web/app/controllers/errorLog.js | 2 +-
.../ui/pig-web/app/controllers/jobResults.js | 58 -
.../app/controllers/modal/confirmAway.js | 47 +
.../app/controllers/modal/confirmDelete.js | 28 +
.../app/controllers/modal/createScript.js | 52 +
.../pig-web/app/controllers/modal/createUdf.js | 34 +
.../pig-web/app/controllers/modal/deleteJob.js | 28 +
.../pig-web/app/controllers/modal/gotoCopy.js | 28 +
.../app/controllers/modal/logDownload.js | 42 +
.../pig-web/app/controllers/modal/pigModal.js | 41 +-
.../app/controllers/modal/resultsDownload.js | 42 +
.../ui/pig-web/app/controllers/page.js | 27 +
.../resources/ui/pig-web/app/controllers/pig.js | 83 +-
.../ui/pig-web/app/controllers/pigAlert.js | 34 +
.../ui/pig-web/app/controllers/pigHistory.js | 26 +-
.../ui/pig-web/app/controllers/pigJob.js | 31 -
.../ui/pig-web/app/controllers/pigScriptEdit.js | 68 -
.../app/controllers/pigScriptEditResults.js | 22 -
.../ui/pig-web/app/controllers/pigScriptList.js | 23 -
.../ui/pig-web/app/controllers/pigScripts.js | 84 ++
.../ui/pig-web/app/controllers/pigUdfs.js | 48 +-
.../ui/pig-web/app/controllers/poll.js | 59 -
.../ui/pig-web/app/controllers/script.js | 117 ++
.../ui/pig-web/app/controllers/scriptEdit.js | 208 +++
.../ui/pig-web/app/controllers/scriptHistory.js | 35 +
.../ui/pig-web/app/controllers/scriptJob.js | 65 +
.../app/controllers/util/pigUtilAlert.js | 34 -
.../main/resources/ui/pig-web/app/initialize.js | 217 ++--
.../ui/pig-web/app/mixins/fileHandler.js | 54 +
.../ui/pig-web/app/mixins/pagination.js | 99 ++
.../resources/ui/pig-web/app/models/pig_job.js | 31 +-
.../ui/pig-web/app/models/pig_script.js | 29 +-
.../src/main/resources/ui/pig-web/app/router.js | 13 +-
.../ui/pig-web/app/routes/jobResults.js | 29 -
.../main/resources/ui/pig-web/app/routes/pig.js | 55 +-
.../ui/pig-web/app/routes/pigHistory.js | 14 +
.../resources/ui/pig-web/app/routes/pigIndex.js | 25 -
.../resources/ui/pig-web/app/routes/pigJob.js | 57 -
.../ui/pig-web/app/routes/pigScriptEdit.js | 74 --
.../ui/pig-web/app/routes/pigScriptEditIndex.js | 27 -
.../pig-web/app/routes/pigScriptEditResults.js | 36 -
.../ui/pig-web/app/routes/pigScriptList.js | 93 --
.../ui/pig-web/app/routes/pigScripts.js | 28 +
.../resources/ui/pig-web/app/routes/pigUdfs.js | 33 -
.../resources/ui/pig-web/app/routes/script.js | 59 +
.../ui/pig-web/app/routes/scriptEdit.js | 47 +
.../ui/pig-web/app/routes/scriptHistory.js | 36 +
.../ui/pig-web/app/routes/scriptJob.js | 58 +
.../resources/ui/pig-web/app/routes/splash.js | 18 +-
.../resources/ui/pig-web/app/styles/style.less | 465 ++++++-
.../app/templates/components/pigHelper.hbs | 34 +
.../app/templates/components/scriptListRow.hbs | 44 +
.../ui/pig-web/app/templates/error.hbs | 35 +
.../ui/pig-web/app/templates/index.hbs | 18 -
.../pig-web/app/templates/modal/confirmAway.hbs | 32 +
.../app/templates/modal/confirmDelete.hbs | 31 +
.../app/templates/modal/createScript.hbs | 42 +
.../pig-web/app/templates/modal/createUdf.hbs | 38 +
.../pig-web/app/templates/modal/deleteJob.hbs | 31 +
.../ui/pig-web/app/templates/modal/gotoCopy.hbs | 31 +
.../pig-web/app/templates/modal/logDownload.hbs | 42 +
.../pig-web/app/templates/modal/modalLayout.hbs | 25 +
.../app/templates/modal/resultsDownload.hbs | 42 +
.../app/templates/partials/alert-content.hbs | 25 +
.../templates/partials/paginationControls.hbs | 42 +
.../resources/ui/pig-web/app/templates/pig.hbs | 32 +-
.../ui/pig-web/app/templates/pig/alert.hbs | 21 +
.../ui/pig-web/app/templates/pig/history.hbs | 58 +-
.../ui/pig-web/app/templates/pig/index.hbs | 19 -
.../ui/pig-web/app/templates/pig/job.hbs | 39 -
.../ui/pig-web/app/templates/pig/jobEdit.hbs | 18 -
.../ui/pig-web/app/templates/pig/jobResults.hbs | 19 -
.../app/templates/pig/jobResultsOutput.hbs | 40 -
.../ui/pig-web/app/templates/pig/jobStatus.hbs | 18 -
.../app/templates/pig/modal/confirmdelete.hbs | 29 -
.../app/templates/pig/modal/createScript.hbs | 40 -
.../app/templates/pig/modal/createUdf.hbs | 36 -
.../app/templates/pig/modal/modalLayout.hbs | 25 -
.../ui/pig-web/app/templates/pig/scriptEdit.hbs | 167 ---
.../app/templates/pig/scriptEditIndex.hbs | 18 -
.../ui/pig-web/app/templates/pig/scriptList.hbs | 62 -
.../pig-web/app/templates/pig/scriptResults.hbs | 18 -
.../app/templates/pig/scriptResultsNav.hbs | 19 -
.../ui/pig-web/app/templates/pig/scripts.hbs | 86 ++
.../ui/pig-web/app/templates/pig/udfs.hbs | 24 +-
.../app/templates/pig/util/alert-content.hbs | 25 -
.../ui/pig-web/app/templates/pig/util/alert.hbs | 21 -
.../app/templates/pig/util/pigHelper.hbs | 34 -
.../app/templates/pig/util/script-nav.hbs | 30 -
.../ui/pig-web/app/templates/script.hbs | 27 +
.../ui/pig-web/app/templates/script/edit.hbs | 146 +++
.../ui/pig-web/app/templates/script/history.hbs | 61 +
.../ui/pig-web/app/templates/script/job.hbs | 153 +++
.../ui/pig-web/app/templates/splash.hbs | 66 +-
.../resources/ui/pig-web/app/translations.js | 67 +-
.../main/resources/ui/pig-web/app/views/pig.js | 49 +-
.../resources/ui/pig-web/app/views/pig/alert.js | 55 +
.../ui/pig-web/app/views/pig/history.js | 46 +
.../ui/pig-web/app/views/pig/jobResults.js | 52 -
.../ui/pig-web/app/views/pig/loading.js | 23 +
.../app/views/pig/modal/confirmDelete.js | 29 -
.../pig-web/app/views/pig/modal/createScript.js | 39 -
.../ui/pig-web/app/views/pig/modal/createUdf.js | 38 -
.../ui/pig-web/app/views/pig/modal/pigModal.js | 35 -
.../ui/pig-web/app/views/pig/pigHistory.js | 36 -
.../ui/pig-web/app/views/pig/pigJob.js | 41 -
.../ui/pig-web/app/views/pig/pigUdfs.js | 23 -
.../ui/pig-web/app/views/pig/scriptEdit.js | 256 ----
.../ui/pig-web/app/views/pig/scriptList.js | 22 -
.../ui/pig-web/app/views/pig/scriptResults.js | 23 -
.../pig-web/app/views/pig/scriptResultsNav.js | 49 -
.../ui/pig-web/app/views/pig/scripts.js | 22 +
.../resources/ui/pig-web/app/views/pig/udfs.js | 23 +
.../pig-web/app/views/pig/util/pigUtilAlert.js | 55 -
.../ui/pig-web/app/views/script/edit.js | 56 +
.../ui/pig-web/app/views/script/job.js | 35 +
.../src/main/resources/ui/pig-web/bower.json | 26 +-
.../src/main/resources/ui/pig-web/config.coffee | 27 +-
.../src/main/resources/ui/pig-web/package.json | 2 +-
.../ui/pig-web/vendor/jquery-ui/core.js | 304 +++++
.../ui/pig-web/vendor/jquery-ui/draggable.js | 1128 +++++++++++++++++
.../ui/pig-web/vendor/jquery-ui/droppable.js | 413 ++++++
.../ui/pig-web/vendor/jquery-ui/mouse.js | 199 +++
.../ui/pig-web/vendor/jquery-ui/position.js | 517 ++++++++
.../ui/pig-web/vendor/jquery-ui/resizable.js | 1179 ++++++++++++++++++
.../ui/pig-web/vendor/jquery-ui/selectable.js | 287 +++++
.../ui/pig-web/vendor/jquery-ui/widget.js | 557 +++++++++
.../ui/pig-web/vendor/moment-duration-format.js | 482 +++++++
contrib/views/pig/src/main/resources/view.xml | 67 +-
.../org/apache/ambari/view/pig/BasePigTest.java | 15 +-
.../org/apache/ambari/view/pig/HDFSTest.java | 3 +-
.../apache/ambari/view/pig/test/FileTest.java | 7 +-
.../apache/ambari/view/pig/test/HelpTest.java | 7 +-
.../ambari/view/pig/test/IntegrationalTest.java | 134 ++
.../apache/ambari/view/pig/test/JobTest.java | 178 ++-
.../apache/ambari/view/pig/test/ScriptTest.java | 273 ++--
.../view/pig/test/ScriptTestHDFSUnmanaged.java | 40 +-
.../view/pig/test/ScriptTestUnmanaged.java | 5 +-
174 files changed, 10659 insertions(+), 3027 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/82e72e51/contrib/views/pig/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/views/pig/pom.xml b/contrib/views/pig/pom.xml
index 809cdd3..7aeab3a 100644
--- a/contrib/views/pig/pom.xml
+++ b/contrib/views/pig/pom.xml
@@ -169,7 +169,6 @@
<arguments>
<argument>node_modules/.bin/brunch</argument>
<argument>build</argument>
- <argument>--production</argument>
</arguments>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/ambari/blob/82e72e51/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/DataStoreStorage.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/DataStoreStorage.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/DataStoreStorage.java
new file mode 100644
index 0000000..d89e7a6
--- /dev/null
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/DataStoreStorage.java
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.pig.persistence;
+
+import com.google.gson.Gson;
+import org.apache.ambari.view.PersistenceException;
+import org.apache.ambari.view.ViewContext;
+import org.apache.ambari.view.pig.persistence.utils.*;
+import org.apache.ambari.view.pig.utils.ServiceFormattedException;
+import org.apache.commons.configuration.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.WebApplicationException;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * Engine for storing objects to context DataStore storage
+ */
+public class DataStoreStorage implements Storage {
+ private final static Logger LOG =
+ LoggerFactory.getLogger(DataStoreStorage.class);
+ protected final Gson gson = new Gson();
+ protected ViewContext context;
+
+ /**
+ * Constructor
+ * @param context View Context instance
+ */
+ public DataStoreStorage(ViewContext context) {
+ this.context = context;
+ }
+
+ @Override
+ public synchronized void store(Indexed obj) {
+ try {
+ if (obj.getId() == null) {
+ int id = nextIdForEntity(context, obj.getClass());
+ obj.setId(String.valueOf(id));
+ }
+ context.getDataStore().store(obj);
+ } catch (PersistenceException e) {
+ throw new ServiceFormattedException("Error while saving object to DataStorage", e);
+ }
+ }
+
+ private static synchronized int nextIdForEntity(ViewContext context, Class aClass) {
+ // auto increment id implementation
+ String lastId = context.getInstanceData(aClass.getName());
+ int newId;
+ if (lastId == null) {
+ newId = 1;
+ } else {
+ newId = Integer.parseInt(lastId) + 1;
+ }
+ context.putInstanceData(aClass.getName(), String.valueOf(newId));
+ return newId;
+ }
+
+ @Override
+ public synchronized <T extends Indexed> T load(Class<T> model, int id) throws ItemNotFound {
+ LOG.debug(String.format("Loading %s #%d", model.getName(), id));
+ try {
+ T obj = context.getDataStore().find(model, String.valueOf(id));
+ if (obj != null) {
+ return obj;
+ } else {
+ throw new ItemNotFound();
+ }
+ } catch (PersistenceException e) {
+ throw new ServiceFormattedException("Error while finding object in DataStorage", e);
+ }
+ }
+
+ @Override
+ public synchronized <T extends Indexed> List<T> loadAll(Class<T> model, FilteringStrategy filter) {
+ LinkedList<T> list = new LinkedList<T>();
+ LOG.debug(String.format("Loading all %s-s", model.getName()));
+ try {
+ for(T item: context.getDataStore().findAll(model, null)) {
+ if ((filter == null) || filter.isConform(item)) {
+ list.add(item);
+ }
+ }
+ } catch (PersistenceException e) {
+ throw new ServiceFormattedException("Error while finding all objects in DataStorage", e);
+ }
+ return list;
+ }
+
+ @Override
+ public synchronized <T extends Indexed> List<T> loadAll(Class<T> model) {
+ return loadAll(model, new OnlyOwnersFilteringStrategy(this.context.getUsername()));
+ }
+
+ @Override
+ public synchronized void delete(Class model, int id) throws ItemNotFound {
+ LOG.debug(String.format("Deleting %s:%d", model.getName(), id));
+ Object obj = load(model, id);
+ try {
+ context.getDataStore().remove(obj);
+ } catch (PersistenceException e) {
+ throw new ServiceFormattedException("Error while removing object from DataStorage", e);
+ }
+ }
+
+ @Override
+ public boolean exists(Class model, int id) {
+ try {
+ return context.getDataStore().find(model, String.valueOf(id)) != null;
+ } catch (PersistenceException e) {
+ throw new ServiceFormattedException("Error while finding object in DataStorage", e);
+ }
+ }
+
+ public static void storageSmokeTest(ViewContext context) {
+ try {
+ SmokeTestEntity entity = new SmokeTestEntity();
+ entity.setData("42");
+ DataStoreStorage storage = new DataStoreStorage(context);
+ storage.store(entity);
+
+ if (entity.getId() == null) throw new ServiceFormattedException("Ambari Views instance data DB doesn't work properly (auto increment id doesn't work)", null);
+ int id = Integer.parseInt(entity.getId());
+ SmokeTestEntity entity2 = storage.load(SmokeTestEntity.class, id);
+ boolean status = entity2.getData().compareTo("42") == 0;
+ storage.delete(SmokeTestEntity.class, id);
+ if (!status) throw new ServiceFormattedException("Ambari Views instance data DB doesn't work properly", null);
+ } catch (WebApplicationException ex) {
+ throw ex;
+ } catch (Exception ex) {
+ throw new ServiceFormattedException(ex.getMessage(), ex);
+ }
+ }
+
+ public static class SmokeTestEntity implements Indexed {
+ private String id = null;
+ private String data = null;
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public String getData() {
+ return data;
+ }
+
+ public void setData(String data) {
+ this.data = data;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/82e72e51/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/InstanceKeyValueStorage.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/InstanceKeyValueStorage.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/InstanceKeyValueStorage.java
index da1ca2f..de00a90 100644
--- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/InstanceKeyValueStorage.java
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/InstanceKeyValueStorage.java
@@ -30,9 +30,9 @@ import javax.ws.rs.WebApplicationException;
/**
* Persistent storage engine for storing java beans to
- * properties file
- * Path to file should be in 'dataworker.storagePath' parameter
+ * instance data
*/
+@Deprecated
public class InstanceKeyValueStorage extends KeyValueStorage {
private final static Logger LOG =
LoggerFactory.getLogger(InstanceKeyValueStorage.class);
http://git-wip-us.apache.org/repos/asf/ambari/blob/82e72e51/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/LocalKeyValueStorage.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/LocalKeyValueStorage.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/LocalKeyValueStorage.java
index c3b87c2..44546c2 100644
--- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/LocalKeyValueStorage.java
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/LocalKeyValueStorage.java
@@ -30,6 +30,7 @@ import org.slf4j.LoggerFactory;
* properties file
* Path to file should be in 'dataworker.storagePath' parameter
*/
+@Deprecated
public class LocalKeyValueStorage extends KeyValueStorage {
private final static Logger LOG =
LoggerFactory.getLogger(LocalKeyValueStorage.class);
http://git-wip-us.apache.org/repos/asf/ambari/blob/82e72e51/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/PersistentConfiguration.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/PersistentConfiguration.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/PersistentConfiguration.java
index c3748c7..e5328cd 100644
--- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/PersistentConfiguration.java
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/PersistentConfiguration.java
@@ -28,6 +28,7 @@ import java.io.File;
* Configuration enables all necessary options for PropertiesConfiguration:
* auto-save, auto-reloading, no delimiter parsing and other
*/
+@Deprecated
public class PersistentConfiguration extends PropertiesConfiguration {
/**
* Constructor
http://git-wip-us.apache.org/repos/asf/ambari/blob/82e72e51/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/Storage.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/Storage.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/Storage.java
index da9cfc3..e2c7db4 100644
--- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/Storage.java
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/Storage.java
@@ -66,7 +66,7 @@ public interface Storage {
* @param model bean class
* @param id identifier
*/
- void delete(Class model, int id);
+ void delete(Class model, int id) throws ItemNotFound;
/**
* Check is object exists
http://git-wip-us.apache.org/repos/asf/ambari/blob/82e72e51/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/ContextConfigurationAdapter.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/ContextConfigurationAdapter.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/ContextConfigurationAdapter.java
index d49a921..a281c39 100644
--- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/ContextConfigurationAdapter.java
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/ContextConfigurationAdapter.java
@@ -25,11 +25,13 @@ import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
/**
* Persistence API to Apache Configuration adapter
*/
+@Deprecated
public class ContextConfigurationAdapter implements Configuration {
private ViewContext context;
@@ -53,7 +55,8 @@ public class ContextConfigurationAdapter implements Configuration {
@Override
public boolean containsKey(String s) {
- return context.getInstanceData().containsKey(s);
+ Map<String, String> data = context.getInstanceData();
+ return data.containsKey(s);
}
@Override
http://git-wip-us.apache.org/repos/asf/ambari/blob/82e72e51/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/StorageUtil.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/StorageUtil.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/StorageUtil.java
index 245ad54..5d283f1 100644
--- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/StorageUtil.java
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/persistence/utils/StorageUtil.java
@@ -19,12 +19,16 @@
package org.apache.ambari.view.pig.persistence.utils;
import org.apache.ambari.view.ViewContext;
+import org.apache.ambari.view.pig.persistence.DataStoreStorage;
import org.apache.ambari.view.pig.persistence.InstanceKeyValueStorage;
import org.apache.ambari.view.pig.persistence.LocalKeyValueStorage;
import org.apache.ambari.view.pig.persistence.Storage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.HashMap;
+import java.util.Map;
+
/**
* Storage factory, creates storage of Local or Persistence API type.
* Type depends on context configuration: if "dataworker.storagePath" is set,
@@ -33,17 +37,38 @@ import org.slf4j.LoggerFactory;
* Storage is singleton.
*/
public class StorageUtil {
- private static Storage storageInstance = null;
+ private Storage storageInstance = null;
protected final static Logger LOG =
LoggerFactory.getLogger(StorageUtil.class);
+
+ private static Map<String, StorageUtil> viewSingletonObjects = new HashMap<String, StorageUtil>();
+ public static StorageUtil getInstance(ViewContext context) {
+ if (!viewSingletonObjects.containsKey(context.getInstanceName()))
+ viewSingletonObjects.put(context.getInstanceName(), new StorageUtil(context));
+ return viewSingletonObjects.get(context.getInstanceName());
+ }
+
+ public static void dropAllConnections() {
+ viewSingletonObjects.clear();
+ }
+
+ private ViewContext context;
+
/**
- * Get storage instance. If one is not created, creates instance.
+ * Constructor of storage util
* @param context View Context instance
+ */
+ public StorageUtil(ViewContext context) {
+ this.context = context;
+ }
+
+ /**
+ * Get storage instance. If one is not created, creates instance.
* @return storage instance
*/
- public synchronized static Storage getStorage(ViewContext context) {
+ public synchronized Storage getStorage() {
if (storageInstance == null) {
String fileName = context.getProperties().get("dataworker.storagePath");
if (fileName != null) {
@@ -53,7 +78,7 @@ public class StorageUtil {
} else {
LOG.debug("Using Persistence API to store data");
// If not specifed, use ambari-views Persistence API
- storageInstance = new InstanceKeyValueStorage(context);
+ storageInstance = new DataStoreStorage(context);
}
}
return storageInstance;
@@ -64,7 +89,7 @@ public class StorageUtil {
* Used in unit tests.
* @param storage storage instance
*/
- public static void setStorage(Storage storage) {
+ public void setStorage(Storage storage) {
storageInstance = storage;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/82e72e51/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/CRUDResourceManager.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/CRUDResourceManager.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/CRUDResourceManager.java
index b4a2059..94c3b5a 100644
--- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/CRUDResourceManager.java
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/CRUDResourceManager.java
@@ -113,7 +113,7 @@ abstract public class CRUDResourceManager<T extends Indexed> {
protected Storage getPigStorage() {
if (storage == null) {
- storage = StorageUtil.getStorage(getContext());
+ storage = StorageUtil.getInstance(getContext()).getStorage();
}
return storage;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/82e72e51/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/PersonalCRUDResourceManager.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/PersonalCRUDResourceManager.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/PersonalCRUDResourceManager.java
index b54ca4b..1f6c2d9 100644
--- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/PersonalCRUDResourceManager.java
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/PersonalCRUDResourceManager.java
@@ -55,7 +55,11 @@ public class PersonalCRUDResourceManager<T extends PersonalResource> extends CRU
@Override
public T save(T object) {
- object.setOwner(this.context.getUsername());
+ if (!ignorePermissions) {
+ // in threads permissions should be ignored,
+ // because context.getUsername doesn't work. See BUG-27093.
+ object.setOwner(this.context.getUsername());
+ }
return super.save(object);
}
@@ -67,7 +71,7 @@ public class PersonalCRUDResourceManager<T extends PersonalResource> extends CRU
}
@Override
- protected ViewContext getContext() {
+ public ViewContext getContext() {
return context;
}
@@ -88,10 +92,14 @@ public class PersonalCRUDResourceManager<T extends PersonalResource> extends CRU
return result;
}
- protected String getUsername() {
+ protected static String getUsername(ViewContext context) {
String userName = context.getProperties().get("dataworker.username");
- if (userName == null)
+ if (userName == null || userName.compareTo("null") == 0 || userName.compareTo("") == 0)
userName = context.getUsername();
return userName;
}
+
+ protected String getUsername() {
+ return getUsername(context);
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/82e72e51/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/files/FileService.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/files/FileService.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/files/FileService.java
index 155c6d9..aaf877c 100644
--- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/files/FileService.java
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/files/FileService.java
@@ -164,7 +164,7 @@ public class FileService extends BaseService {
*/
public static void hdfsSmokeTest(ViewContext context) {
try {
- HdfsApi api = connectToHDFSApi(context);
+ HdfsApi api = HdfsApi.connectToHDFSApi(context);
api.getStatus();
} catch (WebApplicationException ex) {
throw ex;
http://git-wip-us.apache.org/repos/asf/ambari/blob/82e72e51/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobResourceManager.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobResourceManager.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobResourceManager.java
index a43a992..0230625 100644
--- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobResourceManager.java
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobResourceManager.java
@@ -26,6 +26,7 @@ import org.apache.ambari.view.pig.resources.jobs.models.PigJob;
import org.apache.ambari.view.pig.resources.jobs.utils.JobPolling;
import org.apache.ambari.view.pig.services.BaseService;
import org.apache.ambari.view.pig.templeton.client.TempletonApi;
+import org.apache.ambari.view.pig.utils.HdfsApi;
import org.apache.ambari.view.pig.utils.MisconfigurationFormattedException;
import org.apache.ambari.view.pig.utils.ServiceFormattedException;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -89,26 +90,26 @@ public class JobResourceManager extends PersonalCRUDResourceManager<PigJob> {
});
for(PigJob job : notCompleted) {
- JobPolling.pollJob(context, job);
+ JobPolling.pollJob(this, job);
}
}
@Override
public PigJob create(PigJob object) {
- object.setStatus(PigJob.Status.SUBMITTING);
+ object.setStatus(PigJob.PIG_JOB_STATE_SUBMITTING);
PigJob job = super.create(object);
LOG.debug("Submitting job...");
try {
submitJob(object);
} catch (RuntimeException e) {
- object.setStatus(PigJob.Status.SUBMIT_FAILED);
+ object.setStatus(PigJob.PIG_JOB_STATE_SUBMIT_FAILED);
save(object);
LOG.debug("Job submit FAILED");
throw e;
}
LOG.debug("Job submit OK");
- object.setStatus(PigJob.Status.SUBMITTED);
+ object.setStatus(PigJob.PIG_JOB_STATE_SUBMITTED);
save(object);
return job;
}
@@ -121,13 +122,17 @@ public class JobResourceManager extends PersonalCRUDResourceManager<PigJob> {
public void killJob(PigJob object) throws IOException {
LOG.debug("Killing job...");
- try {
- getTempletonApi().killJob(object.getJobId());
- } catch (IOException e) {
- LOG.debug("Job kill FAILED");
- throw e;
+ if (object.getJobId() != null) {
+ try {
+ getTempletonApi().killJob(object.getJobId());
+ } catch (IOException e) {
+ LOG.debug("Job kill FAILED");
+ throw e;
+ }
+ LOG.debug("Job kill OK");
+ } else {
+ LOG.debug("Job was not submitted, ignoring kill request...");
}
- LOG.debug("Job kill OK");
}
/**
@@ -136,15 +141,18 @@ public class JobResourceManager extends PersonalCRUDResourceManager<PigJob> {
*/
private void submitJob(PigJob job) {
String date = new SimpleDateFormat("dd-MM-yyyy-HH-mm-ss").format(new Date());
- String statusdir = String.format(context.getProperties().get("dataworker.jobs.path") +
- "/%s/%s_%s", getUsername(),
- job.getTitle().toLowerCase().replaceAll("[^a-zA-Z0-9 ]+", "").replace(" ", "_"),
- date);
+ String jobsBaseDir = context.getProperties().get("jobs.dir");
+ String storeBaseDir = context.getProperties().get("store.dir");
+ if (storeBaseDir == null || storeBaseDir.compareTo("null") == 0 || storeBaseDir.compareTo("") == 0)
+ storeBaseDir = context.getProperties().get("jobs.dir");
+ String jobNameCleaned = job.getTitle().toLowerCase().replaceAll("[^a-zA-Z0-9 ]+", "").replace(" ", "_");
+ String storedir = String.format(storeBaseDir + "/%s_%s", jobNameCleaned, date);
+ String statusdir = String.format(jobsBaseDir + "/%s_%s", jobNameCleaned, date);
- String newPigScriptPath = statusdir + "/script.pig";
- String newSourceFilePath = statusdir + "/source.pig";
- String newPythonScriptPath = statusdir + "/udf.py";
- String templetonParamsFilePath = statusdir + "/params";
+ String newPigScriptPath = storedir + "/script.pig";
+ String newSourceFilePath = storedir + "/source.pig";
+ String newPythonScriptPath = storedir + "/udf.py";
+ String templetonParamsFilePath = storedir + "/params";
try {
// additional file can be passed to copy into work directory
if (job.getSourceFileContent() != null && !job.getSourceFileContent().isEmpty()) {
@@ -152,13 +160,13 @@ public class JobResourceManager extends PersonalCRUDResourceManager<PigJob> {
job.setSourceFileContent(null); // we should not store content in DB
save(job);
- FSDataOutputStream stream = BaseService.getHdfsApi(context).create(newSourceFilePath, true);
+ FSDataOutputStream stream = HdfsApi.getInstance(context).create(newSourceFilePath, true);
stream.writeBytes(sourceFileContent);
stream.close();
} else {
if (job.getSourceFile() != null && !job.getSourceFile().isEmpty()) {
// otherwise, just copy original file
- if (!BaseService.getHdfsApi(context).copy(job.getSourceFile(), newSourceFilePath)) {
+ if (!HdfsApi.getInstance(context).copy(job.getSourceFile(), newSourceFilePath)) {
throw new ServiceFormattedException("Can't copy source file from " + job.getSourceFile() +
" to " + newPigScriptPath);
}
@@ -176,16 +184,16 @@ public class JobResourceManager extends PersonalCRUDResourceManager<PigJob> {
String forcedContent = job.getForcedContent();
// variable for sourceFile can be passed from front-ent
forcedContent = forcedContent.replace("${sourceFile}",
- context.getProperties().get("dataworker.defaultFs") + newSourceFilePath);
+ context.getProperties().get("webhdfs.url") + newSourceFilePath);
job.setForcedContent(null); // we should not store content in DB
save(job);
- FSDataOutputStream stream = BaseService.getHdfsApi(context).create(newPigScriptPath, true);
+ FSDataOutputStream stream = HdfsApi.getInstance(context).create(newPigScriptPath, true);
stream.writeBytes(forcedContent);
stream.close();
} else {
// otherwise, just copy original file
- if (!BaseService.getHdfsApi(context).copy(job.getPigScript(), newPigScriptPath)) {
+ if (!HdfsApi.getInstance(context).copy(job.getPigScript(), newPigScriptPath)) {
throw new ServiceFormattedException("Can't copy pig script file from " + job.getPigScript() +
" to " + newPigScriptPath);
}
@@ -198,7 +206,7 @@ public class JobResourceManager extends PersonalCRUDResourceManager<PigJob> {
if (job.getPythonScript() != null && !job.getPythonScript().isEmpty()) {
try {
- if (!BaseService.getHdfsApi(context).copy(job.getPythonScript(), newPythonScriptPath)) {
+ if (!HdfsApi.getInstance(context).copy(job.getPythonScript(), newPythonScriptPath)) {
throw new ServiceFormattedException("Can't copy python udf script file from " + job.getPythonScript() +
" to " + newPythonScriptPath);
}
@@ -210,7 +218,7 @@ public class JobResourceManager extends PersonalCRUDResourceManager<PigJob> {
}
try {
- FSDataOutputStream stream = BaseService.getHdfsApi(context).create(templetonParamsFilePath, true);
+ FSDataOutputStream stream = HdfsApi.getInstance(context).create(templetonParamsFilePath, true);
if (job.getTempletonArguments() != null) {
stream.writeBytes(job.getTempletonArguments());
}
@@ -235,7 +243,7 @@ public class JobResourceManager extends PersonalCRUDResourceManager<PigJob> {
}
job.setJobId(data.id);
- JobPolling.pollJob(context, job);
+ JobPolling.pollJob(this, job);
}
/**
@@ -253,29 +261,40 @@ public class JobResourceManager extends PersonalCRUDResourceManager<PigJob> {
if (info.status != null && (info.status.containsKey("runState"))) {
//TODO: retrieve from RM
+ Long time = System.currentTimeMillis() / 1000L;
+ Long currentDuration = time - job.getDateStarted();
int runState = ((Double) info.status.get("runState")).intValue();
+ boolean isStatusChanged = false;
switch (runState) {
- case PigJob.RUN_STATE_KILLED:
+ case RUN_STATE_KILLED:
LOG.debug(String.format("Job KILLED: %s", job.getJobId()));
- job.setStatus(PigJob.Status.KILLED);
+ isStatusChanged = job.getStatus() != PigJob.PIG_JOB_STATE_KILLED;
+ job.setStatus(PigJob.PIG_JOB_STATE_KILLED);
break;
- case PigJob.RUN_STATE_FAILED:
+ case RUN_STATE_FAILED:
LOG.debug(String.format("Job FAILED: %s", job.getJobId()));
- job.setStatus(PigJob.Status.FAILED);
+ isStatusChanged = job.getStatus() != PigJob.PIG_JOB_STATE_FAILED;
+ job.setStatus(PigJob.PIG_JOB_STATE_FAILED);
break;
- case PigJob.RUN_STATE_PREP:
- case PigJob.RUN_STATE_RUNNING:
- job.setStatus(PigJob.Status.RUNNING);
+ case RUN_STATE_PREP:
+ case RUN_STATE_RUNNING:
+ isStatusChanged = job.getStatus() != PigJob.PIG_JOB_STATE_RUNNING;
+ job.setStatus(PigJob.PIG_JOB_STATE_RUNNING);
break;
- case PigJob.RUN_STATE_SUCCEEDED:
+ case RUN_STATE_SUCCEEDED:
LOG.debug(String.format("Job COMPLETED: %s", job.getJobId()));
- job.setStatus(PigJob.Status.COMPLETED);
+ isStatusChanged = job.getStatus() != PigJob.PIG_JOB_STATE_COMPLETED;
+ job.setStatus(PigJob.PIG_JOB_STATE_COMPLETED);
break;
default:
LOG.debug(String.format("Job in unknown state: %s", job.getJobId()));
- job.setStatus(PigJob.Status.UNKNOWN);
+ isStatusChanged = job.getStatus() != PigJob.PIG_JOB_STATE_UNKNOWN;
+ job.setStatus(PigJob.PIG_JOB_STATE_UNKNOWN);
break;
}
+ if (isStatusChanged) {
+ job.setDuration(currentDuration);
+ }
}
Pattern pattern = Pattern.compile("\\d+");
Matcher matcher = null;
@@ -306,13 +325,13 @@ public class JobResourceManager extends PersonalCRUDResourceManager<PigJob> {
}
private static TempletonApi connectToTempletonApi(ViewContext context) {
- String webhcatUrl = context.getProperties().get("dataworker.webhcat.url");
+ String webhcatUrl = context.getProperties().get("webhcat.url");
if (webhcatUrl == null) {
- String message = "dataworker.webhcat.url is not configured!";
+ String message = "webhcat.url is not configured!";
LOG.error(message);
- throw new MisconfigurationFormattedException("dataworker.webhcat.url");
+ throw new MisconfigurationFormattedException("webhcat.url");
}
- return new TempletonApi(context.getProperties().get("dataworker.webhcat.url"),
+ return new TempletonApi(context.getProperties().get("webhcat.url"),
getTempletonUser(context), getTempletonUser(context), context);
}
@@ -322,12 +341,16 @@ public class JobResourceManager extends PersonalCRUDResourceManager<PigJob> {
* @return username in templeton
*/
private static String getTempletonUser(ViewContext context) {
- String username = context.getProperties().get("dataworker.webhcat.user");
- if (username == null) {
- String message = "dataworker.webhcat.user is not configured!";
- LOG.error(message);
- throw new MisconfigurationFormattedException("dataworker.webhcat.user");
+ String username = context.getProperties().get("webhcat.username");
+ if (username == null || username.compareTo("null") == 0 || username.compareTo("") == 0) {
+ username = getUsername(context);
}
return username;
}
+
+ public static final int RUN_STATE_RUNNING = 1;
+ public static final int RUN_STATE_SUCCEEDED = 2;
+ public static final int RUN_STATE_FAILED = 3;
+ public static final int RUN_STATE_PREP = 4;
+ public static final int RUN_STATE_KILLED = 5;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/82e72e51/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobService.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobService.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobService.java
index c4f2d29..9ecbd75 100644
--- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobService.java
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobService.java
@@ -21,6 +21,7 @@ package org.apache.ambari.view.pig.resources.jobs;
import com.google.inject.Inject;
import org.apache.ambari.view.ViewContext;
import org.apache.ambari.view.ViewResourceHandler;
+import org.apache.ambari.view.pig.persistence.utils.Indexed;
import org.apache.ambari.view.pig.persistence.utils.ItemNotFound;
import org.apache.ambari.view.pig.persistence.utils.OnlyOwnersFilteringStrategy;
import org.apache.ambari.view.pig.resources.files.FileResource;
@@ -31,6 +32,8 @@ import org.apache.ambari.view.pig.utils.FilePaginator;
import org.apache.ambari.view.pig.utils.NotFoundFormattedException;
import org.apache.ambari.view.pig.utils.ServiceFormattedException;
import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.*;
@@ -57,6 +60,9 @@ public class JobService extends BaseService {
@Inject
ViewResourceHandler handler;
+ protected final static Logger LOG =
+ LoggerFactory.getLogger(JobService.class);
+
protected JobResourceManager resourceManager = null;
/**
@@ -108,7 +114,8 @@ public class JobService extends BaseService {
*/
@DELETE
@Path("{jobId}")
- public Response killJob(@PathParam("jobId") String jobId) throws IOException {
+ public Response killJob(@PathParam("jobId") String jobId,
+ @QueryParam("remove") final String remove) throws IOException {
try {
PigJob job = null;
try {
@@ -117,6 +124,9 @@ public class JobService extends BaseService {
throw new NotFoundFormattedException(itemNotFound.getMessage(), itemNotFound);
}
getResourceManager().killJob(job);
+ if (remove != null && remove.compareTo("true") == 0) {
+ getResourceManager().delete(jobId);
+ }
return Response.status(204).build();
} catch (WebApplicationException ex) {
throw ex;
@@ -206,10 +216,20 @@ public class JobService extends BaseService {
*/
@GET
@Produces(MediaType.APPLICATION_JSON)
- public Response getJobList(@Context HttpHeaders headers, @Context UriInfo ui) {
+ public Response getJobList(@QueryParam("scriptId") final String scriptId) {
try {
List allJobs = getResourceManager().readAll(
- new OnlyOwnersFilteringStrategy(this.context.getUsername()));
+ new OnlyOwnersFilteringStrategy(this.context.getUsername()) {
+ @Override
+ public boolean isConform(Indexed item) {
+ if (scriptId == null)
+ return super.isConform(item);
+ else {
+ PigJob job = (PigJob) item;
+ return (job.getScriptId() != null && scriptId.compareTo(job.getScriptId()) == 0 && super.isConform(item));
+ }
+ }
+ }); //TODO: move strategy to PersonalCRUDRM
JSONObject object = new JSONObject();
object.put("jobs", allJobs);
http://git-wip-us.apache.org/repos/asf/ambari/blob/82e72e51/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/models/PigJob.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/models/PigJob.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/models/PigJob.java
index f55da93..6f80d6b 100644
--- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/models/PigJob.java
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/models/PigJob.java
@@ -20,7 +20,6 @@ package org.apache.ambari.view.pig.resources.jobs.models;
import org.apache.ambari.view.pig.persistence.utils.PersonalResource;
import org.apache.commons.beanutils.BeanUtils;
-import org.codehaus.jackson.annotate.JsonIgnore;
import java.io.Serializable;
import java.lang.reflect.InvocationTargetException;
@@ -42,23 +41,16 @@ import java.util.Map;
* COMPLETED KILLED FAILED
*/
public class PigJob implements Serializable, PersonalResource {
-
- public enum Status {
- UNKNOWN,
- SUBMITTING, SUBMITTED, RUNNING, // in progress
- SUBMIT_FAILED, COMPLETED, FAILED, KILLED // finished
- }
-
- public boolean isInProgress() {
- return status == Status.SUBMITTED || status == Status.SUBMITTING ||
- status == Status.RUNNING;
- }
-
- public static final int RUN_STATE_RUNNING = 1;
- public static final int RUN_STATE_SUCCEEDED = 2;
- public static final int RUN_STATE_FAILED = 3;
- public static final int RUN_STATE_PREP = 4;
- public static final int RUN_STATE_KILLED = 5;
+ public static final String PIG_JOB_STATE_UNKNOWN = "UNKNOWN";
+ // in progress
+ public static final String PIG_JOB_STATE_SUBMITTING = "SUBMITTING";
+ public static final String PIG_JOB_STATE_SUBMITTED = "SUBMITTED";
+ public static final String PIG_JOB_STATE_RUNNING = "RUNNING";
+ // finished
+ public static final String PIG_JOB_STATE_SUBMIT_FAILED = "SUBMIT_FAILED";
+ public static final String PIG_JOB_STATE_COMPLETED = "COMPLETED";
+ public static final String PIG_JOB_STATE_FAILED = "FAILED";
+ public static final String PIG_JOB_STATE_KILLED = "KILLED";
public PigJob() {
}
@@ -96,10 +88,11 @@ public class PigJob implements Serializable, PersonalResource {
private String statusDir;
private Long dateStarted = 0L;
+ private Long duration = 0L;
private String jobId = null;
// status fields (not reliable)
- private Status status = Status.UNKNOWN;
+ private String status = PIG_JOB_STATE_UNKNOWN;
private Integer percentComplete = null;
@Override
@@ -119,6 +112,11 @@ public class PigJob implements Serializable, PersonalResource {
return id.hashCode();
}
+ public boolean isInProgress() {
+ return status.equals(PIG_JOB_STATE_SUBMITTED) || status.equals(PIG_JOB_STATE_SUBMITTING) ||
+ status.equals(PIG_JOB_STATE_RUNNING);
+ }
+
@Override
public String getId() {
return id;
@@ -139,11 +137,11 @@ public class PigJob implements Serializable, PersonalResource {
this.owner = owner;
}
- public Status getStatus() {
+ public String getStatus() {
return status;
}
- public void setStatus(Status status) {
+ public void setStatus(String status) {
this.status = status;
}
@@ -203,6 +201,14 @@ public class PigJob implements Serializable, PersonalResource {
this.dateStarted = dateStarted;
}
+ public Long getDuration() {
+ return duration;
+ }
+
+ public void setDuration(Long duration) {
+ this.duration = duration;
+ }
+
public Integer getPercentComplete() {
return percentComplete;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/82e72e51/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/utils/JobPolling.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/utils/JobPolling.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/utils/JobPolling.java
index bc633b2..8a60a1c 100644
--- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/utils/JobPolling.java
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/utils/JobPolling.java
@@ -18,7 +18,8 @@
package org.apache.ambari.view.pig.resources.jobs.utils;
-import org.apache.ambari.view.ViewContext;
+import org.apache.ambari.view.pig.persistence.utils.FilteringStrategy;
+import org.apache.ambari.view.pig.persistence.utils.Indexed;
import org.apache.ambari.view.pig.persistence.utils.ItemNotFound;
import org.apache.ambari.view.pig.resources.jobs.JobResourceManager;
import org.apache.ambari.view.pig.resources.jobs.models.PigJob;
@@ -26,12 +27,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
-import java.util.Observable;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.*;
/**
* Polling manager
@@ -49,67 +47,71 @@ public class JobPolling implements Runnable {
*/
private static final int WORKER_COUNT = 2;
- private static final int POLLING_DELAY = 10*60; // 10 minutes
+ private static final int POLLING_DELAY = 60; // 1 minutes
/**
* In LONG_JOB_THRESHOLD seconds job reschedules polling from POLLING_DELAY to LONG_POLLING_DELAY
*/
- private static final int LONG_POLLING_DELAY = 60*60; // 1 hour
- private static final int LONG_JOB_THRESHOLD = 60*60; // 1 hour
+ private static final int LONG_POLLING_DELAY = 10*60; // 10 minutes
+ private static final int LONG_JOB_THRESHOLD = 10*60; // 10 minutes
private static final ScheduledExecutorService pollWorkersPool = Executors.newScheduledThreadPool(WORKER_COUNT);
private static final Map<String, JobPolling> jobPollers = new HashMap<String, JobPolling>();
private JobResourceManager resourceManager = null;
- private final ViewContext context;
private PigJob job;
private volatile ScheduledFuture<?> thisFuture;
- private JobPolling(ViewContext context, PigJob job) {
- this.context = context;
+ private JobPolling(JobResourceManager resourceManager, PigJob job) {
+ this.resourceManager = resourceManager;
this.job = job;
}
- protected synchronized JobResourceManager getResourceManager() {
- if (resourceManager == null) {
- resourceManager = new JobResourceManager(context);
- }
- return resourceManager;
- }
-
/**
* Do polling
*/
public void run() {
- LOG.debug("Polling job status " + job.getJobId() + " #" + job.getId());
try {
- job = getResourceManager().read(job.getId());
- } catch (ItemNotFound itemNotFound) {
- LOG.error("Job " + job.getJobId() + " does not exist! Polling canceled");
- thisFuture.cancel(false);
- return;
- }
- getResourceManager().retrieveJobStatus(job);
-
- Long time = System.currentTimeMillis() / 1000L;
- if (time - job.getDateStarted() > LONG_JOB_THRESHOLD) {
- LOG.debug("Job becomes long.. Rescheduling polling to longer period");
- // If job running longer than LONG_JOB_THRESHOLD, reschedule
- // it to poll every LONG_POLLING_DELAY instead of POLLING_DELAY
- thisFuture.cancel(false);
- scheduleJobPolling(true);
- }
-
- switch (job.getStatus()) {
- case SUBMIT_FAILED:
- case COMPLETED:
- case FAILED:
- case KILLED:
- LOG.debug("Job finished. Polling canceled");
- thisFuture.cancel(false);
- break;
- default:
+ // Hack to make permission check work. It is based on
+ // context.getUsername(), but it doesn't work in another thread. See BUG-27093.
+ resourceManager.ignorePermissions(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ LOG.debug("Polling job status " + job.getJobId() + " #" + job.getId());
+ try {
+ job = resourceManager.read(job.getId());
+ } catch (ItemNotFound itemNotFound) {
+ LOG.error("Job " + job.getId() + " does not exist! Polling canceled");
+ thisFuture.cancel(false);
+ return null;
+ }
+ resourceManager.retrieveJobStatus(job);
+
+ Long time = System.currentTimeMillis() / 1000L;
+ if (time - job.getDateStarted() > LONG_JOB_THRESHOLD) {
+ LOG.debug("Job becomes long.. Rescheduling polling to longer period");
+ // If job running longer than LONG_JOB_THRESHOLD, reschedule
+ // it to poll every LONG_POLLING_DELAY instead of POLLING_DELAY
+ thisFuture.cancel(false);
+ scheduleJobPolling(true);
+ }
+
+ if (job.getStatus().equals(PigJob.PIG_JOB_STATE_SUBMIT_FAILED) ||
+ job.getStatus().equals(PigJob.PIG_JOB_STATE_COMPLETED) ||
+ job.getStatus().equals(PigJob.PIG_JOB_STATE_FAILED) ||
+ job.getStatus().equals(PigJob.PIG_JOB_STATE_KILLED)) {
+ LOG.debug("Job finished. Polling canceled");
+ thisFuture.cancel(false);
+
+ } else {
+ }
+ return null;
+ }
+ });
+ } catch (Exception e) {
+ LOG.error("Exception during handling job polling: " + e.getMessage());
+ e.printStackTrace();
}
}
@@ -129,14 +131,13 @@ public class JobPolling implements Runnable {
/**
* Schedule job polling
- * @param context ViewContext of web app
* @param job job instance
* @return returns false if already scheduled
*/
- public static boolean pollJob(ViewContext context, PigJob job) {
+ public static boolean pollJob(JobResourceManager resourceManager, PigJob job) {
if (jobPollers.get(job.getJobId()) == null) {
LOG.debug("Setting up polling for " + job.getJobId());
- JobPolling polling = new JobPolling(context, job);
+ JobPolling polling = new JobPolling(resourceManager, job);
polling.scheduleJobPolling();
jobPollers.put(job.getJobId(), polling);
return true;
http://git-wip-us.apache.org/repos/asf/ambari/blob/82e72e51/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/scripts/ScriptResourceManager.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/scripts/ScriptResourceManager.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/scripts/ScriptResourceManager.java
index c213a6e..9a56bb4 100644
--- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/scripts/ScriptResourceManager.java
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/scripts/ScriptResourceManager.java
@@ -23,6 +23,7 @@ import org.apache.ambari.view.pig.persistence.utils.ItemNotFound;
import org.apache.ambari.view.pig.resources.PersonalCRUDResourceManager;
import org.apache.ambari.view.pig.resources.scripts.models.PigScript;
import org.apache.ambari.view.pig.services.BaseService;
+import org.apache.ambari.view.pig.utils.HdfsApi;
import org.apache.ambari.view.pig.utils.MisconfigurationFormattedException;
import org.apache.ambari.view.pig.utils.ServiceFormattedException;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -59,11 +60,11 @@ public class ScriptResourceManager extends PersonalCRUDResourceManager<PigScript
}
private void createDefaultScriptFile(PigScript object) {
- String userScriptsPath = context.getProperties().get("dataworker.scripts.path");
+ String userScriptsPath = context.getProperties().get("scripts.dir");
if (userScriptsPath == null) {
- String msg = "dataworker.scripts.path is not configured!";
+ String msg = "scripts.dir is not configured!";
LOG.error(msg);
- throw new MisconfigurationFormattedException("dataworker.scripts.path");
+ throw new MisconfigurationFormattedException("scripts.dir");
}
int checkId = 0;
@@ -73,12 +74,11 @@ public class ScriptResourceManager extends PersonalCRUDResourceManager<PigScript
String normalizedName = object.getTitle().replaceAll("[^a-zA-Z0-9 ]+", "").replaceAll(" ", "_").toLowerCase();
String timestamp = new SimpleDateFormat("yyyy-MM-dd_hh-mm").format(new Date());
newFilePath = String.format(userScriptsPath +
- "/%s/%s-%s%s.pig", getUsername(),
- normalizedName, timestamp, (checkId == 0)?"":"_"+checkId);
+ "/%s-%s%s.pig", normalizedName, timestamp, (checkId == 0)?"":"_"+checkId);
LOG.debug("Trying to create new file " + newFilePath);
try {
- FSDataOutputStream stream = BaseService.getHdfsApi(context).create(newFilePath, false);
+ FSDataOutputStream stream = HdfsApi.getInstance(context).create(newFilePath, false);
stream.close();
fileCreated = true;
LOG.debug("File created successfully!");
http://git-wip-us.apache.org/repos/asf/ambari/blob/82e72e51/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/scripts/ScriptService.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/scripts/ScriptService.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/scripts/ScriptService.java
index 0e7536e..46e6247 100644
--- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/scripts/ScriptService.java
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/scripts/ScriptService.java
@@ -111,7 +111,7 @@ public class ScriptService extends BaseService {
try {
LOG.debug("Getting all scripts");
List allScripts = getResourceManager().readAll(
- new OnlyOwnersFilteringStrategy(this.context.getUsername()));
+ new OnlyOwnersFilteringStrategy(this.context.getUsername())); //TODO: move strategy to PersonalCRUDRM
JSONObject object = new JSONObject();
object.put("scripts", allScripts);
http://git-wip-us.apache.org/repos/asf/ambari/blob/82e72e51/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/services/BaseService.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/services/BaseService.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/services/BaseService.java
index 741e32f..ec15c37 100644
--- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/services/BaseService.java
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/services/BaseService.java
@@ -42,14 +42,14 @@ public class BaseService {
@Inject
protected ViewContext context;
- protected final static Logger LOG =
+ private final static Logger LOG =
LoggerFactory.getLogger(BaseService.class);
private Storage storage;
protected Storage getStorage() {
if (this.storage == null) {
- storage = StorageUtil.getStorage(context);
+ storage = StorageUtil.getInstance(context).getStorage();
}
return storage;
}
@@ -58,62 +58,19 @@ public class BaseService {
this.storage = storage;
}
- private static HdfsApi hdfsApi = null;
-
- /**
- * Returns HdfsApi object
- * @param context View Context instance
- * @return Hdfs business delegate object
- */
- public static HdfsApi getHdfsApi(ViewContext context) {
- if (hdfsApi == null) {
- hdfsApi = connectToHDFSApi(context);
- }
- return hdfsApi;
- }
-
- protected static HdfsApi connectToHDFSApi(ViewContext context) {
- HdfsApi api = null;
- Thread.currentThread().setContextClassLoader(null);
-
- String defaultFS = context.getProperties().get("dataworker.defaultFs");
- if (defaultFS == null) {
- String message = "dataworker.defaultFs is not configured!";
- LOG.error(message);
- throw new MisconfigurationFormattedException("dataworker.defaultFs");
- }
-
- try {
- api = new HdfsApi(defaultFS, getHdfsUsername(context));
- LOG.info("HdfsApi connected OK");
- } catch (IOException e) {
- String message = "HdfsApi IO error: " + e.getMessage();
- LOG.error(message);
- throw new ServiceFormattedException(message, e);
- } catch (InterruptedException e) {
- String message = "HdfsApi Interrupted error: " + e.getMessage();
- LOG.error(message);
- throw new ServiceFormattedException(message, e);
- }
- return api;
- }
-
- public static String getHdfsUsername(ViewContext context) {
- String userName = context.getProperties().get("dataworker.hdfs.username");
- if (userName == null)
- userName = context.getUsername();
- return userName;
- }
+ private HdfsApi hdfsApi = null;
protected HdfsApi getHdfsApi() {
- return getHdfsApi(context);
+ if (hdfsApi == null)
+ hdfsApi = HdfsApi.getInstance(context);
+ return hdfsApi;
}
/**
* Set HdfsApi delegate
* @param api HdfsApi instance
*/
- public static void setHdfsApi(HdfsApi api) {
+ public void setHdfsApi(HdfsApi api) {
hdfsApi = api;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/82e72e51/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/services/HelpService.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/services/HelpService.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/services/HelpService.java
index 1a0e6dd..eb363a0 100644
--- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/services/HelpService.java
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/services/HelpService.java
@@ -20,6 +20,7 @@ package org.apache.ambari.view.pig.services;
import org.apache.ambari.view.ViewContext;
import org.apache.ambari.view.ViewResourceHandler;
+import org.apache.ambari.view.pig.persistence.DataStoreStorage;
import org.apache.ambari.view.pig.persistence.InstanceKeyValueStorage;
import org.apache.ambari.view.pig.resources.files.FileService;
import org.apache.ambari.view.pig.resources.jobs.JobResourceManager;
@@ -59,8 +60,8 @@ public class HelpService extends BaseService {
@Produces(MediaType.APPLICATION_JSON)
public Response config(){
JSONObject object = new JSONObject();
- String fs = context.getProperties().get("dataworker.defaultFs");
- object.put("dataworker.defaultFs", fs);
+ String fs = context.getProperties().get("webhdfs.url");
+ object.put("webhdfs.url", fs);
return Response.ok(object).build();
}
@@ -111,7 +112,7 @@ public class HelpService extends BaseService {
@Path("/storageStatus")
@Produces(MediaType.APPLICATION_JSON)
public Response storageStatus(){
- InstanceKeyValueStorage.storageSmokeTest(context);
+ DataStoreStorage.storageSmokeTest(context);
return getOKResponse();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/82e72e51/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/Request.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/Request.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/Request.java
index a23f008..521bfad 100644
--- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/Request.java
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/Request.java
@@ -69,6 +69,7 @@ public class Request<RESPONSE> {
InputStream inputStream = context.getURLStreamProvider().readFrom(resource.toString(), "GET",
null, new HashMap<String, String>());
+ LOG.info(String.format("curl \"" + resource.toString() + "\""));
String responseJson = IOUtils.toString(inputStream);
LOG.debug(String.format("RESPONSE => %s", responseJson));
return gson.fromJson(responseJson, responseClass);
@@ -100,10 +101,14 @@ public class Request<RESPONSE> {
LOG.debug("POST " + resource.toString());
LOG.debug("data: " + data.toString());
+ StringBuilder curlBuilder = new StringBuilder();
+
UriBuilder builder = UriBuilder.fromPath("host/");
for(String key : data.keySet()) {
- for(String value : data.get(key))
+ for(String value : data.get(key)) {
builder.queryParam(key, value);
+ curlBuilder.append(String.format("-d %s=\"%s\"", key, value.replace("\"", "\\\"")));
+ }
}
if (data != null)
@@ -112,6 +117,7 @@ public class Request<RESPONSE> {
Map<String, String> headers = new HashMap<String, String>();
headers.put("Content-Type", "application/x-www-form-urlencoded");
+ LOG.info(String.format("curl " + curlBuilder.toString() + " \"" + resource.toString() + "\""));
InputStream inputStream = context.getURLStreamProvider().readFrom(resource.toString(),
"POST", builder.build().getRawQuery(), headers);
String responseJson = IOUtils.toString(inputStream);
@@ -150,10 +156,14 @@ public class Request<RESPONSE> {
public RESPONSE put(WebResource resource, MultivaluedMapImpl data) throws IOException {
LOG.debug("PUT " + resource.toString());
+ StringBuilder curlBuilder = new StringBuilder();
+
UriBuilder builder = UriBuilder.fromPath("host/");
for(String key : data.keySet()) {
- for(String value : data.get(key))
+ for(String value : data.get(key)) {
builder.queryParam(key, value);
+ curlBuilder.append(String.format("-d %s=\"%s\"", key, value.replace("\"", "\\\"")));
+ }
}
if (data != null)
@@ -162,6 +172,8 @@ public class Request<RESPONSE> {
Map<String, String> headers = new HashMap<String, String>();
headers.put("Content-Type", "application/x-www-form-urlencoded");
+ LOG.info(String.format("curl -X PUT " + curlBuilder.toString() + " \"" + resource.toString() + "\""));
+
InputStream inputStream = context.getURLStreamProvider().readFrom(resource.toString(),
"PUT", builder.build().getRawQuery(), headers);
String responseJson = IOUtils.toString(inputStream);
@@ -200,10 +212,14 @@ public class Request<RESPONSE> {
public RESPONSE delete(WebResource resource, MultivaluedMapImpl data) throws IOException {
LOG.debug("DELETE " + resource.toString());
+ StringBuilder curlBuilder = new StringBuilder();
+
UriBuilder builder = UriBuilder.fromPath("host/");
for(String key : data.keySet()) {
- for(String value : data.get(key))
+ for(String value : data.get(key)) {
builder.queryParam(key, value);
+ curlBuilder.append(String.format("-d %s=\"%s\"", key, value.replace("\"", "\\\"")));
+ }
}
if (data != null)
@@ -212,6 +228,8 @@ public class Request<RESPONSE> {
Map<String, String> headers = new HashMap<String, String>();
headers.put("Content-Type", "application/x-www-form-urlencoded");
+ LOG.info(String.format("curl -X DELETE " + curlBuilder.toString() + " \"" + resource.toString() + "\""));
+
InputStream inputStream = context.getURLStreamProvider().readFrom(resource.toString(),
"DELETE", builder.build().getRawQuery(), headers);
String responseJson = IOUtils.toString(inputStream);
http://git-wip-us.apache.org/repos/asf/ambari/blob/82e72e51/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/TempletonApi.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/TempletonApi.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/TempletonApi.java
index 9ad54ac..d45b55c 100644
--- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/TempletonApi.java
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/TempletonApi.java
@@ -52,7 +52,7 @@ public class TempletonApi {
/**
* TempletonApi constructor
- * @param api dataworker.webhcat.url
+ * @param api webhcat.url
* @param username templeton username
* @param doAs doAs argument
* @param context context with URLStreamProvider
http://git-wip-us.apache.org/repos/asf/ambari/blob/82e72e51/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/utils/FilePaginator.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/utils/FilePaginator.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/utils/FilePaginator.java
index 2d49c97..9ebbb80 100644
--- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/utils/FilePaginator.java
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/utils/FilePaginator.java
@@ -44,7 +44,7 @@ public class FilePaginator {
*/
public FilePaginator(String filePath, ViewContext context) {
this.filePath = filePath;
- hdfsApi = BaseService.getHdfsApi(context);
+ hdfsApi = HdfsApi.getInstance(context);
}
/**
http://git-wip-us.apache.org/repos/asf/ambari/blob/82e72e51/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/utils/HdfsApi.java
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/utils/HdfsApi.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/utils/HdfsApi.java
index 845b459..f9dfb35 100644
--- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/utils/HdfsApi.java
+++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/utils/HdfsApi.java
@@ -18,6 +18,7 @@
package org.apache.ambari.view.pig.utils;
+import org.apache.ambari.view.ViewContext;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -29,10 +30,13 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
+import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.security.UserGroupInformation;
import org.json.simple.JSONArray;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.LinkedHashMap;
@@ -45,6 +49,9 @@ public class HdfsApi {
private FileSystem fs;
private UserGroupInformation ugi;
+ private final static Logger LOG =
+ LoggerFactory.getLogger(HdfsApi.class);
+
/**
* Constructor
* @param defaultFs hdfs uri
@@ -284,4 +291,57 @@ public class HdfsApi {
return json;
}
+
+ private static Map<String, HdfsApi> viewSingletonObjects = new HashMap<String, HdfsApi>();
+ /**
+ * Returns HdfsApi object specific to instance
+ * @param context View Context instance
+ * @return Hdfs business delegate object
+ */
+ public static HdfsApi getInstance(ViewContext context) {
+ if (!viewSingletonObjects.containsKey(context.getInstanceName()))
+ viewSingletonObjects.put(context.getInstanceName(), connectToHDFSApi(context));
+ return viewSingletonObjects.get(context.getInstanceName());
+ }
+
+ public static void setInstance(ViewContext context, HdfsApi api) {
+ viewSingletonObjects.put(context.getInstanceName(), api);
+ }
+
+ public static HdfsApi connectToHDFSApi(ViewContext context) {
+ HdfsApi api = null;
+ Thread.currentThread().setContextClassLoader(null);
+
+ String defaultFS = context.getProperties().get("webhdfs.url");
+ if (defaultFS == null) {
+ String message = "webhdfs.url is not configured!";
+ LOG.error(message);
+ throw new MisconfigurationFormattedException("webhdfs.url");
+ }
+
+ try {
+ api = new HdfsApi(defaultFS, getHdfsUsername(context));
+ LOG.info("HdfsApi connected OK");
+ } catch (IOException e) {
+ String message = "HdfsApi IO error: " + e.getMessage();
+ LOG.error(message);
+ throw new ServiceFormattedException(message, e);
+ } catch (InterruptedException e) {
+ String message = "HdfsApi Interrupted error: " + e.getMessage();
+ LOG.error(message);
+ throw new ServiceFormattedException(message, e);
+ }
+ return api;
+ }
+
+ public static String getHdfsUsername(ViewContext context) {
+ String userName = context.getProperties().get("webhdfs.username");
+ if (userName == null || userName.compareTo("null") == 0 || userName.compareTo("") == 0)
+ userName = context.getUsername();
+ return userName;
+ }
+
+ public static void dropAllConnections() {
+ viewSingletonObjects.clear();
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/82e72e51/contrib/views/pig/src/main/resources/ui/pig-web/app/assets/data/pigHelpers.json
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/resources/ui/pig-web/app/assets/data/pigHelpers.json b/contrib/views/pig/src/main/resources/ui/pig-web/app/assets/data/pigHelpers.json
new file mode 100644
index 0000000..dbc83cb
--- /dev/null
+++ b/contrib/views/pig/src/main/resources/ui/pig-web/app/assets/data/pigHelpers.json
@@ -0,0 +1,128 @@
+[
+ {
+ "title":"Eval Functions",
+ "helpers":[
+ "AVG(%VAR%)",
+ "CONCAT(%VAR1%, %VAR2%)",
+ "COUNT(%VAR%)",
+ "COUNT_START(%VAR%)",
+ "IsEmpty(%VAR%)",
+ "DIFF(%VAR1%, %VAR2%)",
+ "MAX(%VAR%)",
+ "MIN(%VAR%)",
+ "SIZE(%VAR%)",
+ "SUM(%VAR%)",
+ "TOKENIZE(%VAR%, %DELIM%)"
+ ]
+ },
+ {
+ "title":"Relational Operators",
+ "helpers":[
+ "COGROUP %VAR% BY %VAR%",
+ "CROSS %VAR1%, %VAR2%;",
+ "DISTINCT %VAR%;",
+ "FILTER %VAR% BY %COND%",
+ "FLATTEN(%VAR%)",
+ "FOREACH %DATA% GENERATE %NEW_DATA%",
+ "FOREACH %DATA% {%NESTED_BLOCK%}",
+ "GROUP %VAR% BY %VAR%",
+ "GROUP %VAR% ALL",
+ "JOIN %VAR% BY ",
+ "LIMIT %VAR% %N%",
+ "ORDER %VAR% BY %FIELD%",
+ "SAMPLE %VAR% %SIZE%",
+ "SPLIT %VAR1% INTO %VAR2% IF %EXPRESSIONS%",
+ "UNION %VAR1%, %VAR2%"
+ ]
+ },
+ {
+ "title":"I/0",
+ "helpers":[
+ "LOAD \"%FILE%\";",
+ "DUMP %VAR%;",
+ "STORE %VAR% INTO %PATH%;"
+ ]
+ },
+ {
+ "title":"Debug",
+ "helpers":[
+ "EXPLAIN %VAR%;",
+ "ILLUSTRATE %VAR%;",
+ "DESCRIBE %VAR%;"
+ ]
+ },
+ {
+ "title":"HCatalog",
+ "helpers":[
+ "LOAD \"%TABLE%\" USING org.apache.hcatalog.pig.HCatLoader();"
+ ]
+ },
+ {
+ "title":"Math",
+ "helpers":[
+ "ABS(%VAR%)",
+ "ACOS(%VAR%)",
+ "ASIN(%VAR%)",
+ "ATAN(%VAR%)",
+ "CBRT(%VAR%)",
+ "CEIL(%VAR%)",
+ "COS(%VAR%)",
+ "COSH(%VAR%)",
+ "EXP(%VAR%)",
+ "FLOOR(%VAR%)",
+ "LOG(%VAR%)",
+ "LOG10(%VAR%)",
+ "RANDOM(%VAR%)",
+ "ROUND(%VAR%)",
+ "SIN(%VAR%)",
+ "SINH(%VAR%)",
+ "SQRT(%VAR%)",
+ "TAN(%VAR%)",
+ "TANH(%VAR%)"
+ ]
+ },
+ {
+ "title":"Tuple, Bag, Map Functions",
+ "helpers":[
+ "TOTUPLE(%VAR%)",
+ "TOBAG(%VAR%)",
+ "TOMAP(%KEY%, %VALUE%)",
+ "TOP(%topN%, %COLUMN%, %RELATION%)"
+ ]
+ },
+ {
+ "title":"String Functions",
+ "helpers":[
+ "INDEXOF(%STRING%, \"%CHARACTER%\", %STARTINDEX%)",
+ "LAST_INDEX_OF(%STRING%, \"%CHARACTER%\", %STARTINDEX%)",
+ "LOWER(%STRING%)",
+ "REGEX_EXTRACT(%STRING%, %REGEX%, %INDEX%)",
+ "REGEX_EXTRACT_ALL(%STRING%, %REGEX%)",
+ "REPLACE(%STRING%, \"%oldChar%\", \"%newChar%\")",
+ "STRSPLIT(%STRING%, %REGEX%, %LIMIT%)",
+ "SUBSTRING(%STRING%, %STARTINDEX%, %STOPINDEX%)",
+ "TRIM(%STRING%)",
+ "UCFIRST(%STRING%)",
+ "UPPER(%STRING%)"
+ ]
+ },
+ {
+ "title":"Macros",
+ "helpers":[
+ "IMPORT \"%PATH_TO_MACRO%\";"
+ ]
+ },
+ {
+ "title":"HBase",
+ "helpers":[
+ "LOAD \"hbase://%TABLE%\" USING org.apache.pig.backend.hadoop.hbase.HBaseStorage(\"%columnList%\")",
+ "STORE %VAR% INTO \"hbase://%TABLE%\" USING org.apache.pig.backend.hadoop.hbase.HBaseStorage(\"%columnList%\")"
+ ]
+ },
+ {
+ "title":"Python UDF",
+ "helpers":[
+ "REGISTER \"python_udf.py\" USING jython AS myfuncs;"
+ ]
+ }
+ ]
http://git-wip-us.apache.org/repos/asf/ambari/blob/82e72e51/contrib/views/pig/src/main/resources/ui/pig-web/app/assets/static/fonts/FontAwesome.otf
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/resources/ui/pig-web/app/assets/static/fonts/FontAwesome.otf b/contrib/views/pig/src/main/resources/ui/pig-web/app/assets/static/fonts/FontAwesome.otf
new file mode 100644
index 0000000..81c9ad9
Binary files /dev/null and b/contrib/views/pig/src/main/resources/ui/pig-web/app/assets/static/fonts/FontAwesome.otf differ
http://git-wip-us.apache.org/repos/asf/ambari/blob/82e72e51/contrib/views/pig/src/main/resources/ui/pig-web/app/assets/static/fonts/fontawesome-webfont.eot
----------------------------------------------------------------------
diff --git a/contrib/views/pig/src/main/resources/ui/pig-web/app/assets/static/fonts/fontawesome-webfont.eot b/contrib/views/pig/src/main/resources/ui/pig-web/app/assets/static/fonts/fontawesome-webfont.eot
new file mode 100644
index 0000000..84677bc0
Binary files /dev/null and b/contrib/views/pig/src/main/resources/ui/pig-web/app/assets/static/fonts/fontawesome-webfont.eot differ