You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2014/08/11 21:08:56 UTC
git commit: AMBARI-6533 - Add API to get file log location in the
task execution output,
currently hardcoded in UI (Alejandro Fernandez via jonathanhurley)
Repository: ambari
Updated Branches:
refs/heads/trunk 88b170d49 -> a2086b1fc
AMBARI-6533 - Add API to get file log location in the task execution output, currently hardcoded in UI (Alejandro Fernandez via jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a2086b1f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a2086b1f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a2086b1f
Branch: refs/heads/trunk
Commit: a2086b1fc88ccc85b9765a58d53f48f6c14d8ce1
Parents: 88b170d
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Mon Aug 11 14:39:36 2014 -0400
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Mon Aug 11 15:08:37 2014 -0400
----------------------------------------------------------------------
.../apache/ambari/server/orm/dao/DaoUtils.java | 2 +-
.../server/orm/dao/HostRoleCommandDAO.java | 1 -
.../orm/entities/HostRoleCommandEntity_.java | 51 +++++++++++++
.../server/upgrade/UpgradeCatalog170.java | 80 +++++++++++++++++---
.../server/upgrade/UpgradeCatalog170Test.java | 74 ++++++++++++------
5 files changed, 173 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a2086b1f/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/DaoUtils.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/DaoUtils.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/DaoUtils.java
index b4e29b9..d92491f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/DaoUtils.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/DaoUtils.java
@@ -31,7 +31,7 @@ import java.util.Collections;
import java.util.List;
@Singleton
-class DaoUtils {
+public class DaoUtils {
public <T> List<T> selectAll(EntityManager entityManager, Class<T> entityClass) {
CriteriaBuilder criteriaBuilder = entityManager.getCriteriaBuilder();
http://git-wip-us.apache.org/repos/asf/ambari/blob/a2086b1f/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java
index 71c27cb..56dd304 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostRoleCommandDAO.java
@@ -266,5 +266,4 @@ public class HostRoleCommandDAO {
public void removeByPK(int taskId) {
remove(findByPK(taskId));
}
-
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a2086b1f/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostRoleCommandEntity_.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostRoleCommandEntity_.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostRoleCommandEntity_.java
new file mode 100644
index 0000000..4dad21a
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostRoleCommandEntity_.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.orm.entities;
+
+import javax.persistence.metamodel.SingularAttribute;
+
+
+/**
+ * This class exists so that JPQL can use static singular attributes that are strongly typed
+ * as opposed to Java reflection like HostRoleCommandEntity.get("fieldname")
+ */
+@javax.persistence.metamodel.StaticMetamodel(HostRoleCommandEntity.class)
+public class HostRoleCommandEntity_ {
+ public static volatile SingularAttribute<HostRoleCommandEntity, Long> taskId;
+ public static volatile SingularAttribute<HostRoleCommandEntity, Long> requestId;
+ public static volatile SingularAttribute<HostRoleCommandEntity, Long> stageId;
+ public static volatile SingularAttribute<HostRoleCommandEntity, String> hostName;
+ public static volatile SingularAttribute<HostRoleCommandEntity, String> role;
+ public static volatile SingularAttribute<HostRoleCommandEntity, String> event;
+ public static volatile SingularAttribute<HostRoleCommandEntity, Integer> exitcode;
+ public static volatile SingularAttribute<HostRoleCommandEntity, String> status;
+ public static volatile SingularAttribute<HostRoleCommandEntity, byte[]> stdError;
+ public static volatile SingularAttribute<HostRoleCommandEntity, byte[]> stdOut;
+ public static volatile SingularAttribute<HostRoleCommandEntity, String> outputLog;
+ public static volatile SingularAttribute<HostRoleCommandEntity, String> errorLog;
+ public static volatile SingularAttribute<HostRoleCommandEntity, byte[]> structuredOut;
+ public static volatile SingularAttribute<HostRoleCommandEntity, Long> startTime;
+ public static volatile SingularAttribute<HostRoleCommandEntity, Long> endTime;
+ public static volatile SingularAttribute<HostRoleCommandEntity, Long> lastAttemptTime;
+ public static volatile SingularAttribute<HostRoleCommandEntity, Short> attemptCount;
+ public static volatile SingularAttribute<HostRoleCommandEntity, String> roleCommand;
+ public static volatile SingularAttribute<HostRoleCommandEntity, String> commandDetail;
+ public static volatile SingularAttribute<HostRoleCommandEntity, String> customCommandName;
+}
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/a2086b1f/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java
index 31123ce..758efe9 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java
@@ -28,9 +28,14 @@ import java.util.Map.Entry;
import java.util.Set;
import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.actionmanager.HostRoleCommand;
import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.orm.DBAccessor.DBColumnInfo;
+import org.apache.ambari.server.orm.dao.DaoUtils;
+import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
+import org.apache.ambari.server.orm.entities.HostRoleCommandEntity;
+import org.apache.ambari.server.orm.entities.HostRoleCommandEntity_;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.Config;
@@ -41,6 +46,10 @@ import org.slf4j.LoggerFactory;
import com.google.inject.Inject;
import com.google.inject.Injector;
+import javax.persistence.EntityManager;
+import javax.persistence.TypedQuery;
+import javax.persistence.criteria.*;
+
/**
* Upgrade catalog for version 1.7.0.
*/
@@ -82,6 +91,8 @@ public class UpgradeCatalog170 extends AbstractUpgradeCatalog {
this.injector = injector;
}
+ @Inject
+ DaoUtils daoUtils;
// ----- AbstractUpgradeCatalog --------------------------------------------
@@ -194,17 +205,6 @@ public class UpgradeCatalog170 extends AbstractUpgradeCatalog {
dbAccessor.addColumn("host_role_command", new DBColumnInfo("error_log",
String.class, 255, null, true));
- // Update historic records with the log paths, but only enough so as to not prolong the upgrade process
- if (dbType.equals(Configuration.POSTGRES_DB_NAME) || dbType.equals(Configuration.ORACLE_DB_NAME)) {
- // Postgres and Oracle use a different concatenation operator.
- dbAccessor.executeQuery("UPDATE host_role_command SET output_log = ('/var/lib/ambari-agent/data/output-' || CAST(task_id AS VARCHAR(20)) || '.txt') WHERE task_id IN (SELECT task_id FROM host_role_command WHERE output_log IS NULL OR output_log = '' ORDER BY task_id DESC LIMIT 1000);");
- dbAccessor.executeQuery("UPDATE host_role_command SET error_log = ('/var/lib/ambari-agent/data/errors-' || CAST(task_id AS VARCHAR(20)) || '.txt') WHERE task_id IN (SELECT task_id FROM host_role_command WHERE error_log IS NULL OR error_log = '' ORDER BY task_id DESC LIMIT 1000);");
- } else if (dbType.equals(Configuration.MYSQL_DB_NAME)) {
- // MySQL uses a different concatenation operator.
- dbAccessor.executeQuery("UPDATE host_role_command SET output_log = CONCAT('/var/lib/ambari-agent/data/output-', task_id, '.txt') WHERE task_id IN (SELECT task_id FROM host_role_command WHERE output_log IS NULL OR output_log = '' ORDER BY task_id DESC LIMIT 1000);");
- dbAccessor.executeQuery("UPDATE host_role_command SET error_log = CONCAT('/var/lib/ambari-agent/data/errors-', task_id, '.txt') WHERE task_id IN (SELECT task_id FROM host_role_command WHERE error_log IS NULL OR error_log = '' ORDER BY task_id DESC LIMIT 1000);");
- }
-
addAlertingFrameworkDDL();
//service config versions changes
@@ -333,6 +333,64 @@ public class UpgradeCatalog170 extends AbstractUpgradeCatalog {
dbAccessor.executeQuery("INSERT INTO ambari_sequences(sequence_name, "
+ valueColumnName + ") " + "VALUES('alert_current_id_seq', 0)", false);
+
+ // Update historic records with the log paths, but only enough so as to not prolong the upgrade process
+ executeInTransaction(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ HostRoleCommandDAO hostRoleCommandDAO = injector.getInstance(HostRoleCommandDAO.class);
+ EntityManager em = getEntityManagerProvider().get();
+ CriteriaBuilder cb = em.getCriteriaBuilder();
+ CriteriaQuery<HostRoleCommandEntity> cq1 = cb.createQuery(HostRoleCommandEntity.class);
+ CriteriaQuery<HostRoleCommandEntity> cq2 = cb.createQuery(HostRoleCommandEntity.class);
+ Root<HostRoleCommandEntity> hrc1 = cq1.from(HostRoleCommandEntity.class);
+ Root<HostRoleCommandEntity> hrc2 = cq1.from(HostRoleCommandEntity.class);
+
+ // Rather than using Java reflection, which is more susceptible to breaking, use the classname_.field canonical model
+ // that is safer because it exposes the persistent attributes statically.
+ Expression<Long> taskID1 = hrc1.get(HostRoleCommandEntity_.taskId);
+ Expression<Long> taskID2 = hrc2.get(HostRoleCommandEntity_.taskId);
+ Expression<String> outputLog = hrc1.get(HostRoleCommandEntity_.outputLog);
+ Expression<String> errorLog = hrc2.get(HostRoleCommandEntity_.errorLog);
+
+ Predicate p1 = cb.isNull(outputLog);
+ Predicate p2 = cb.equal(outputLog, "");
+ Predicate p1_or_2 = cb.or(p1, p2);
+
+ Predicate p3 = cb.isNull(errorLog);
+ Predicate p4 = cb.equal(errorLog, "");
+ Predicate p3_or_4 = cb.or(p3, p4);
+
+ if (daoUtils == null) {
+ daoUtils = new DaoUtils();
+ }
+
+ // Update output_log
+ cq1.select(hrc1).where(p1_or_2).orderBy(cb.desc(taskID1));
+ TypedQuery<HostRoleCommandEntity> q1 = em.createQuery(cq1);
+ q1.setMaxResults(1000);
+ List<HostRoleCommandEntity> r1 = daoUtils.selectList(q1);
+ for (HostRoleCommandEntity entity : r1) {
+ entity.setOutputLog("/var/lib/ambari-agent/data/output-" + entity.getTaskId() + ".txt");
+ hostRoleCommandDAO.merge(entity);
+ }
+
+ // Update error_log
+ cq2.select(hrc2).where(p3_or_4).orderBy(cb.desc(taskID2));
+ TypedQuery<HostRoleCommandEntity> q2 = em.createQuery(cq2);
+ q2.setMaxResults(1000);
+ List<HostRoleCommandEntity> r2 = daoUtils.selectList(q2);
+ for (HostRoleCommandEntity entity : r2) {
+ entity.setErrorLog("/var/lib/ambari-agent/data/errors-" + entity.getTaskId() + ".txt");
+ hostRoleCommandDAO.merge(entity);
+ }
+ } catch (Exception e) {
+ LOG.warn("Could not populate historic records with output_log and error_log in host_role_command table. ", e);
+ }
+ }
+ });
+
moveGlobalsToEnv();
addEnvContentFields();
addMissingConfigs();
http://git-wip-us.apache.org/repos/asf/ambari/blob/a2086b1f/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java
index a0f1e65..63b64e4 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java
@@ -21,31 +21,19 @@ package org.apache.ambari.server.upgrade;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertNull;
import static junit.framework.Assert.assertTrue;
-import static org.easymock.EasyMock.anyObject;
-import static org.easymock.EasyMock.capture;
-import static org.easymock.EasyMock.createMockBuilder;
-import static org.easymock.EasyMock.createNiceMock;
-import static org.easymock.EasyMock.createStrictMock;
-import static org.easymock.EasyMock.eq;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.expectLastCall;
-import static org.easymock.EasyMock.replay;
-import static org.easymock.EasyMock.verify;
+import static org.easymock.EasyMock.*;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.sql.ResultSet;
import java.sql.SQLException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.orm.DBAccessor;
+import org.apache.ambari.server.orm.dao.DaoUtils;
+import org.apache.ambari.server.orm.entities.HostRoleCommandEntity;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.Config;
@@ -53,18 +41,31 @@ import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.StackId;
import org.easymock.Capture;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
-import com.google.inject.Binder;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.Module;
+import com.google.inject.*;
+import javax.persistence.EntityManager;
+import javax.persistence.EntityTransaction;
+import javax.persistence.TypedQuery;
+import javax.persistence.criteria.*;
+import javax.persistence.metamodel.SingularAttribute;
/**
* UpgradeCatalog170 unit tests.
*/
public class UpgradeCatalog170Test {
+ Provider<EntityManager> entityManagerProvider = createStrictMock(Provider.class);
+ EntityManager entityManager = createStrictMock(EntityManager.class);
+
+ @Before
+ public void init() {
+ reset(entityManagerProvider);
+ expect(entityManagerProvider.get()).andReturn(entityManager).anyTimes();
+ replay(entityManagerProvider);
+ }
+
@Test
public void testExecuteDDLUpdates() throws Exception {
@@ -155,11 +156,23 @@ public class UpgradeCatalog170Test {
Clusters clusters = createStrictMock(Clusters.class);
Config config = createStrictMock(Config.class);
+ EntityTransaction trans = createNiceMock(EntityTransaction.class);
+ CriteriaBuilder cb = createNiceMock(CriteriaBuilder.class);
+ CriteriaQuery<HostRoleCommandEntity> cq = createNiceMock(CriteriaQuery.class);
+ Root<HostRoleCommandEntity> hrc = (Root<HostRoleCommandEntity>) createNiceMock(Root.class);
+ Path<Long> taskId = null;
+ Path<String> outputLog = null;
+ Path<String> errorLog = null;
+ Order o = createNiceMock(Order.class);
+ TypedQuery<HostRoleCommandEntity> q = createNiceMock(TypedQuery.class);
+ List<HostRoleCommandEntity> r = new ArrayList<HostRoleCommandEntity>();
+
Method m = AbstractUpgradeCatalog.class.getDeclaredMethod
("updateConfigurationProperties", String.class, Map.class, boolean.class, boolean.class);
+ Method n = AbstractUpgradeCatalog.class.getDeclaredMethod("getEntityManagerProvider");
UpgradeCatalog170 upgradeCatalog = createMockBuilder(UpgradeCatalog170.class)
- .addMockedMethod(m).createMock();
+ .addMockedMethod(m).addMockedMethod(n).createMock();
Map<String, Cluster> clustersMap = new HashMap<String, Cluster>();
clustersMap.put("c1", cluster);
@@ -191,6 +204,22 @@ public class UpgradeCatalog170Test {
Collections.singletonMap("hbase_regionserver_xmn_ratio", "0.2"), false, false);
expectLastCall();
+ expect(entityManager.getTransaction()).andReturn(trans).anyTimes();
+ expect(entityManager.getCriteriaBuilder()).andReturn(cb).anyTimes();
+ expect(entityManager.createQuery(cq)).andReturn(q).anyTimes();
+ expect(trans.isActive()).andReturn(true).anyTimes();
+ expect(upgradeCatalog.getEntityManagerProvider()).andReturn(entityManagerProvider).anyTimes();
+ expect(cb.createQuery(HostRoleCommandEntity.class)).andReturn(cq).anyTimes();
+ expect(cb.desc(taskId)).andReturn(o).anyTimes();
+ expect(cq.from(HostRoleCommandEntity.class)).andReturn(hrc).anyTimes();
+ expect(cq.select(hrc)).andReturn(cq).anyTimes();
+ expect(cq.where(anyObject(Predicate.class))).andReturn(cq).anyTimes();
+ expect(hrc.get(isA(SingularAttribute.class))).andReturn(taskId).times(2);
+ expect(hrc.get(isA(SingularAttribute.class))).andReturn(outputLog).once();
+ expect(hrc.get(isA(SingularAttribute.class))).andReturn(errorLog).once();
+ expect(q.setMaxResults(1000)).andReturn(q).anyTimes();
+ expect(q.getResultList()).andReturn(r).anyTimes();
+
expect(configuration.getDatabaseUrl()).andReturn(Configuration.JDBC_IN_MEMORY_URL).anyTimes();
expect(injector.getInstance(ConfigHelper.class)).andReturn(configHelper).anyTimes();
expect(injector.getInstance(AmbariManagementController.class)).andReturn(amc).anyTimes();
@@ -204,7 +233,8 @@ public class UpgradeCatalog170Test {
expect(configHelper.findConfigTypesByPropertyName(new StackId("HDP", "2.1"), "content")).andReturn(envDicts).once();
expect(configHelper.getPropertyValueFromStackDefenitions(cluster, "hadoop-env", "content")).andReturn("env file contents").once();
- replay(upgradeCatalog, dbAccessor, configuration, injector, cluster, clusters, amc, config, configHelper);
+ replay(entityManager, trans, upgradeCatalog, cb, cq, hrc, q);
+ replay(dbAccessor, configuration, injector, cluster, clusters, amc, config, configHelper);
Class<?> c = AbstractUpgradeCatalog.class;
Field f = c.getDeclaredField("configuration");