You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2015/02/15 17:30:50 UTC

[1/2] ambari git commit: AMBARI-9652 AMS : Hosts metrics - "mem_shared" metrics array contains "null" (dsen)

Repository: ambari
Updated Branches:
  refs/heads/trunk 075acd71f -> 1be56cb9e


AMBARI-9652 AMS : Hosts metrics - "mem_shared" metrics array contains "null" (dsen)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/58597876
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/58597876
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/58597876

Branch: refs/heads/trunk
Commit: 58597876ef6893ef32a59f03be36c03e7dd7cb10
Parents: 075acd7
Author: Dmytro Sen <ds...@apache.org>
Authored: Sun Feb 15 18:22:08 2015 +0200
Committer: Dmytro Sen <ds...@apache.org>
Committed: Sun Feb 15 18:22:08 2015 +0200

----------------------------------------------------------------------
 .../metrics2/sink/timeline/TimelineMetric.java  |  8 ++-
 .../deserialize/IgnoringNullsLinkedHashMap.java | 34 ++++++++++++
 .../IgnoringNullsLinkedHashMapTest.java         | 39 ++++++++++++++
 .../webapp/AHSWebApp.java                       |  5 +-
 .../webapp/YarnAMSJacksonJaxbJsonProvider.java  | 57 ++++++++++++++++++++
 .../metrics2/sink/timeline/TimelineMetric.java  | 10 ++--
 .../deserialize/IgnoringNullsLinkedHashMap.java | 35 ++++++++++++
 7 files changed, 180 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/58597876/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java
index 68b4be8..36e40fa 100644
--- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java
+++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java
@@ -19,6 +19,10 @@ package org.apache.hadoop.metrics2.sink.timeline;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.metrics2.sink.timeline.deserialize
+  .IgnoringNullsLinkedHashMap;
+import org.codehaus.jackson.map.annotate.JsonDeserialize;
+
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlElement;
@@ -39,7 +43,9 @@ public class TimelineMetric implements Comparable<TimelineMetric> {
   private long timestamp;
   private long startTime;
   private String type;
-  private Map<Long, Double> metricValues = new TreeMap<Long, Double>();
+
+  @JsonDeserialize(as = IgnoringNullsLinkedHashMap.class)
+  private Map<Long, Double> metricValues = new IgnoringNullsLinkedHashMap<Long, Double>();
 
   @XmlElement(name = "metricname")
   public String getMetricName() {

http://git-wip-us.apache.org/repos/asf/ambari/blob/58597876/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/deserialize/IgnoringNullsLinkedHashMap.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/deserialize/IgnoringNullsLinkedHashMap.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/deserialize/IgnoringNullsLinkedHashMap.java
new file mode 100644
index 0000000..331f2b3
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/deserialize/IgnoringNullsLinkedHashMap.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.metrics2.sink.timeline.deserialize;
+
+import java.util.LinkedHashMap;
+import java.util.TreeMap;
+
+/**
+ * This Map does not store neither null values. When null value is
+ * inserted it is ignored.
+ */
+public class IgnoringNullsLinkedHashMap<K, V> extends TreeMap<K, V> {
+  @Override
+  public V put(K key, V value) {
+    if (value == null) return null;
+
+    return super.put(key, value);
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/58597876/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/deserialize/IgnoringNullsLinkedHashMapTest.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/deserialize/IgnoringNullsLinkedHashMapTest.java b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/deserialize/IgnoringNullsLinkedHashMapTest.java
new file mode 100644
index 0000000..0156b87
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-common/src/test/java/org/apache/hadoop/metrics2/sink/timeline/deserialize/IgnoringNullsLinkedHashMapTest.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.metrics2.sink.timeline.deserialize;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class IgnoringNullsLinkedHashMapTest {
+
+  @Test
+  public void testPut() throws Exception {
+    IgnoringNullsLinkedHashMap<Integer, String> map
+      = new IgnoringNullsLinkedHashMap<Integer, String>();
+
+    map.put(1, null);
+    map.put(2, "two");
+    map.put(3, null);
+    map.put(4, "four");
+    map.put(5, "five");
+
+    assertEquals(3,map.size());
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/58597876/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java
index 8cff741..995f0f0 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java
@@ -25,7 +25,6 @@ import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.
 import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
 import org.apache.hadoop.yarn.webapp.WebApp;
-import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
 import org.apache.hadoop.yarn.webapp.YarnWebParams;
 
 public class AHSWebApp extends WebApp implements YarnWebParams {
@@ -43,7 +42,7 @@ public class AHSWebApp extends WebApp implements YarnWebParams {
 
   @Override
   public void setup() {
-    bind(YarnJacksonJaxbJsonProvider.class);
+    bind(YarnAMSJacksonJaxbJsonProvider.class);
     bind(AHSWebServices.class);
     bind(TimelineWebServices.class);
     bind(GenericExceptionHandler.class);
@@ -60,4 +59,4 @@ public class AHSWebApp extends WebApp implements YarnWebParams {
       pajoin("/logs", NM_NODENAME, CONTAINER_ID, ENTITY_STRING, APP_OWNER,
         CONTAINER_LOG_TYPE), AHSController.class, "logs");
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/58597876/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/YarnAMSJacksonJaxbJsonProvider.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/YarnAMSJacksonJaxbJsonProvider.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/YarnAMSJacksonJaxbJsonProvider.java
new file mode 100644
index 0000000..1211a37
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/YarnAMSJacksonJaxbJsonProvider.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp;
+
+import com.google.inject.Singleton;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.codehaus.jackson.jaxrs.JacksonJaxbJsonProvider;
+import org.codehaus.jackson.map.AnnotationIntrospector;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+import org.codehaus.jackson.map.introspect.JacksonAnnotationIntrospector;
+import org.codehaus.jackson.xc.JaxbAnnotationIntrospector;
+
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.ext.Provider;
+
+/**
+ *
+ */
+@Singleton
+@Provider
+@InterfaceStability.Unstable
+@InterfaceAudience.Private
+public class YarnAMSJacksonJaxbJsonProvider extends JacksonJaxbJsonProvider {
+
+  public YarnAMSJacksonJaxbJsonProvider() {
+    super();
+  }
+
+  @Override
+  public ObjectMapper locateMapper(Class<?> type, MediaType mediaType) {
+    ObjectMapper mapper = super.locateMapper(type, mediaType);
+    configObjectMapper(mapper);
+    return mapper;
+  }
+
+  public static void configObjectMapper(ObjectMapper mapper) {
+    mapper.setSerializationInclusion(JsonSerialize.Inclusion.NON_NULL);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/58597876/ambari-server/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java b/ambari-server/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java
index 843aecd..ce30033 100644
--- a/ambari-server/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java
+++ b/ambari-server/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java
@@ -17,15 +17,15 @@
  */
 package org.apache.hadoop.metrics2.sink.timeline;
 
+import org.apache.hadoop.metrics2.sink.timeline.deserialize
+  .IgnoringNullsLinkedHashMap;
+import org.codehaus.jackson.map.annotate.JsonDeserialize;
+
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
-import java.text.DecimalFormat;
-import java.util.Collections;
-import java.util.HashSet;
 import java.util.Map;
-import java.util.Set;
 import java.util.TreeMap;
 
 @XmlRootElement(name = "metric")
@@ -39,6 +39,8 @@ public class TimelineMetric implements Comparable<TimelineMetric> {
   private long timestamp;
   private long startTime;
   private String type;
+
+  @JsonDeserialize(as = IgnoringNullsLinkedHashMap.class)
   private Map<Long, Double> metricValues = new TreeMap<Long, Double>();
 
   @XmlElement(name = "metricname")

http://git-wip-us.apache.org/repos/asf/ambari/blob/58597876/ambari-server/src/main/java/org/apache/hadoop/metrics2/sink/timeline/deserialize/IgnoringNullsLinkedHashMap.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/hadoop/metrics2/sink/timeline/deserialize/IgnoringNullsLinkedHashMap.java b/ambari-server/src/main/java/org/apache/hadoop/metrics2/sink/timeline/deserialize/IgnoringNullsLinkedHashMap.java
new file mode 100644
index 0000000..c31a64e
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/hadoop/metrics2/sink/timeline/deserialize/IgnoringNullsLinkedHashMap.java
@@ -0,0 +1,35 @@
+ /**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.metrics2.sink.timeline.deserialize;
+
+
+ import java.util.LinkedHashMap;
+
+ /**
+ * This Map does not store neither null values. When null value is
+ * inserted it is ignored.
+ */
+public class IgnoringNullsLinkedHashMap<K, V> extends LinkedHashMap<K, V> {
+  @Override
+  public V put(K key, V value) {
+    if (value == null) return null;
+
+    return super.put(key, value);
+  }
+}


[2/2] ambari git commit: AMBARI-9643 Zookeeper Client shell fails to start in kerberized cluster (dsen)

Posted by ds...@apache.org.
AMBARI-9643 Zookeeper Client shell fails to start in kerberized cluster (dsen)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1be56cb9
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1be56cb9
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1be56cb9

Branch: refs/heads/trunk
Commit: 1be56cb9eee19b313ea202bd5cfa6d5a801b24e0
Parents: 5859787
Author: Dmytro Sen <ds...@apache.org>
Authored: Sun Feb 15 18:22:57 2015 +0200
Committer: Dmytro Sen <ds...@apache.org>
Committed: Sun Feb 15 18:22:57 2015 +0200

----------------------------------------------------------------------
 .../3.4.5.2.0/package/scripts/zookeeper_server.py        | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/1be56cb9/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_server.py b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_server.py
index 59bd9fd..e402eaa 100644
--- a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_server.py
+++ b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_server.py
@@ -33,7 +33,7 @@ from resource_management.core.shell import call
 
 from zookeeper import zookeeper
 from zookeeper_service import zookeeper_service
-
+import random
 
 @retry(times=10, sleep_time=2, err_class=Fail)
 def call_and_match_output(command, regex_expression, err_message):
@@ -81,12 +81,13 @@ class ZookeeperServer(Script):
     Logger.info("Executing Rolling Upgrade post-restart")
     import params
     env.set_params(params)
-
+    zk_server_host = random.choice(params.zookeeper_hosts)
+    cli_shell = format("{zk_cli_shell} -server {zk_server_host}:{client_port}")
     # Ensure that a quorum is still formed.
     unique = get_unique_id_and_date()
-    create_command = format("echo 'create /{unique} mydata' | {zk_cli_shell}")
-    list_command = format("echo 'ls /' | {zk_cli_shell}")
-    delete_command = format("echo 'delete /{unique} ' | {zk_cli_shell}")
+    create_command = format("echo 'create /{unique} mydata' | {cli_shell}")
+    list_command = format("echo 'ls /' | {cli_shell}")
+    delete_command = format("echo 'delete /{unique} ' | {cli_shell}")
 
     quorum_err_message = "Failed to establish zookeeper quorum"
     call_and_match_output(create_command, 'Created', quorum_err_message)