You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2015/05/07 19:37:51 UTC
ambari git commit: AMBARI-10987. Error loading deferred resources
when making metric API requests with null padding. (swagle)
Repository: ambari
Updated Branches:
refs/heads/trunk 7b822e42b -> 34c1e9b2c
AMBARI-10987. Error loading deferred resources when making metric API requests with null padding. (swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/34c1e9b2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/34c1e9b2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/34c1e9b2
Branch: refs/heads/trunk
Commit: 34c1e9b2c7726a92d18de68d5b292cb87e564740
Parents: 7b822e4
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Thu May 7 10:34:33 2015 -0700
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Thu May 7 10:37:50 2015 -0700
----------------------------------------------------------------------
.../metrics/timeline/PhoenixHBaseAccessor.java | 2 +
.../query/render/MetricsPaddingRenderer.java | 46 +++++++++++++
.../api/resources/BaseResourceDefinition.java | 5 ++
.../MetricsDownsamplingMethodFactory.java | 14 ++--
.../metrics/MetricsPropertyProvider.java | 20 ++++--
.../metrics/MetricsPropertyProviderProxy.java | 13 ++++
.../render/MetricsPaddingRendererTest.java | 69 ++++++++++++++++++++
7 files changed, 155 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/34c1e9b2/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
index e27d9a9..7258cad 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
@@ -309,6 +309,8 @@ public class PhoenixHBaseAccessor {
clusterDailyTtl));
conn.commit();
+
+ LOG.info("Metrics schema initialized.");
} catch (SQLException sql) {
if (sql.getErrorCode() ==
SQLExceptionCode.SET_UNSUPPORTED_PROP_ON_ALTER_TABLE.getErrorCode()) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/34c1e9b2/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/MetricsPaddingRenderer.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/MetricsPaddingRenderer.java b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/MetricsPaddingRenderer.java
new file mode 100644
index 0000000..ff776c7
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/MetricsPaddingRenderer.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.api.query.render;
+
+import org.apache.ambari.server.api.query.QueryInfo;
+import org.apache.ambari.server.api.util.TreeNode;
+import java.util.Set;
+
+import static org.apache.ambari.server.controller.metrics.MetricsPaddingMethod.PADDING_STRATEGY;
+
+public class MetricsPaddingRenderer extends DefaultRenderer {
+ PADDING_STRATEGY paddingMethod = PADDING_STRATEGY.ZEROS;
+
+ public MetricsPaddingRenderer(String paddingMethod) {
+ if (paddingMethod.equalsIgnoreCase("null_padding")) {
+ this.paddingMethod = PADDING_STRATEGY.NULLS;
+ } else if (paddingMethod.equalsIgnoreCase("no_padding")) {
+ this.paddingMethod = PADDING_STRATEGY.NONE;
+ }
+ }
+
+ @Override
+ public TreeNode<Set<String>> finalizeProperties(TreeNode<QueryInfo> queryProperties,
+ boolean isCollection) {
+ Set<String> properties = queryProperties.getObject().getProperties();
+ if (properties != null) {
+ properties.add("params/padding/" + paddingMethod.name());
+ }
+ return super.finalizeProperties(queryProperties, isCollection);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/34c1e9b2/ambari-server/src/main/java/org/apache/ambari/server/api/resources/BaseResourceDefinition.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/BaseResourceDefinition.java b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/BaseResourceDefinition.java
index e63f7eb..c8bf87d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/BaseResourceDefinition.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/BaseResourceDefinition.java
@@ -20,6 +20,7 @@ package org.apache.ambari.server.api.resources;
import org.apache.ambari.server.api.query.render.DefaultRenderer;
+import org.apache.ambari.server.api.query.render.MetricsPaddingRenderer;
import org.apache.ambari.server.api.query.render.MinimalRenderer;
import org.apache.ambari.server.api.query.render.Renderer;
import org.apache.ambari.server.api.services.Request;
@@ -125,6 +126,10 @@ public abstract class BaseResourceDefinition implements ResourceDefinition {
return new DefaultRenderer();
} else if (name.equals("minimal")) {
return new MinimalRenderer();
+ } else if (name.contains("null_padding")
+ || name.contains("no_padding")
+ || name.contains("zero_padding")) {
+ return new MetricsPaddingRenderer(name);
} else {
throw new IllegalArgumentException("Invalid renderer name: " + name +
" for resource of type: " + m_type);
http://git-wip-us.apache.org/repos/asf/ambari/blob/34c1e9b2/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsDownsamplingMethodFactory.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsDownsamplingMethodFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsDownsamplingMethodFactory.java
index a53e26d..2d22643 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsDownsamplingMethodFactory.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsDownsamplingMethodFactory.java
@@ -18,7 +18,6 @@
package org.apache.ambari.server.controller.metrics;
import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Map;
@@ -64,9 +63,9 @@ class MetricNoDownsampling extends MetricsDownsamplingMethod {
class MetricsAveragePerSecondDownsampling extends MetricsDownsamplingMethod {
class Accumulo {
public long ts;
- public double val;
+ public Double val;
- public Accumulo(long t, double v) {
+ public Accumulo(long t, Double v) {
this.ts = t;
this.val = v;
}
@@ -79,7 +78,7 @@ class MetricsAveragePerSecondDownsampling extends MetricsDownsamplingMethod {
if (ci.hasNext()) {
Map.Entry<Long, Double> e0 = ci.next();
long t0 = e0.getKey() / 1000;
- double s0 = e0.getValue();
+ Double s0 = e0.getValue();
int nSamples = 1;
while(ci.hasNext()) {
@@ -87,18 +86,19 @@ class MetricsAveragePerSecondDownsampling extends MetricsDownsamplingMethod {
long t = e0.getKey() / 1000;
if (t != t0) {
- cache.add(new Accumulo(t0, dataTransferMethod.getData(s0 / nSamples)));
+ cache.add(new Accumulo(t0, s0 != null ? dataTransferMethod.getData(s0 / nSamples) : null));
t0 = t;
s0 = e0.getValue();
nSamples = 1;
} else {
- s0 += e0.getValue();
+ // Zero value contributes to nothing for downsampling method
+ s0 += e0.getValue() != null ? e0.getValue() : 0;
nSamples++;
}
}
//Add the last entry into the cache
- cache.add(new Accumulo(t0, dataTransferMethod.getData(s0 / nSamples)));
+ cache.add(new Accumulo(t0, s0 != null ? dataTransferMethod.getData(s0 / nSamples) : null));
}
Number[][] datapointsArray = new Number[cache.size()][2];
http://git-wip-us.apache.org/repos/asf/ambari/blob/34c1e9b2/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPropertyProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPropertyProvider.java
index 8360f5e..569bb17 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPropertyProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPropertyProvider.java
@@ -52,7 +52,9 @@ public abstract class MetricsPropertyProvider extends AbstractPropertyProvider {
protected final ComponentSSLConfiguration configuration;
- protected MetricsPaddingMethod metricsPaddingMethod =
+ protected MetricsPaddingMethod metricsPaddingMethod;
+
+ private static final MetricsPaddingMethod DEFAULT_PADDING_METHOD =
new MetricsPaddingMethod(MetricsPaddingMethod.PADDING_STRATEGY.ZEROS);
protected MetricsPropertyProvider(Map<String, Map<String,
@@ -129,12 +131,16 @@ public abstract class MetricsPropertyProvider extends AbstractPropertyProvider {
return resources;
}
- Map<String, Object> predicateProperties = PredicateHelper.getProperties(predicate);
- if (predicateProperties != null && predicateProperties.keySet().contains(ZERO_PADDING_PARAM)) {
- String paddingStr = (String) predicateProperties.get(ZERO_PADDING_PARAM);
- for (MetricsPaddingMethod.PADDING_STRATEGY strategy : MetricsPaddingMethod.PADDING_STRATEGY.values()) {
- if (paddingStr.equalsIgnoreCase(strategy.name())) {
- metricsPaddingMethod = new MetricsPaddingMethod(strategy);
+ // Re-initialize in case of reuse.
+ metricsPaddingMethod = DEFAULT_PADDING_METHOD;
+
+ Set<String> requestPropertyIds = request.getPropertyIds();
+ if (requestPropertyIds != null && !requestPropertyIds.isEmpty()) {
+ for (String propertyId : requestPropertyIds) {
+ if (propertyId.startsWith(ZERO_PADDING_PARAM)) {
+ String paddingStrategyStr = propertyId.substring(ZERO_PADDING_PARAM.length() + 1);
+ metricsPaddingMethod = new MetricsPaddingMethod(
+ MetricsPaddingMethod.PADDING_STRATEGY.valueOf(paddingStrategyStr));
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/34c1e9b2/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPropertyProviderProxy.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPropertyProviderProxy.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPropertyProviderProxy.java
index 24f1851..7fa9c5e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPropertyProviderProxy.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPropertyProviderProxy.java
@@ -35,6 +35,8 @@ import org.apache.ambari.server.controller.spi.SystemException;
import org.apache.ambari.server.controller.utilities.StreamProvider;
import java.util.Map;
import java.util.Set;
+
+import static org.apache.ambari.server.controller.metrics.MetricsPaddingMethod.ZERO_PADDING_PARAM;
import static org.apache.ambari.server.controller.metrics.MetricsServiceProvider.MetricsService;
import static org.apache.ambari.server.controller.metrics.MetricsServiceProvider.MetricsService.GANGLIA;
import static org.apache.ambari.server.controller.metrics.MetricsServiceProvider.MetricsService.TIMELINE_METRICS;
@@ -81,6 +83,17 @@ public class MetricsPropertyProviderProxy extends AbstractPropertyProvider {
}
}
+ @Override
+ public Set<String> checkPropertyIds(Set<String> propertyIds) {
+ Set<String> checkedPropertyIds = super.checkPropertyIds(propertyIds);
+ for (String propertyId : checkedPropertyIds) {
+ if (propertyId.startsWith(ZERO_PADDING_PARAM)) {
+ checkedPropertyIds.remove(propertyId);
+ }
+ }
+ return checkedPropertyIds;
+ }
+
private void createHostPropertyProviders(Map<String, Map<String, PropertyInfo>> componentPropertyInfoMap,
StreamProvider streamProvider,
ComponentSSLConfiguration configuration,
http://git-wip-us.apache.org/repos/asf/ambari/blob/34c1e9b2/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/MetricsPaddingRendererTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/MetricsPaddingRendererTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/MetricsPaddingRendererTest.java
new file mode 100644
index 0000000..dafc605
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/MetricsPaddingRendererTest.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.api.query.render;
+
+import org.apache.ambari.server.api.query.QueryInfo;
+import org.apache.ambari.server.api.resources.ServiceResourceDefinition;
+import org.apache.ambari.server.api.util.TreeNode;
+import org.apache.ambari.server.api.util.TreeNodeImpl;
+import org.apache.ambari.server.controller.spi.Resource;
+import org.apache.ambari.server.controller.spi.Schema;
+import org.apache.ambari.server.controller.spi.SchemaFactory;
+import org.junit.Test;
+import java.util.HashSet;
+import java.util.Set;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class MetricsPaddingRendererTest {
+ @Test
+ public void testFinalizeProperties__NullPadding_property() {
+ SchemaFactory schemaFactory = createNiceMock(SchemaFactory.class);
+ Schema schema = createNiceMock(Schema.class);
+
+ // schema expectations
+ expect(schemaFactory.getSchema(Resource.Type.Service)).andReturn(schema).anyTimes();
+ expect(schema.getKeyPropertyId(Resource.Type.Service)).andReturn("ServiceInfo/service_name").anyTimes();
+ expect(schema.getKeyPropertyId(Resource.Type.Cluster)).andReturn("ServiceInfo/cluster_name").anyTimes();
+
+ replay(schemaFactory, schema);
+
+ HashSet<String> serviceProperties = new HashSet<String>();
+ serviceProperties.add("foo/bar");
+ QueryInfo rootQuery = new QueryInfo(new ServiceResourceDefinition(), serviceProperties);
+ TreeNode<QueryInfo> queryTree = new TreeNodeImpl<QueryInfo>(null, rootQuery, "Service");
+
+ MetricsPaddingRenderer renderer = new MetricsPaddingRenderer("null_padding");
+ renderer.init(schemaFactory);
+ TreeNode<Set<String>> propertyTree = renderer.finalizeProperties(queryTree, false);
+
+ assertEquals(4, propertyTree.getObject().size());
+ assertTrue(propertyTree.getObject().contains("ServiceInfo/service_name"));
+ assertTrue(propertyTree.getObject().contains("ServiceInfo/cluster_name"));
+ assertTrue(propertyTree.getObject().contains("foo/bar"));
+ assertTrue(propertyTree.getObject().contains("params/padding/NULLS"));
+ assertEquals(0, propertyTree.getChildren().size());
+
+
+ verify(schemaFactory, schema);
+ }
+}