You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2015/03/11 13:01:52 UTC
incubator-lens git commit: LENS-281: Timeline should have same
contract in toProperties() and initFromProperties() (Rajat Khandelwal via
amareshwari)
Repository: incubator-lens
Updated Branches:
refs/heads/master 85b3d9e9e -> 76218cb14
LENS-281: Timeline should have same contract in toProperties() and initFromProperties() (Rajat Khandelwal via amareshwari)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/76218cb1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/76218cb1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/76218cb1
Branch: refs/heads/master
Commit: 76218cb14260d068c1fb941646b72063d28fde7b
Parents: 85b3d9e
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Wed Mar 11 17:31:42 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Wed Mar 11 17:31:42 2015 +0530
----------------------------------------------------------------------
.../lens/cube/metadata/CubeMetastoreClient.java | 5 +-
.../timeline/EndsAndHolesPartitionTimeline.java | 17 +++--
.../metadata/timeline/PartitionTimeline.java | 5 ++
.../timeline/StoreAllPartitionTimeline.java | 1 +
.../timeline/TestPartitionTimelines.java | 74 ++++++++++++++++++++
5 files changed, 97 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/76218cb1/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index 2e03df1..8797bfe 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -172,6 +172,7 @@ public class CubeMetastoreClient {
throws HiveException, LensException {
String storageTableName = MetastoreUtil.getStorageTableName(fact, Storage.getPrefix(storage));
if (get(storageTableName) == null) {
+ log.info("loading timeline from all partitions for storage table: " + storageTableName);
// not found in memory, try loading from table properties.
Table storageTable = getTable(storageTableName);
if (storageTable.getParameters().get(MetastoreUtil.getPartitoinTimelineCachePresenceKey()) == null) {
@@ -256,7 +257,9 @@ public class CubeMetastoreClient {
get(storageTable).get(updatePeriod).put(partitionColumn, PartitionTimelineFactory.get(
CubeMetastoreClient.this, storageTable, updatePeriod, partitionColumn));
}
- return get(storageTable).get(updatePeriod).get(partitionColumn);
+ PartitionTimeline ret = get(storageTable).get(updatePeriod).get(partitionColumn);
+ log.info("ensured entry " + ret);
+ return ret;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/76218cb1/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java
index 57dae9e..d442bc0 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java
@@ -43,7 +43,7 @@ import lombok.NonNull;
@Data
public class EndsAndHolesPartitionTimeline extends PartitionTimeline {
private TimePartition first;
- private TreeSet<TimePartition> holes = new TreeSet<TimePartition>();
+ private TreeSet<TimePartition> holes = Sets.newTreeSet();
private TimePartition latest;
public EndsAndHolesPartitionTimeline(CubeMetastoreClient client, String storageTableName, UpdatePeriod updatePeriod,
@@ -122,10 +122,19 @@ public class EndsAndHolesPartitionTimeline extends PartitionTimeline {
@Override
public boolean initFromProperties(Map<String, String> properties) throws LensException {
- first = TimePartition.of(getUpdatePeriod(), properties.get("first"));
- latest = TimePartition.of(getUpdatePeriod(), properties.get("latest"));
- holes = Sets.newTreeSet();
+ first = null;
+ latest = null;
+ holes.clear();
+ String firstStr = properties.get("first");
+ String latestStr = properties.get("latest");
String holesStr = properties.get("holes");
+ if (!Strings.isNullOrEmpty(firstStr)) {
+ first = TimePartition.of(getUpdatePeriod(), firstStr);
+ }
+ if (!Strings.isNullOrEmpty(latestStr)) {
+ latest = TimePartition.of(getUpdatePeriod(), latestStr);
+ }
+ holes = Sets.newTreeSet();
if (!Strings.isNullOrEmpty(holesStr)) {
for (String hole : properties.get("holes").split("\\s*,\\s*")) {
holes.add(TimePartition.of(getUpdatePeriod(), hole));
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/76218cb1/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java
index 64771d0..30921df 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java
@@ -31,6 +31,7 @@ import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import lombok.Data;
import lombok.NonNull;
+import lombok.extern.apachecommons.CommonsLog;
/**
* Represents the in-memory data structure that represents timeline of all existing partitions for a given storage
@@ -40,6 +41,7 @@ import lombok.NonNull;
* @see org.apache.lens.cube.metadata.timeline.StoreAllPartitionTimeline
*/
@Data
+@CommonsLog
public abstract class PartitionTimeline implements Iterable<TimePartition> {
private final CubeMetastoreClient client;
private final String storageTableName;
@@ -84,7 +86,9 @@ public abstract class PartitionTimeline implements Iterable<TimePartition> {
props.put(entry.getKey().substring(prefix.length()), entry.getValue());
}
}
+ log.info("initializing timeline: " + getStorageTableName() + ", " + getUpdatePeriod() + ", " + getPartCol());
initFromProperties(props);
+ log.info("initialized to " + toProperties());
}
/**
@@ -111,6 +115,7 @@ public abstract class PartitionTimeline implements Iterable<TimePartition> {
}
boolean result = add(getAll());
all = null;
+ log.info("after commit batch additions, timeline is: " + this);
return result;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/76218cb1/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java
index 8a9450c..c79c9ce 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java
@@ -78,6 +78,7 @@ public class StoreAllPartitionTimeline extends PartitionTimeline {
@Override
public boolean initFromProperties(Map<String, String> properties) throws LensException {
+ allPartitions.clear();
String partitionsStr = properties.get("partitions");
if (partitionsStr == null) {
return true;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/76218cb1/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java
new file mode 100644
index 0000000..0027e64
--- /dev/null
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.metadata.timeline;
+
+import java.util.Date;
+import java.util.Map;
+
+import org.apache.lens.api.LensException;
+import org.apache.lens.cube.metadata.CubeMetastoreClient;
+import org.apache.lens.cube.metadata.TimePartition;
+import org.apache.lens.cube.metadata.UpdatePeriod;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class TestPartitionTimelines {
+ CubeMetastoreClient client = null;
+ private static final String TABLE_NAME = "storage_fact";
+ private static final UpdatePeriod PERIOD = UpdatePeriod.HOURLY;
+ private static final String PART_COL = "pt";
+ private static final Date DATE = new Date();
+
+ @Test
+ public void testPropertiesContractsForAllSubclasses() throws LensException {
+ testPropertiesContract(StoreAllPartitionTimeline.class);
+ testPropertiesContract(EndsAndHolesPartitionTimeline.class);
+ }
+
+ private <T extends PartitionTimeline> T getInstance(Class<T> clz) {
+ try {
+ return clz.getConstructor(CubeMetastoreClient.class, String.class, UpdatePeriod.class, String.class)
+ .newInstance(client, TABLE_NAME, PERIOD, PART_COL);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return null;
+ }
+
+ private <T extends PartitionTimeline> void testPropertiesContract(Class<T> clz) throws LensException {
+ T inst1 = getInstance(clz);
+ T inst2 = getInstance(clz);
+ Map<String, String> props = inst1.toProperties();
+ Assert.assertTrue(inst2.initFromProperties(props));
+ Assert.assertEquals(inst1, inst2);
+ Assert.assertTrue(inst1.isEmpty());
+ Assert.assertTrue(inst2.isEmpty());
+ Assert.assertTrue(inst1.add(TimePartition.of(PERIOD, DATE)));
+ Assert.assertFalse(inst1.equals(inst2));
+ Assert.assertTrue(inst2.add(TimePartition.of(PERIOD, DATE)));
+ Assert.assertTrue(inst1.isConsistent());
+ Assert.assertTrue(inst2.isConsistent());
+ Assert.assertEquals(inst1, inst2);
+ Assert.assertTrue(inst2.initFromProperties(props));
+ Assert.assertFalse(inst1.equals(inst2));
+ Assert.assertTrue(inst2.initFromProperties(inst1.toProperties()));
+ Assert.assertEquals(inst1, inst2);
+ }
+}