You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2018/02/23 19:44:52 UTC
[11/50] [abbrv] hadoop git commit: YARN-7919. Refactor
timelineservice-hbase module into submodules. Contributed by Haibo Chen.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKey.java
new file mode 100644
index 0000000..fb1f774
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKey.java
@@ -0,0 +1,290 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication;
+
+import java.util.List;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverterToString;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+
+/**
+ * Represents a rowkey for the sub app table.
+ */
+public class SubApplicationRowKey {
+ private final String subAppUserId;
+ private final String clusterId;
+ private final String entityType;
+ private final Long entityIdPrefix;
+ private final String entityId;
+ private final String userId;
+ private final SubApplicationRowKeyConverter subAppRowKeyConverter =
+ new SubApplicationRowKeyConverter();
+
+ public SubApplicationRowKey(String subAppUserId, String clusterId,
+ String entityType, Long entityIdPrefix, String entityId, String userId) {
+ this.subAppUserId = subAppUserId;
+ this.clusterId = clusterId;
+ this.entityType = entityType;
+ this.entityIdPrefix = entityIdPrefix;
+ this.entityId = entityId;
+ this.userId = userId;
+ }
+
+ public String getClusterId() {
+ return clusterId;
+ }
+
+ public String getSubAppUserId() {
+ return subAppUserId;
+ }
+
+ public String getEntityType() {
+ return entityType;
+ }
+
+ public String getEntityId() {
+ return entityId;
+ }
+
+ public Long getEntityIdPrefix() {
+ return entityIdPrefix;
+ }
+
+ public String getUserId() {
+ return userId;
+ }
+
+ /**
+ * Constructs a row key for the sub app table as follows:
+ * {@code subAppUserId!clusterId!entityType
+ * !entityPrefix!entityId!userId}.
+ * Typically used while querying a specific sub app.
+ *
+ * subAppUserId is usually the doAsUser.
+ * userId is the yarn user that the AM runs as.
+ *
+ * @return byte array with the row key.
+ */
+ public byte[] getRowKey() {
+ return subAppRowKeyConverter.encode(this);
+ }
+
+ /**
+ * Given the raw row key as bytes, returns the row key as an object.
+ *
+ * @param rowKey byte representation of row key.
+ * @return An <cite>SubApplicationRowKey</cite> object.
+ */
+ public static SubApplicationRowKey parseRowKey(byte[] rowKey) {
+ return new SubApplicationRowKeyConverter().decode(rowKey);
+ }
+
+ /**
+ * Constructs a row key for the sub app table as follows:
+ * <p>
+ * {@code subAppUserId!clusterId!
+ * entityType!entityIdPrefix!entityId!userId}.
+ *
+ * subAppUserId is usually the doAsUser.
+ * userId is the yarn user that that the AM runs as.
+ *
+ * </p>
+ *
+ * @return String representation of row key.
+ */
+ public String getRowKeyAsString() {
+ return subAppRowKeyConverter.encodeAsString(this);
+ }
+
+ /**
+ * Given the encoded row key as string, returns the row key as an object.
+ *
+ * @param encodedRowKey String representation of row key.
+ * @return A <cite>SubApplicationRowKey</cite> object.
+ */
+ public static SubApplicationRowKey parseRowKeyFromString(
+ String encodedRowKey) {
+ return new SubApplicationRowKeyConverter().decodeFromString(encodedRowKey);
+ }
+
+ /**
+ * Encodes and decodes row key for sub app table.
+ * The row key is of the form :
+ * subAppUserId!clusterId!flowRunId!appId!entityType!entityId!userId
+ *
+ * subAppUserId is usually the doAsUser.
+ * userId is the yarn user that the AM runs as.
+ *
+ * <p>
+ */
+ final private static class SubApplicationRowKeyConverter
+ implements KeyConverter<SubApplicationRowKey>,
+ KeyConverterToString<SubApplicationRowKey> {
+
+ private SubApplicationRowKeyConverter() {
+ }
+
+ /**
+ * sub app row key is of the form
+ * subAppUserId!clusterId!entityType!entityPrefix!entityId!userId
+ * w. each segment separated by !.
+ *
+ * subAppUserId is usually the doAsUser.
+ * userId is the yarn user that the AM runs as.
+ *
+ * The sizes below indicate sizes of each one of these
+ * segments in sequence. clusterId, subAppUserId, entityType,
+ * entityId and userId are strings.
+ * entity prefix is a long hence 8 bytes in size. Strings are
+ * variable in size (i.e. end whenever separator is encountered).
+ * This is used while decoding and helps in determining where to split.
+ */
+ private static final int[] SEGMENT_SIZES = {Separator.VARIABLE_SIZE,
+ Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
+ Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE};
+
+ /*
+ * (non-Javadoc)
+ *
+ * Encodes SubApplicationRowKey object into a byte array with each
+ * component/field in SubApplicationRowKey separated by
+ * Separator#QUALIFIERS.
+ * This leads to an sub app table row key of the form
+ * subAppUserId!clusterId!entityType!entityPrefix!entityId!userId
+ *
+ * subAppUserId is usually the doAsUser.
+ * userId is the yarn user that the AM runs as.
+ *
+ * If entityType in passed SubApplicationRowKey object is null (and the
+ * fields preceding it are not null i.e. clusterId, subAppUserId), this
+ * returns a row key prefix of the form subAppUserId!clusterId!
+ * If entityId in SubApplicationRowKey is null
+ * (other components are not null), this returns a row key prefix
+ * of the form subAppUserId!clusterId!entityType!
+ *
+ * @see org.apache.hadoop.yarn.server.timelineservice.storage.common
+ * .KeyConverter#encode(java.lang.Object)
+ */
+ @Override
+ public byte[] encode(SubApplicationRowKey rowKey) {
+ byte[] subAppUser = Separator.encode(rowKey.getSubAppUserId(),
+ Separator.SPACE, Separator.TAB, Separator.QUALIFIERS);
+ byte[] cluster = Separator.encode(rowKey.getClusterId(), Separator.SPACE,
+ Separator.TAB, Separator.QUALIFIERS);
+ byte[] first = Separator.QUALIFIERS.join(subAppUser, cluster);
+ if (rowKey.getEntityType() == null) {
+ return first;
+ }
+ byte[] entityType = Separator.encode(rowKey.getEntityType(),
+ Separator.SPACE, Separator.TAB, Separator.QUALIFIERS);
+
+ if (rowKey.getEntityIdPrefix() == null) {
+ return Separator.QUALIFIERS.join(first, entityType,
+ Separator.EMPTY_BYTES);
+ }
+
+ byte[] entityIdPrefix = Bytes.toBytes(rowKey.getEntityIdPrefix());
+
+ if (rowKey.getEntityId() == null) {
+ return Separator.QUALIFIERS.join(first, entityType, entityIdPrefix,
+ Separator.EMPTY_BYTES);
+ }
+
+ byte[] entityId = Separator.encode(rowKey.getEntityId(), Separator.SPACE,
+ Separator.TAB, Separator.QUALIFIERS);
+
+ byte[] userId = Separator.encode(rowKey.getUserId(),
+ Separator.SPACE, Separator.TAB, Separator.QUALIFIERS);
+
+ byte[] second = Separator.QUALIFIERS.join(entityType, entityIdPrefix,
+ entityId, userId);
+
+ return Separator.QUALIFIERS.join(first, second);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * Decodes a sub application row key of the form
+ * subAppUserId!clusterId!entityType!entityPrefix!entityId!userId
+ *
+ * subAppUserId is usually the doAsUser.
+ * userId is the yarn user that the AM runs as.
+ *
+ * represented in byte format
+ * and converts it into an SubApplicationRowKey object.
+ *
+ * @see org.apache.hadoop.yarn.server.timelineservice.storage.common
+ * .KeyConverter#decode(byte[])
+ */
+ @Override
+ public SubApplicationRowKey decode(byte[] rowKey) {
+ byte[][] rowKeyComponents =
+ Separator.QUALIFIERS.split(rowKey, SEGMENT_SIZES);
+ if (rowKeyComponents.length != 6) {
+ throw new IllegalArgumentException(
+ "the row key is not valid for " + "a sub app");
+ }
+ String subAppUserId =
+ Separator.decode(Bytes.toString(rowKeyComponents[0]),
+ Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
+ String clusterId = Separator.decode(Bytes.toString(rowKeyComponents[1]),
+ Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
+ String entityType = Separator.decode(Bytes.toString(rowKeyComponents[2]),
+ Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
+
+ Long entityPrefixId = Bytes.toLong(rowKeyComponents[3]);
+
+ String entityId = Separator.decode(Bytes.toString(rowKeyComponents[4]),
+ Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
+ String userId =
+ Separator.decode(Bytes.toString(rowKeyComponents[5]),
+ Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
+
+ return new SubApplicationRowKey(subAppUserId, clusterId, entityType,
+ entityPrefixId, entityId, userId);
+ }
+
+ @Override
+ public String encodeAsString(SubApplicationRowKey key) {
+ if (key.subAppUserId == null || key.clusterId == null
+ || key.entityType == null || key.entityIdPrefix == null
+ || key.entityId == null || key.userId == null) {
+ throw new IllegalArgumentException();
+ }
+ return TimelineReaderUtils.joinAndEscapeStrings(
+ new String[] {key.subAppUserId, key.clusterId, key.entityType,
+ key.entityIdPrefix.toString(), key.entityId, key.userId});
+ }
+
+ @Override
+ public SubApplicationRowKey decodeFromString(String encodedRowKey) {
+ List<String> split = TimelineReaderUtils.split(encodedRowKey);
+ if (split == null || split.size() != 6) {
+ throw new IllegalArgumentException(
+ "Invalid row key for sub app table.");
+ }
+ Long entityIdPrefix = Long.valueOf(split.get(3));
+ return new SubApplicationRowKey(split.get(0), split.get(1),
+ split.get(2), entityIdPrefix, split.get(4), split.get(5));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKeyPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKeyPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKeyPrefix.java
new file mode 100644
index 0000000..0c04959
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKeyPrefix.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication;
+
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.RowKeyPrefix;
+
+/**
+ * Represents a partial rowkey without the entityId or without entityType and
+ * entityId for the sub application table.
+ *
+ */
+public class SubApplicationRowKeyPrefix extends SubApplicationRowKey
+ implements RowKeyPrefix<SubApplicationRowKey> {
+
+ /**
+ * Creates a prefix which generates the following rowKeyPrefixes for the sub
+ * application table:
+ * {@code subAppUserId!clusterId!entityType!entityPrefix!userId}.
+ *
+ * @param subAppUserId
+ * identifying the subApp User
+ * @param clusterId
+ * identifying the cluster
+ * @param entityType
+ * which entity type
+ * @param entityIdPrefix
+ * for entityId
+ * @param entityId
+ * for an entity
+ * @param userId
+ * for the user who runs the AM
+ *
+ * subAppUserId is usually the doAsUser.
+ * userId is the yarn user that the AM runs as.
+ *
+ */
+ public SubApplicationRowKeyPrefix(String subAppUserId, String clusterId,
+ String entityType, Long entityIdPrefix, String entityId,
+ String userId) {
+ super(subAppUserId, clusterId, entityType, entityIdPrefix, entityId,
+ userId);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.
+ * RowKeyPrefix#getRowKeyPrefix()
+ */
+ public byte[] getRowKeyPrefix() {
+ return super.getRowKey();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationTable.java
new file mode 100644
index 0000000..de7dd4d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationTable.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication;
+
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+
+/**
+ * The sub application table has column families:
+ * info, config and metrics.
+ * Info stores information about a timeline entity object
+ * config stores configuration data of a timeline entity object
+ * metrics stores the metrics of a timeline entity object
+ *
+ * Example sub application table record:
+ *
+ * <pre>
+ * |-------------------------------------------------------------------------|
+ * | Row | Column Family | Column Family| Column Family|
+ * | key | info | metrics | config |
+ * |-------------------------------------------------------------------------|
+ * | subAppUserId! | id:entityId | metricId1: | configKey1: |
+ * | clusterId! | type:entityType | metricValue1 | configValue1 |
+ * | entityType! | | @timestamp1 | |
+ * | idPrefix!| | | | configKey2: |
+ * | entityId! | created_time: | metricId1: | configValue2 |
+ * | userId | 1392993084018 | metricValue2 | |
+ * | | | @timestamp2 | |
+ * | | i!infoKey: | | |
+ * | | infoValue | metricId1: | |
+ * | | | metricValue1 | |
+ * | | | @timestamp2 | |
+ * | | e!eventId=timestamp= | | |
+ * | | infoKey: | | |
+ * | | eventInfoValue | | |
+ * | | | | |
+ * | | r!relatesToKey: | | |
+ * | | id3=id4=id5 | | |
+ * | | | | |
+ * | | s!isRelatedToKey | | |
+ * | | id7=id9=id6 | | |
+ * | | | | |
+ * | | flowVersion: | | |
+ * | | versionValue | | |
+ * |-------------------------------------------------------------------------|
+ * </pre>
+ */
+public final class SubApplicationTable extends BaseTable<SubApplicationTable> {
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/package-info.java
new file mode 100644
index 0000000..52cc399
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication
+ * contains classes related to implementation for subapplication table.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestCustomApplicationIdConversion.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestCustomApplicationIdConversion.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestCustomApplicationIdConversion.java
new file mode 100644
index 0000000..0dc344f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestCustomApplicationIdConversion.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test for HBaseTimelineStorageUtils.convertApplicationIdToString(),
+ * a custom conversion from ApplicationId to String that avoids the
+ * incompatibility issue caused by mixing hadoop-common 2.5.1 and
+ * hadoop-yarn-api 3.0. See YARN-6905.
+ */
+public class TestCustomApplicationIdConversion {
+ @Test
+ public void testConvertAplicationIdToString() {
+ ApplicationId applicationId = ApplicationId.newInstance(0, 1);
+ String applicationIdStr =
+ HBaseTimelineSchemaUtils.convertApplicationIdToString(applicationId);
+ Assert.assertEquals(applicationId,
+ ApplicationId.fromString(applicationIdStr));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestKeyConverters.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestKeyConverters.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestKeyConverters.java
new file mode 100644
index 0000000..1bd363f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestKeyConverters.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.junit.Test;
+
+/**
+ * Unit tests for key converters for various tables' row keys.
+ *
+ */
+public class TestKeyConverters {
+
+ @Test
+ public void testAppIdKeyConverter() {
+ AppIdKeyConverter appIdKeyConverter = new AppIdKeyConverter();
+ long currentTs = System.currentTimeMillis();
+ ApplicationId appId1 = ApplicationId.newInstance(currentTs, 1);
+ ApplicationId appId2 = ApplicationId.newInstance(currentTs, 2);
+ ApplicationId appId3 = ApplicationId.newInstance(currentTs + 300, 1);
+ String appIdStr1 = appId1.toString();
+ String appIdStr2 = appId2.toString();
+ String appIdStr3 = appId3.toString();
+ byte[] appIdBytes1 = appIdKeyConverter.encode(appIdStr1);
+ byte[] appIdBytes2 = appIdKeyConverter.encode(appIdStr2);
+ byte[] appIdBytes3 = appIdKeyConverter.encode(appIdStr3);
+ // App ids' should be encoded in a manner wherein descending order
+ // is maintained.
+ assertTrue(
+ "Ordering of app ids' is incorrect",
+ Bytes.compareTo(appIdBytes1, appIdBytes2) > 0
+ && Bytes.compareTo(appIdBytes1, appIdBytes3) > 0
+ && Bytes.compareTo(appIdBytes2, appIdBytes3) > 0);
+ String decodedAppId1 = appIdKeyConverter.decode(appIdBytes1);
+ String decodedAppId2 = appIdKeyConverter.decode(appIdBytes2);
+ String decodedAppId3 = appIdKeyConverter.decode(appIdBytes3);
+ assertTrue("Decoded app id is not same as the app id encoded",
+ appIdStr1.equals(decodedAppId1));
+ assertTrue("Decoded app id is not same as the app id encoded",
+ appIdStr2.equals(decodedAppId2));
+ assertTrue("Decoded app id is not same as the app id encoded",
+ appIdStr3.equals(decodedAppId3));
+ }
+
+ @Test
+ public void testEventColumnNameConverter() {
+ String eventId = "=foo_=eve=nt=";
+ byte[] valSepBytes = Bytes.toBytes(Separator.VALUES.getValue());
+ byte[] maxByteArr =
+ Bytes.createMaxByteArray(Bytes.SIZEOF_LONG - valSepBytes.length);
+ byte[] ts = Bytes.add(valSepBytes, maxByteArr);
+ Long eventTs = Bytes.toLong(ts);
+ byte[] byteEventColName =
+ new EventColumnName(eventId, eventTs, null).getColumnQualifier();
+ KeyConverter<EventColumnName> eventColumnNameConverter =
+ new EventColumnNameConverter();
+ EventColumnName eventColName =
+ eventColumnNameConverter.decode(byteEventColName);
+ assertEquals(eventId, eventColName.getId());
+ assertEquals(eventTs, eventColName.getTimestamp());
+ assertNull(eventColName.getInfoKey());
+
+ String infoKey = "f=oo_event_in=fo=_key";
+ byteEventColName =
+ new EventColumnName(eventId, eventTs, infoKey).getColumnQualifier();
+ eventColName = eventColumnNameConverter.decode(byteEventColName);
+ assertEquals(eventId, eventColName.getId());
+ assertEquals(eventTs, eventColName.getTimestamp());
+ assertEquals(infoKey, eventColName.getInfoKey());
+ }
+
+ @Test
+ public void testLongKeyConverter() {
+ LongKeyConverter longKeyConverter = new LongKeyConverter();
+ confirmLongKeyConverter(longKeyConverter, Long.MIN_VALUE);
+ confirmLongKeyConverter(longKeyConverter, -1234567890L);
+ confirmLongKeyConverter(longKeyConverter, -128L);
+ confirmLongKeyConverter(longKeyConverter, -127L);
+ confirmLongKeyConverter(longKeyConverter, -1L);
+ confirmLongKeyConverter(longKeyConverter, 0L);
+ confirmLongKeyConverter(longKeyConverter, 1L);
+ confirmLongKeyConverter(longKeyConverter, 127L);
+ confirmLongKeyConverter(longKeyConverter, 128L);
+ confirmLongKeyConverter(longKeyConverter, 1234567890L);
+ confirmLongKeyConverter(longKeyConverter, Long.MAX_VALUE);
+ }
+
+ private void confirmLongKeyConverter(LongKeyConverter longKeyConverter,
+ Long testValue) {
+ Long decoded = longKeyConverter.decode(longKeyConverter.encode(testValue));
+ assertEquals(testValue, decoded);
+ }
+
+ @Test
+ public void testStringKeyConverter() {
+ StringKeyConverter stringKeyConverter = new StringKeyConverter();
+ String phrase = "QuackAttack now!";
+
+ for (int i = 0; i < phrase.length(); i++) {
+ String sub = phrase.substring(i, phrase.length());
+ confirmStrignKeyConverter(stringKeyConverter, sub);
+ confirmStrignKeyConverter(stringKeyConverter, sub + sub);
+ }
+ }
+
+ private void confirmStrignKeyConverter(StringKeyConverter stringKeyConverter,
+ String testValue) {
+ String decoded =
+ stringKeyConverter.decode(stringKeyConverter.encode(testValue));
+ assertEquals(testValue, decoded);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java
new file mode 100644
index 0000000..d05cbad
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java
@@ -0,0 +1,276 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKeyPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKeyPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKeyPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationRowKey;
+import org.junit.Test;
+
+
+/**
+ * Class to test the row key structures for various tables.
+ *
+ */
+public class TestRowKeys {
+
+ private final static String QUALIFIER_SEP = Separator.QUALIFIERS.getValue();
+ private final static byte[] QUALIFIER_SEP_BYTES = Bytes
+ .toBytes(QUALIFIER_SEP);
+ private final static String CLUSTER = "cl" + QUALIFIER_SEP + "uster";
+ private final static String USER = QUALIFIER_SEP + "user";
+ private final static String SUB_APP_USER = QUALIFIER_SEP + "subAppUser";
+ private final static String FLOW_NAME = "dummy_" + QUALIFIER_SEP + "flow"
+ + QUALIFIER_SEP;
+ private final static Long FLOW_RUN_ID;
+ private final static String APPLICATION_ID;
+ static {
+ long runid = Long.MAX_VALUE - 900L;
+ byte[] longMaxByteArr = Bytes.toBytes(Long.MAX_VALUE);
+ byte[] byteArr = Bytes.toBytes(runid);
+ int sepByteLen = QUALIFIER_SEP_BYTES.length;
+ if (sepByteLen <= byteArr.length) {
+ for (int i = 0; i < sepByteLen; i++) {
+ byteArr[i] = (byte) (longMaxByteArr[i] - QUALIFIER_SEP_BYTES[i]);
+ }
+ }
+ FLOW_RUN_ID = Bytes.toLong(byteArr);
+ long clusterTs = System.currentTimeMillis();
+ byteArr = Bytes.toBytes(clusterTs);
+ if (sepByteLen <= byteArr.length) {
+ for (int i = 0; i < sepByteLen; i++) {
+ byteArr[byteArr.length - sepByteLen + i] =
+ (byte) (longMaxByteArr[byteArr.length - sepByteLen + i] -
+ QUALIFIER_SEP_BYTES[i]);
+ }
+ }
+ clusterTs = Bytes.toLong(byteArr);
+ int seqId = 222;
+ APPLICATION_ID = ApplicationId.newInstance(clusterTs, seqId).toString();
+ }
+
+ private static void verifyRowPrefixBytes(byte[] byteRowKeyPrefix) {
+ int sepLen = QUALIFIER_SEP_BYTES.length;
+ for (int i = 0; i < sepLen; i++) {
+ assertTrue(
+ "Row key prefix not encoded properly.",
+ byteRowKeyPrefix[byteRowKeyPrefix.length - sepLen + i] ==
+ QUALIFIER_SEP_BYTES[i]);
+ }
+ }
+
+ @Test
+ public void testApplicationRowKey() {
+ byte[] byteRowKey =
+ new ApplicationRowKey(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID,
+ APPLICATION_ID).getRowKey();
+ ApplicationRowKey rowKey = ApplicationRowKey.parseRowKey(byteRowKey);
+ assertEquals(CLUSTER, rowKey.getClusterId());
+ assertEquals(USER, rowKey.getUserId());
+ assertEquals(FLOW_NAME, rowKey.getFlowName());
+ assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId());
+ assertEquals(APPLICATION_ID, rowKey.getAppId());
+
+ byte[] byteRowKeyPrefix =
+ new ApplicationRowKeyPrefix(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID)
+ .getRowKeyPrefix();
+ byte[][] splits =
+ Separator.QUALIFIERS.split(byteRowKeyPrefix,
+ new int[] {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
+ Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
+ Separator.VARIABLE_SIZE});
+ assertEquals(5, splits.length);
+ assertEquals(0, splits[4].length);
+ assertEquals(FLOW_NAME,
+ Separator.QUALIFIERS.decode(Bytes.toString(splits[2])));
+ assertEquals(FLOW_RUN_ID,
+ (Long) LongConverter.invertLong(Bytes.toLong(splits[3])));
+ verifyRowPrefixBytes(byteRowKeyPrefix);
+
+ byteRowKeyPrefix =
+ new ApplicationRowKeyPrefix(CLUSTER, USER, FLOW_NAME).getRowKeyPrefix();
+ splits =
+ Separator.QUALIFIERS.split(byteRowKeyPrefix, new int[] {
+ Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
+ Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE });
+ assertEquals(4, splits.length);
+ assertEquals(0, splits[3].length);
+ assertEquals(FLOW_NAME,
+ Separator.QUALIFIERS.decode(Bytes.toString(splits[2])));
+ verifyRowPrefixBytes(byteRowKeyPrefix);
+ }
+
+ /**
+ * Tests the converters indirectly through the public methods of the
+ * corresponding rowkey.
+ */
+ @Test
+ public void testAppToFlowRowKey() {
+ byte[] byteRowKey = new AppToFlowRowKey(APPLICATION_ID).getRowKey();
+ AppToFlowRowKey rowKey = AppToFlowRowKey.parseRowKey(byteRowKey);
+ assertEquals(APPLICATION_ID, rowKey.getAppId());
+ }
+
+ @Test
+ public void testEntityRowKey() {
+ TimelineEntity entity = new TimelineEntity();
+ entity.setId("!ent!ity!!id!");
+ entity.setType("entity!Type");
+ entity.setIdPrefix(54321);
+
+ byte[] byteRowKey =
+ new EntityRowKey(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID, APPLICATION_ID,
+ entity.getType(), entity.getIdPrefix(),
+ entity.getId()).getRowKey();
+ EntityRowKey rowKey = EntityRowKey.parseRowKey(byteRowKey);
+ assertEquals(CLUSTER, rowKey.getClusterId());
+ assertEquals(USER, rowKey.getUserId());
+ assertEquals(FLOW_NAME, rowKey.getFlowName());
+ assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId());
+ assertEquals(APPLICATION_ID, rowKey.getAppId());
+ assertEquals(entity.getType(), rowKey.getEntityType());
+ assertEquals(entity.getIdPrefix(), rowKey.getEntityIdPrefix().longValue());
+ assertEquals(entity.getId(), rowKey.getEntityId());
+
+ byte[] byteRowKeyPrefix =
+ new EntityRowKeyPrefix(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID,
+ APPLICATION_ID, entity.getType(), null, null)
+ .getRowKeyPrefix();
+ byte[][] splits =
+ Separator.QUALIFIERS.split(
+ byteRowKeyPrefix,
+ new int[] {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
+ Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
+ AppIdKeyConverter.getKeySize(), Separator.VARIABLE_SIZE,
+ Bytes.SIZEOF_LONG, Separator.VARIABLE_SIZE });
+ assertEquals(7, splits.length);
+ assertEquals(APPLICATION_ID, new AppIdKeyConverter().decode(splits[4]));
+ assertEquals(entity.getType(),
+ Separator.QUALIFIERS.decode(Bytes.toString(splits[5])));
+ verifyRowPrefixBytes(byteRowKeyPrefix);
+
+ byteRowKeyPrefix =
+ new EntityRowKeyPrefix(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID,
+ APPLICATION_ID).getRowKeyPrefix();
+ splits =
+ Separator.QUALIFIERS.split(
+ byteRowKeyPrefix,
+ new int[] {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
+ Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
+ AppIdKeyConverter.getKeySize(), Separator.VARIABLE_SIZE});
+ assertEquals(6, splits.length);
+ assertEquals(0, splits[5].length);
+ AppIdKeyConverter appIdKeyConverter = new AppIdKeyConverter();
+ assertEquals(APPLICATION_ID, appIdKeyConverter.decode(splits[4]));
+ verifyRowPrefixBytes(byteRowKeyPrefix);
+ }
+
+ @Test
+ public void testFlowActivityRowKey() {
+ Long ts = 1459900830000L;
+ Long dayTimestamp = HBaseTimelineSchemaUtils.getTopOfTheDayTimestamp(ts);
+ byte[] byteRowKey =
+ new FlowActivityRowKey(CLUSTER, ts, USER, FLOW_NAME).getRowKey();
+ FlowActivityRowKey rowKey = FlowActivityRowKey.parseRowKey(byteRowKey);
+ assertEquals(CLUSTER, rowKey.getClusterId());
+ assertEquals(dayTimestamp, rowKey.getDayTimestamp());
+ assertEquals(USER, rowKey.getUserId());
+ assertEquals(FLOW_NAME, rowKey.getFlowName());
+
+ byte[] byteRowKeyPrefix =
+ new FlowActivityRowKeyPrefix(CLUSTER).getRowKeyPrefix();
+ byte[][] splits =
+ Separator.QUALIFIERS.split(byteRowKeyPrefix, new int[] {
+ Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE });
+ assertEquals(2, splits.length);
+ assertEquals(0, splits[1].length);
+ assertEquals(CLUSTER,
+ Separator.QUALIFIERS.decode(Bytes.toString(splits[0])));
+ verifyRowPrefixBytes(byteRowKeyPrefix);
+
+ byteRowKeyPrefix =
+ new FlowActivityRowKeyPrefix(CLUSTER, ts).getRowKeyPrefix();
+ splits =
+ Separator.QUALIFIERS.split(byteRowKeyPrefix,
+ new int[] {Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
+ Separator.VARIABLE_SIZE});
+ assertEquals(3, splits.length);
+ assertEquals(0, splits[2].length);
+ assertEquals(CLUSTER,
+ Separator.QUALIFIERS.decode(Bytes.toString(splits[0])));
+ assertEquals(ts,
+ (Long) LongConverter.invertLong(Bytes.toLong(splits[1])));
+ verifyRowPrefixBytes(byteRowKeyPrefix);
+ }
+
+ @Test
+ public void testFlowRunRowKey() {
+ byte[] byteRowKey =
+ new FlowRunRowKey(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID).getRowKey();
+ FlowRunRowKey rowKey = FlowRunRowKey.parseRowKey(byteRowKey);
+ assertEquals(CLUSTER, rowKey.getClusterId());
+ assertEquals(USER, rowKey.getUserId());
+ assertEquals(FLOW_NAME, rowKey.getFlowName());
+ assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId());
+
+ byte[] byteRowKeyPrefix =
+ new FlowRunRowKey(CLUSTER, USER, FLOW_NAME, null).getRowKey();
+ byte[][] splits =
+ Separator.QUALIFIERS.split(byteRowKeyPrefix, new int[] {
+ Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
+ Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE });
+ assertEquals(4, splits.length);
+ assertEquals(0, splits[3].length);
+ assertEquals(FLOW_NAME,
+ Separator.QUALIFIERS.decode(Bytes.toString(splits[2])));
+ verifyRowPrefixBytes(byteRowKeyPrefix);
+ }
+
+ @Test
+ public void testSubAppRowKey() {
+ TimelineEntity entity = new TimelineEntity();
+ entity.setId("entity1");
+ entity.setType("DAG");
+ entity.setIdPrefix(54321);
+
+ byte[] byteRowKey =
+ new SubApplicationRowKey(SUB_APP_USER, CLUSTER,
+ entity.getType(), entity.getIdPrefix(),
+ entity.getId(), USER).getRowKey();
+ SubApplicationRowKey rowKey = SubApplicationRowKey.parseRowKey(byteRowKey);
+ assertEquals(CLUSTER, rowKey.getClusterId());
+ assertEquals(SUB_APP_USER, rowKey.getSubAppUserId());
+ assertEquals(entity.getType(), rowKey.getEntityType());
+ assertEquals(entity.getIdPrefix(), rowKey.getEntityIdPrefix().longValue());
+ assertEquals(entity.getId(), rowKey.getEntityId());
+ assertEquals(USER, rowKey.getUserId());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeysAsString.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeysAsString.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeysAsString.java
new file mode 100644
index 0000000..c4d07c7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeysAsString.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationRowKey;
+import org.junit.Test;
+
+/**
+ * Test for row key as string.
+ */
+public class TestRowKeysAsString {
+
+ private final static String CLUSTER =
+ "cl" + TimelineReaderUtils.DEFAULT_DELIMITER_CHAR + "uster"
+ + TimelineReaderUtils.DEFAULT_ESCAPE_CHAR;
+ private final static String USER =
+ TimelineReaderUtils.DEFAULT_ESCAPE_CHAR + "user";
+ private final static String SUB_APP_USER =
+ TimelineReaderUtils.DEFAULT_ESCAPE_CHAR + "subAppUser";
+
+ private final static String FLOW_NAME =
+ "dummy_" + TimelineReaderUtils.DEFAULT_DELIMITER_CHAR
+ + TimelineReaderUtils.DEFAULT_ESCAPE_CHAR + "flow"
+ + TimelineReaderUtils.DEFAULT_DELIMITER_CHAR;
+ private final static Long FLOW_RUN_ID = System.currentTimeMillis();
+ private final static String APPLICATION_ID =
+ ApplicationId.newInstance(System.currentTimeMillis(), 1).toString();
+
+ @Test(timeout = 10000)
+ public void testApplicationRow() {
+ String rowKeyAsString = new ApplicationRowKey(CLUSTER, USER, FLOW_NAME,
+ FLOW_RUN_ID, APPLICATION_ID).getRowKeyAsString();
+ ApplicationRowKey rowKey =
+ ApplicationRowKey.parseRowKeyFromString(rowKeyAsString);
+ assertEquals(CLUSTER, rowKey.getClusterId());
+ assertEquals(USER, rowKey.getUserId());
+ assertEquals(FLOW_NAME, rowKey.getFlowName());
+ assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId());
+ assertEquals(APPLICATION_ID, rowKey.getAppId());
+ }
+
+ @Test(timeout = 10000)
+ public void testEntityRowKey() {
+ char del = TimelineReaderUtils.DEFAULT_DELIMITER_CHAR;
+ char esc = TimelineReaderUtils.DEFAULT_ESCAPE_CHAR;
+ String id = del + esc + "ent" + esc + del + "ity" + esc + del + esc + "id"
+ + esc + del + esc;
+ String type = "entity" + esc + del + esc + "Type";
+ TimelineEntity entity = new TimelineEntity();
+ entity.setId(id);
+ entity.setType(type);
+ entity.setIdPrefix(54321);
+
+ String rowKeyAsString =
+ new EntityRowKey(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID, APPLICATION_ID,
+ entity.getType(), entity.getIdPrefix(), entity.getId())
+ .getRowKeyAsString();
+ EntityRowKey rowKey = EntityRowKey.parseRowKeyFromString(rowKeyAsString);
+ assertEquals(CLUSTER, rowKey.getClusterId());
+ assertEquals(USER, rowKey.getUserId());
+ assertEquals(FLOW_NAME, rowKey.getFlowName());
+ assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId());
+ assertEquals(APPLICATION_ID, rowKey.getAppId());
+ assertEquals(entity.getType(), rowKey.getEntityType());
+ assertEquals(entity.getIdPrefix(), rowKey.getEntityIdPrefix().longValue());
+ assertEquals(entity.getId(), rowKey.getEntityId());
+
+ }
+
+ @Test(timeout = 10000)
+ public void testFlowActivityRowKey() {
+ Long ts = 1459900830000L;
+ Long dayTimestamp = HBaseTimelineSchemaUtils.getTopOfTheDayTimestamp(ts);
+ String rowKeyAsString = new FlowActivityRowKey(CLUSTER, ts, USER, FLOW_NAME)
+ .getRowKeyAsString();
+ FlowActivityRowKey rowKey =
+ FlowActivityRowKey.parseRowKeyFromString(rowKeyAsString);
+ assertEquals(CLUSTER, rowKey.getClusterId());
+ assertEquals(dayTimestamp, rowKey.getDayTimestamp());
+ assertEquals(USER, rowKey.getUserId());
+ assertEquals(FLOW_NAME, rowKey.getFlowName());
+ }
+
+ @Test(timeout = 10000)
+ public void testFlowRunRowKey() {
+ String rowKeyAsString =
+ new FlowRunRowKey(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID)
+ .getRowKeyAsString();
+ FlowRunRowKey rowKey = FlowRunRowKey.parseRowKeyFromString(rowKeyAsString);
+ assertEquals(CLUSTER, rowKey.getClusterId());
+ assertEquals(USER, rowKey.getUserId());
+ assertEquals(FLOW_NAME, rowKey.getFlowName());
+ assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId());
+ }
+
+ @Test(timeout = 10000)
+ public void testSubApplicationRowKey() {
+ char del = TimelineReaderUtils.DEFAULT_DELIMITER_CHAR;
+ char esc = TimelineReaderUtils.DEFAULT_ESCAPE_CHAR;
+ String id = del + esc + "ent" + esc + del + "ity" + esc + del + esc + "id"
+ + esc + del + esc;
+ String type = "entity" + esc + del + esc + "Type";
+ TimelineEntity entity = new TimelineEntity();
+ entity.setId(id);
+ entity.setType(type);
+ entity.setIdPrefix(54321);
+
+ String rowKeyAsString = new SubApplicationRowKey(SUB_APP_USER, CLUSTER,
+ entity.getType(), entity.getIdPrefix(), entity.getId(), USER)
+ .getRowKeyAsString();
+ SubApplicationRowKey rowKey = SubApplicationRowKey
+ .parseRowKeyFromString(rowKeyAsString);
+ assertEquals(SUB_APP_USER, rowKey.getSubAppUserId());
+ assertEquals(CLUSTER, rowKey.getClusterId());
+ assertEquals(entity.getType(), rowKey.getEntityType());
+ assertEquals(entity.getIdPrefix(), rowKey.getEntityIdPrefix().longValue());
+ assertEquals(entity.getId(), rowKey.getEntityId());
+ assertEquals(USER, rowKey.getUserId());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestSeparator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestSeparator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestSeparator.java
new file mode 100644
index 0000000..7d37206
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestSeparator.java
@@ -0,0 +1,215 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Test;
+
+import com.google.common.collect.Iterables;
+
+public class TestSeparator {
+
+ private static String villain = "Dr. Heinz Doofenshmirtz";
+ private static String special =
+ ". * | ? + \t ( ) [ ] { } ^ $ \\ \" %";
+
+ /**
+ *
+ */
+ @Test
+ public void testEncodeDecodeString() {
+
+ for (Separator separator : Separator.values()) {
+ testEncodeDecode(separator, "");
+ testEncodeDecode(separator, " ");
+ testEncodeDecode(separator, "!");
+ testEncodeDecode(separator, "?");
+ testEncodeDecode(separator, "&");
+ testEncodeDecode(separator, "+");
+ testEncodeDecode(separator, "\t");
+ testEncodeDecode(separator, "Dr.");
+ testEncodeDecode(separator, "Heinz");
+ testEncodeDecode(separator, "Doofenshmirtz");
+ testEncodeDecode(separator, villain);
+ testEncodeDecode(separator, special);
+
+ assertNull(separator.encode(null));
+
+ }
+ }
+
+ private void testEncodeDecode(Separator separator, String token) {
+ String encoded = separator.encode(token);
+ String decoded = separator.decode(encoded);
+ String msg = "token:" + token + " separator:" + separator + ".";
+ assertEquals(msg, token, decoded);
+ }
+
+ @Test
+ public void testEncodeDecode() {
+ testEncodeDecode("Dr.", Separator.QUALIFIERS);
+ testEncodeDecode("Heinz", Separator.QUALIFIERS, Separator.QUALIFIERS);
+ testEncodeDecode("Doofenshmirtz", Separator.QUALIFIERS, null,
+ Separator.QUALIFIERS);
+ testEncodeDecode("&Perry", Separator.QUALIFIERS, Separator.VALUES, null);
+ testEncodeDecode("the ", Separator.QUALIFIERS, Separator.SPACE);
+ testEncodeDecode("Platypus...", (Separator) null);
+ testEncodeDecode("The what now ?!?", Separator.QUALIFIERS,
+ Separator.VALUES, Separator.SPACE);
+
+ }
+ @Test
+ public void testEncodedValues() {
+ testEncodeDecode("Double-escape %2$ and %9$ or %%2$ or %%3$, nor %%%2$" +
+ "= no problem!",
+ Separator.QUALIFIERS, Separator.VALUES, Separator.SPACE, Separator.TAB);
+ }
+
+ @Test
+ public void testSplits() {
+ byte[] maxLongBytes = Bytes.toBytes(Long.MAX_VALUE);
+ byte[] maxIntBytes = Bytes.toBytes(Integer.MAX_VALUE);
+ for (Separator separator : Separator.values()) {
+ String str1 = "cl" + separator.getValue() + "us";
+ String str2 = separator.getValue() + "rst";
+ byte[] sepByteArr = Bytes.toBytes(separator.getValue());
+ byte[] longVal1Arr = Bytes.add(sepByteArr, Bytes.copy(maxLongBytes,
+ sepByteArr.length, Bytes.SIZEOF_LONG - sepByteArr.length));
+ byte[] intVal1Arr = Bytes.add(sepByteArr, Bytes.copy(maxIntBytes,
+ sepByteArr.length, Bytes.SIZEOF_INT - sepByteArr.length));
+ byte[] arr = separator.join(
+ Bytes.toBytes(separator.encode(str1)), longVal1Arr,
+ Bytes.toBytes(separator.encode(str2)), intVal1Arr);
+ int[] sizes = {Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
+ Separator.VARIABLE_SIZE, Bytes.SIZEOF_INT};
+ byte[][] splits = separator.split(arr, sizes);
+ assertEquals(4, splits.length);
+ assertEquals(str1, separator.decode(Bytes.toString(splits[0])));
+ assertEquals(Bytes.toLong(longVal1Arr), Bytes.toLong(splits[1]));
+ assertEquals(str2, separator.decode(Bytes.toString(splits[2])));
+ assertEquals(Bytes.toInt(intVal1Arr), Bytes.toInt(splits[3]));
+
+ longVal1Arr = Bytes.add(Bytes.copy(maxLongBytes, 0, Bytes.SIZEOF_LONG -
+ sepByteArr.length), sepByteArr);
+ intVal1Arr = Bytes.add(Bytes.copy(maxIntBytes, 0, Bytes.SIZEOF_INT -
+ sepByteArr.length), sepByteArr);
+ arr = separator.join(Bytes.toBytes(separator.encode(str1)), longVal1Arr,
+ Bytes.toBytes(separator.encode(str2)), intVal1Arr);
+ splits = separator.split(arr, sizes);
+ assertEquals(4, splits.length);
+ assertEquals(str1, separator.decode(Bytes.toString(splits[0])));
+ assertEquals(Bytes.toLong(longVal1Arr), Bytes.toLong(splits[1]));
+ assertEquals(str2, separator.decode(Bytes.toString(splits[2])));
+ assertEquals(Bytes.toInt(intVal1Arr), Bytes.toInt(splits[3]));
+
+ longVal1Arr = Bytes.add(sepByteArr, Bytes.copy(maxLongBytes,
+ sepByteArr.length, 4 - sepByteArr.length), sepByteArr);
+ longVal1Arr = Bytes.add(longVal1Arr, Bytes.copy(maxLongBytes, 4, 3 -
+ sepByteArr.length), sepByteArr);
+ arr = separator.join(Bytes.toBytes(separator.encode(str1)), longVal1Arr,
+ Bytes.toBytes(separator.encode(str2)), intVal1Arr);
+ splits = separator.split(arr, sizes);
+ assertEquals(4, splits.length);
+ assertEquals(str1, separator.decode(Bytes.toString(splits[0])));
+ assertEquals(Bytes.toLong(longVal1Arr), Bytes.toLong(splits[1]));
+ assertEquals(str2, separator.decode(Bytes.toString(splits[2])));
+ assertEquals(Bytes.toInt(intVal1Arr), Bytes.toInt(splits[3]));
+
+ arr = separator.join(Bytes.toBytes(separator.encode(str1)),
+ Bytes.toBytes(separator.encode(str2)), intVal1Arr, longVal1Arr);
+ int[] sizes1 = {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
+ Bytes.SIZEOF_INT, Bytes.SIZEOF_LONG};
+ splits = separator.split(arr, sizes1);
+ assertEquals(4, splits.length);
+ assertEquals(str1, separator.decode(Bytes.toString(splits[0])));
+ assertEquals(str2, separator.decode(Bytes.toString(splits[1])));
+ assertEquals(Bytes.toInt(intVal1Arr), Bytes.toInt(splits[2]));
+ assertEquals(Bytes.toLong(longVal1Arr), Bytes.toLong(splits[3]));
+
+ try {
+ int[] sizes2 = {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
+ Bytes.SIZEOF_INT, 7};
+ splits = separator.split(arr, sizes2);
+ fail("Exception should have been thrown.");
+ } catch (IllegalArgumentException e) {}
+
+ try {
+ int[] sizes2 = {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE, 2,
+ Bytes.SIZEOF_LONG};
+ splits = separator.split(arr, sizes2);
+ fail("Exception should have been thrown.");
+ } catch (IllegalArgumentException e) {}
+ }
+ }
+
+ /**
+ * Simple test to encode and decode using the same separators and confirm that
+ * we end up with the same as what we started with.
+ *
+ * @param token
+ * @param separators
+ */
+ private static void testEncodeDecode(String token, Separator... separators) {
+ byte[] encoded = Separator.encode(token, separators);
+ String decoded = Separator.decode(encoded, separators);
+ assertEquals(token, decoded);
+ }
+
+ @Test
+ public void testJoinStripped() {
+ List<String> stringList = new ArrayList<String>(0);
+ stringList.add("nothing");
+
+ String joined = Separator.VALUES.joinEncoded(stringList);
+ Iterable<String> split = Separator.VALUES.splitEncoded(joined);
+ assertTrue(Iterables.elementsEqual(stringList, split));
+
+ stringList = new ArrayList<String>(3);
+ stringList.add("a");
+ stringList.add("b?");
+ stringList.add("c");
+
+ joined = Separator.VALUES.joinEncoded(stringList);
+ split = Separator.VALUES.splitEncoded(joined);
+ assertTrue(Iterables.elementsEqual(stringList, split));
+
+ String[] stringArray1 = {"else"};
+ joined = Separator.VALUES.joinEncoded(stringArray1);
+ split = Separator.VALUES.splitEncoded(joined);
+ assertTrue(Iterables.elementsEqual(Arrays.asList(stringArray1), split));
+
+ String[] stringArray2 = {"d", "e?", "f"};
+ joined = Separator.VALUES.joinEncoded(stringArray2);
+ split = Separator.VALUES.splitEncoded(joined);
+ assertTrue(Iterables.elementsEqual(Arrays.asList(stringArray2), split));
+
+ List<String> empty = new ArrayList<String>(0);
+ split = Separator.VALUES.splitEncoded(null);
+ assertTrue(Iterables.elementsEqual(empty, split));
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/pom.xml
new file mode 100644
index 0000000..d06907d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/pom.xml
@@ -0,0 +1,161 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+ http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>hadoop-yarn-server-timelineservice-hbase</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <version>3.2.0-SNAPSHOT</version>
+ </parent>
+
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hadoop-yarn-server-timelineservice-hbase-server</artifactId>
+ <name>Apache Hadoop YARN TimelineService HBase Server</name>
+ <version>3.2.0-SNAPSHOT</version>
+
+ <properties>
+ <!-- Needed for generating FindBugs warnings using parent pom -->
+ <yarn.basedir>${project.parent.parent.parent.basedir}</yarn.basedir>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-timelineservice-hbase-common</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-sslengine</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptor>src/assembly/coprocessor.xml</descriptor>
+ <attach>true</attach>
+ </configuration>
+ <executions>
+ <execution>
+ <id>create-coprocessor-jar</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/assembly/coprocessor.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/assembly/coprocessor.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/assembly/coprocessor.xml
new file mode 100644
index 0000000..01ff0dd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/assembly/coprocessor.xml
@@ -0,0 +1,37 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.01
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
+ <id>coprocessor</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>/</outputDirectory>
+ <useProjectArtifact>true</useProjectArtifact>
+ <unpack>true</unpack>
+ <scope>runtime</scope>
+ <includes>
+ <include>org.apache.hadoop:hadoop-yarn-server-timelineservice-hbase-common</include>
+ <include>org.apache.hadoop:hadoop-yarn-server-timelineservice-hbase-server</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+</assembly>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineServerUtils.java
new file mode 100644
index 0000000..5c07670
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineServerUtils.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.Tag;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationCompactionDimension;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationOperation;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A utility class used by hbase-server module.
+ */
+public final class HBaseTimelineServerUtils {
+ private HBaseTimelineServerUtils() {
+ }
+
+ /**
+ * Creates a {@link Tag} from the input attribute.
+ *
+ * @param attribute Attribute from which tag has to be fetched.
+ * @return a HBase Tag.
+ */
+ public static Tag getTagFromAttribute(Map.Entry<String, byte[]> attribute) {
+ // attribute could be either an Aggregation Operation or
+ // an Aggregation Dimension
+ // Get the Tag type from either
+ AggregationOperation aggOp = AggregationOperation
+ .getAggregationOperation(attribute.getKey());
+ if (aggOp != null) {
+ Tag t = new Tag(aggOp.getTagType(), attribute.getValue());
+ return t;
+ }
+
+ AggregationCompactionDimension aggCompactDim =
+ AggregationCompactionDimension.getAggregationCompactionDimension(
+ attribute.getKey());
+ if (aggCompactDim != null) {
+ Tag t = new Tag(aggCompactDim.getTagType(), attribute.getValue());
+ return t;
+ }
+ return null;
+ }
+
+ /**
+ * creates a new cell based on the input cell but with the new value.
+ *
+ * @param origCell Original cell
+ * @param newValue new cell value
+ * @return cell
+ * @throws IOException while creating new cell.
+ */
+ public static Cell createNewCell(Cell origCell, byte[] newValue)
+ throws IOException {
+ return CellUtil.createCell(CellUtil.cloneRow(origCell),
+ CellUtil.cloneFamily(origCell), CellUtil.cloneQualifier(origCell),
+ origCell.getTimestamp(), KeyValue.Type.Put.getCode(), newValue);
+ }
+
+ /**
+ * creates a cell with the given inputs.
+ *
+ * @param row row of the cell to be created
+ * @param family column family name of the new cell
+ * @param qualifier qualifier for the new cell
+ * @param ts timestamp of the new cell
+ * @param newValue value of the new cell
+ * @param tags tags in the new cell
+ * @return cell
+ * @throws IOException while creating the cell.
+ */
+ public static Cell createNewCell(byte[] row, byte[] family, byte[] qualifier,
+ long ts, byte[] newValue, byte[] tags) throws IOException {
+ return CellUtil.createCell(row, family, qualifier, ts, KeyValue.Type.Put,
+ newValue, tags);
+ }
+
+ /**
+ * returns app id from the list of tags.
+ *
+ * @param tags cell tags to be looked into
+ * @return App Id as the AggregationCompactionDimension
+ */
+ public static String getAggregationCompactionDimension(List<Tag> tags) {
+ String appId = null;
+ for (Tag t : tags) {
+ if (AggregationCompactionDimension.APPLICATION_ID.getTagType() == t
+ .getType()) {
+ appId = Bytes.toString(t.getValue());
+ return appId;
+ }
+ }
+ return appId;
+ }
+
+ /**
+ * Returns the first seen aggregation operation as seen in the list of input
+ * tags or null otherwise.
+ *
+ * @param tags list of HBase tags.
+ * @return AggregationOperation
+ */
+ public static AggregationOperation getAggregationOperationFromTagsList(
+ List<Tag> tags) {
+ for (AggregationOperation aggOp : AggregationOperation.values()) {
+ for (Tag tag : tags) {
+ if (tag.getType() == aggOp.getTagType()) {
+ return aggOp;
+ }
+ }
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java
new file mode 100644
index 0000000..0df5b8a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.storage.common contains
+ * a set of utility classes used across backend storage reader and writer.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org