You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ro...@apache.org on 2018/02/18 08:09:33 UTC
[01/18] hadoop git commit: YARN-7919. Refactor timelineservice-hbase
module into submodules. Contributed by Haibo Chen.
Repository: hadoop
Updated Branches:
refs/heads/trunk a1e56a628 -> 9af30d46c
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationTable.java
deleted file mode 100644
index 785a243..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationTable.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.regionserver.BloomType;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBaseSchemaConstants;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * The sub application table has column families:
- * info, config and metrics.
- * Info stores information about a timeline entity object
- * config stores configuration data of a timeline entity object
- * metrics stores the metrics of a timeline entity object
- *
- * Example sub application table record:
- *
- * <pre>
- * |-------------------------------------------------------------------------|
- * | Row | Column Family | Column Family| Column Family|
- * | key | info | metrics | config |
- * |-------------------------------------------------------------------------|
- * | subAppUserId! | id:entityId | metricId1: | configKey1: |
- * | clusterId! | type:entityType | metricValue1 | configValue1 |
- * | entityType! | | @timestamp1 | |
- * | idPrefix!| | | | configKey2: |
- * | entityId! | created_time: | metricId1: | configValue2 |
- * | userId | 1392993084018 | metricValue2 | |
- * | | | @timestamp2 | |
- * | | i!infoKey: | | |
- * | | infoValue | metricId1: | |
- * | | | metricValue1 | |
- * | | | @timestamp2 | |
- * | | e!eventId=timestamp= | | |
- * | | infoKey: | | |
- * | | eventInfoValue | | |
- * | | | | |
- * | | r!relatesToKey: | | |
- * | | id3=id4=id5 | | |
- * | | | | |
- * | | s!isRelatedToKey | | |
- * | | id7=id9=id6 | | |
- * | | | | |
- * | | flowVersion: | | |
- * | | versionValue | | |
- * |-------------------------------------------------------------------------|
- * </pre>
- */
-public class SubApplicationTable extends BaseTable<SubApplicationTable> {
- /** sub app prefix. */
- private static final String PREFIX =
- YarnConfiguration.TIMELINE_SERVICE_PREFIX + "subapplication";
-
- /** config param name that specifies the subapplication table name. */
- public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
-
- /**
- * config param name that specifies the TTL for metrics column family in
- * subapplication table.
- */
- private static final String METRICS_TTL_CONF_NAME = PREFIX
- + ".table.metrics.ttl";
-
- /**
- * config param name that specifies max-versions for
- * metrics column family in subapplication table.
- */
- private static final String METRICS_MAX_VERSIONS =
- PREFIX + ".table.metrics.max-versions";
-
- /** default value for subapplication table name. */
- public static final String DEFAULT_TABLE_NAME =
- "timelineservice.subapplication";
-
- /** default TTL is 30 days for metrics timeseries. */
- private static final int DEFAULT_METRICS_TTL = 2592000;
-
- /** default max number of versions. */
- private static final int DEFAULT_METRICS_MAX_VERSIONS = 10000;
-
- private static final Logger LOG = LoggerFactory.getLogger(
- SubApplicationTable.class);
-
- public SubApplicationTable() {
- super(TABLE_NAME_CONF_NAME, DEFAULT_TABLE_NAME);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.BaseTable#createTable
- * (org.apache.hadoop.hbase.client.Admin,
- * org.apache.hadoop.conf.Configuration)
- */
- public void createTable(Admin admin, Configuration hbaseConf)
- throws IOException {
-
- TableName table = getTableName(hbaseConf);
- if (admin.tableExists(table)) {
- // do not disable / delete existing table
- // similar to the approach taken by map-reduce jobs when
- // output directory exists
- throw new IOException("Table " + table.getNameAsString()
- + " already exists.");
- }
-
- HTableDescriptor subAppTableDescp = new HTableDescriptor(table);
- HColumnDescriptor infoCF =
- new HColumnDescriptor(SubApplicationColumnFamily.INFO.getBytes());
- infoCF.setBloomFilterType(BloomType.ROWCOL);
- subAppTableDescp.addFamily(infoCF);
-
- HColumnDescriptor configCF =
- new HColumnDescriptor(SubApplicationColumnFamily.CONFIGS.getBytes());
- configCF.setBloomFilterType(BloomType.ROWCOL);
- configCF.setBlockCacheEnabled(true);
- subAppTableDescp.addFamily(configCF);
-
- HColumnDescriptor metricsCF =
- new HColumnDescriptor(SubApplicationColumnFamily.METRICS.getBytes());
- subAppTableDescp.addFamily(metricsCF);
- metricsCF.setBlockCacheEnabled(true);
- // always keep 1 version (the latest)
- metricsCF.setMinVersions(1);
- metricsCF.setMaxVersions(
- hbaseConf.getInt(METRICS_MAX_VERSIONS, DEFAULT_METRICS_MAX_VERSIONS));
- metricsCF.setTimeToLive(hbaseConf.getInt(METRICS_TTL_CONF_NAME,
- DEFAULT_METRICS_TTL));
- subAppTableDescp.setRegionSplitPolicyClassName(
- "org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy");
- subAppTableDescp.setValue("KeyPrefixRegionSplitPolicy.prefix_length",
- TimelineHBaseSchemaConstants.USERNAME_SPLIT_KEY_PREFIX_LENGTH);
- admin.createTable(subAppTableDescp,
- TimelineHBaseSchemaConstants.getUsernameSplits());
- LOG.info("Status of table creation for " + table.getNameAsString() + "="
- + admin.tableExists(table));
- }
-
- /**
- * @param metricsTTL time to live parameter for the metricss in this table.
- * @param hbaseConf configururation in which to set the metrics TTL config
- * variable.
- */
- public void setMetricsTTL(int metricsTTL, Configuration hbaseConf) {
- hbaseConf.setInt(METRICS_TTL_CONF_NAME, metricsTTL);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/package-info.java
deleted file mode 100644
index 52cc399..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/package-info.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication
- * contains classes related to implementation for subapplication table.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestCustomApplicationIdConversion.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestCustomApplicationIdConversion.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestCustomApplicationIdConversion.java
deleted file mode 100644
index 73bc29e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestCustomApplicationIdConversion.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * Test for HBaseTimelineStorageUtils.convertApplicationIdToString(),
- * a custom conversion from ApplicationId to String that avoids the
- * incompatibility issue caused by mixing hadoop-common 2.5.1 and
- * hadoop-yarn-api 3.0. See YARN-6905.
- */
-public class TestCustomApplicationIdConversion {
- @Test
- public void testConvertAplicationIdToString() {
- ApplicationId applicationId = ApplicationId.newInstance(0, 1);
- String applicationIdStr =
- HBaseTimelineStorageUtils.convertApplicationIdToString(applicationId);
- Assert.assertEquals(applicationId,
- ApplicationId.fromString(applicationIdStr));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java
deleted file mode 100644
index 402a89b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import org.junit.Test;
-
-/**
- * Unit tests for HBaseTimelineStorageUtils static methos.
- */
-public class TestHBaseTimelineStorageUtils {
-
- @Test(expected=NullPointerException.class)
- public void testGetTimelineServiceHBaseConfNullArgument() throws Exception {
- HBaseTimelineStorageUtils.getTimelineServiceHBaseConf(null);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestKeyConverters.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestKeyConverters.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestKeyConverters.java
deleted file mode 100644
index 1bd363f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestKeyConverters.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.junit.Test;
-
-/**
- * Unit tests for key converters for various tables' row keys.
- *
- */
-public class TestKeyConverters {
-
- @Test
- public void testAppIdKeyConverter() {
- AppIdKeyConverter appIdKeyConverter = new AppIdKeyConverter();
- long currentTs = System.currentTimeMillis();
- ApplicationId appId1 = ApplicationId.newInstance(currentTs, 1);
- ApplicationId appId2 = ApplicationId.newInstance(currentTs, 2);
- ApplicationId appId3 = ApplicationId.newInstance(currentTs + 300, 1);
- String appIdStr1 = appId1.toString();
- String appIdStr2 = appId2.toString();
- String appIdStr3 = appId3.toString();
- byte[] appIdBytes1 = appIdKeyConverter.encode(appIdStr1);
- byte[] appIdBytes2 = appIdKeyConverter.encode(appIdStr2);
- byte[] appIdBytes3 = appIdKeyConverter.encode(appIdStr3);
- // App ids' should be encoded in a manner wherein descending order
- // is maintained.
- assertTrue(
- "Ordering of app ids' is incorrect",
- Bytes.compareTo(appIdBytes1, appIdBytes2) > 0
- && Bytes.compareTo(appIdBytes1, appIdBytes3) > 0
- && Bytes.compareTo(appIdBytes2, appIdBytes3) > 0);
- String decodedAppId1 = appIdKeyConverter.decode(appIdBytes1);
- String decodedAppId2 = appIdKeyConverter.decode(appIdBytes2);
- String decodedAppId3 = appIdKeyConverter.decode(appIdBytes3);
- assertTrue("Decoded app id is not same as the app id encoded",
- appIdStr1.equals(decodedAppId1));
- assertTrue("Decoded app id is not same as the app id encoded",
- appIdStr2.equals(decodedAppId2));
- assertTrue("Decoded app id is not same as the app id encoded",
- appIdStr3.equals(decodedAppId3));
- }
-
- @Test
- public void testEventColumnNameConverter() {
- String eventId = "=foo_=eve=nt=";
- byte[] valSepBytes = Bytes.toBytes(Separator.VALUES.getValue());
- byte[] maxByteArr =
- Bytes.createMaxByteArray(Bytes.SIZEOF_LONG - valSepBytes.length);
- byte[] ts = Bytes.add(valSepBytes, maxByteArr);
- Long eventTs = Bytes.toLong(ts);
- byte[] byteEventColName =
- new EventColumnName(eventId, eventTs, null).getColumnQualifier();
- KeyConverter<EventColumnName> eventColumnNameConverter =
- new EventColumnNameConverter();
- EventColumnName eventColName =
- eventColumnNameConverter.decode(byteEventColName);
- assertEquals(eventId, eventColName.getId());
- assertEquals(eventTs, eventColName.getTimestamp());
- assertNull(eventColName.getInfoKey());
-
- String infoKey = "f=oo_event_in=fo=_key";
- byteEventColName =
- new EventColumnName(eventId, eventTs, infoKey).getColumnQualifier();
- eventColName = eventColumnNameConverter.decode(byteEventColName);
- assertEquals(eventId, eventColName.getId());
- assertEquals(eventTs, eventColName.getTimestamp());
- assertEquals(infoKey, eventColName.getInfoKey());
- }
-
- @Test
- public void testLongKeyConverter() {
- LongKeyConverter longKeyConverter = new LongKeyConverter();
- confirmLongKeyConverter(longKeyConverter, Long.MIN_VALUE);
- confirmLongKeyConverter(longKeyConverter, -1234567890L);
- confirmLongKeyConverter(longKeyConverter, -128L);
- confirmLongKeyConverter(longKeyConverter, -127L);
- confirmLongKeyConverter(longKeyConverter, -1L);
- confirmLongKeyConverter(longKeyConverter, 0L);
- confirmLongKeyConverter(longKeyConverter, 1L);
- confirmLongKeyConverter(longKeyConverter, 127L);
- confirmLongKeyConverter(longKeyConverter, 128L);
- confirmLongKeyConverter(longKeyConverter, 1234567890L);
- confirmLongKeyConverter(longKeyConverter, Long.MAX_VALUE);
- }
-
- private void confirmLongKeyConverter(LongKeyConverter longKeyConverter,
- Long testValue) {
- Long decoded = longKeyConverter.decode(longKeyConverter.encode(testValue));
- assertEquals(testValue, decoded);
- }
-
- @Test
- public void testStringKeyConverter() {
- StringKeyConverter stringKeyConverter = new StringKeyConverter();
- String phrase = "QuackAttack now!";
-
- for (int i = 0; i < phrase.length(); i++) {
- String sub = phrase.substring(i, phrase.length());
- confirmStrignKeyConverter(stringKeyConverter, sub);
- confirmStrignKeyConverter(stringKeyConverter, sub + sub);
- }
- }
-
- private void confirmStrignKeyConverter(StringKeyConverter stringKeyConverter,
- String testValue) {
- String decoded =
- stringKeyConverter.decode(stringKeyConverter.encode(testValue));
- assertEquals(testValue, decoded);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java
deleted file mode 100644
index 4770238..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java
+++ /dev/null
@@ -1,276 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKeyPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKeyPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKeyPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationRowKey;
-import org.junit.Test;
-
-
-/**
- * Class to test the row key structures for various tables.
- *
- */
-public class TestRowKeys {
-
- private final static String QUALIFIER_SEP = Separator.QUALIFIERS.getValue();
- private final static byte[] QUALIFIER_SEP_BYTES = Bytes
- .toBytes(QUALIFIER_SEP);
- private final static String CLUSTER = "cl" + QUALIFIER_SEP + "uster";
- private final static String USER = QUALIFIER_SEP + "user";
- private final static String SUB_APP_USER = QUALIFIER_SEP + "subAppUser";
- private final static String FLOW_NAME = "dummy_" + QUALIFIER_SEP + "flow"
- + QUALIFIER_SEP;
- private final static Long FLOW_RUN_ID;
- private final static String APPLICATION_ID;
- static {
- long runid = Long.MAX_VALUE - 900L;
- byte[] longMaxByteArr = Bytes.toBytes(Long.MAX_VALUE);
- byte[] byteArr = Bytes.toBytes(runid);
- int sepByteLen = QUALIFIER_SEP_BYTES.length;
- if (sepByteLen <= byteArr.length) {
- for (int i = 0; i < sepByteLen; i++) {
- byteArr[i] = (byte) (longMaxByteArr[i] - QUALIFIER_SEP_BYTES[i]);
- }
- }
- FLOW_RUN_ID = Bytes.toLong(byteArr);
- long clusterTs = System.currentTimeMillis();
- byteArr = Bytes.toBytes(clusterTs);
- if (sepByteLen <= byteArr.length) {
- for (int i = 0; i < sepByteLen; i++) {
- byteArr[byteArr.length - sepByteLen + i] =
- (byte) (longMaxByteArr[byteArr.length - sepByteLen + i] -
- QUALIFIER_SEP_BYTES[i]);
- }
- }
- clusterTs = Bytes.toLong(byteArr);
- int seqId = 222;
- APPLICATION_ID = ApplicationId.newInstance(clusterTs, seqId).toString();
- }
-
- private static void verifyRowPrefixBytes(byte[] byteRowKeyPrefix) {
- int sepLen = QUALIFIER_SEP_BYTES.length;
- for (int i = 0; i < sepLen; i++) {
- assertTrue(
- "Row key prefix not encoded properly.",
- byteRowKeyPrefix[byteRowKeyPrefix.length - sepLen + i] ==
- QUALIFIER_SEP_BYTES[i]);
- }
- }
-
- @Test
- public void testApplicationRowKey() {
- byte[] byteRowKey =
- new ApplicationRowKey(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID,
- APPLICATION_ID).getRowKey();
- ApplicationRowKey rowKey = ApplicationRowKey.parseRowKey(byteRowKey);
- assertEquals(CLUSTER, rowKey.getClusterId());
- assertEquals(USER, rowKey.getUserId());
- assertEquals(FLOW_NAME, rowKey.getFlowName());
- assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId());
- assertEquals(APPLICATION_ID, rowKey.getAppId());
-
- byte[] byteRowKeyPrefix =
- new ApplicationRowKeyPrefix(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID)
- .getRowKeyPrefix();
- byte[][] splits =
- Separator.QUALIFIERS.split(byteRowKeyPrefix,
- new int[] {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
- Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
- Separator.VARIABLE_SIZE});
- assertEquals(5, splits.length);
- assertEquals(0, splits[4].length);
- assertEquals(FLOW_NAME,
- Separator.QUALIFIERS.decode(Bytes.toString(splits[2])));
- assertEquals(FLOW_RUN_ID,
- (Long) LongConverter.invertLong(Bytes.toLong(splits[3])));
- verifyRowPrefixBytes(byteRowKeyPrefix);
-
- byteRowKeyPrefix =
- new ApplicationRowKeyPrefix(CLUSTER, USER, FLOW_NAME).getRowKeyPrefix();
- splits =
- Separator.QUALIFIERS.split(byteRowKeyPrefix, new int[] {
- Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
- Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE });
- assertEquals(4, splits.length);
- assertEquals(0, splits[3].length);
- assertEquals(FLOW_NAME,
- Separator.QUALIFIERS.decode(Bytes.toString(splits[2])));
- verifyRowPrefixBytes(byteRowKeyPrefix);
- }
-
- /**
- * Tests the converters indirectly through the public methods of the
- * corresponding rowkey.
- */
- @Test
- public void testAppToFlowRowKey() {
- byte[] byteRowKey = new AppToFlowRowKey(APPLICATION_ID).getRowKey();
- AppToFlowRowKey rowKey = AppToFlowRowKey.parseRowKey(byteRowKey);
- assertEquals(APPLICATION_ID, rowKey.getAppId());
- }
-
- @Test
- public void testEntityRowKey() {
- TimelineEntity entity = new TimelineEntity();
- entity.setId("!ent!ity!!id!");
- entity.setType("entity!Type");
- entity.setIdPrefix(54321);
-
- byte[] byteRowKey =
- new EntityRowKey(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID, APPLICATION_ID,
- entity.getType(), entity.getIdPrefix(),
- entity.getId()).getRowKey();
- EntityRowKey rowKey = EntityRowKey.parseRowKey(byteRowKey);
- assertEquals(CLUSTER, rowKey.getClusterId());
- assertEquals(USER, rowKey.getUserId());
- assertEquals(FLOW_NAME, rowKey.getFlowName());
- assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId());
- assertEquals(APPLICATION_ID, rowKey.getAppId());
- assertEquals(entity.getType(), rowKey.getEntityType());
- assertEquals(entity.getIdPrefix(), rowKey.getEntityIdPrefix().longValue());
- assertEquals(entity.getId(), rowKey.getEntityId());
-
- byte[] byteRowKeyPrefix =
- new EntityRowKeyPrefix(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID,
- APPLICATION_ID, entity.getType(), null, null)
- .getRowKeyPrefix();
- byte[][] splits =
- Separator.QUALIFIERS.split(
- byteRowKeyPrefix,
- new int[] {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
- Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
- AppIdKeyConverter.getKeySize(), Separator.VARIABLE_SIZE,
- Bytes.SIZEOF_LONG, Separator.VARIABLE_SIZE });
- assertEquals(7, splits.length);
- assertEquals(APPLICATION_ID, new AppIdKeyConverter().decode(splits[4]));
- assertEquals(entity.getType(),
- Separator.QUALIFIERS.decode(Bytes.toString(splits[5])));
- verifyRowPrefixBytes(byteRowKeyPrefix);
-
- byteRowKeyPrefix =
- new EntityRowKeyPrefix(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID,
- APPLICATION_ID).getRowKeyPrefix();
- splits =
- Separator.QUALIFIERS.split(
- byteRowKeyPrefix,
- new int[] {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
- Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
- AppIdKeyConverter.getKeySize(), Separator.VARIABLE_SIZE});
- assertEquals(6, splits.length);
- assertEquals(0, splits[5].length);
- AppIdKeyConverter appIdKeyConverter = new AppIdKeyConverter();
- assertEquals(APPLICATION_ID, appIdKeyConverter.decode(splits[4]));
- verifyRowPrefixBytes(byteRowKeyPrefix);
- }
-
- @Test
- public void testFlowActivityRowKey() {
- Long ts = 1459900830000L;
- Long dayTimestamp = HBaseTimelineStorageUtils.getTopOfTheDayTimestamp(ts);
- byte[] byteRowKey =
- new FlowActivityRowKey(CLUSTER, ts, USER, FLOW_NAME).getRowKey();
- FlowActivityRowKey rowKey = FlowActivityRowKey.parseRowKey(byteRowKey);
- assertEquals(CLUSTER, rowKey.getClusterId());
- assertEquals(dayTimestamp, rowKey.getDayTimestamp());
- assertEquals(USER, rowKey.getUserId());
- assertEquals(FLOW_NAME, rowKey.getFlowName());
-
- byte[] byteRowKeyPrefix =
- new FlowActivityRowKeyPrefix(CLUSTER).getRowKeyPrefix();
- byte[][] splits =
- Separator.QUALIFIERS.split(byteRowKeyPrefix, new int[] {
- Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE });
- assertEquals(2, splits.length);
- assertEquals(0, splits[1].length);
- assertEquals(CLUSTER,
- Separator.QUALIFIERS.decode(Bytes.toString(splits[0])));
- verifyRowPrefixBytes(byteRowKeyPrefix);
-
- byteRowKeyPrefix =
- new FlowActivityRowKeyPrefix(CLUSTER, ts).getRowKeyPrefix();
- splits =
- Separator.QUALIFIERS.split(byteRowKeyPrefix,
- new int[] {Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
- Separator.VARIABLE_SIZE});
- assertEquals(3, splits.length);
- assertEquals(0, splits[2].length);
- assertEquals(CLUSTER,
- Separator.QUALIFIERS.decode(Bytes.toString(splits[0])));
- assertEquals(ts,
- (Long) LongConverter.invertLong(Bytes.toLong(splits[1])));
- verifyRowPrefixBytes(byteRowKeyPrefix);
- }
-
- @Test
- public void testFlowRunRowKey() {
- byte[] byteRowKey =
- new FlowRunRowKey(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID).getRowKey();
- FlowRunRowKey rowKey = FlowRunRowKey.parseRowKey(byteRowKey);
- assertEquals(CLUSTER, rowKey.getClusterId());
- assertEquals(USER, rowKey.getUserId());
- assertEquals(FLOW_NAME, rowKey.getFlowName());
- assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId());
-
- byte[] byteRowKeyPrefix =
- new FlowRunRowKey(CLUSTER, USER, FLOW_NAME, null).getRowKey();
- byte[][] splits =
- Separator.QUALIFIERS.split(byteRowKeyPrefix, new int[] {
- Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
- Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE });
- assertEquals(4, splits.length);
- assertEquals(0, splits[3].length);
- assertEquals(FLOW_NAME,
- Separator.QUALIFIERS.decode(Bytes.toString(splits[2])));
- verifyRowPrefixBytes(byteRowKeyPrefix);
- }
-
- @Test
- public void testSubAppRowKey() {
- TimelineEntity entity = new TimelineEntity();
- entity.setId("entity1");
- entity.setType("DAG");
- entity.setIdPrefix(54321);
-
- byte[] byteRowKey =
- new SubApplicationRowKey(SUB_APP_USER, CLUSTER,
- entity.getType(), entity.getIdPrefix(),
- entity.getId(), USER).getRowKey();
- SubApplicationRowKey rowKey = SubApplicationRowKey.parseRowKey(byteRowKey);
- assertEquals(CLUSTER, rowKey.getClusterId());
- assertEquals(SUB_APP_USER, rowKey.getSubAppUserId());
- assertEquals(entity.getType(), rowKey.getEntityType());
- assertEquals(entity.getIdPrefix(), rowKey.getEntityIdPrefix().longValue());
- assertEquals(entity.getId(), rowKey.getEntityId());
- assertEquals(USER, rowKey.getUserId());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeysAsString.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeysAsString.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeysAsString.java
deleted file mode 100644
index 148cf56..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeysAsString.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import static org.junit.Assert.assertEquals;
-
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationRowKey;
-import org.junit.Test;
-
-/**
- * Test for row key as string.
- */
-public class TestRowKeysAsString {
-
- private final static String CLUSTER =
- "cl" + TimelineReaderUtils.DEFAULT_DELIMITER_CHAR + "uster"
- + TimelineReaderUtils.DEFAULT_ESCAPE_CHAR;
- private final static String USER =
- TimelineReaderUtils.DEFAULT_ESCAPE_CHAR + "user";
- private final static String SUB_APP_USER =
- TimelineReaderUtils.DEFAULT_ESCAPE_CHAR + "subAppUser";
-
- private final static String FLOW_NAME =
- "dummy_" + TimelineReaderUtils.DEFAULT_DELIMITER_CHAR
- + TimelineReaderUtils.DEFAULT_ESCAPE_CHAR + "flow"
- + TimelineReaderUtils.DEFAULT_DELIMITER_CHAR;
- private final static Long FLOW_RUN_ID = System.currentTimeMillis();
- private final static String APPLICATION_ID =
- ApplicationId.newInstance(System.currentTimeMillis(), 1).toString();
-
- @Test(timeout = 10000)
- public void testApplicationRow() {
- String rowKeyAsString = new ApplicationRowKey(CLUSTER, USER, FLOW_NAME,
- FLOW_RUN_ID, APPLICATION_ID).getRowKeyAsString();
- ApplicationRowKey rowKey =
- ApplicationRowKey.parseRowKeyFromString(rowKeyAsString);
- assertEquals(CLUSTER, rowKey.getClusterId());
- assertEquals(USER, rowKey.getUserId());
- assertEquals(FLOW_NAME, rowKey.getFlowName());
- assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId());
- assertEquals(APPLICATION_ID, rowKey.getAppId());
- }
-
- @Test(timeout = 10000)
- public void testEntityRowKey() {
- char del = TimelineReaderUtils.DEFAULT_DELIMITER_CHAR;
- char esc = TimelineReaderUtils.DEFAULT_ESCAPE_CHAR;
- String id = del + esc + "ent" + esc + del + "ity" + esc + del + esc + "id"
- + esc + del + esc;
- String type = "entity" + esc + del + esc + "Type";
- TimelineEntity entity = new TimelineEntity();
- entity.setId(id);
- entity.setType(type);
- entity.setIdPrefix(54321);
-
- String rowKeyAsString =
- new EntityRowKey(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID, APPLICATION_ID,
- entity.getType(), entity.getIdPrefix(), entity.getId())
- .getRowKeyAsString();
- EntityRowKey rowKey = EntityRowKey.parseRowKeyFromString(rowKeyAsString);
- assertEquals(CLUSTER, rowKey.getClusterId());
- assertEquals(USER, rowKey.getUserId());
- assertEquals(FLOW_NAME, rowKey.getFlowName());
- assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId());
- assertEquals(APPLICATION_ID, rowKey.getAppId());
- assertEquals(entity.getType(), rowKey.getEntityType());
- assertEquals(entity.getIdPrefix(), rowKey.getEntityIdPrefix().longValue());
- assertEquals(entity.getId(), rowKey.getEntityId());
-
- }
-
- @Test(timeout = 10000)
- public void testFlowActivityRowKey() {
- Long ts = 1459900830000L;
- Long dayTimestamp = HBaseTimelineStorageUtils.getTopOfTheDayTimestamp(ts);
- String rowKeyAsString = new FlowActivityRowKey(CLUSTER, ts, USER, FLOW_NAME)
- .getRowKeyAsString();
- FlowActivityRowKey rowKey =
- FlowActivityRowKey.parseRowKeyFromString(rowKeyAsString);
- assertEquals(CLUSTER, rowKey.getClusterId());
- assertEquals(dayTimestamp, rowKey.getDayTimestamp());
- assertEquals(USER, rowKey.getUserId());
- assertEquals(FLOW_NAME, rowKey.getFlowName());
- }
-
- @Test(timeout = 10000)
- public void testFlowRunRowKey() {
- String rowKeyAsString =
- new FlowRunRowKey(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID)
- .getRowKeyAsString();
- FlowRunRowKey rowKey = FlowRunRowKey.parseRowKeyFromString(rowKeyAsString);
- assertEquals(CLUSTER, rowKey.getClusterId());
- assertEquals(USER, rowKey.getUserId());
- assertEquals(FLOW_NAME, rowKey.getFlowName());
- assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId());
- }
-
- @Test(timeout = 10000)
- public void testSubApplicationRowKey() {
- char del = TimelineReaderUtils.DEFAULT_DELIMITER_CHAR;
- char esc = TimelineReaderUtils.DEFAULT_ESCAPE_CHAR;
- String id = del + esc + "ent" + esc + del + "ity" + esc + del + esc + "id"
- + esc + del + esc;
- String type = "entity" + esc + del + esc + "Type";
- TimelineEntity entity = new TimelineEntity();
- entity.setId(id);
- entity.setType(type);
- entity.setIdPrefix(54321);
-
- String rowKeyAsString = new SubApplicationRowKey(SUB_APP_USER, CLUSTER,
- entity.getType(), entity.getIdPrefix(), entity.getId(), USER)
- .getRowKeyAsString();
- SubApplicationRowKey rowKey = SubApplicationRowKey
- .parseRowKeyFromString(rowKeyAsString);
- assertEquals(SUB_APP_USER, rowKey.getSubAppUserId());
- assertEquals(CLUSTER, rowKey.getClusterId());
- assertEquals(entity.getType(), rowKey.getEntityType());
- assertEquals(entity.getIdPrefix(), rowKey.getEntityIdPrefix().longValue());
- assertEquals(entity.getId(), rowKey.getEntityId());
- assertEquals(USER, rowKey.getUserId());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestSeparator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestSeparator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestSeparator.java
deleted file mode 100644
index 7d37206..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestSeparator.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.Test;
-
-import com.google.common.collect.Iterables;
-
-public class TestSeparator {
-
- private static String villain = "Dr. Heinz Doofenshmirtz";
- private static String special =
- ". * | ? + \t ( ) [ ] { } ^ $ \\ \" %";
-
- /**
- *
- */
- @Test
- public void testEncodeDecodeString() {
-
- for (Separator separator : Separator.values()) {
- testEncodeDecode(separator, "");
- testEncodeDecode(separator, " ");
- testEncodeDecode(separator, "!");
- testEncodeDecode(separator, "?");
- testEncodeDecode(separator, "&");
- testEncodeDecode(separator, "+");
- testEncodeDecode(separator, "\t");
- testEncodeDecode(separator, "Dr.");
- testEncodeDecode(separator, "Heinz");
- testEncodeDecode(separator, "Doofenshmirtz");
- testEncodeDecode(separator, villain);
- testEncodeDecode(separator, special);
-
- assertNull(separator.encode(null));
-
- }
- }
-
- private void testEncodeDecode(Separator separator, String token) {
- String encoded = separator.encode(token);
- String decoded = separator.decode(encoded);
- String msg = "token:" + token + " separator:" + separator + ".";
- assertEquals(msg, token, decoded);
- }
-
- @Test
- public void testEncodeDecode() {
- testEncodeDecode("Dr.", Separator.QUALIFIERS);
- testEncodeDecode("Heinz", Separator.QUALIFIERS, Separator.QUALIFIERS);
- testEncodeDecode("Doofenshmirtz", Separator.QUALIFIERS, null,
- Separator.QUALIFIERS);
- testEncodeDecode("&Perry", Separator.QUALIFIERS, Separator.VALUES, null);
- testEncodeDecode("the ", Separator.QUALIFIERS, Separator.SPACE);
- testEncodeDecode("Platypus...", (Separator) null);
- testEncodeDecode("The what now ?!?", Separator.QUALIFIERS,
- Separator.VALUES, Separator.SPACE);
-
- }
- @Test
- public void testEncodedValues() {
- testEncodeDecode("Double-escape %2$ and %9$ or %%2$ or %%3$, nor %%%2$" +
- "= no problem!",
- Separator.QUALIFIERS, Separator.VALUES, Separator.SPACE, Separator.TAB);
- }
-
- @Test
- public void testSplits() {
- byte[] maxLongBytes = Bytes.toBytes(Long.MAX_VALUE);
- byte[] maxIntBytes = Bytes.toBytes(Integer.MAX_VALUE);
- for (Separator separator : Separator.values()) {
- String str1 = "cl" + separator.getValue() + "us";
- String str2 = separator.getValue() + "rst";
- byte[] sepByteArr = Bytes.toBytes(separator.getValue());
- byte[] longVal1Arr = Bytes.add(sepByteArr, Bytes.copy(maxLongBytes,
- sepByteArr.length, Bytes.SIZEOF_LONG - sepByteArr.length));
- byte[] intVal1Arr = Bytes.add(sepByteArr, Bytes.copy(maxIntBytes,
- sepByteArr.length, Bytes.SIZEOF_INT - sepByteArr.length));
- byte[] arr = separator.join(
- Bytes.toBytes(separator.encode(str1)), longVal1Arr,
- Bytes.toBytes(separator.encode(str2)), intVal1Arr);
- int[] sizes = {Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
- Separator.VARIABLE_SIZE, Bytes.SIZEOF_INT};
- byte[][] splits = separator.split(arr, sizes);
- assertEquals(4, splits.length);
- assertEquals(str1, separator.decode(Bytes.toString(splits[0])));
- assertEquals(Bytes.toLong(longVal1Arr), Bytes.toLong(splits[1]));
- assertEquals(str2, separator.decode(Bytes.toString(splits[2])));
- assertEquals(Bytes.toInt(intVal1Arr), Bytes.toInt(splits[3]));
-
- longVal1Arr = Bytes.add(Bytes.copy(maxLongBytes, 0, Bytes.SIZEOF_LONG -
- sepByteArr.length), sepByteArr);
- intVal1Arr = Bytes.add(Bytes.copy(maxIntBytes, 0, Bytes.SIZEOF_INT -
- sepByteArr.length), sepByteArr);
- arr = separator.join(Bytes.toBytes(separator.encode(str1)), longVal1Arr,
- Bytes.toBytes(separator.encode(str2)), intVal1Arr);
- splits = separator.split(arr, sizes);
- assertEquals(4, splits.length);
- assertEquals(str1, separator.decode(Bytes.toString(splits[0])));
- assertEquals(Bytes.toLong(longVal1Arr), Bytes.toLong(splits[1]));
- assertEquals(str2, separator.decode(Bytes.toString(splits[2])));
- assertEquals(Bytes.toInt(intVal1Arr), Bytes.toInt(splits[3]));
-
- longVal1Arr = Bytes.add(sepByteArr, Bytes.copy(maxLongBytes,
- sepByteArr.length, 4 - sepByteArr.length), sepByteArr);
- longVal1Arr = Bytes.add(longVal1Arr, Bytes.copy(maxLongBytes, 4, 3 -
- sepByteArr.length), sepByteArr);
- arr = separator.join(Bytes.toBytes(separator.encode(str1)), longVal1Arr,
- Bytes.toBytes(separator.encode(str2)), intVal1Arr);
- splits = separator.split(arr, sizes);
- assertEquals(4, splits.length);
- assertEquals(str1, separator.decode(Bytes.toString(splits[0])));
- assertEquals(Bytes.toLong(longVal1Arr), Bytes.toLong(splits[1]));
- assertEquals(str2, separator.decode(Bytes.toString(splits[2])));
- assertEquals(Bytes.toInt(intVal1Arr), Bytes.toInt(splits[3]));
-
- arr = separator.join(Bytes.toBytes(separator.encode(str1)),
- Bytes.toBytes(separator.encode(str2)), intVal1Arr, longVal1Arr);
- int[] sizes1 = {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
- Bytes.SIZEOF_INT, Bytes.SIZEOF_LONG};
- splits = separator.split(arr, sizes1);
- assertEquals(4, splits.length);
- assertEquals(str1, separator.decode(Bytes.toString(splits[0])));
- assertEquals(str2, separator.decode(Bytes.toString(splits[1])));
- assertEquals(Bytes.toInt(intVal1Arr), Bytes.toInt(splits[2]));
- assertEquals(Bytes.toLong(longVal1Arr), Bytes.toLong(splits[3]));
-
- try {
- int[] sizes2 = {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
- Bytes.SIZEOF_INT, 7};
- splits = separator.split(arr, sizes2);
- fail("Exception should have been thrown.");
- } catch (IllegalArgumentException e) {}
-
- try {
- int[] sizes2 = {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE, 2,
- Bytes.SIZEOF_LONG};
- splits = separator.split(arr, sizes2);
- fail("Exception should have been thrown.");
- } catch (IllegalArgumentException e) {}
- }
- }
-
- /**
- * Simple test to encode and decode using the same separators and confirm that
- * we end up with the same as what we started with.
- *
- * @param token
- * @param separators
- */
- private static void testEncodeDecode(String token, Separator... separators) {
- byte[] encoded = Separator.encode(token, separators);
- String decoded = Separator.decode(encoded, separators);
- assertEquals(token, decoded);
- }
-
- @Test
- public void testJoinStripped() {
- List<String> stringList = new ArrayList<String>(0);
- stringList.add("nothing");
-
- String joined = Separator.VALUES.joinEncoded(stringList);
- Iterable<String> split = Separator.VALUES.splitEncoded(joined);
- assertTrue(Iterables.elementsEqual(stringList, split));
-
- stringList = new ArrayList<String>(3);
- stringList.add("a");
- stringList.add("b?");
- stringList.add("c");
-
- joined = Separator.VALUES.joinEncoded(stringList);
- split = Separator.VALUES.splitEncoded(joined);
- assertTrue(Iterables.elementsEqual(stringList, split));
-
- String[] stringArray1 = {"else"};
- joined = Separator.VALUES.joinEncoded(stringArray1);
- split = Separator.VALUES.splitEncoded(joined);
- assertTrue(Iterables.elementsEqual(Arrays.asList(stringArray1), split));
-
- String[] stringArray2 = {"d", "e?", "f"};
- joined = Separator.VALUES.joinEncoded(stringArray2);
- split = Separator.VALUES.splitEncoded(joined);
- assertTrue(Iterables.elementsEqual(Arrays.asList(stringArray2), split));
-
- List<String> empty = new ArrayList<String>(0);
- split = Separator.VALUES.splitEncoded(null);
- assertTrue(Iterables.elementsEqual(empty, split));
-
- }
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/18] hadoop git commit: YARN-7919. Refactor timelineservice-hbase
module into submodules. Contributed by Haibo Chen.
Posted by ro...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
new file mode 100644
index 0000000..e89a6a7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
@@ -0,0 +1,251 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.application;
+
+import java.util.List;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.AppIdKeyConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverterToString;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+
+/**
+ * Represents a rowkey for the application table.
+ */
+public class ApplicationRowKey {
+ private final String clusterId;
+ private final String userId;
+ private final String flowName;
+ private final Long flowRunId;
+ private final String appId;
+ private final ApplicationRowKeyConverter appRowKeyConverter =
+ new ApplicationRowKeyConverter();
+
+ public ApplicationRowKey(String clusterId, String userId, String flowName,
+ Long flowRunId, String appId) {
+ this.clusterId = clusterId;
+ this.userId = userId;
+ this.flowName = flowName;
+ this.flowRunId = flowRunId;
+ this.appId = appId;
+ }
+
+ public String getClusterId() {
+ return clusterId;
+ }
+
+ public String getUserId() {
+ return userId;
+ }
+
+ public String getFlowName() {
+ return flowName;
+ }
+
+ public Long getFlowRunId() {
+ return flowRunId;
+ }
+
+ public String getAppId() {
+ return appId;
+ }
+
+ /**
+ * Constructs a row key for the application table as follows:
+ * {@code clusterId!userName!flowName!flowRunId!AppId}.
+ *
+ * @return byte array with the row key
+ */
+ public byte[] getRowKey() {
+ return appRowKeyConverter.encode(this);
+ }
+
+ /**
+ * Given the raw row key as bytes, returns the row key as an object.
+ *
+ * @param rowKey Byte representation of row key.
+ * @return An <cite>ApplicationRowKey</cite> object.
+ */
+ public static ApplicationRowKey parseRowKey(byte[] rowKey) {
+ return new ApplicationRowKeyConverter().decode(rowKey);
+ }
+
+ /**
+ * Constructs a row key for the application table as follows:
+ * {@code clusterId!userName!flowName!flowRunId!AppId}.
+ * @return String representation of row key.
+ */
+ public String getRowKeyAsString() {
+ return appRowKeyConverter.encodeAsString(this);
+ }
+
+ /**
+ * Given the encoded row key as string, returns the row key as an object.
+ * @param encodedRowKey String representation of row key.
+ * @return A <cite>ApplicationRowKey</cite> object.
+ */
+ public static ApplicationRowKey parseRowKeyFromString(String encodedRowKey) {
+ return new ApplicationRowKeyConverter().decodeFromString(encodedRowKey);
+ }
+
+ /**
+ * Encodes and decodes row key for application table. The row key is of the
+ * form: clusterId!userName!flowName!flowRunId!appId. flowRunId is a long,
+ * appId is encoded and decoded using {@link AppIdKeyConverter} and rest are
+ * strings.
+ * <p>
+ */
+ final private static class ApplicationRowKeyConverter implements
+ KeyConverter<ApplicationRowKey>, KeyConverterToString<ApplicationRowKey> {
+
+ private final KeyConverter<String> appIDKeyConverter =
+ new AppIdKeyConverter();
+
+ /**
+ * Intended for use in ApplicationRowKey only.
+ */
+ private ApplicationRowKeyConverter() {
+ }
+
+ /**
+ * Application row key is of the form
+ * clusterId!userName!flowName!flowRunId!appId with each segment separated
+ * by !. The sizes below indicate sizes of each one of these segements in
+ * sequence. clusterId, userName and flowName are strings. flowrunId is a
+ * long hence 8 bytes in size. app id is represented as 12 bytes with
+ * cluster timestamp part of appid takes 8 bytes(long) and seq id takes 4
+ * bytes(int). Strings are variable in size (i.e. end whenever separator is
+ * encountered). This is used while decoding and helps in determining where
+ * to split.
+ */
+ private static final int[] SEGMENT_SIZES = {Separator.VARIABLE_SIZE,
+ Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
+ AppIdKeyConverter.getKeySize() };
+
+ /*
+ * (non-Javadoc)
+ *
+ * Encodes ApplicationRowKey object into a byte array with each
+ * component/field in ApplicationRowKey separated by Separator#QUALIFIERS.
+ * This leads to an application table row key of the form
+ * clusterId!userName!flowName!flowRunId!appId If flowRunId in passed
+ * ApplicationRowKey object is null (and the fields preceding it i.e.
+ * clusterId, userId and flowName are not null), this returns a row key
+ * prefix of the form clusterId!userName!flowName! and if appId in
+ * ApplicationRowKey is null (other 4 components all are not null), this
+ * returns a row key prefix of the form
+ * clusterId!userName!flowName!flowRunId! flowRunId is inverted while
+ * encoding as it helps maintain a descending order for row keys in the
+ * application table.
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common
+ * .KeyConverter#encode(java.lang.Object)
+ */
+ @Override
+ public byte[] encode(ApplicationRowKey rowKey) {
+ byte[] cluster =
+ Separator.encode(rowKey.getClusterId(), Separator.SPACE,
+ Separator.TAB, Separator.QUALIFIERS);
+ byte[] user =
+ Separator.encode(rowKey.getUserId(), Separator.SPACE, Separator.TAB,
+ Separator.QUALIFIERS);
+ byte[] flow =
+ Separator.encode(rowKey.getFlowName(), Separator.SPACE,
+ Separator.TAB, Separator.QUALIFIERS);
+ byte[] first = Separator.QUALIFIERS.join(cluster, user, flow);
+ // Note that flowRunId is a long, so we can't encode them all at the same
+ // time.
+ if (rowKey.getFlowRunId() == null) {
+ return Separator.QUALIFIERS.join(first, Separator.EMPTY_BYTES);
+ }
+ byte[] second =
+ Bytes.toBytes(LongConverter.invertLong(
+ rowKey.getFlowRunId()));
+ if (rowKey.getAppId() == null || rowKey.getAppId().isEmpty()) {
+ return Separator.QUALIFIERS.join(first, second, Separator.EMPTY_BYTES);
+ }
+ byte[] third = appIDKeyConverter.encode(rowKey.getAppId());
+ return Separator.QUALIFIERS.join(first, second, third);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * Decodes an application row key of the form
+ * clusterId!userName!flowName!flowRunId!appId represented in byte format
+ * and converts it into an ApplicationRowKey object.flowRunId is inverted
+ * while decoding as it was inverted while encoding.
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common
+ * .KeyConverter#decode(byte[])
+ */
+ @Override
+ public ApplicationRowKey decode(byte[] rowKey) {
+ byte[][] rowKeyComponents =
+ Separator.QUALIFIERS.split(rowKey, SEGMENT_SIZES);
+ if (rowKeyComponents.length != 5) {
+ throw new IllegalArgumentException("the row key is not valid for "
+ + "an application");
+ }
+ String clusterId =
+ Separator.decode(Bytes.toString(rowKeyComponents[0]),
+ Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
+ String userId =
+ Separator.decode(Bytes.toString(rowKeyComponents[1]),
+ Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
+ String flowName =
+ Separator.decode(Bytes.toString(rowKeyComponents[2]),
+ Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
+ Long flowRunId =
+ LongConverter.invertLong(Bytes.toLong(rowKeyComponents[3]));
+ String appId = appIDKeyConverter.decode(rowKeyComponents[4]);
+ return new ApplicationRowKey(clusterId, userId, flowName, flowRunId,
+ appId);
+ }
+
+ @Override
+ public String encodeAsString(ApplicationRowKey key) {
+ if (key.clusterId == null || key.userId == null || key.flowName == null
+ || key.flowRunId == null || key.appId == null) {
+ throw new IllegalArgumentException();
+ }
+ return TimelineReaderUtils
+ .joinAndEscapeStrings(new String[] {key.clusterId, key.userId,
+ key.flowName, key.flowRunId.toString(), key.appId});
+ }
+
+ @Override
+ public ApplicationRowKey decodeFromString(String encodedRowKey) {
+ List<String> split = TimelineReaderUtils.split(encodedRowKey);
+ if (split == null || split.size() != 5) {
+ throw new IllegalArgumentException(
+ "Invalid row key for application table.");
+ }
+ Long flowRunId = Long.valueOf(split.get(3));
+ return new ApplicationRowKey(split.get(0), split.get(1), split.get(2),
+ flowRunId, split.get(4));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKeyPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKeyPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKeyPrefix.java
new file mode 100644
index 0000000..f61b0e9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKeyPrefix.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.application;
+
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.RowKeyPrefix;
+
+/**
+ * Represents a partial rowkey (without flowName or without flowName and
+ * flowRunId) for the application table.
+ */
+public class ApplicationRowKeyPrefix extends ApplicationRowKey implements
+ RowKeyPrefix<ApplicationRowKey> {
+
+ /**
+ * Creates a prefix which generates the following rowKeyPrefixes for the
+ * application table: {@code clusterId!userName!flowName!}.
+ *
+ * @param clusterId the cluster on which applications ran
+ * @param userId the user that ran applications
+ * @param flowName the name of the flow that was run by the user on the
+ * cluster
+ */
+ public ApplicationRowKeyPrefix(String clusterId, String userId,
+ String flowName) {
+ super(clusterId, userId, flowName, null, null);
+ }
+
+ /**
+ * Creates a prefix which generates the following rowKeyPrefixes for the
+ * application table: {@code clusterId!userName!flowName!flowRunId!}.
+ *
+ * @param clusterId identifying the cluster
+ * @param userId identifying the user
+ * @param flowName identifying the flow
+ * @param flowRunId identifying the instance of this flow
+ */
+ public ApplicationRowKeyPrefix(String clusterId, String userId,
+ String flowName, Long flowRunId) {
+ super(clusterId, userId, flowName, flowRunId, null);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.application.
+ * RowKeyPrefix#getRowKeyPrefix()
+ */
+ @Override
+ public byte[] getRowKeyPrefix() {
+ return super.getRowKey();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
new file mode 100644
index 0000000..16ab5fa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.application;
+
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+
+/**
+ * The application table as column families info, config and metrics. Info
+ * stores information about a YARN application entity, config stores
+ * configuration data of a YARN application, metrics stores the metrics of a
+ * YARN application. This table is entirely analogous to the entity table but
+ * created for better performance.
+ *
+ * Example application table record:
+ *
+ * <pre>
+ * |-------------------------------------------------------------------------|
+ * | Row | Column Family | Column Family| Column Family|
+ * | key | info | metrics | config |
+ * |-------------------------------------------------------------------------|
+ * | clusterId! | id:appId | metricId1: | configKey1: |
+ * | userName! | | metricValue1 | configValue1 |
+ * | flowName! | created_time: | @timestamp1 | |
+ * | flowRunId! | 1392993084018 | | configKey2: |
+ * | AppId | | metriciD1: | configValue2 |
+ * | | i!infoKey: | metricValue2 | |
+ * | | infoValue | @timestamp2 | |
+ * | | | | |
+ * | | r!relatesToKey: | metricId2: | |
+ * | | id3=id4=id5 | metricValue1 | |
+ * | | | @timestamp2 | |
+ * | | s!isRelatedToKey: | | |
+ * | | id7=id9=id6 | | |
+ * | | | | |
+ * | | e!eventId=timestamp=infoKey: | | |
+ * | | eventInfoValue | | |
+ * | | | | |
+ * | | flowVersion: | | |
+ * | | versionValue | | |
+ * |-------------------------------------------------------------------------|
+ * </pre>
+ */
+public final class ApplicationTable extends BaseTable<ApplicationTable> {
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/package-info.java
new file mode 100644
index 0000000..03f508f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.storage.application
+ * contains classes related to implementation for application table.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.application;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java
new file mode 100644
index 0000000..0065f07
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow;
+
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Column;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.GenericConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
+
+/**
+ * Identifies fully qualified columns for the {@link AppToFlowTable}.
+ */
+public enum AppToFlowColumn implements Column<AppToFlowTable> {
+
+ /**
+ * The flow ID.
+ */
+ FLOW_ID(AppToFlowColumnFamily.MAPPING, "flow_id"),
+
+ /**
+ * The flow run ID.
+ */
+ FLOW_RUN_ID(AppToFlowColumnFamily.MAPPING, "flow_run_id"),
+
+ /**
+ * The user.
+ */
+ USER_ID(AppToFlowColumnFamily.MAPPING, "user_id");
+
+ private final ColumnFamily<AppToFlowTable> columnFamily;
+ private final String columnQualifier;
+ private final byte[] columnQualifierBytes;
+ private final ValueConverter valueConverter;
+
+ AppToFlowColumn(ColumnFamily<AppToFlowTable> columnFamily,
+ String columnQualifier) {
+ this.columnFamily = columnFamily;
+ this.columnQualifier = columnQualifier;
+ // Future-proof by ensuring the right column prefix hygiene.
+ this.columnQualifierBytes =
+ Bytes.toBytes(Separator.SPACE.encode(columnQualifier));
+ this.valueConverter = GenericConverter.getInstance();
+ }
+
+ /**
+ * @return the column name value
+ */
+ private String getColumnQualifier() {
+ return columnQualifier;
+ }
+
+ @Override
+ public byte[] getColumnQualifierBytes() {
+ return columnQualifierBytes.clone();
+ }
+
+ @Override
+ public byte[] getColumnFamilyBytes() {
+ return columnFamily.getBytes();
+ }
+
+ @Override
+ public ValueConverter getValueConverter() {
+ return valueConverter;
+ }
+
+ @Override
+ public Attribute[] getCombinedAttrsWithAggr(Attribute... attributes) {
+ return attributes;
+ }
+
+ @Override
+ public boolean supplementCellTimestamp() {
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnFamily.java
new file mode 100644
index 0000000..f3f045e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnFamily.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+
+/**
+ * Represents the app_flow table column families.
+ */
+public enum AppToFlowColumnFamily implements ColumnFamily<AppToFlowTable> {
+ /**
+ * Mapping column family houses known columns such as flowName and flowRunId.
+ */
+ MAPPING("m");
+
+ /**
+ * Byte representation of this column family.
+ */
+ private final byte[] bytes;
+
+ /**
+ * @param value create a column family with this name. Must be lower case and
+ * without spaces.
+ */
+ AppToFlowColumnFamily(String value) {
+ // column families should be lower case and not contain any spaces.
+ this.bytes = Bytes.toBytes(Separator.SPACE.encode(value));
+ }
+
+ public byte[] getBytes() {
+ return Bytes.copy(bytes);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnPrefix.java
new file mode 100644
index 0000000..9540129
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnPrefix.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.GenericConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
+
+/**
+ * Identifies partially qualified columns for the app-to-flow table.
+ */
+public enum AppToFlowColumnPrefix implements ColumnPrefix<AppToFlowTable> {
+
+ /**
+ * The flow name.
+ */
+ FLOW_NAME(AppToFlowColumnFamily.MAPPING, "flow_name"),
+
+ /**
+ * The flow run ID.
+ */
+ FLOW_RUN_ID(AppToFlowColumnFamily.MAPPING, "flow_run_id"),
+
+ /**
+ * The user.
+ */
+ USER_ID(AppToFlowColumnFamily.MAPPING, "user_id");
+
+ private final ColumnFamily<AppToFlowTable> columnFamily;
+ private final String columnPrefix;
+ private final byte[] columnPrefixBytes;
+ private final ValueConverter valueConverter;
+
+ AppToFlowColumnPrefix(ColumnFamily<AppToFlowTable> columnFamily,
+ String columnPrefix) {
+ this.columnFamily = columnFamily;
+ this.columnPrefix = columnPrefix;
+ if (columnPrefix == null) {
+ this.columnPrefixBytes = null;
+ } else {
+ // Future-proof by ensuring the right column prefix hygiene.
+ this.columnPrefixBytes =
+ Bytes.toBytes(Separator.SPACE.encode(columnPrefix));
+ }
+ this.valueConverter = GenericConverter.getInstance();
+ }
+
+ @Override
+ public byte[] getColumnPrefixBytes(String qualifierPrefix) {
+ return ColumnHelper.getColumnQualifier(
+ columnPrefixBytes, qualifierPrefix);
+ }
+
+ @Override
+ public byte[] getColumnPrefixBytes(byte[] qualifierPrefix) {
+ return ColumnHelper.getColumnQualifier(
+ columnPrefixBytes, qualifierPrefix);
+ }
+
+ @Override
+ public byte[] getColumnPrefixInBytes() {
+ return columnPrefixBytes != null ? columnPrefixBytes.clone() : null;
+ }
+
+ @Override
+ public byte[] getColumnFamilyBytes() {
+ return columnFamily.getBytes();
+ }
+
+ @Override
+ public ValueConverter getValueConverter() {
+ return valueConverter;
+ }
+
+ @Override
+ public Attribute[] getCombinedAttrsWithAggr(Attribute... attributes) {
+ return attributes;
+ }
+
+ @Override
+ public boolean supplementCellTimeStamp() {
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java
new file mode 100644
index 0000000..146c475
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow;
+
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.AppIdKeyConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
+
+/**
+ * Represents a row key for the app_flow table, which is the app id.
+ */
+public class AppToFlowRowKey {
+ private final String appId;
+ private final KeyConverter<String> appIdKeyConverter =
+ new AppIdKeyConverter();
+
+ public AppToFlowRowKey(String appId) {
+ this.appId = appId;
+ }
+
+ public String getAppId() {
+ return appId;
+ }
+
+ /**
+ * Constructs a row key prefix for the app_flow table.
+ *
+ * @return byte array with the row key
+ */
+ public byte[] getRowKey() {
+ return appIdKeyConverter.encode(appId);
+ }
+
+ /**
+ * Given the raw row key as bytes, returns the row key as an object.
+ *
+ * @param rowKey a rowkey represented as a byte array.
+ * @return an <cite>AppToFlowRowKey</cite> object.
+ */
+ public static AppToFlowRowKey parseRowKey(byte[] rowKey) {
+ String appId = new AppIdKeyConverter().decode(rowKey);
+ return new AppToFlowRowKey(appId);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java
new file mode 100644
index 0000000..e184288
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow;
+
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+
+/**
+ * The app_flow table as column families mapping. Mapping stores
+ * appId to flowName and flowRunId mapping information
+ *
+ * Example app_flow table record:
+ *
+ * <pre>
+ * |--------------------------------------|
+ * | Row | Column Family |
+ * | key | mapping |
+ * |--------------------------------------|
+ * | appId | flow_name!cluster1: |
+ * | | foo@daily_hive_report |
+ * | | |
+ * | | flow_run_id!cluster1: |
+ * | | 1452828720457 |
+ * | | |
+ * | | user_id!cluster1: |
+ * | | admin |
+ * | | |
+ * | | flow_name!cluster2: |
+ * | | bar@ad_hoc_query |
+ * | | |
+ * | | flow_run_id!cluster2: |
+ * | | 1452828498752 |
+ * | | |
+ * | | user_id!cluster2: |
+ * | | joe |
+ * | | |
+ * |--------------------------------------|
+ * </pre>
+ *
+ * It is possible (although unlikely) in a multi-cluster environment that there
+ * may be more than one applications for a given app id. Different clusters are
+ * recorded as different sets of columns.
+ */
+public final class AppToFlowTable extends BaseTable<AppToFlowTable> {
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/package-info.java
new file mode 100644
index 0000000..f01d982
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow
+ * contains classes related to implementation for app to flow table.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/AppIdKeyConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/AppIdKeyConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/AppIdKeyConverter.java
new file mode 100644
index 0000000..d934f74
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/AppIdKeyConverter.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+/**
+ * Encodes and decodes {@link ApplicationId} for row keys.
+ * App ID is stored in row key as 12 bytes, cluster timestamp section of app id
+ * (long - 8 bytes) followed by sequence id section of app id (int - 4 bytes).
+ */
+public final class AppIdKeyConverter implements KeyConverter<String> {
+
+ public AppIdKeyConverter() {
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * Converts/encodes a string app Id into a byte representation for (row) keys.
+ * For conversion, we extract cluster timestamp and sequence id from the
+ * string app id (calls ConverterUtils#toApplicationId(String) for
+ * conversion) and then store it in a byte array of length 12 (8 bytes (long)
+ * for cluster timestamp followed 4 bytes(int) for sequence id). Both cluster
+ * timestamp and sequence id are inverted so that the most recent cluster
+ * timestamp and highest sequence id appears first in the table (i.e.
+ * application id appears in a descending order).
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter
+ * #encode(java.lang.Object)
+ */
+ @Override
+ public byte[] encode(String appIdStr) {
+ ApplicationId appId = ApplicationId.fromString(appIdStr);
+ byte[] appIdBytes = new byte[getKeySize()];
+ byte[] clusterTs = Bytes.toBytes(
+ LongConverter.invertLong(appId.getClusterTimestamp()));
+ System.arraycopy(clusterTs, 0, appIdBytes, 0, Bytes.SIZEOF_LONG);
+ byte[] seqId = Bytes.toBytes(
+ HBaseTimelineSchemaUtils.invertInt(appId.getId()));
+ System.arraycopy(seqId, 0, appIdBytes, Bytes.SIZEOF_LONG, Bytes.SIZEOF_INT);
+ return appIdBytes;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * Converts/decodes a 12 byte representation of app id for (row) keys to an
+ * app id in string format which can be returned back to client.
+ * For decoding, 12 bytes are interpreted as 8 bytes of inverted cluster
+ * timestamp(long) followed by 4 bytes of inverted sequence id(int). Calls
+ * ApplicationId#toString to generate string representation of app id.
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter
+ * #decode(byte[])
+ */
+ @Override
+ public String decode(byte[] appIdBytes) {
+ if (appIdBytes.length != getKeySize()) {
+ throw new IllegalArgumentException("Invalid app id in byte format");
+ }
+ long clusterTs = LongConverter.invertLong(
+ Bytes.toLong(appIdBytes, 0, Bytes.SIZEOF_LONG));
+ int seqId = HBaseTimelineSchemaUtils.invertInt(
+ Bytes.toInt(appIdBytes, Bytes.SIZEOF_LONG, Bytes.SIZEOF_INT));
+ return HBaseTimelineSchemaUtils.convertApplicationIdToString(
+ ApplicationId.newInstance(clusterTs, seqId));
+ }
+
+ /**
+ * Returns the size of app id after encoding.
+ *
+ * @return size of app id after encoding.
+ */
+ public static int getKeySize() {
+ return Bytes.SIZEOF_LONG + Bytes.SIZEOF_INT;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
new file mode 100644
index 0000000..433b352
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+
+/**
+ * The base type of tables.
+ * @param T table type
+ */
+public abstract class BaseTable<T> {
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java
new file mode 100644
index 0000000..2b50252
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
+
+/**
+ * A Column represents the way to store a fully qualified column in a specific
+ * table.
+ */
+public interface Column<T extends BaseTable<T>> {
+ /**
+ * Returns column family name(as bytes) associated with this column.
+ * @return a byte array encoding column family for this column qualifier.
+ */
+ byte[] getColumnFamilyBytes();
+
+ /**
+ * Get byte representation for this column qualifier.
+ * @return a byte array representing column qualifier.
+ */
+ byte[] getColumnQualifierBytes();
+
+ /**
+ * Returns value converter implementation associated with this column.
+ * @return a {@link ValueConverter} implementation.
+ */
+ ValueConverter getValueConverter();
+
+ /**
+ * Return attributed combined with aggregations, if any.
+ * @return an array of Attributes
+ */
+ Attribute[] getCombinedAttrsWithAggr(Attribute... attributes);
+
+ /**
+ * Return true if the cell timestamp needs to be supplemented.
+ * @return true if the cell timestamp needs to be supplemented
+ */
+ boolean supplementCellTimestamp();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnFamily.java
new file mode 100644
index 0000000..c5c8cb4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnFamily.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+/**
+ * Type safe column family.
+ *
+ * @param <T> refers to the table for which this column family is used for.
+ */
+public interface ColumnFamily<T extends BaseTable<T>> {
+
+ /**
+ * Keep a local copy if you need to avoid overhead of repeated cloning.
+ *
+ * @return a clone of the byte representation of the column family.
+ */
+ byte[] getBytes();
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
new file mode 100644
index 0000000..b173ef2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * This class is meant to be used only by explicit Columns, and not directly to
+ * write by clients.
+ */
+public final class ColumnHelper {
+
+ private ColumnHelper() {
+ }
+
+
+ /**
+ * @param columnPrefixBytes The byte representation for the column prefix.
+ * Should not contain {@link Separator#QUALIFIERS}.
+ * @param qualifier for the remainder of the column.
+ * {@link Separator#QUALIFIERS} is permissible in the qualifier
+ * as it is joined only with the column prefix bytes.
+ * @return fully sanitized column qualifier that is a combination of prefix
+ * and qualifier. If prefix is null, the result is simply the encoded
+ * qualifier without any separator.
+ */
+ public static byte[] getColumnQualifier(byte[] columnPrefixBytes,
+ String qualifier) {
+
+ // We don't want column names to have spaces / tabs.
+ byte[] encodedQualifier =
+ Separator.encode(qualifier, Separator.SPACE, Separator.TAB);
+ if (columnPrefixBytes == null) {
+ return encodedQualifier;
+ }
+
+ // Convert qualifier to lower case, strip of separators and tag on column
+ // prefix.
+ byte[] columnQualifier =
+ Separator.QUALIFIERS.join(columnPrefixBytes, encodedQualifier);
+ return columnQualifier;
+ }
+
+ /**
+ * @param columnPrefixBytes The byte representation for the column prefix.
+ * Should not contain {@link Separator#QUALIFIERS}.
+ * @param qualifier for the remainder of the column.
+ * @return fully sanitized column qualifier that is a combination of prefix
+ * and qualifier. If prefix is null, the result is simply the encoded
+ * qualifier without any separator.
+ */
+ public static byte[] getColumnQualifier(byte[] columnPrefixBytes,
+ long qualifier) {
+
+ if (columnPrefixBytes == null) {
+ return Bytes.toBytes(qualifier);
+ }
+
+ // Convert qualifier to lower case, strip of separators and tag on column
+ // prefix.
+ byte[] columnQualifier =
+ Separator.QUALIFIERS.join(columnPrefixBytes, Bytes.toBytes(qualifier));
+ return columnQualifier;
+ }
+
+ /**
+ * @param columnPrefixBytes The byte representation for the column prefix.
+ * Should not contain {@link Separator#QUALIFIERS}.
+ * @param qualifier the byte representation for the remainder of the column.
+ * @return fully sanitized column qualifier that is a combination of prefix
+ * and qualifier. If prefix is null, the result is simply the encoded
+ * qualifier without any separator.
+ */
+ public static byte[] getColumnQualifier(byte[] columnPrefixBytes,
+ byte[] qualifier) {
+
+ if (columnPrefixBytes == null) {
+ return qualifier;
+ }
+
+ byte[] columnQualifier =
+ Separator.QUALIFIERS.join(columnPrefixBytes, qualifier);
+ return columnQualifier;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
new file mode 100644
index 0000000..cbcd936
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
+
+/**
+ * Used to represent a partially qualified column, where the actual column name
+ * will be composed of a prefix and the remainder of the column qualifier. The
+ * prefix can be null, in which case the column qualifier will be completely
+ * determined when the values are stored.
+ */
+public interface ColumnPrefix<T extends BaseTable<T>> {
+
+ /**
+ * @param qualifierPrefix Column qualifier or prefix of qualifier.
+ * @return a byte array encoding column prefix and qualifier/prefix passed.
+ */
+ byte[] getColumnPrefixBytes(String qualifierPrefix);
+
+ /**
+ * @param qualifierPrefix Column qualifier or prefix of qualifier.
+ * @return a byte array encoding column prefix and qualifier/prefix passed.
+ */
+ byte[] getColumnPrefixBytes(byte[] qualifierPrefix);
+
+ /**
+ * Get the column prefix in bytes.
+ * @return column prefix in bytes
+ */
+ byte[] getColumnPrefixInBytes();
+
+ /**
+ * Returns column family name(as bytes) associated with this column prefix.
+ * @return a byte array encoding column family for this prefix.
+ */
+ byte[] getColumnFamilyBytes();
+
+ /**
+ * Returns value converter implementation associated with this column prefix.
+ * @return a {@link ValueConverter} implementation.
+ */
+ ValueConverter getValueConverter();
+
+ /**
+ * Return attributed combined with aggregations, if any.
+ * @return an array of Attributes
+ */
+ Attribute[] getCombinedAttrsWithAggr(Attribute... attributes);
+
+ /**
+ * Return true if the cell timestamp needs to be supplemented.
+ * @return true if the cell timestamp needs to be supplemented
+ */
+ boolean supplementCellTimeStamp();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/EventColumnName.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/EventColumnName.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/EventColumnName.java
new file mode 100644
index 0000000..8445575
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/EventColumnName.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+/**
+ * Encapsulates information about Event column names for application and entity
+ * tables. Used while encoding/decoding event column names.
+ */
+public class EventColumnName {
+
+ private final String id;
+ private final Long timestamp;
+ private final String infoKey;
+ private final KeyConverter<EventColumnName> eventColumnNameConverter =
+ new EventColumnNameConverter();
+
+ public EventColumnName(String id, Long timestamp, String infoKey) {
+ this.id = id;
+ this.timestamp = timestamp;
+ this.infoKey = infoKey;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public Long getTimestamp() {
+ return timestamp;
+ }
+
+ public String getInfoKey() {
+ return infoKey;
+ }
+
+ /**
+ * @return a byte array with each components/fields separated by
+ * Separator#VALUES. This leads to an event column name of the form
+ * eventId=timestamp=infokey. If both timestamp and infokey are null,
+ * then a qualifier of the form eventId=timestamp= is returned. If
+ * only infokey is null, then a qualifier of the form eventId= is
+ * returned. These prefix forms are useful for queries that intend to
+ * retrieve more than one specific column name.
+ */
+ public byte[] getColumnQualifier() {
+ return eventColumnNameConverter.encode(this);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/EventColumnNameConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/EventColumnNameConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/EventColumnNameConverter.java
new file mode 100644
index 0000000..d3ef897
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/EventColumnNameConverter.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Encodes and decodes event column names for application and entity tables.
+ * The event column name is of the form : eventId=timestamp=infokey.
+ * If info is not associated with the event, event column name is of the form :
+ * eventId=timestamp=
+ * Event timestamp is long and rest are strings.
+ * Column prefixes are not part of the eventcolumn name passed for encoding. It
+ * is added later, if required in the associated ColumnPrefix implementations.
+ */
+public final class EventColumnNameConverter
+ implements KeyConverter<EventColumnName> {
+
+ public EventColumnNameConverter() {
+ }
+
+ // eventId=timestamp=infokey are of types String, Long String
+ // Strings are variable in size (i.e. end whenever separator is encountered).
+ // This is used while decoding and helps in determining where to split.
+ private static final int[] SEGMENT_SIZES = {
+ Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG, Separator.VARIABLE_SIZE };
+
+ /*
+ * (non-Javadoc)
+ *
+ * Encodes EventColumnName into a byte array with each component/field in
+ * EventColumnName separated by Separator#VALUES. This leads to an event
+ * column name of the form eventId=timestamp=infokey.
+ * If timestamp in passed EventColumnName object is null (eventId is not null)
+ * this returns a column prefix of the form eventId= and if infokey in
+ * EventColumnName is null (other 2 components are not null), this returns a
+ * column name of the form eventId=timestamp=
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter
+ * #encode(java.lang.Object)
+ */
+ @Override
+ public byte[] encode(EventColumnName key) {
+ byte[] first = Separator.encode(key.getId(), Separator.SPACE, Separator.TAB,
+ Separator.VALUES);
+ if (key.getTimestamp() == null) {
+ return Separator.VALUES.join(first, Separator.EMPTY_BYTES);
+ }
+ byte[] second = Bytes.toBytes(
+ LongConverter.invertLong(key.getTimestamp()));
+ if (key.getInfoKey() == null) {
+ return Separator.VALUES.join(first, second, Separator.EMPTY_BYTES);
+ }
+ return Separator.VALUES.join(first, second, Separator.encode(
+ key.getInfoKey(), Separator.SPACE, Separator.TAB, Separator.VALUES));
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * Decodes an event column name of the form eventId=timestamp= or
+ * eventId=timestamp=infoKey represented in byte format and converts it into
+ * an EventColumnName object.
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter
+ * #decode(byte[])
+ */
+ @Override
+ public EventColumnName decode(byte[] bytes) {
+ byte[][] components = Separator.VALUES.split(bytes, SEGMENT_SIZES);
+ if (components.length != 3) {
+ throw new IllegalArgumentException("the column name is not valid");
+ }
+ String id = Separator.decode(Bytes.toString(components[0]),
+ Separator.VALUES, Separator.TAB, Separator.SPACE);
+ Long ts = LongConverter.invertLong(Bytes.toLong(components[1]));
+ String infoKey = components[2].length == 0 ? null :
+ Separator.decode(Bytes.toString(components[2]),
+ Separator.VALUES, Separator.TAB, Separator.SPACE);
+ return new EventColumnName(id, ts, infoKey);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/GenericConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/GenericConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/GenericConverter.java
new file mode 100644
index 0000000..c34bfcb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/GenericConverter.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import java.io.IOException;
+
+import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
+
+/**
+ * Uses GenericObjectMapper to encode objects as bytes and decode bytes as
+ * objects.
+ */
+public final class GenericConverter implements ValueConverter {
+ private static final GenericConverter INSTANCE = new GenericConverter();
+
+ private GenericConverter() {
+ }
+
+ public static GenericConverter getInstance() {
+ return INSTANCE;
+ }
+
+ @Override
+ public byte[] encodeValue(Object value) throws IOException {
+ return GenericObjectMapper.write(value);
+ }
+
+ @Override
+ public Object decodeValue(byte[] bytes) throws IOException {
+ return GenericObjectMapper.read(bytes);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineSchemaUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineSchemaUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineSchemaUtils.java
new file mode 100644
index 0000000..e5f92cc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineSchemaUtils.java
@@ -0,0 +1,156 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationOperation;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
+
+import java.text.NumberFormat;
+
+/**
+ * A bunch of utility functions used in HBase TimelineService common module.
+ */
+public final class HBaseTimelineSchemaUtils {
+ /** milliseconds in one day. */
+ public static final long MILLIS_ONE_DAY = 86400000L;
+
+ private static final ThreadLocal<NumberFormat> APP_ID_FORMAT =
+ new ThreadLocal<NumberFormat>() {
+ @Override
+ public NumberFormat initialValue() {
+ NumberFormat fmt = NumberFormat.getInstance();
+ fmt.setGroupingUsed(false);
+ fmt.setMinimumIntegerDigits(4);
+ return fmt;
+ }
+ };
+
+ private HBaseTimelineSchemaUtils() {
+ }
+
+ /**
+ * Combines the input array of attributes and the input aggregation operation
+ * into a new array of attributes.
+ *
+ * @param attributes Attributes to be combined.
+ * @param aggOp Aggregation operation.
+ * @return array of combined attributes.
+ */
+ public static Attribute[] combineAttributes(Attribute[] attributes,
+ AggregationOperation aggOp) {
+ int newLength = getNewLengthCombinedAttributes(attributes, aggOp);
+ Attribute[] combinedAttributes = new Attribute[newLength];
+
+ if (attributes != null) {
+ System.arraycopy(attributes, 0, combinedAttributes, 0, attributes.length);
+ }
+
+ if (aggOp != null) {
+ Attribute a2 = aggOp.getAttribute();
+ combinedAttributes[newLength - 1] = a2;
+ }
+ return combinedAttributes;
+ }
+
+ /**
+ * Returns a number for the new array size. The new array is the combination
+ * of input array of attributes and the input aggregation operation.
+ *
+ * @param attributes Attributes.
+ * @param aggOp Aggregation operation.
+ * @return the size for the new array
+ */
+ private static int getNewLengthCombinedAttributes(Attribute[] attributes,
+ AggregationOperation aggOp) {
+ int oldLength = getAttributesLength(attributes);
+ int aggLength = getAppOpLength(aggOp);
+ return oldLength + aggLength;
+ }
+
+ private static int getAppOpLength(AggregationOperation aggOp) {
+ if (aggOp != null) {
+ return 1;
+ }
+ return 0;
+ }
+
+ private static int getAttributesLength(Attribute[] attributes) {
+ if (attributes != null) {
+ return attributes.length;
+ }
+ return 0;
+ }
+
+ /**
+ * Converts an int into it's inverse int to be used in (row) keys
+ * where we want to have the largest int value in the top of the table
+ * (scans start at the largest int first).
+ *
+ * @param key value to be inverted so that the latest version will be first in
+ * a scan.
+ * @return inverted int
+ */
+ public static int invertInt(int key) {
+ return Integer.MAX_VALUE - key;
+ }
+
+ /**
+ * returns the timestamp of that day's start (which is midnight 00:00:00 AM)
+ * for a given input timestamp.
+ *
+ * @param ts Timestamp.
+ * @return timestamp of that day's beginning (midnight)
+ */
+ public static long getTopOfTheDayTimestamp(long ts) {
+ long dayTimestamp = ts - (ts % MILLIS_ONE_DAY);
+ return dayTimestamp;
+ }
+
+ /**
+ * Checks if passed object is of integral type(Short/Integer/Long).
+ *
+ * @param obj Object to be checked.
+ * @return true if object passed is of type Short or Integer or Long, false
+ * otherwise.
+ */
+ public static boolean isIntegralValue(Object obj) {
+ return (obj instanceof Short) || (obj instanceof Integer) ||
+ (obj instanceof Long);
+ }
+
+ /**
+ * A utility method that converts ApplicationId to string without using
+ * FastNumberFormat in order to avoid the incompatibility issue caused
+ * by mixing hadoop-common 2.5.1 and hadoop-yarn-api 3.0 in this module.
+ * This is a work-around implementation as discussed in YARN-6905.
+ *
+ * @param appId application id
+ * @return the string representation of the given application id
+ *
+ */
+ public static String convertApplicationIdToString(ApplicationId appId) {
+ StringBuilder sb = new StringBuilder(64);
+ sb.append(ApplicationId.appIdStrPrefix);
+ sb.append("_");
+ sb.append(appId.getClusterTimestamp());
+ sb.append('_');
+ sb.append(APP_ID_FORMAT.get().format(appId.getId()));
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/KeyConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/KeyConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/KeyConverter.java
new file mode 100644
index 0000000..4229e81
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/KeyConverter.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+/**
+ * Interface which has to be implemented for encoding and decoding row keys and
+ * columns.
+ */
+public interface KeyConverter<T> {
+ /**
+ * Encodes a key as a byte array.
+ *
+ * @param key key to be encoded.
+ * @return a byte array.
+ */
+ byte[] encode(T key);
+
+ /**
+ * Decodes a byte array and returns a key of type T.
+ *
+ * @param bytes byte representation
+ * @return an object(key) of type T which has been constructed after decoding
+ * the bytes.
+ */
+ T decode(byte[] bytes);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/KeyConverterToString.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/KeyConverterToString.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/KeyConverterToString.java
new file mode 100644
index 0000000..1f52a7b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/KeyConverterToString.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+/**
+ * Interface which has to be implemented for encoding and decoding row keys or
+ * column qualifiers as string.
+ */
+public interface KeyConverterToString<T> {
+ /**
+ * Encode key as string.
+ * @param key of type T to be encoded as string.
+ * @return encoded value as string.
+ */
+ String encodeAsString(T key);
+
+ /**
+ * Decode row key from string to a key of type T.
+ * @param encodedKey string representation of row key
+ * @return type T which has been constructed after decoding string.
+ */
+ T decodeFromString(String encodedKey);
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/18] hadoop git commit: YARN-7919. Refactor timelineservice-hbase
module into submodules. Contributed by Haibo Chen.
Posted by ro...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/SubApplicationEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/SubApplicationEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/SubApplicationEntityReader.java
deleted file mode 100644
index e780dcc..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/SubApplicationEntityReader.java
+++ /dev/null
@@ -1,488 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.Set;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Query;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.filter.BinaryComparator;
-import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
-import org.apache.hadoop.hbase.filter.FamilyFilter;
-import org.apache.hadoop.hbase.filter.FilterList;
-import org.apache.hadoop.hbase.filter.FilterList.Operator;
-import org.apache.hadoop.hbase.filter.PageFilter;
-import org.apache.hadoop.hbase.filter.QualifierFilter;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.RowKeyPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationColumn;
-import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationRowKeyPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationTable;
-import org.apache.hadoop.yarn.webapp.BadRequestException;
-
-import com.google.common.base.Preconditions;
-
-class SubApplicationEntityReader extends GenericEntityReader {
- private static final SubApplicationTable SUB_APPLICATION_TABLE =
- new SubApplicationTable();
-
- SubApplicationEntityReader(TimelineReaderContext ctxt,
- TimelineEntityFilters entityFilters, TimelineDataToRetrieve toRetrieve) {
- super(ctxt, entityFilters, toRetrieve);
- }
-
- SubApplicationEntityReader(TimelineReaderContext ctxt,
- TimelineDataToRetrieve toRetrieve) {
- super(ctxt, toRetrieve);
- }
-
- /**
- * Uses the {@link SubApplicationTable}.
- */
- protected BaseTable<?> getTable() {
- return SUB_APPLICATION_TABLE;
- }
-
- @Override
- protected FilterList constructFilterListBasedOnFilters() throws IOException {
- // Filters here cannot be null for multiple entity reads as they are set in
- // augmentParams if null.
- FilterList listBasedOnFilters = new FilterList();
- TimelineEntityFilters filters = getFilters();
- // Create filter list based on created time range and add it to
- // listBasedOnFilters.
- long createdTimeBegin = filters.getCreatedTimeBegin();
- long createdTimeEnd = filters.getCreatedTimeEnd();
- if (createdTimeBegin != 0 || createdTimeEnd != Long.MAX_VALUE) {
- listBasedOnFilters.addFilter(TimelineFilterUtils
- .createSingleColValueFiltersByRange(SubApplicationColumn.CREATED_TIME,
- createdTimeBegin, createdTimeEnd));
- }
- // Create filter list based on metric filters and add it to
- // listBasedOnFilters.
- TimelineFilterList metricFilters = filters.getMetricFilters();
- if (metricFilters != null && !metricFilters.getFilterList().isEmpty()) {
- listBasedOnFilters.addFilter(TimelineFilterUtils.createHBaseFilterList(
- SubApplicationColumnPrefix.METRIC, metricFilters));
- }
- // Create filter list based on config filters and add it to
- // listBasedOnFilters.
- TimelineFilterList configFilters = filters.getConfigFilters();
- if (configFilters != null && !configFilters.getFilterList().isEmpty()) {
- listBasedOnFilters.addFilter(TimelineFilterUtils.createHBaseFilterList(
- SubApplicationColumnPrefix.CONFIG, configFilters));
- }
- // Create filter list based on info filters and add it to listBasedOnFilters
- TimelineFilterList infoFilters = filters.getInfoFilters();
- if (infoFilters != null && !infoFilters.getFilterList().isEmpty()) {
- listBasedOnFilters.addFilter(TimelineFilterUtils
- .createHBaseFilterList(SubApplicationColumnPrefix.INFO, infoFilters));
- }
- return listBasedOnFilters;
- }
-
- /**
- * Add {@link QualifierFilter} filters to filter list for each column of
- * entity table.
- *
- * @param list filter list to which qualifier filters have to be added.
- */
- protected void updateFixedColumns(FilterList list) {
- for (SubApplicationColumn column : SubApplicationColumn.values()) {
- list.addFilter(new QualifierFilter(CompareOp.EQUAL,
- new BinaryComparator(column.getColumnQualifierBytes())));
- }
- }
-
- /**
- * Creates a filter list which indicates that only some of the column
- * qualifiers in the info column family will be returned in result.
- *
- * @param isApplication If true, it means operations are to be performed for
- * application table, otherwise for entity table.
- * @return filter list.
- * @throws IOException if any problem occurs while creating filter list.
- */
- private FilterList createFilterListForColsOfInfoFamily() throws IOException {
- FilterList infoFamilyColsFilter = new FilterList(Operator.MUST_PASS_ONE);
- // Add filters for each column in entity table.
- updateFixedColumns(infoFamilyColsFilter);
- EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
- // If INFO field has to be retrieved, add a filter for fetching columns
- // with INFO column prefix.
- if (hasField(fieldsToRetrieve, Field.INFO)) {
- infoFamilyColsFilter.addFilter(
- TimelineFilterUtils.createHBaseQualifierFilter(CompareOp.EQUAL,
- SubApplicationColumnPrefix.INFO));
- }
- TimelineFilterList relatesTo = getFilters().getRelatesTo();
- if (hasField(fieldsToRetrieve, Field.RELATES_TO)) {
- // If RELATES_TO field has to be retrieved, add a filter for fetching
- // columns with RELATES_TO column prefix.
- infoFamilyColsFilter.addFilter(
- TimelineFilterUtils.createHBaseQualifierFilter(CompareOp.EQUAL,
- SubApplicationColumnPrefix.RELATES_TO));
- } else if (relatesTo != null && !relatesTo.getFilterList().isEmpty()) {
- // Even if fields to retrieve does not contain RELATES_TO, we still
- // need to have a filter to fetch some of the column qualifiers if
- // relatesTo filters are specified. relatesTo filters will then be
- // matched after fetching rows from HBase.
- Set<String> relatesToCols =
- TimelineFilterUtils.fetchColumnsFromFilterList(relatesTo);
- infoFamilyColsFilter.addFilter(createFiltersFromColumnQualifiers(
- SubApplicationColumnPrefix.RELATES_TO, relatesToCols));
- }
- TimelineFilterList isRelatedTo = getFilters().getIsRelatedTo();
- if (hasField(fieldsToRetrieve, Field.IS_RELATED_TO)) {
- // If IS_RELATED_TO field has to be retrieved, add a filter for fetching
- // columns with IS_RELATED_TO column prefix.
- infoFamilyColsFilter.addFilter(
- TimelineFilterUtils.createHBaseQualifierFilter(CompareOp.EQUAL,
- SubApplicationColumnPrefix.IS_RELATED_TO));
- } else if (isRelatedTo != null && !isRelatedTo.getFilterList().isEmpty()) {
- // Even if fields to retrieve does not contain IS_RELATED_TO, we still
- // need to have a filter to fetch some of the column qualifiers if
- // isRelatedTo filters are specified. isRelatedTo filters will then be
- // matched after fetching rows from HBase.
- Set<String> isRelatedToCols =
- TimelineFilterUtils.fetchColumnsFromFilterList(isRelatedTo);
- infoFamilyColsFilter.addFilter(createFiltersFromColumnQualifiers(
- SubApplicationColumnPrefix.IS_RELATED_TO, isRelatedToCols));
- }
- TimelineFilterList eventFilters = getFilters().getEventFilters();
- if (hasField(fieldsToRetrieve, Field.EVENTS)) {
- // If EVENTS field has to be retrieved, add a filter for fetching columns
- // with EVENT column prefix.
- infoFamilyColsFilter.addFilter(
- TimelineFilterUtils.createHBaseQualifierFilter(CompareOp.EQUAL,
- SubApplicationColumnPrefix.EVENT));
- } else if (eventFilters != null
- && !eventFilters.getFilterList().isEmpty()) {
- // Even if fields to retrieve does not contain EVENTS, we still need to
- // have a filter to fetch some of the column qualifiers on the basis of
- // event filters specified. Event filters will then be matched after
- // fetching rows from HBase.
- Set<String> eventCols =
- TimelineFilterUtils.fetchColumnsFromFilterList(eventFilters);
- infoFamilyColsFilter.addFilter(createFiltersFromColumnQualifiers(
- SubApplicationColumnPrefix.EVENT, eventCols));
- }
- return infoFamilyColsFilter;
- }
-
- /**
- * Exclude column prefixes via filters which are not required(based on fields
- * to retrieve) from info column family. These filters are added to filter
- * list which contains a filter for getting info column family.
- *
- * @param infoColFamilyList filter list for info column family.
- */
- private void excludeFieldsFromInfoColFamily(FilterList infoColFamilyList) {
- EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
- // Events not required.
- if (!hasField(fieldsToRetrieve, Field.EVENTS)) {
- infoColFamilyList.addFilter(
- TimelineFilterUtils.createHBaseQualifierFilter(CompareOp.NOT_EQUAL,
- SubApplicationColumnPrefix.EVENT));
- }
- // info not required.
- if (!hasField(fieldsToRetrieve, Field.INFO)) {
- infoColFamilyList.addFilter(
- TimelineFilterUtils.createHBaseQualifierFilter(CompareOp.NOT_EQUAL,
- SubApplicationColumnPrefix.INFO));
- }
- // is related to not required.
- if (!hasField(fieldsToRetrieve, Field.IS_RELATED_TO)) {
- infoColFamilyList.addFilter(
- TimelineFilterUtils.createHBaseQualifierFilter(CompareOp.NOT_EQUAL,
- SubApplicationColumnPrefix.IS_RELATED_TO));
- }
- // relates to not required.
- if (!hasField(fieldsToRetrieve, Field.RELATES_TO)) {
- infoColFamilyList.addFilter(
- TimelineFilterUtils.createHBaseQualifierFilter(CompareOp.NOT_EQUAL,
- SubApplicationColumnPrefix.RELATES_TO));
- }
- }
-
- /**
- * Updates filter list based on fields for confs and metrics to retrieve.
- *
- * @param listBasedOnFields filter list based on fields.
- * @throws IOException if any problem occurs while updating filter list.
- */
- private void updateFilterForConfsAndMetricsToRetrieve(
- FilterList listBasedOnFields) throws IOException {
- TimelineDataToRetrieve dataToRetrieve = getDataToRetrieve();
- // Please note that if confsToRetrieve is specified, we would have added
- // CONFS to fields to retrieve in augmentParams() even if not specified.
- if (dataToRetrieve.getFieldsToRetrieve().contains(Field.CONFIGS)) {
- // Create a filter list for configs.
- listBasedOnFields.addFilter(
- TimelineFilterUtils.createFilterForConfsOrMetricsToRetrieve(
- dataToRetrieve.getConfsToRetrieve(),
- SubApplicationColumnFamily.CONFIGS,
- SubApplicationColumnPrefix.CONFIG));
- }
-
- // Please note that if metricsToRetrieve is specified, we would have added
- // METRICS to fields to retrieve in augmentParams() even if not specified.
- if (dataToRetrieve.getFieldsToRetrieve().contains(Field.METRICS)) {
- // Create a filter list for metrics.
- listBasedOnFields.addFilter(
- TimelineFilterUtils.createFilterForConfsOrMetricsToRetrieve(
- dataToRetrieve.getMetricsToRetrieve(),
- SubApplicationColumnFamily.METRICS,
- SubApplicationColumnPrefix.METRIC));
- }
- }
-
- @Override
- protected FilterList constructFilterListBasedOnFields() throws IOException {
- if (!needCreateFilterListBasedOnFields()) {
- // Fetch all the columns. No need of a filter.
- return null;
- }
- FilterList listBasedOnFields = new FilterList(Operator.MUST_PASS_ONE);
- FilterList infoColFamilyList = new FilterList();
- // By default fetch everything in INFO column family.
- FamilyFilter infoColumnFamily = new FamilyFilter(CompareOp.EQUAL,
- new BinaryComparator(SubApplicationColumnFamily.INFO.getBytes()));
- infoColFamilyList.addFilter(infoColumnFamily);
- if (fetchPartialColsFromInfoFamily()) {
- // We can fetch only some of the columns from info family.
- infoColFamilyList.addFilter(createFilterListForColsOfInfoFamily());
- } else {
- // Exclude column prefixes in info column family which are not required
- // based on fields to retrieve.
- excludeFieldsFromInfoColFamily(infoColFamilyList);
- }
- listBasedOnFields.addFilter(infoColFamilyList);
- updateFilterForConfsAndMetricsToRetrieve(listBasedOnFields);
- return listBasedOnFields;
- }
-
- @Override
- protected void validateParams() {
- Preconditions.checkNotNull(getContext(), "context shouldn't be null");
- Preconditions.checkNotNull(getDataToRetrieve(),
- "data to retrieve shouldn't be null");
- Preconditions.checkNotNull(getContext().getClusterId(),
- "clusterId shouldn't be null");
- Preconditions.checkNotNull(getContext().getDoAsUser(),
- "DoAsUser shouldn't be null");
- Preconditions.checkNotNull(getContext().getEntityType(),
- "entityType shouldn't be null");
- }
-
- @Override
- protected void augmentParams(Configuration hbaseConf, Connection conn)
- throws IOException {
- getDataToRetrieve().addFieldsBasedOnConfsAndMetricsToRetrieve();
- createFiltersIfNull();
- }
-
- private void setMetricsTimeRange(Query query) {
- // Set time range for metric values.
- HBaseTimelineStorageUtils.setMetricsTimeRange(query,
- SubApplicationColumnFamily.METRICS.getBytes(),
- getDataToRetrieve().getMetricsTimeBegin(),
- getDataToRetrieve().getMetricsTimeEnd());
- }
-
- @Override
- protected ResultScanner getResults(Configuration hbaseConf, Connection conn,
- FilterList filterList) throws IOException {
-
- // Scan through part of the table to find the entities belong to one app
- // and one type
- Scan scan = new Scan();
- TimelineReaderContext context = getContext();
- if (context.getDoAsUser() == null) {
- throw new BadRequestException("Invalid user!");
- }
-
- RowKeyPrefix<SubApplicationRowKey> subApplicationRowKeyPrefix = null;
- // default mode, will always scans from beginning of entity type.
- if (getFilters() == null || getFilters().getFromId() == null) {
- subApplicationRowKeyPrefix = new SubApplicationRowKeyPrefix(
- context.getDoAsUser(), context.getClusterId(),
- context.getEntityType(), null, null, null);
- scan.setRowPrefixFilter(subApplicationRowKeyPrefix.getRowKeyPrefix());
- } else { // pagination mode, will scan from given entityIdPrefix!enitityId
-
- SubApplicationRowKey entityRowKey = null;
- try {
- entityRowKey = SubApplicationRowKey
- .parseRowKeyFromString(getFilters().getFromId());
- } catch (IllegalArgumentException e) {
- throw new BadRequestException("Invalid filter fromid is provided.");
- }
- if (!context.getClusterId().equals(entityRowKey.getClusterId())) {
- throw new BadRequestException(
- "fromid doesn't belong to clusterId=" + context.getClusterId());
- }
-
- // set start row
- scan.setStartRow(entityRowKey.getRowKey());
-
- // get the bytes for stop row
- subApplicationRowKeyPrefix = new SubApplicationRowKeyPrefix(
- context.getDoAsUser(), context.getClusterId(),
- context.getEntityType(), null, null, null);
-
- // set stop row
- scan.setStopRow(
- HBaseTimelineStorageUtils.calculateTheClosestNextRowKeyForPrefix(
- subApplicationRowKeyPrefix.getRowKeyPrefix()));
-
- // set page filter to limit. This filter has to set only in pagination
- // mode.
- filterList.addFilter(new PageFilter(getFilters().getLimit()));
- }
- setMetricsTimeRange(scan);
- scan.setMaxVersions(getDataToRetrieve().getMetricsLimit());
- if (filterList != null && !filterList.getFilters().isEmpty()) {
- scan.setFilter(filterList);
- }
- return getTable().getResultScanner(hbaseConf, conn, scan);
- }
-
- @Override
- protected Result getResult(Configuration hbaseConf, Connection conn,
- FilterList filterList) throws IOException {
- throw new UnsupportedOperationException(
- "we don't support a single entity query");
- }
-
- @Override
- protected TimelineEntity parseEntity(Result result) throws IOException {
- if (result == null || result.isEmpty()) {
- return null;
- }
- TimelineEntity entity = new TimelineEntity();
- SubApplicationRowKey parseRowKey =
- SubApplicationRowKey.parseRowKey(result.getRow());
- entity.setType(parseRowKey.getEntityType());
- entity.setId(parseRowKey.getEntityId());
- entity.setIdPrefix(parseRowKey.getEntityIdPrefix().longValue());
-
- TimelineEntityFilters filters = getFilters();
- // fetch created time
- Long createdTime =
- (Long) SubApplicationColumn.CREATED_TIME.readResult(result);
- entity.setCreatedTime(createdTime);
-
- EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
- // fetch is related to entities and match isRelatedTo filter. If isRelatedTo
- // filters do not match, entity would be dropped. We have to match filters
- // locally as relevant HBase filters to filter out rows on the basis of
- // isRelatedTo are not set in HBase scan.
- boolean checkIsRelatedTo =
- filters.getIsRelatedTo() != null
- && filters.getIsRelatedTo().getFilterList().size() > 0;
- if (hasField(fieldsToRetrieve, Field.IS_RELATED_TO) || checkIsRelatedTo) {
- readRelationship(entity, result, SubApplicationColumnPrefix.IS_RELATED_TO,
- true);
- if (checkIsRelatedTo && !TimelineStorageUtils.matchIsRelatedTo(entity,
- filters.getIsRelatedTo())) {
- return null;
- }
- if (!hasField(fieldsToRetrieve, Field.IS_RELATED_TO)) {
- entity.getIsRelatedToEntities().clear();
- }
- }
-
- // fetch relates to entities and match relatesTo filter. If relatesTo
- // filters do not match, entity would be dropped. We have to match filters
- // locally as relevant HBase filters to filter out rows on the basis of
- // relatesTo are not set in HBase scan.
- boolean checkRelatesTo =
- !isSingleEntityRead() && filters.getRelatesTo() != null
- && filters.getRelatesTo().getFilterList().size() > 0;
- if (hasField(fieldsToRetrieve, Field.RELATES_TO) || checkRelatesTo) {
- readRelationship(entity, result, SubApplicationColumnPrefix.RELATES_TO,
- false);
- if (checkRelatesTo && !TimelineStorageUtils.matchRelatesTo(entity,
- filters.getRelatesTo())) {
- return null;
- }
- if (!hasField(fieldsToRetrieve, Field.RELATES_TO)) {
- entity.getRelatesToEntities().clear();
- }
- }
-
- // fetch info if fieldsToRetrieve contains INFO or ALL.
- if (hasField(fieldsToRetrieve, Field.INFO)) {
- readKeyValuePairs(entity, result, SubApplicationColumnPrefix.INFO, false);
- }
-
- // fetch configs if fieldsToRetrieve contains CONFIGS or ALL.
- if (hasField(fieldsToRetrieve, Field.CONFIGS)) {
- readKeyValuePairs(entity, result, SubApplicationColumnPrefix.CONFIG,
- true);
- }
-
- // fetch events and match event filters if they exist. If event filters do
- // not match, entity would be dropped. We have to match filters locally
- // as relevant HBase filters to filter out rows on the basis of events
- // are not set in HBase scan.
- boolean checkEvents =
- !isSingleEntityRead() && filters.getEventFilters() != null
- && filters.getEventFilters().getFilterList().size() > 0;
- if (hasField(fieldsToRetrieve, Field.EVENTS) || checkEvents) {
- readEvents(entity, result, SubApplicationColumnPrefix.EVENT);
- if (checkEvents && !TimelineStorageUtils.matchEventFilters(entity,
- filters.getEventFilters())) {
- return null;
- }
- if (!hasField(fieldsToRetrieve, Field.EVENTS)) {
- entity.getEvents().clear();
- }
- }
-
- // fetch metrics if fieldsToRetrieve contains METRICS or ALL.
- if (hasField(fieldsToRetrieve, Field.METRICS)) {
- readMetrics(entity, result, SubApplicationColumnPrefix.METRIC);
- }
-
- entity.getInfo().put(TimelineReaderUtils.FROMID_KEY,
- parseRowKey.getRowKeyAsString());
- return entity;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
deleted file mode 100644
index 07e8423..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
+++ /dev/null
@@ -1,459 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.Set;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.filter.BinaryPrefixComparator;
-import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
-import org.apache.hadoop.hbase.filter.FilterList;
-import org.apache.hadoop.hbase.filter.FilterList.Operator;
-import org.apache.hadoop.hbase.filter.QualifierFilter;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.EventColumnName;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.EventColumnNameConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.StringKeyConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * The base class for reading and deserializing timeline entities from the
- * HBase storage. Different types can be defined for different types of the
- * entities that are being requested.
- */
-public abstract class TimelineEntityReader extends
- AbstractTimelineStorageReader {
- private static final Logger LOG =
- LoggerFactory.getLogger(TimelineEntityReader.class);
-
- private final boolean singleEntityRead;
- private TimelineDataToRetrieve dataToRetrieve;
- // used only for multiple entity read mode
- private TimelineEntityFilters filters;
-
- /**
- * Main table the entity reader uses.
- */
- private BaseTable<?> table;
-
- /**
- * Used to convert strings key components to and from storage format.
- */
- private final KeyConverter<String> stringKeyConverter =
- new StringKeyConverter();
-
- /**
- * Instantiates a reader for multiple-entity reads.
- *
- * @param ctxt Reader context which defines the scope in which query has to be
- * made.
- * @param entityFilters Filters which limit the entities returned.
- * @param toRetrieve Data to retrieve for each entity.
- */
- protected TimelineEntityReader(TimelineReaderContext ctxt,
- TimelineEntityFilters entityFilters, TimelineDataToRetrieve toRetrieve) {
- super(ctxt);
- this.singleEntityRead = false;
- this.dataToRetrieve = toRetrieve;
- this.filters = entityFilters;
-
- this.setTable(getTable());
- }
-
- /**
- * Instantiates a reader for single-entity reads.
- *
- * @param ctxt Reader context which defines the scope in which query has to be
- * made.
- * @param toRetrieve Data to retrieve for each entity.
- */
- protected TimelineEntityReader(TimelineReaderContext ctxt,
- TimelineDataToRetrieve toRetrieve) {
- super(ctxt);
- this.singleEntityRead = true;
- this.dataToRetrieve = toRetrieve;
-
- this.setTable(getTable());
- }
-
- /**
- * Creates a {@link FilterList} based on fields, confs and metrics to
- * retrieve. This filter list will be set in Scan/Get objects to trim down
- * results fetched from HBase back-end storage. This is called only for
- * multiple entity reads.
- *
- * @return a {@link FilterList} object.
- * @throws IOException if any problem occurs while creating filter list.
- */
- protected abstract FilterList constructFilterListBasedOnFields()
- throws IOException;
-
- /**
- * Creates a {@link FilterList} based on info, config and metric filters. This
- * filter list will be set in HBase Get to trim down results fetched from
- * HBase back-end storage.
- *
- * @return a {@link FilterList} object.
- * @throws IOException if any problem occurs while creating filter list.
- */
- protected abstract FilterList constructFilterListBasedOnFilters()
- throws IOException;
-
- /**
- * Combines filter lists created based on fields and based on filters.
- *
- * @return a {@link FilterList} object if it can be constructed. Returns null,
- * if filter list cannot be created either on the basis of filters or on the
- * basis of fields.
- * @throws IOException if any problem occurs while creating filter list.
- */
- private FilterList createFilterList() throws IOException {
- FilterList listBasedOnFilters = constructFilterListBasedOnFilters();
- boolean hasListBasedOnFilters = listBasedOnFilters != null &&
- !listBasedOnFilters.getFilters().isEmpty();
- FilterList listBasedOnFields = constructFilterListBasedOnFields();
- boolean hasListBasedOnFields = listBasedOnFields != null &&
- !listBasedOnFields.getFilters().isEmpty();
- // If filter lists based on both filters and fields can be created,
- // combine them in a new filter list and return it.
- // If either one of them has been created, return that filter list.
- // Return null, if none of the filter lists can be created. This indicates
- // that no filter list needs to be added to HBase Scan as filters are not
- // specified for the query or only the default view of entity needs to be
- // returned.
- if (hasListBasedOnFilters && hasListBasedOnFields) {
- FilterList list = new FilterList();
- list.addFilter(listBasedOnFilters);
- list.addFilter(listBasedOnFields);
- return list;
- } else if (hasListBasedOnFilters) {
- return listBasedOnFilters;
- } else if (hasListBasedOnFields) {
- return listBasedOnFields;
- }
- return null;
- }
-
- protected TimelineDataToRetrieve getDataToRetrieve() {
- return dataToRetrieve;
- }
-
- protected TimelineEntityFilters getFilters() {
- return filters;
- }
-
- /**
- * Create a {@link TimelineEntityFilters} object with default values for
- * filters.
- */
- protected void createFiltersIfNull() {
- if (filters == null) {
- filters = new TimelineEntityFilters.Builder().build();
- }
- }
-
- /**
- * Reads and deserializes a single timeline entity from the HBase storage.
- *
- * @param hbaseConf HBase Configuration.
- * @param conn HBase Connection.
- * @return A <cite>TimelineEntity</cite> object.
- * @throws IOException if there is any exception encountered while reading
- * entity.
- */
- public TimelineEntity readEntity(Configuration hbaseConf, Connection conn)
- throws IOException {
- validateParams();
- augmentParams(hbaseConf, conn);
-
- FilterList filterList = constructFilterListBasedOnFields();
- if (LOG.isDebugEnabled() && filterList != null) {
- LOG.debug("FilterList created for get is - " + filterList);
- }
- Result result = getResult(hbaseConf, conn, filterList);
- if (result == null || result.isEmpty()) {
- // Could not find a matching row.
- LOG.info("Cannot find matching entity of type " +
- getContext().getEntityType());
- return null;
- }
- return parseEntity(result);
- }
-
- /**
- * Reads and deserializes a set of timeline entities from the HBase storage.
- * It goes through all the results available, and returns the number of
- * entries as specified in the limit in the entity's natural sort order.
- *
- * @param hbaseConf HBase Configuration.
- * @param conn HBase Connection.
- * @return a set of <cite>TimelineEntity</cite> objects.
- * @throws IOException if any exception is encountered while reading entities.
- */
- public Set<TimelineEntity> readEntities(Configuration hbaseConf,
- Connection conn) throws IOException {
- validateParams();
- augmentParams(hbaseConf, conn);
-
- Set<TimelineEntity> entities = new LinkedHashSet<>();
- FilterList filterList = createFilterList();
- if (LOG.isDebugEnabled() && filterList != null) {
- LOG.debug("FilterList created for scan is - " + filterList);
- }
- ResultScanner results = getResults(hbaseConf, conn, filterList);
- try {
- for (Result result : results) {
- TimelineEntity entity = parseEntity(result);
- if (entity == null) {
- continue;
- }
- entities.add(entity);
- if (entities.size() == filters.getLimit()) {
- break;
- }
- }
- return entities;
- } finally {
- results.close();
- }
- }
-
- /**
- * Returns the main table to be used by the entity reader.
- *
- * @return A reference to the table.
- */
- protected BaseTable<?> getTable() {
- return table;
- }
-
- /**
- * Fetches a {@link Result} instance for a single-entity read.
- *
- * @param hbaseConf HBase Configuration.
- * @param conn HBase Connection.
- * @param filterList filter list which will be applied to HBase Get.
- * @return the {@link Result} instance or null if no such record is found.
- * @throws IOException if any exception is encountered while getting result.
- */
- protected abstract Result getResult(Configuration hbaseConf, Connection conn,
- FilterList filterList) throws IOException;
-
- /**
- * Fetches a {@link ResultScanner} for a multi-entity read.
- *
- * @param hbaseConf HBase Configuration.
- * @param conn HBase Connection.
- * @param filterList filter list which will be applied to HBase Scan.
- * @return the {@link ResultScanner} instance.
- * @throws IOException if any exception is encountered while getting results.
- */
- protected abstract ResultScanner getResults(Configuration hbaseConf,
- Connection conn, FilterList filterList) throws IOException;
-
- /**
- * Parses the result retrieved from HBase backend and convert it into a
- * {@link TimelineEntity} object.
- *
- * @param result Single row result of a Get/Scan.
- * @return the <cite>TimelineEntity</cite> instance or null if the entity is
- * filtered.
- * @throws IOException if any exception is encountered while parsing entity.
- */
- protected abstract TimelineEntity parseEntity(Result result)
- throws IOException;
-
- /**
- * Helper method for reading and deserializing {@link TimelineMetric} objects
- * using the specified column prefix. The timeline metrics then are added to
- * the given timeline entity.
- *
- * @param entity {@link TimelineEntity} object.
- * @param result {@link Result} object retrieved from backend.
- * @param columnPrefix Metric column prefix
- * @throws IOException if any exception is encountered while reading metrics.
- */
- protected void readMetrics(TimelineEntity entity, Result result,
- ColumnPrefix<?> columnPrefix) throws IOException {
- NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
- columnPrefix.readResultsWithTimestamps(
- result, stringKeyConverter);
- for (Map.Entry<String, NavigableMap<Long, Number>> metricResult:
- metricsResult.entrySet()) {
- TimelineMetric metric = new TimelineMetric();
- metric.setId(metricResult.getKey());
- // Simply assume that if the value set contains more than 1 elements, the
- // metric is a TIME_SERIES metric, otherwise, it's a SINGLE_VALUE metric
- TimelineMetric.Type metricType = metricResult.getValue().size() > 1 ?
- TimelineMetric.Type.TIME_SERIES : TimelineMetric.Type.SINGLE_VALUE;
- metric.setType(metricType);
- metric.addValues(metricResult.getValue());
- entity.addMetric(metric);
- }
- }
-
- /**
- * Checks whether the reader has been created to fetch single entity or
- * multiple entities.
- *
- * @return true, if query is for single entity, false otherwise.
- */
- public boolean isSingleEntityRead() {
- return singleEntityRead;
- }
-
- protected void setTable(BaseTable<?> baseTable) {
- this.table = baseTable;
- }
-
- /**
- * Check if we have a certain field amongst fields to retrieve. This method
- * checks against {@link Field#ALL} as well because that would mean field
- * passed needs to be matched.
- *
- * @param fieldsToRetrieve fields to be retrieved.
- * @param requiredField fields to be checked in fieldsToRetrieve.
- * @return true if has the required field, false otherwise.
- */
- protected boolean hasField(EnumSet<Field> fieldsToRetrieve,
- Field requiredField) {
- return fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(requiredField);
- }
-
- /**
- * Create a filter list of qualifier filters based on passed set of columns.
- *
- * @param <T> Describes the type of column prefix.
- * @param colPrefix Column Prefix.
- * @param columns set of column qualifiers.
- * @return filter list.
- */
- protected <T> FilterList createFiltersFromColumnQualifiers(
- ColumnPrefix<T> colPrefix, Set<String> columns) {
- FilterList list = new FilterList(Operator.MUST_PASS_ONE);
- for (String column : columns) {
- // For columns which have compound column qualifiers (eg. events), we need
- // to include the required separator.
- byte[] compoundColQual = createColQualifierPrefix(colPrefix, column);
- list.addFilter(new QualifierFilter(CompareOp.EQUAL,
- new BinaryPrefixComparator(colPrefix
- .getColumnPrefixBytes(compoundColQual))));
- }
- return list;
- }
-
- protected <T> byte[] createColQualifierPrefix(ColumnPrefix<T> colPrefix,
- String column) {
- if (colPrefix == ApplicationColumnPrefix.EVENT
- || colPrefix == EntityColumnPrefix.EVENT) {
- return new EventColumnName(column, null, null).getColumnQualifier();
- } else {
- return stringKeyConverter.encode(column);
- }
- }
-
- /**
- * Helper method for reading relationship.
- *
- * @param <T> Describes the type of column prefix.
- * @param entity entity to fill.
- * @param result result from HBase.
- * @param prefix column prefix.
- * @param isRelatedTo if true, means relationship is to be added to
- * isRelatedTo, otherwise its added to relatesTo.
- * @throws IOException if any problem is encountered while reading result.
- */
- protected <T> void readRelationship(TimelineEntity entity, Result result,
- ColumnPrefix<T> prefix, boolean isRelatedTo) throws IOException {
- // isRelatedTo and relatesTo are of type Map<String, Set<String>>
- Map<String, Object> columns =
- prefix.readResults(result, stringKeyConverter);
- for (Map.Entry<String, Object> column : columns.entrySet()) {
- for (String id : Separator.VALUES.splitEncoded(column.getValue()
- .toString())) {
- if (isRelatedTo) {
- entity.addIsRelatedToEntity(column.getKey(), id);
- } else {
- entity.addRelatesToEntity(column.getKey(), id);
- }
- }
- }
- }
-
- /**
- * Read events from the entity table or the application table. The column name
- * is of the form "eventId=timestamp=infoKey" where "infoKey" may be omitted
- * if there is no info associated with the event.
- *
- * @param <T> Describes the type of column prefix.
- * @param entity entity to fill.
- * @param result HBase Result.
- * @param prefix column prefix.
- * @throws IOException if any problem is encountered while reading result.
- */
- protected static <T> void readEvents(TimelineEntity entity, Result result,
- ColumnPrefix<T> prefix) throws IOException {
- Map<String, TimelineEvent> eventsMap = new HashMap<>();
- Map<EventColumnName, Object> eventsResult =
- prefix.readResults(result, new EventColumnNameConverter());
- for (Map.Entry<EventColumnName, Object>
- eventResult : eventsResult.entrySet()) {
- EventColumnName eventColumnName = eventResult.getKey();
- String key = eventColumnName.getId() +
- Long.toString(eventColumnName.getTimestamp());
- // Retrieve previously seen event to add to it
- TimelineEvent event = eventsMap.get(key);
- if (event == null) {
- // First time we're seeing this event, add it to the eventsMap
- event = new TimelineEvent();
- event.setId(eventColumnName.getId());
- event.setTimestamp(eventColumnName.getTimestamp());
- eventsMap.put(key, event);
- }
- if (eventColumnName.getInfoKey() != null) {
- event.addInfo(eventColumnName.getInfoKey(), eventResult.getValue());
- }
- }
- Set<TimelineEvent> eventsSet = new HashSet<>(eventsMap.values());
- entity.addEvents(eventsSet);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
deleted file mode 100644
index fa16077..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
-
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
-
-/**
- * Factory methods for instantiating a timeline entity reader.
- */
-public final class TimelineEntityReaderFactory {
- private TimelineEntityReaderFactory() {
- }
-
- /**
- * Creates a timeline entity reader instance for reading a single entity with
- * the specified input.
- *
- * @param context Reader context which defines the scope in which query has to
- * be made.
- * @param dataToRetrieve Data to retrieve for each entity.
- * @return An implementation of <cite>TimelineEntityReader</cite> object
- * depending on entity type.
- */
- public static TimelineEntityReader createSingleEntityReader(
- TimelineReaderContext context, TimelineDataToRetrieve dataToRetrieve) {
- // currently the types that are handled separate from the generic entity
- // table are application, flow run, and flow activity entities
- if (TimelineEntityType.YARN_APPLICATION.matches(context.getEntityType())) {
- return new ApplicationEntityReader(context, dataToRetrieve);
- } else if (TimelineEntityType.
- YARN_FLOW_RUN.matches(context.getEntityType())) {
- return new FlowRunEntityReader(context, dataToRetrieve);
- } else if (TimelineEntityType.
- YARN_FLOW_ACTIVITY.matches(context.getEntityType())) {
- return new FlowActivityEntityReader(context, dataToRetrieve);
- } else {
- // assume we're dealing with a generic entity read
- return new GenericEntityReader(context, dataToRetrieve);
- }
- }
-
- /**
- * Creates a timeline entity reader instance for reading set of entities with
- * the specified input and predicates.
- *
- * @param context Reader context which defines the scope in which query has to
- * be made.
- * @param filters Filters which limit the entities returned.
- * @param dataToRetrieve Data to retrieve for each entity.
- * @return An implementation of <cite>TimelineEntityReader</cite> object
- * depending on entity type.
- */
- public static TimelineEntityReader createMultipleEntitiesReader(
- TimelineReaderContext context, TimelineEntityFilters filters,
- TimelineDataToRetrieve dataToRetrieve) {
- // currently the types that are handled separate from the generic entity
- // table are application, flow run, and flow activity entities
- if (TimelineEntityType.YARN_APPLICATION.matches(context.getEntityType())) {
- return new ApplicationEntityReader(context, filters, dataToRetrieve);
- } else if (TimelineEntityType.
- YARN_FLOW_ACTIVITY.matches(context.getEntityType())) {
- return new FlowActivityEntityReader(context, filters, dataToRetrieve);
- } else if (TimelineEntityType.
- YARN_FLOW_RUN.matches(context.getEntityType())) {
- return new FlowRunEntityReader(context, filters, dataToRetrieve);
- } else {
- if (context.getDoAsUser() != null) {
- return new SubApplicationEntityReader(context, filters, dataToRetrieve);
- }
- // assume we're dealing with a generic entity read
- return new GenericEntityReader(context, filters, dataToRetrieve);
- }
- }
-
- /**
- * Creates a timeline entity type reader that will read all available entity
- * types within the specified context.
- *
- * @param context Reader context which defines the scope in which query has to
- * be made. Limited to application level only.
- * @return an <cite>EntityTypeReader</cite> object
- */
- public static EntityTypeReader createEntityTypeReader(
- TimelineReaderContext context) {
- return new EntityTypeReader(context);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/package-info.java
deleted file mode 100644
index 9814d6d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/package-info.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Package org.apache.hadoop.yarn.server.timelineservice.storage.reader
- * contains classes used to read entities from backend based on query type.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumn.java
deleted file mode 100644
index 46b0cc9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumn.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Column;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.GenericConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
-
-/**
- * Identifies fully qualified columns for the {@link SubApplicationTable}.
- */
-public enum SubApplicationColumn implements Column<SubApplicationTable> {
-
- /**
- * Identifier for the sub application.
- */
- ID(SubApplicationColumnFamily.INFO, "id"),
-
- /**
- * The type of sub application.
- */
- TYPE(SubApplicationColumnFamily.INFO, "type"),
-
- /**
- * When the sub application was created.
- */
- CREATED_TIME(SubApplicationColumnFamily.INFO, "created_time",
- new LongConverter()),
-
- /**
- * The version of the flow that this sub application belongs to.
- */
- FLOW_VERSION(SubApplicationColumnFamily.INFO, "flow_version");
-
- private final ColumnHelper<SubApplicationTable> column;
- private final ColumnFamily<SubApplicationTable> columnFamily;
- private final String columnQualifier;
- private final byte[] columnQualifierBytes;
-
- SubApplicationColumn(ColumnFamily<SubApplicationTable> columnFamily,
- String columnQualifier) {
- this(columnFamily, columnQualifier, GenericConverter.getInstance());
- }
-
- SubApplicationColumn(ColumnFamily<SubApplicationTable> columnFamily,
- String columnQualifier, ValueConverter converter) {
- this.columnFamily = columnFamily;
- this.columnQualifier = columnQualifier;
- // Future-proof by ensuring the right column prefix hygiene.
- this.columnQualifierBytes =
- Bytes.toBytes(Separator.SPACE.encode(columnQualifier));
- this.column = new ColumnHelper<SubApplicationTable>(columnFamily,
- converter);
- }
-
-
- public void store(byte[] rowKey,
- TypedBufferedMutator<SubApplicationTable> tableMutator, Long timestamp,
- Object inputValue, Attribute... attributes) throws IOException {
- column.store(rowKey, tableMutator, columnQualifierBytes, timestamp,
- inputValue, attributes);
- }
-
- public Object readResult(Result result) throws IOException {
- return column.readResult(result, columnQualifierBytes);
- }
-
- @Override
- public byte[] getColumnQualifierBytes() {
- return columnQualifierBytes.clone();
- }
-
- @Override
- public byte[] getColumnFamilyBytes() {
- return columnFamily.getBytes();
- }
-
- @Override
- public ValueConverter getValueConverter() {
- return column.getValueConverter();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumnFamily.java
deleted file mode 100644
index 1d7f8fd..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumnFamily.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication;
-
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-
-/**
- * Represents the sub application table column families.
- */
-public enum SubApplicationColumnFamily
- implements ColumnFamily<SubApplicationTable> {
-
- /**
- * Info column family houses known columns, specifically ones included in
- * columnfamily filters.
- */
- INFO("i"),
-
- /**
- * Configurations are in a separate column family for two reasons:
- * a) the size of the config values can be very large and
- * b) we expect that config values
- * are often separately accessed from other metrics and info columns.
- */
- CONFIGS("c"),
-
- /**
- * Metrics have a separate column family, because they have a separate TTL.
- */
- METRICS("m");
-
- /**
- * Byte representation of this column family.
- */
- private final byte[] bytes;
-
- /**
- * @param value
- * create a column family with this name. Must be lower case and
- * without spaces.
- */
- SubApplicationColumnFamily(String value) {
- // column families should be lower case and not contain any spaces.
- this.bytes = Bytes.toBytes(Separator.SPACE.encode(value));
- }
-
- public byte[] getBytes() {
- return Bytes.copy(bytes);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumnPrefix.java
deleted file mode 100644
index 06ecced..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumnPrefix.java
+++ /dev/null
@@ -1,250 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.NavigableMap;
-
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.GenericConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
-
-/**
- * Identifies partially qualified columns for the sub app table.
- */
-public enum SubApplicationColumnPrefix
- implements ColumnPrefix<SubApplicationTable> {
-
- /**
- * To store TimelineEntity getIsRelatedToEntities values.
- */
- IS_RELATED_TO(SubApplicationColumnFamily.INFO, "s"),
-
- /**
- * To store TimelineEntity getRelatesToEntities values.
- */
- RELATES_TO(SubApplicationColumnFamily.INFO, "r"),
-
- /**
- * To store TimelineEntity info values.
- */
- INFO(SubApplicationColumnFamily.INFO, "i"),
-
- /**
- * Lifecycle events for an entity.
- */
- EVENT(SubApplicationColumnFamily.INFO, "e", true),
-
- /**
- * Config column stores configuration with config key as the column name.
- */
- CONFIG(SubApplicationColumnFamily.CONFIGS, null),
-
- /**
- * Metrics are stored with the metric name as the column name.
- */
- METRIC(SubApplicationColumnFamily.METRICS, null, new LongConverter());
-
- private final ColumnHelper<SubApplicationTable> column;
- private final ColumnFamily<SubApplicationTable> columnFamily;
-
- /**
- * Can be null for those cases where the provided column qualifier is the
- * entire column name.
- */
- private final String columnPrefix;
- private final byte[] columnPrefixBytes;
-
- /**
- * Private constructor, meant to be used by the enum definition.
- *
- * @param columnFamily that this column is stored in.
- * @param columnPrefix for this column.
- */
- SubApplicationColumnPrefix(ColumnFamily<SubApplicationTable> columnFamily,
- String columnPrefix) {
- this(columnFamily, columnPrefix, false, GenericConverter.getInstance());
- }
-
- SubApplicationColumnPrefix(ColumnFamily<SubApplicationTable> columnFamily,
- String columnPrefix, boolean compondColQual) {
- this(columnFamily, columnPrefix, compondColQual,
- GenericConverter.getInstance());
- }
-
- SubApplicationColumnPrefix(ColumnFamily<SubApplicationTable> columnFamily,
- String columnPrefix, ValueConverter converter) {
- this(columnFamily, columnPrefix, false, converter);
- }
-
- /**
- * Private constructor, meant to be used by the enum definition.
- *
- * @param columnFamily that this column is stored in.
- * @param columnPrefix for this column.
- * @param converter used to encode/decode values to be stored in HBase for
- * this column prefix.
- */
- SubApplicationColumnPrefix(ColumnFamily<SubApplicationTable> columnFamily,
- String columnPrefix, boolean compondColQual, ValueConverter converter) {
- column = new ColumnHelper<SubApplicationTable>(columnFamily, converter);
- this.columnFamily = columnFamily;
- this.columnPrefix = columnPrefix;
- if (columnPrefix == null) {
- this.columnPrefixBytes = null;
- } else {
- // Future-proof by ensuring the right column prefix hygiene.
- this.columnPrefixBytes =
- Bytes.toBytes(Separator.SPACE.encode(columnPrefix));
- }
- }
-
- /**
- * @return the column name value
- */
- public String getColumnPrefix() {
- return columnPrefix;
- }
-
- @Override
- public byte[] getColumnPrefixBytes(byte[] qualifierPrefix) {
- return ColumnHelper.getColumnQualifier(
- this.columnPrefixBytes, qualifierPrefix);
- }
-
- @Override
- public byte[] getColumnPrefixBytes(String qualifierPrefix) {
- return ColumnHelper.getColumnQualifier(
- this.columnPrefixBytes, qualifierPrefix);
- }
-
- @Override
- public byte[] getColumnFamilyBytes() {
- return columnFamily.getBytes();
- }
-
- @Override
- public ValueConverter getValueConverter() {
- return column.getValueConverter();
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
- * #store(byte[],
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.
- * TypedBufferedMutator, java.lang.String, java.lang.Long, java.lang.Object,
- * org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute[])
- */
- public void store(byte[] rowKey,
- TypedBufferedMutator<SubApplicationTable> tableMutator, String qualifier,
- Long timestamp, Object inputValue, Attribute... attributes)
- throws IOException {
-
- // Null check
- if (qualifier == null) {
- throw new IOException("Cannot store column with null qualifier in "
- + tableMutator.getName().getNameAsString());
- }
-
- byte[] columnQualifier = getColumnPrefixBytes(qualifier);
-
- column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue,
- attributes);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
- * #store(byte[],
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.
- * TypedBufferedMutator, java.lang.String, java.lang.Long, java.lang.Object)
- */
- public void store(byte[] rowKey,
- TypedBufferedMutator<SubApplicationTable> tableMutator, byte[] qualifier,
- Long timestamp, Object inputValue, Attribute... attributes)
- throws IOException {
-
- // Null check
- if (qualifier == null) {
- throw new IOException("Cannot store column with null qualifier in "
- + tableMutator.getName().getNameAsString());
- }
-
- byte[] columnQualifier = getColumnPrefixBytes(qualifier);
-
- column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue,
- attributes);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
- * #readResult(org.apache.hadoop.hbase.client.Result, java.lang.String)
- */
- public Object readResult(Result result, String qualifier) throws IOException {
- byte[] columnQualifier =
- ColumnHelper.getColumnQualifier(this.columnPrefixBytes, qualifier);
- return column.readResult(result, columnQualifier);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
- * #readResults(org.apache.hadoop.hbase.client.Result,
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter)
- */
- public <K> Map<K, Object> readResults(Result result,
- KeyConverter<K> keyConverter) throws IOException {
- return column.readResults(result, columnPrefixBytes, keyConverter);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
- * #readResultsWithTimestamps(org.apache.hadoop.hbase.client.Result,
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter)
- */
- public <K, V> NavigableMap<K, NavigableMap<Long, V>>
- readResultsWithTimestamps(Result result, KeyConverter<K> keyConverter)
- throws IOException {
- return column.readResultsWithTimestamps(result, columnPrefixBytes,
- keyConverter);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKey.java
deleted file mode 100644
index fb1f774..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKey.java
+++ /dev/null
@@ -1,290 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication;
-
-import java.util.List;
-
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverterToString;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-
-/**
- * Represents a rowkey for the sub app table.
- */
-public class SubApplicationRowKey {
- private final String subAppUserId;
- private final String clusterId;
- private final String entityType;
- private final Long entityIdPrefix;
- private final String entityId;
- private final String userId;
- private final SubApplicationRowKeyConverter subAppRowKeyConverter =
- new SubApplicationRowKeyConverter();
-
- public SubApplicationRowKey(String subAppUserId, String clusterId,
- String entityType, Long entityIdPrefix, String entityId, String userId) {
- this.subAppUserId = subAppUserId;
- this.clusterId = clusterId;
- this.entityType = entityType;
- this.entityIdPrefix = entityIdPrefix;
- this.entityId = entityId;
- this.userId = userId;
- }
-
- public String getClusterId() {
- return clusterId;
- }
-
- public String getSubAppUserId() {
- return subAppUserId;
- }
-
- public String getEntityType() {
- return entityType;
- }
-
- public String getEntityId() {
- return entityId;
- }
-
- public Long getEntityIdPrefix() {
- return entityIdPrefix;
- }
-
- public String getUserId() {
- return userId;
- }
-
- /**
- * Constructs a row key for the sub app table as follows:
- * {@code subAppUserId!clusterId!entityType
- * !entityPrefix!entityId!userId}.
- * Typically used while querying a specific sub app.
- *
- * subAppUserId is usually the doAsUser.
- * userId is the yarn user that the AM runs as.
- *
- * @return byte array with the row key.
- */
- public byte[] getRowKey() {
- return subAppRowKeyConverter.encode(this);
- }
-
- /**
- * Given the raw row key as bytes, returns the row key as an object.
- *
- * @param rowKey byte representation of row key.
- * @return An <cite>SubApplicationRowKey</cite> object.
- */
- public static SubApplicationRowKey parseRowKey(byte[] rowKey) {
- return new SubApplicationRowKeyConverter().decode(rowKey);
- }
-
- /**
- * Constructs a row key for the sub app table as follows:
- * <p>
- * {@code subAppUserId!clusterId!
- * entityType!entityIdPrefix!entityId!userId}.
- *
- * subAppUserId is usually the doAsUser.
- * userId is the yarn user that that the AM runs as.
- *
- * </p>
- *
- * @return String representation of row key.
- */
- public String getRowKeyAsString() {
- return subAppRowKeyConverter.encodeAsString(this);
- }
-
- /**
- * Given the encoded row key as string, returns the row key as an object.
- *
- * @param encodedRowKey String representation of row key.
- * @return A <cite>SubApplicationRowKey</cite> object.
- */
- public static SubApplicationRowKey parseRowKeyFromString(
- String encodedRowKey) {
- return new SubApplicationRowKeyConverter().decodeFromString(encodedRowKey);
- }
-
- /**
- * Encodes and decodes row key for sub app table.
- * The row key is of the form :
- * subAppUserId!clusterId!flowRunId!appId!entityType!entityId!userId
- *
- * subAppUserId is usually the doAsUser.
- * userId is the yarn user that the AM runs as.
- *
- * <p>
- */
- final private static class SubApplicationRowKeyConverter
- implements KeyConverter<SubApplicationRowKey>,
- KeyConverterToString<SubApplicationRowKey> {
-
- private SubApplicationRowKeyConverter() {
- }
-
- /**
- * sub app row key is of the form
- * subAppUserId!clusterId!entityType!entityPrefix!entityId!userId
- * w. each segment separated by !.
- *
- * subAppUserId is usually the doAsUser.
- * userId is the yarn user that the AM runs as.
- *
- * The sizes below indicate sizes of each one of these
- * segments in sequence. clusterId, subAppUserId, entityType,
- * entityId and userId are strings.
- * entity prefix is a long hence 8 bytes in size. Strings are
- * variable in size (i.e. end whenever separator is encountered).
- * This is used while decoding and helps in determining where to split.
- */
- private static final int[] SEGMENT_SIZES = {Separator.VARIABLE_SIZE,
- Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
- Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE};
-
- /*
- * (non-Javadoc)
- *
- * Encodes SubApplicationRowKey object into a byte array with each
- * component/field in SubApplicationRowKey separated by
- * Separator#QUALIFIERS.
- * This leads to an sub app table row key of the form
- * subAppUserId!clusterId!entityType!entityPrefix!entityId!userId
- *
- * subAppUserId is usually the doAsUser.
- * userId is the yarn user that the AM runs as.
- *
- * If entityType in passed SubApplicationRowKey object is null (and the
- * fields preceding it are not null i.e. clusterId, subAppUserId), this
- * returns a row key prefix of the form subAppUserId!clusterId!
- * If entityId in SubApplicationRowKey is null
- * (other components are not null), this returns a row key prefix
- * of the form subAppUserId!clusterId!entityType!
- *
- * @see org.apache.hadoop.yarn.server.timelineservice.storage.common
- * .KeyConverter#encode(java.lang.Object)
- */
- @Override
- public byte[] encode(SubApplicationRowKey rowKey) {
- byte[] subAppUser = Separator.encode(rowKey.getSubAppUserId(),
- Separator.SPACE, Separator.TAB, Separator.QUALIFIERS);
- byte[] cluster = Separator.encode(rowKey.getClusterId(), Separator.SPACE,
- Separator.TAB, Separator.QUALIFIERS);
- byte[] first = Separator.QUALIFIERS.join(subAppUser, cluster);
- if (rowKey.getEntityType() == null) {
- return first;
- }
- byte[] entityType = Separator.encode(rowKey.getEntityType(),
- Separator.SPACE, Separator.TAB, Separator.QUALIFIERS);
-
- if (rowKey.getEntityIdPrefix() == null) {
- return Separator.QUALIFIERS.join(first, entityType,
- Separator.EMPTY_BYTES);
- }
-
- byte[] entityIdPrefix = Bytes.toBytes(rowKey.getEntityIdPrefix());
-
- if (rowKey.getEntityId() == null) {
- return Separator.QUALIFIERS.join(first, entityType, entityIdPrefix,
- Separator.EMPTY_BYTES);
- }
-
- byte[] entityId = Separator.encode(rowKey.getEntityId(), Separator.SPACE,
- Separator.TAB, Separator.QUALIFIERS);
-
- byte[] userId = Separator.encode(rowKey.getUserId(),
- Separator.SPACE, Separator.TAB, Separator.QUALIFIERS);
-
- byte[] second = Separator.QUALIFIERS.join(entityType, entityIdPrefix,
- entityId, userId);
-
- return Separator.QUALIFIERS.join(first, second);
- }
-
- /*
- * (non-Javadoc)
- *
- * Decodes a sub application row key of the form
- * subAppUserId!clusterId!entityType!entityPrefix!entityId!userId
- *
- * subAppUserId is usually the doAsUser.
- * userId is the yarn user that the AM runs as.
- *
- * represented in byte format
- * and converts it into an SubApplicationRowKey object.
- *
- * @see org.apache.hadoop.yarn.server.timelineservice.storage.common
- * .KeyConverter#decode(byte[])
- */
- @Override
- public SubApplicationRowKey decode(byte[] rowKey) {
- byte[][] rowKeyComponents =
- Separator.QUALIFIERS.split(rowKey, SEGMENT_SIZES);
- if (rowKeyComponents.length != 6) {
- throw new IllegalArgumentException(
- "the row key is not valid for " + "a sub app");
- }
- String subAppUserId =
- Separator.decode(Bytes.toString(rowKeyComponents[0]),
- Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
- String clusterId = Separator.decode(Bytes.toString(rowKeyComponents[1]),
- Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
- String entityType = Separator.decode(Bytes.toString(rowKeyComponents[2]),
- Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
-
- Long entityPrefixId = Bytes.toLong(rowKeyComponents[3]);
-
- String entityId = Separator.decode(Bytes.toString(rowKeyComponents[4]),
- Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
- String userId =
- Separator.decode(Bytes.toString(rowKeyComponents[5]),
- Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
-
- return new SubApplicationRowKey(subAppUserId, clusterId, entityType,
- entityPrefixId, entityId, userId);
- }
-
- @Override
- public String encodeAsString(SubApplicationRowKey key) {
- if (key.subAppUserId == null || key.clusterId == null
- || key.entityType == null || key.entityIdPrefix == null
- || key.entityId == null || key.userId == null) {
- throw new IllegalArgumentException();
- }
- return TimelineReaderUtils.joinAndEscapeStrings(
- new String[] {key.subAppUserId, key.clusterId, key.entityType,
- key.entityIdPrefix.toString(), key.entityId, key.userId});
- }
-
- @Override
- public SubApplicationRowKey decodeFromString(String encodedRowKey) {
- List<String> split = TimelineReaderUtils.split(encodedRowKey);
- if (split == null || split.size() != 6) {
- throw new IllegalArgumentException(
- "Invalid row key for sub app table.");
- }
- Long entityIdPrefix = Long.valueOf(split.get(3));
- return new SubApplicationRowKey(split.get(0), split.get(1),
- split.get(2), entityIdPrefix, split.get(4), split.get(5));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKeyPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKeyPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKeyPrefix.java
deleted file mode 100644
index 0c04959..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKeyPrefix.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication;
-
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.RowKeyPrefix;
-
-/**
- * Represents a partial rowkey without the entityId or without entityType and
- * entityId for the sub application table.
- *
- */
-public class SubApplicationRowKeyPrefix extends SubApplicationRowKey
- implements RowKeyPrefix<SubApplicationRowKey> {
-
- /**
- * Creates a prefix which generates the following rowKeyPrefixes for the sub
- * application table:
- * {@code subAppUserId!clusterId!entityType!entityPrefix!userId}.
- *
- * @param subAppUserId
- * identifying the subApp User
- * @param clusterId
- * identifying the cluster
- * @param entityType
- * which entity type
- * @param entityIdPrefix
- * for entityId
- * @param entityId
- * for an entity
- * @param userId
- * for the user who runs the AM
- *
- * subAppUserId is usually the doAsUser.
- * userId is the yarn user that the AM runs as.
- *
- */
- public SubApplicationRowKeyPrefix(String subAppUserId, String clusterId,
- String entityType, Long entityIdPrefix, String entityId,
- String userId) {
- super(subAppUserId, clusterId, entityType, entityIdPrefix, entityId,
- userId);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.
- * RowKeyPrefix#getRowKeyPrefix()
- */
- public byte[] getRowKeyPrefix() {
- return super.getRowKey();
- }
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/18] hadoop git commit: YARN-7919. Refactor timelineservice-hbase
module into submodules. Contributed by Haibo Chen.
Posted by ro...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
deleted file mode 100644
index d385108..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.NavigableMap;
-
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.GenericConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
-
-/**
- * Identifies partially qualified columns for the entity table.
- */
-public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
-
- /**
- * To store TimelineEntity getIsRelatedToEntities values.
- */
- IS_RELATED_TO(EntityColumnFamily.INFO, "s"),
-
- /**
- * To store TimelineEntity getRelatesToEntities values.
- */
- RELATES_TO(EntityColumnFamily.INFO, "r"),
-
- /**
- * To store TimelineEntity info values.
- */
- INFO(EntityColumnFamily.INFO, "i"),
-
- /**
- * Lifecycle events for an entity.
- */
- EVENT(EntityColumnFamily.INFO, "e", true),
-
- /**
- * Config column stores configuration with config key as the column name.
- */
- CONFIG(EntityColumnFamily.CONFIGS, null),
-
- /**
- * Metrics are stored with the metric name as the column name.
- */
- METRIC(EntityColumnFamily.METRICS, null, new LongConverter());
-
- private final ColumnHelper<EntityTable> column;
- private final ColumnFamily<EntityTable> columnFamily;
-
- /**
- * Can be null for those cases where the provided column qualifier is the
- * entire column name.
- */
- private final String columnPrefix;
- private final byte[] columnPrefixBytes;
-
- /**
- * Private constructor, meant to be used by the enum definition.
- *
- * @param columnFamily that this column is stored in.
- * @param columnPrefix for this column.
- */
- EntityColumnPrefix(ColumnFamily<EntityTable> columnFamily,
- String columnPrefix) {
- this(columnFamily, columnPrefix, false, GenericConverter.getInstance());
- }
-
- EntityColumnPrefix(ColumnFamily<EntityTable> columnFamily,
- String columnPrefix, boolean compondColQual) {
- this(columnFamily, columnPrefix, compondColQual,
- GenericConverter.getInstance());
- }
-
- EntityColumnPrefix(ColumnFamily<EntityTable> columnFamily,
- String columnPrefix, ValueConverter converter) {
- this(columnFamily, columnPrefix, false, converter);
- }
-
- /**
- * Private constructor, meant to be used by the enum definition.
- *
- * @param columnFamily that this column is stored in.
- * @param columnPrefix for this column.
- * @param converter used to encode/decode values to be stored in HBase for
- * this column prefix.
- */
- EntityColumnPrefix(ColumnFamily<EntityTable> columnFamily,
- String columnPrefix, boolean compondColQual, ValueConverter converter) {
- column = new ColumnHelper<EntityTable>(columnFamily, converter);
- this.columnFamily = columnFamily;
- this.columnPrefix = columnPrefix;
- if (columnPrefix == null) {
- this.columnPrefixBytes = null;
- } else {
- // Future-proof by ensuring the right column prefix hygiene.
- this.columnPrefixBytes =
- Bytes.toBytes(Separator.SPACE.encode(columnPrefix));
- }
- }
-
- /**
- * @return the column name value
- */
- public String getColumnPrefix() {
- return columnPrefix;
- }
-
- @Override
- public byte[] getColumnPrefixBytes(byte[] qualifierPrefix) {
- return ColumnHelper.getColumnQualifier(
- this.columnPrefixBytes, qualifierPrefix);
- }
-
- @Override
- public byte[] getColumnPrefixBytes(String qualifierPrefix) {
- return ColumnHelper.getColumnQualifier(
- this.columnPrefixBytes, qualifierPrefix);
- }
-
- @Override
- public byte[] getColumnFamilyBytes() {
- return columnFamily.getBytes();
- }
-
- @Override
- public ValueConverter getValueConverter() {
- return column.getValueConverter();
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
- * #store(byte[],
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.
- * TypedBufferedMutator, java.lang.String, java.lang.Long, java.lang.Object,
- * org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute[])
- */
- public void store(byte[] rowKey,
- TypedBufferedMutator<EntityTable> tableMutator, String qualifier,
- Long timestamp, Object inputValue, Attribute... attributes)
- throws IOException {
-
- // Null check
- if (qualifier == null) {
- throw new IOException("Cannot store column with null qualifier in "
- + tableMutator.getName().getNameAsString());
- }
-
- byte[] columnQualifier = getColumnPrefixBytes(qualifier);
-
- column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue,
- attributes);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
- * #store(byte[],
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.
- * TypedBufferedMutator, java.lang.String, java.lang.Long, java.lang.Object)
- */
- public void store(byte[] rowKey,
- TypedBufferedMutator<EntityTable> tableMutator, byte[] qualifier,
- Long timestamp, Object inputValue, Attribute... attributes)
- throws IOException {
-
- // Null check
- if (qualifier == null) {
- throw new IOException("Cannot store column with null qualifier in "
- + tableMutator.getName().getNameAsString());
- }
-
- byte[] columnQualifier = getColumnPrefixBytes(qualifier);
-
- column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue,
- attributes);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
- * #readResult(org.apache.hadoop.hbase.client.Result, java.lang.String)
- */
- public Object readResult(Result result, String qualifier) throws IOException {
- byte[] columnQualifier =
- ColumnHelper.getColumnQualifier(this.columnPrefixBytes, qualifier);
- return column.readResult(result, columnQualifier);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
- * #readResults(org.apache.hadoop.hbase.client.Result,
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter)
- */
- public <K> Map<K, Object> readResults(Result result,
- KeyConverter<K> keyConverter) throws IOException {
- return column.readResults(result, columnPrefixBytes, keyConverter);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
- * #readResultsWithTimestamps(org.apache.hadoop.hbase.client.Result,
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter)
- */
- public <K, V> NavigableMap<K, NavigableMap<Long, V>>
- readResultsWithTimestamps(Result result, KeyConverter<K> keyConverter)
- throws IOException {
- return column.readResultsWithTimestamps(result, columnPrefixBytes,
- keyConverter);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
deleted file mode 100644
index b85a9b0..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
+++ /dev/null
@@ -1,299 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
-
-import java.util.List;
-
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.AppIdKeyConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverterToString;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-
-/**
- * Represents a rowkey for the entity table.
- */
-public class EntityRowKey {
- private final String clusterId;
- private final String userId;
- private final String flowName;
- private final Long flowRunId;
- private final String appId;
- private final String entityType;
- private final Long entityIdPrefix;
- private final String entityId;
- private final EntityRowKeyConverter entityRowKeyConverter =
- new EntityRowKeyConverter();
-
- public EntityRowKey(String clusterId, String userId, String flowName,
- Long flowRunId, String appId, String entityType, Long entityIdPrefix,
- String entityId) {
- this.clusterId = clusterId;
- this.userId = userId;
- this.flowName = flowName;
- this.flowRunId = flowRunId;
- this.appId = appId;
- this.entityType = entityType;
- this.entityIdPrefix = entityIdPrefix;
- this.entityId = entityId;
- }
-
- public String getClusterId() {
- return clusterId;
- }
-
- public String getUserId() {
- return userId;
- }
-
- public String getFlowName() {
- return flowName;
- }
-
- public Long getFlowRunId() {
- return flowRunId;
- }
-
- public String getAppId() {
- return appId;
- }
-
- public String getEntityType() {
- return entityType;
- }
-
- public String getEntityId() {
- return entityId;
- }
-
- public Long getEntityIdPrefix() {
- return entityIdPrefix;
- }
-
- /**
- * Constructs a row key for the entity table as follows:
- * {@code userName!clusterId!flowName!flowRunId!AppId!entityType!entityId}.
- * Typically used while querying a specific entity.
- *
- * @return byte array with the row key.
- */
- public byte[] getRowKey() {
- return entityRowKeyConverter.encode(this);
- }
-
- /**
- * Given the raw row key as bytes, returns the row key as an object.
- * @param rowKey byte representation of row key.
- * @return An <cite>EntityRowKey</cite> object.
- */
- public static EntityRowKey parseRowKey(byte[] rowKey) {
- return new EntityRowKeyConverter().decode(rowKey);
- }
-
- /**
- * Constructs a row key for the entity table as follows:
- * <p>
- * {@code userName!clusterId!flowName!flowRunId!AppId!
- * entityType!entityIdPrefix!entityId}.
- * </p>
- * @return String representation of row key.
- */
- public String getRowKeyAsString() {
- return entityRowKeyConverter.encodeAsString(this);
- }
-
- /**
- * Given the encoded row key as string, returns the row key as an object.
- * @param encodedRowKey String representation of row key.
- * @return A <cite>EntityRowKey</cite> object.
- */
- public static EntityRowKey parseRowKeyFromString(String encodedRowKey) {
- return new EntityRowKeyConverter().decodeFromString(encodedRowKey);
- }
-
- /**
- * Encodes and decodes row key for entity table. The row key is of the form :
- * userName!clusterId!flowName!flowRunId!appId!entityType!entityId. flowRunId
- * is a long, appId is encoded/decoded using {@link AppIdKeyConverter} and
- * rest are strings.
- * <p>
- */
- final private static class EntityRowKeyConverter implements
- KeyConverter<EntityRowKey>, KeyConverterToString<EntityRowKey> {
-
- private final AppIdKeyConverter appIDKeyConverter = new AppIdKeyConverter();
-
- private EntityRowKeyConverter() {
- }
-
- /**
- * Entity row key is of the form
- * userName!clusterId!flowName!flowRunId!appId!entityType!entityId w. each
- * segment separated by !. The sizes below indicate sizes of each one of
- * these segments in sequence. clusterId, userName, flowName, entityType and
- * entityId are strings. flowrunId is a long hence 8 bytes in size. app id
- * is represented as 12 bytes with cluster timestamp part of appid being 8
- * bytes (long) and seq id being 4 bytes(int). Strings are variable in size
- * (i.e. end whenever separator is encountered). This is used while decoding
- * and helps in determining where to split.
- */
- private static final int[] SEGMENT_SIZES = {Separator.VARIABLE_SIZE,
- Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
- AppIdKeyConverter.getKeySize(), Separator.VARIABLE_SIZE,
- Bytes.SIZEOF_LONG, Separator.VARIABLE_SIZE };
-
- /*
- * (non-Javadoc)
- *
- * Encodes EntityRowKey object into a byte array with each component/field
- * in EntityRowKey separated by Separator#QUALIFIERS. This leads to an
- * entity table row key of the form
- * userName!clusterId!flowName!flowRunId!appId!entityType!entityId If
- * entityType in passed EntityRowKey object is null (and the fields
- * preceding it i.e. clusterId, userId and flowName, flowRunId and appId
- * are not null), this returns a row key prefix of the form
- * userName!clusterId!flowName!flowRunId!appId! and if entityId in
- * EntityRowKey is null (other 6 components are not null), this returns a
- * row key prefix of the form
- * userName!clusterId!flowName!flowRunId!appId!entityType! flowRunId is
- * inverted while encoding as it helps maintain a descending order for row
- * keys in entity table.
- *
- * @see org.apache.hadoop.yarn.server.timelineservice.storage.common
- * .KeyConverter#encode(java.lang.Object)
- */
- @Override
- public byte[] encode(EntityRowKey rowKey) {
- byte[] user =
- Separator.encode(rowKey.getUserId(), Separator.SPACE, Separator.TAB,
- Separator.QUALIFIERS);
- byte[] cluster =
- Separator.encode(rowKey.getClusterId(), Separator.SPACE,
- Separator.TAB, Separator.QUALIFIERS);
- byte[] flow =
- Separator.encode(rowKey.getFlowName(), Separator.SPACE,
- Separator.TAB, Separator.QUALIFIERS);
- byte[] first = Separator.QUALIFIERS.join(user, cluster, flow);
- // Note that flowRunId is a long, so we can't encode them all at the same
- // time.
- byte[] second =
- Bytes.toBytes(LongConverter.invertLong(rowKey.getFlowRunId()));
- byte[] third = appIDKeyConverter.encode(rowKey.getAppId());
- if (rowKey.getEntityType() == null) {
- return Separator.QUALIFIERS.join(first, second, third,
- Separator.EMPTY_BYTES);
- }
- byte[] entityType =
- Separator.encode(rowKey.getEntityType(), Separator.SPACE,
- Separator.TAB, Separator.QUALIFIERS);
-
- if (rowKey.getEntityIdPrefix() == null) {
- return Separator.QUALIFIERS.join(first, second, third, entityType,
- Separator.EMPTY_BYTES);
- }
-
- byte[] entityIdPrefix = Bytes.toBytes(rowKey.getEntityIdPrefix());
-
- if (rowKey.getEntityId() == null) {
- return Separator.QUALIFIERS.join(first, second, third, entityType,
- entityIdPrefix, Separator.EMPTY_BYTES);
- }
-
- byte[] entityId = Separator.encode(rowKey.getEntityId(), Separator.SPACE,
- Separator.TAB, Separator.QUALIFIERS);
-
- byte[] fourth =
- Separator.QUALIFIERS.join(entityType, entityIdPrefix, entityId);
-
- return Separator.QUALIFIERS.join(first, second, third, fourth);
- }
-
- /*
- * (non-Javadoc)
- *
- * Decodes an application row key of the form
- * userName!clusterId!flowName!flowRunId!appId!entityType!entityId
- * represented in byte format and converts it into an EntityRowKey object.
- * flowRunId is inverted while decoding as it was inverted while encoding.
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common
- * .KeyConverter#decode(byte[])
- */
- @Override
- public EntityRowKey decode(byte[] rowKey) {
- byte[][] rowKeyComponents =
- Separator.QUALIFIERS.split(rowKey, SEGMENT_SIZES);
- if (rowKeyComponents.length != 8) {
- throw new IllegalArgumentException("the row key is not valid for "
- + "an entity");
- }
- String userId =
- Separator.decode(Bytes.toString(rowKeyComponents[0]),
- Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
- String clusterId =
- Separator.decode(Bytes.toString(rowKeyComponents[1]),
- Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
- String flowName =
- Separator.decode(Bytes.toString(rowKeyComponents[2]),
- Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
- Long flowRunId =
- LongConverter.invertLong(Bytes.toLong(rowKeyComponents[3]));
- String appId = appIDKeyConverter.decode(rowKeyComponents[4]);
- String entityType =
- Separator.decode(Bytes.toString(rowKeyComponents[5]),
- Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
-
- Long entityPrefixId = Bytes.toLong(rowKeyComponents[6]);
-
- String entityId =
- Separator.decode(Bytes.toString(rowKeyComponents[7]),
- Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
- return new EntityRowKey(clusterId, userId, flowName, flowRunId, appId,
- entityType, entityPrefixId, entityId);
- }
-
- @Override
- public String encodeAsString(EntityRowKey key) {
- if (key.clusterId == null || key.userId == null || key.flowName == null
- || key.flowRunId == null || key.appId == null
- || key.entityType == null || key.entityIdPrefix == null
- || key.entityId == null) {
- throw new IllegalArgumentException();
- }
- return TimelineReaderUtils
- .joinAndEscapeStrings(new String[] {key.clusterId, key.userId,
- key.flowName, key.flowRunId.toString(), key.appId, key.entityType,
- key.entityIdPrefix.toString(), key.entityId});
- }
-
- @Override
- public EntityRowKey decodeFromString(String encodedRowKey) {
- List<String> split = TimelineReaderUtils.split(encodedRowKey);
- if (split == null || split.size() != 8) {
- throw new IllegalArgumentException("Invalid row key for entity table.");
- }
- Long flowRunId = Long.valueOf(split.get(3));
- Long entityIdPrefix = Long.valueOf(split.get(6));
- return new EntityRowKey(split.get(0), split.get(1), split.get(2),
- flowRunId, split.get(4), split.get(5), entityIdPrefix, split.get(7));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKeyPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKeyPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKeyPrefix.java
deleted file mode 100644
index 47a1789..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKeyPrefix.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
-
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.RowKeyPrefix;
-
-/**
- * Represents a partial rowkey without the entityId or without entityType and
- * entityId for the entity table.
- *
- */
-public class EntityRowKeyPrefix extends EntityRowKey implements
- RowKeyPrefix<EntityRowKey> {
-
- /**
- * Creates a prefix which generates the following rowKeyPrefixes for the
- * entity table:
- * {@code userName!clusterId!flowName!flowRunId!AppId!entityType!}.
- * @param clusterId identifying the cluster
- * @param userId identifying the user
- * @param flowName identifying the flow
- * @param flowRunId identifying the individual run of this flow
- * @param appId identifying the application
- * @param entityType which entity type
- * @param entityIdPrefix for entityId
- * @param entityId for an entity
- */
- public EntityRowKeyPrefix(String clusterId, String userId, String flowName,
- Long flowRunId, String appId, String entityType, Long entityIdPrefix,
- String entityId) {
- super(clusterId, userId, flowName, flowRunId, appId, entityType,
- entityIdPrefix, entityId);
- }
-
- /**
- * Creates a prefix which generates the following rowKeyPrefixes for the
- * entity table:
- * {@code userName!clusterId!flowName!flowRunId!AppId!entityType!entityId}.
- *
- * @param clusterId identifying the cluster
- * @param userId identifying the user
- * @param flowName identifying the flow
- * @param flowRunId identifying the individual run of this flow
- * @param appId identifying the application
- */
- public EntityRowKeyPrefix(String clusterId, String userId, String flowName,
- Long flowRunId, String appId) {
- this(clusterId, userId, flowName, flowRunId, appId, null, null, null);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.application.
- * RowKeyPrefix#getRowKeyPrefix()
- */
- public byte[] getRowKeyPrefix() {
- return super.getRowKey();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
deleted file mode 100644
index 988bba2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.regionserver.BloomType;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBaseSchemaConstants;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * The entity table as column families info, config and metrics. Info stores
- * information about a timeline entity object config stores configuration data
- * of a timeline entity object metrics stores the metrics of a timeline entity
- * object
- *
- * Example entity table record:
- *
- * <pre>
- * |-------------------------------------------------------------------------|
- * | Row | Column Family | Column Family| Column Family|
- * | key | info | metrics | config |
- * |-------------------------------------------------------------------------|
- * | userName! | id:entityId | metricId1: | configKey1: |
- * | clusterId! | | metricValue1 | configValue1 |
- * | flowName! | type:entityType | @timestamp1 | |
- * | flowRunId! | | | configKey2: |
- * | AppId! | created_time: | metricId1: | configValue2 |
- * | entityType!| 1392993084018 | metricValue2 | |
- * | idPrefix! | | @timestamp2 | |
- * | entityId | i!infoKey: | | |
- * | | infoValue | metricId1: | |
- * | | | metricValue1 | |
- * | | r!relatesToKey: | @timestamp2 | |
- * | | id3=id4=id5 | | |
- * | | | | |
- * | | s!isRelatedToKey | | |
- * | | id7=id9=id6 | | |
- * | | | | |
- * | | e!eventId=timestamp=infoKey: | | |
- * | | eventInfoValue | | |
- * | | | | |
- * | | flowVersion: | | |
- * | | versionValue | | |
- * |-------------------------------------------------------------------------|
- * </pre>
- */
-public class EntityTable extends BaseTable<EntityTable> {
- /** entity prefix. */
- private static final String PREFIX =
- YarnConfiguration.TIMELINE_SERVICE_PREFIX + "entity";
-
- /** config param name that specifies the entity table name. */
- public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
-
- /**
- * config param name that specifies the TTL for metrics column family in
- * entity table.
- */
- private static final String METRICS_TTL_CONF_NAME = PREFIX
- + ".table.metrics.ttl";
-
- /**
- * config param name that specifies max-versions for metrics column family in
- * entity table.
- */
- private static final String METRICS_MAX_VERSIONS =
- PREFIX + ".table.metrics.max-versions";
-
- /** default value for entity table name. */
- public static final String DEFAULT_TABLE_NAME = "timelineservice.entity";
-
- /** default TTL is 30 days for metrics timeseries. */
- private static final int DEFAULT_METRICS_TTL = 2592000;
-
- /** default max number of versions. */
- private static final int DEFAULT_METRICS_MAX_VERSIONS = 10000;
-
- private static final Logger LOG =
- LoggerFactory.getLogger(EntityTable.class);
-
- public EntityTable() {
- super(TABLE_NAME_CONF_NAME, DEFAULT_TABLE_NAME);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.BaseTable#createTable
- * (org.apache.hadoop.hbase.client.Admin,
- * org.apache.hadoop.conf.Configuration)
- */
- public void createTable(Admin admin, Configuration hbaseConf)
- throws IOException {
-
- TableName table = getTableName(hbaseConf);
- if (admin.tableExists(table)) {
- // do not disable / delete existing table
- // similar to the approach taken by map-reduce jobs when
- // output directory exists
- throw new IOException("Table " + table.getNameAsString()
- + " already exists.");
- }
-
- HTableDescriptor entityTableDescp = new HTableDescriptor(table);
- HColumnDescriptor infoCF =
- new HColumnDescriptor(EntityColumnFamily.INFO.getBytes());
- infoCF.setBloomFilterType(BloomType.ROWCOL);
- entityTableDescp.addFamily(infoCF);
-
- HColumnDescriptor configCF =
- new HColumnDescriptor(EntityColumnFamily.CONFIGS.getBytes());
- configCF.setBloomFilterType(BloomType.ROWCOL);
- configCF.setBlockCacheEnabled(true);
- entityTableDescp.addFamily(configCF);
-
- HColumnDescriptor metricsCF =
- new HColumnDescriptor(EntityColumnFamily.METRICS.getBytes());
- entityTableDescp.addFamily(metricsCF);
- metricsCF.setBlockCacheEnabled(true);
- // always keep 1 version (the latest)
- metricsCF.setMinVersions(1);
- metricsCF.setMaxVersions(
- hbaseConf.getInt(METRICS_MAX_VERSIONS, DEFAULT_METRICS_MAX_VERSIONS));
- metricsCF.setTimeToLive(hbaseConf.getInt(METRICS_TTL_CONF_NAME,
- DEFAULT_METRICS_TTL));
- entityTableDescp.setRegionSplitPolicyClassName(
- "org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy");
- entityTableDescp.setValue("KeyPrefixRegionSplitPolicy.prefix_length",
- TimelineHBaseSchemaConstants.USERNAME_SPLIT_KEY_PREFIX_LENGTH);
- admin.createTable(entityTableDescp,
- TimelineHBaseSchemaConstants.getUsernameSplits());
- LOG.info("Status of table creation for " + table.getNameAsString() + "="
- + admin.tableExists(table));
- }
-
- /**
- * @param metricsTTL time to live parameter for the metricss in this table.
- * @param hbaseConf configururation in which to set the metrics TTL config
- * variable.
- */
- public void setMetricsTTL(int metricsTTL, Configuration hbaseConf) {
- hbaseConf.setInt(METRICS_TTL_CONF_NAME, metricsTTL);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/package-info.java
deleted file mode 100644
index bb0e331..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/package-info.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Package org.apache.hadoop.yarn.server.timelineservice.storage.entity
- * contains classes related to implementation for entity table.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationCompactionDimension.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationCompactionDimension.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationCompactionDimension.java
deleted file mode 100644
index 4e2cf2d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationCompactionDimension.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
-
-import org.apache.hadoop.hbase.util.Bytes;
-
-/**
- * Identifies the compaction dimensions for the data in the {@link FlowRunTable}
- * .
- */
-public enum AggregationCompactionDimension {
-
- /**
- * the application id.
- */
- APPLICATION_ID((byte) 101);
-
- private byte tagType;
- private byte[] inBytes;
-
- private AggregationCompactionDimension(byte tagType) {
- this.tagType = tagType;
- this.inBytes = Bytes.toBytes(this.name());
- }
-
- public Attribute getAttribute(String attributeValue) {
- return new Attribute(this.name(), Bytes.toBytes(attributeValue));
- }
-
- public byte getTagType() {
- return tagType;
- }
-
- public byte[] getInBytes() {
- return this.inBytes.clone();
- }
-
- public static AggregationCompactionDimension
- getAggregationCompactionDimension(String aggCompactDimStr) {
- for (AggregationCompactionDimension aggDim : AggregationCompactionDimension
- .values()) {
- if (aggDim.name().equals(aggCompactDimStr)) {
- return aggDim;
- }
- }
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationOperation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationOperation.java
deleted file mode 100644
index 40cdd2c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationOperation.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
-
-import org.apache.hadoop.hbase.util.Bytes;
-
-/**
- * Identifies the attributes to be set for puts into the {@link FlowRunTable}.
- * The numbers used for tagType are prime numbers.
- */
-public enum AggregationOperation {
-
- /**
- * When the flow was started.
- */
- GLOBAL_MIN((byte) 71),
-
- /**
- * When it ended.
- */
- GLOBAL_MAX((byte) 73),
-
- /**
- * The metrics of the flow.
- */
- SUM((byte) 79),
-
- /**
- * application running.
- */
- SUM_FINAL((byte) 83),
-
- /**
- * Min value as per the latest timestamp
- * seen for a given app.
- */
- LATEST_MIN((byte) 89),
-
- /**
- * Max value as per the latest timestamp
- * seen for a given app.
- */
- LATEST_MAX((byte) 97);
-
- private byte tagType;
- private byte[] inBytes;
-
- private AggregationOperation(byte tagType) {
- this.tagType = tagType;
- this.inBytes = Bytes.toBytes(this.name());
- }
-
- public Attribute getAttribute() {
- return new Attribute(this.name(), this.inBytes);
- }
-
- public byte getTagType() {
- return tagType;
- }
-
- public byte[] getInBytes() {
- return this.inBytes.clone();
- }
-
- /**
- * returns the AggregationOperation enum that represents that string.
- * @param aggOpStr Aggregation operation.
- * @return the AggregationOperation enum that represents that string
- */
- public static AggregationOperation getAggregationOperation(String aggOpStr) {
- for (AggregationOperation aggOp : AggregationOperation.values()) {
- if (aggOp.name().equals(aggOpStr)) {
- return aggOp;
- }
- }
- return null;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/Attribute.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/Attribute.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/Attribute.java
deleted file mode 100644
index d3de518..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/Attribute.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
-
-/**
- * Defines the attribute tuple to be set for puts into the {@link FlowRunTable}.
- */
-public class Attribute {
- private final String name;
- private final byte[] value;
-
- public Attribute(String name, byte[] value) {
- this.name = name;
- this.value = value.clone();
- }
-
- public String getName() {
- return name;
- }
-
- public byte[] getValue() {
- return value.clone();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnFamily.java
deleted file mode 100644
index f9eb5b4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnFamily.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
-
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-
-/**
- * Represents the flow run table column families.
- */
-public enum FlowActivityColumnFamily
- implements ColumnFamily<FlowActivityTable> {
-
- /**
- * Info column family houses known columns, specifically ones included in
- * columnfamily filters.
- */
- INFO("i");
-
- /**
- * Byte representation of this column family.
- */
- private final byte[] bytes;
-
- /**
- * @param value
- * create a column family with this name. Must be lower case and
- * without spaces.
- */
- private FlowActivityColumnFamily(String value) {
- // column families should be lower case and not contain any spaces.
- this.bytes = Bytes.toBytes(Separator.SPACE.encode(value));
- }
-
- public byte[] getBytes() {
- return Bytes.copy(bytes);
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java
deleted file mode 100644
index 706b002..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.NavigableMap;
-
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
-
-/**
- * Identifies partially qualified columns for the {@link FlowActivityTable}.
- */
-public enum FlowActivityColumnPrefix
- implements ColumnPrefix<FlowActivityTable> {
-
- /**
- * To store run ids of the flows.
- */
- RUN_ID(FlowActivityColumnFamily.INFO, "r", null);
-
- private final ColumnHelper<FlowActivityTable> column;
- private final ColumnFamily<FlowActivityTable> columnFamily;
-
- /**
- * Can be null for those cases where the provided column qualifier is the
- * entire column name.
- */
- private final String columnPrefix;
- private final byte[] columnPrefixBytes;
-
- private final AggregationOperation aggOp;
-
- /**
- * Private constructor, meant to be used by the enum definition.
- *
- * @param columnFamily
- * that this column is stored in.
- * @param columnPrefix
- * for this column.
- */
- private FlowActivityColumnPrefix(
- ColumnFamily<FlowActivityTable> columnFamily, String columnPrefix,
- AggregationOperation aggOp) {
- this(columnFamily, columnPrefix, aggOp, false);
- }
-
- private FlowActivityColumnPrefix(
- ColumnFamily<FlowActivityTable> columnFamily, String columnPrefix,
- AggregationOperation aggOp, boolean compoundColQual) {
- column = new ColumnHelper<FlowActivityTable>(columnFamily);
- this.columnFamily = columnFamily;
- this.columnPrefix = columnPrefix;
- if (columnPrefix == null) {
- this.columnPrefixBytes = null;
- } else {
- // Future-proof by ensuring the right column prefix hygiene.
- this.columnPrefixBytes = Bytes.toBytes(Separator.SPACE
- .encode(columnPrefix));
- }
- this.aggOp = aggOp;
- }
-
- /**
- * @return the column name value
- */
- public String getColumnPrefix() {
- return columnPrefix;
- }
-
- @Override
- public byte[] getColumnPrefixBytes(byte[] qualifierPrefix) {
- return ColumnHelper.getColumnQualifier(
- this.columnPrefixBytes, qualifierPrefix);
- }
-
- @Override
- public byte[] getColumnPrefixBytes(String qualifierPrefix) {
- return ColumnHelper.getColumnQualifier(
- this.columnPrefixBytes, qualifierPrefix);
- }
-
- public byte[] getColumnPrefixBytes() {
- return columnPrefixBytes.clone();
- }
-
- @Override
- public byte[] getColumnFamilyBytes() {
- return columnFamily.getBytes();
- }
-
- @Override
- public ValueConverter getValueConverter() {
- return column.getValueConverter();
- }
-
- public AggregationOperation getAttribute() {
- return aggOp;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
- * #store(byte[],
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.
- * TypedBufferedMutator, byte[], java.lang.Long, java.lang.Object,
- * org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute[])
- */
- @Override
- public void store(byte[] rowKey,
- TypedBufferedMutator<FlowActivityTable> tableMutator, byte[] qualifier,
- Long timestamp, Object inputValue, Attribute... attributes)
- throws IOException {
- // Null check
- if (qualifier == null) {
- throw new IOException("Cannot store column with null qualifier in "
- + tableMutator.getName().getNameAsString());
- }
-
- byte[] columnQualifier = getColumnPrefixBytes(qualifier);
- Attribute[] combinedAttributes =
- HBaseTimelineStorageUtils.combineAttributes(attributes, this.aggOp);
- column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue,
- combinedAttributes);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
- * #readResult(org.apache.hadoop.hbase.client.Result, java.lang.String)
- */
- public Object readResult(Result result, String qualifier) throws IOException {
- byte[] columnQualifier = ColumnHelper.getColumnQualifier(
- this.columnPrefixBytes, qualifier);
- return column.readResult(result, columnQualifier);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
- * #readResults(org.apache.hadoop.hbase.client.Result,
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter)
- */
- public <K> Map<K, Object> readResults(Result result,
- KeyConverter<K> keyConverter) throws IOException {
- return column.readResults(result, columnPrefixBytes, keyConverter);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
- * #readResultsWithTimestamps(org.apache.hadoop.hbase.client.Result,
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter)
- */
- public <K, V> NavigableMap<K, NavigableMap<Long, V>>
- readResultsWithTimestamps(Result result, KeyConverter<K> keyConverter)
- throws IOException {
- return column.readResultsWithTimestamps(result, columnPrefixBytes,
- keyConverter);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
- * #store(byte[],
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.
- * TypedBufferedMutator, java.lang.String, java.lang.Long, java.lang.Object,
- * org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute[])
- */
- @Override
- public void store(byte[] rowKey,
- TypedBufferedMutator<FlowActivityTable> tableMutator, String qualifier,
- Long timestamp, Object inputValue, Attribute... attributes)
- throws IOException {
- // Null check
- if (qualifier == null) {
- throw new IOException("Cannot store column with null qualifier in "
- + tableMutator.getName().getNameAsString());
- }
-
- byte[] columnQualifier = getColumnPrefixBytes(qualifier);
- Attribute[] combinedAttributes =
- HBaseTimelineStorageUtils.combineAttributes(attributes, this.aggOp);
- column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue,
- combinedAttributes);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
deleted file mode 100644
index b8a5dba..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
+++ /dev/null
@@ -1,247 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
-
-import java.util.List;
-
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverterToString;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-
-/**
- * Represents a rowkey for the flow activity table.
- */
-public class FlowActivityRowKey {
-
- private final String clusterId;
- private final Long dayTs;
- private final String userId;
- private final String flowName;
- private final FlowActivityRowKeyConverter
- flowActivityRowKeyConverter = new FlowActivityRowKeyConverter();
-
- /**
- * @param clusterId identifying the cluster
- * @param dayTs to be converted to the top of the day timestamp
- * @param userId identifying user
- * @param flowName identifying the flow
- */
- public FlowActivityRowKey(String clusterId, Long dayTs, String userId,
- String flowName) {
- this(clusterId, dayTs, userId, flowName, true);
- }
-
- /**
- * @param clusterId identifying the cluster
- * @param timestamp when the flow activity happened. May be converted to the
- * top of the day depending on the convertDayTsToTopOfDay argument.
- * @param userId identifying user
- * @param flowName identifying the flow
- * @param convertDayTsToTopOfDay if true and timestamp isn't null, then
- * timestamp will be converted to the top-of-the day timestamp
- */
- protected FlowActivityRowKey(String clusterId, Long timestamp, String userId,
- String flowName, boolean convertDayTsToTopOfDay) {
- this.clusterId = clusterId;
- if (convertDayTsToTopOfDay && (timestamp != null)) {
- this.dayTs = HBaseTimelineStorageUtils.getTopOfTheDayTimestamp(timestamp);
- } else {
- this.dayTs = timestamp;
- }
- this.userId = userId;
- this.flowName = flowName;
- }
-
- public String getClusterId() {
- return clusterId;
- }
-
- public Long getDayTimestamp() {
- return dayTs;
- }
-
- public String getUserId() {
- return userId;
- }
-
- public String getFlowName() {
- return flowName;
- }
-
- /**
- * Constructs a row key for the flow activity table as follows:
- * {@code clusterId!dayTimestamp!user!flowName}.
- *
- * @return byte array for the row key
- */
- public byte[] getRowKey() {
- return flowActivityRowKeyConverter.encode(this);
- }
-
- /**
- * Given the raw row key as bytes, returns the row key as an object.
- *
- * @param rowKey Byte representation of row key.
- * @return A <cite>FlowActivityRowKey</cite> object.
- */
- public static FlowActivityRowKey parseRowKey(byte[] rowKey) {
- return new FlowActivityRowKeyConverter().decode(rowKey);
- }
-
- /**
- * Constructs a row key for the flow activity table as follows:
- * {@code clusterId!dayTimestamp!user!flowName}.
- * @return String representation of row key
- */
- public String getRowKeyAsString() {
- return flowActivityRowKeyConverter.encodeAsString(this);
- }
-
- /**
- * Given the raw row key as string, returns the row key as an object.
- * @param encodedRowKey String representation of row key.
- * @return A <cite>FlowActivityRowKey</cite> object.
- */
- public static FlowActivityRowKey parseRowKeyFromString(String encodedRowKey) {
- return new FlowActivityRowKeyConverter().decodeFromString(encodedRowKey);
- }
-
- /**
- * Encodes and decodes row key for flow activity table. The row key is of the
- * form : clusterId!dayTimestamp!user!flowName. dayTimestamp(top of the day
- * timestamp) is a long and rest are strings.
- * <p>
- */
- final private static class FlowActivityRowKeyConverter
- implements KeyConverter<FlowActivityRowKey>,
- KeyConverterToString<FlowActivityRowKey> {
-
- private FlowActivityRowKeyConverter() {
- }
-
- /**
- * The flow activity row key is of the form
- * clusterId!dayTimestamp!user!flowName with each segment separated by !.
- * The sizes below indicate sizes of each one of these segements in
- * sequence. clusterId, user and flowName are strings. Top of the day
- * timestamp is a long hence 8 bytes in size. Strings are variable in size
- * (i.e. they end whenever separator is encountered). This is used while
- * decoding and helps in determining where to split.
- */
- private static final int[] SEGMENT_SIZES = {Separator.VARIABLE_SIZE,
- Bytes.SIZEOF_LONG, Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE };
-
- /*
- * (non-Javadoc)
- *
- * Encodes FlowActivityRowKey object into a byte array with each
- * component/field in FlowActivityRowKey separated by Separator#QUALIFIERS.
- * This leads to an flow activity table row key of the form
- * clusterId!dayTimestamp!user!flowName. If dayTimestamp in passed
- * FlowActivityRowKey object is null and clusterId is not null, then this
- * returns a row key prefix as clusterId! and if userId in
- * FlowActivityRowKey is null (and the fields preceding it i.e. clusterId
- * and dayTimestamp are not null), this returns a row key prefix as
- * clusterId!dayTimeStamp! dayTimestamp is inverted while encoding as it
- * helps maintain a descending order for row keys in flow activity table.
- *
- * @see org.apache.hadoop.yarn.server.timelineservice.storage.common
- * .KeyConverter#encode(java.lang.Object)
- */
- @Override
- public byte[] encode(FlowActivityRowKey rowKey) {
- if (rowKey.getDayTimestamp() == null) {
- return Separator.QUALIFIERS.join(Separator.encode(
- rowKey.getClusterId(), Separator.SPACE, Separator.TAB,
- Separator.QUALIFIERS), Separator.EMPTY_BYTES);
- }
- if (rowKey.getUserId() == null) {
- return Separator.QUALIFIERS.join(Separator.encode(
- rowKey.getClusterId(), Separator.SPACE, Separator.TAB,
- Separator.QUALIFIERS), Bytes.toBytes(LongConverter
- .invertLong(rowKey.getDayTimestamp())), Separator.EMPTY_BYTES);
- }
- return Separator.QUALIFIERS.join(Separator.encode(rowKey.getClusterId(),
- Separator.SPACE, Separator.TAB, Separator.QUALIFIERS), Bytes
- .toBytes(LongConverter.invertLong(rowKey.getDayTimestamp())),
- Separator.encode(rowKey.getUserId(), Separator.SPACE, Separator.TAB,
- Separator.QUALIFIERS), Separator.encode(rowKey.getFlowName(),
- Separator.SPACE, Separator.TAB, Separator.QUALIFIERS));
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common
- * .KeyConverter#decode(byte[])
- */
- @Override
- public FlowActivityRowKey decode(byte[] rowKey) {
- byte[][] rowKeyComponents =
- Separator.QUALIFIERS.split(rowKey, SEGMENT_SIZES);
- if (rowKeyComponents.length != 4) {
- throw new IllegalArgumentException("the row key is not valid for "
- + "a flow activity");
- }
- String clusterId =
- Separator.decode(Bytes.toString(rowKeyComponents[0]),
- Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
- Long dayTs = LongConverter.invertLong(Bytes.toLong(rowKeyComponents[1]));
- String userId =
- Separator.decode(Bytes.toString(rowKeyComponents[2]),
- Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
- String flowName =
- Separator.decode(Bytes.toString(rowKeyComponents[3]),
- Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
- return new FlowActivityRowKey(clusterId, dayTs, userId, flowName);
- }
-
- @Override
- public String encodeAsString(FlowActivityRowKey key) {
- if (key.getDayTimestamp() == null) {
- return TimelineReaderUtils
- .joinAndEscapeStrings(new String[] {key.clusterId});
- } else if (key.getUserId() == null) {
- return TimelineReaderUtils.joinAndEscapeStrings(
- new String[] {key.clusterId, key.dayTs.toString()});
- } else if (key.getFlowName() == null) {
- return TimelineReaderUtils.joinAndEscapeStrings(
- new String[] {key.clusterId, key.dayTs.toString(), key.userId});
- }
- return TimelineReaderUtils.joinAndEscapeStrings(new String[] {
- key.clusterId, key.dayTs.toString(), key.userId, key.flowName});
- }
-
- @Override
- public FlowActivityRowKey decodeFromString(String encodedRowKey) {
- List<String> split = TimelineReaderUtils.split(encodedRowKey);
- if (split == null || split.size() != 4) {
- throw new IllegalArgumentException(
- "Invalid row key for flow activity.");
- }
- Long dayTs = Long.valueOf(split.get(1));
- return new FlowActivityRowKey(split.get(0), dayTs, split.get(2),
- split.get(3));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKeyPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKeyPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKeyPrefix.java
deleted file mode 100644
index eb88e54..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKeyPrefix.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
-
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.RowKeyPrefix;
-
-/**
- * A prefix partial rowkey for flow activities.
- */
-public class FlowActivityRowKeyPrefix extends FlowActivityRowKey implements
- RowKeyPrefix<FlowActivityRowKey> {
-
- /**
- * Constructs a row key prefix for the flow activity table as follows:
- * {@code clusterId!dayTimestamp!}.
- *
- * @param clusterId Cluster Id.
- * @param dayTs Start of the day timestamp.
- */
- public FlowActivityRowKeyPrefix(String clusterId, Long dayTs) {
- super(clusterId, dayTs, null, null, false);
- }
-
- /**
- * Constructs a row key prefix for the flow activity table as follows:
- * {@code clusterId!}.
- *
- * @param clusterId identifying the cluster
- */
- public FlowActivityRowKeyPrefix(String clusterId) {
- super(clusterId, null, null, null, false);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.application.
- * RowKeyPrefix#getRowKeyPrefix()
- */
- public byte[] getRowKeyPrefix() {
- return super.getRowKey();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityTable.java
deleted file mode 100644
index e646eb2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityTable.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.regionserver.BloomType;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * The flow activity table has column family info
- * Stores the daily activity record for flows
- * Useful as a quick lookup of what flows were
- * running on a given day
- *
- * Example flow activity table record:
- *
- * <pre>
- * |-------------------------------------------|
- * | Row key | Column Family |
- * | | info |
- * |-------------------------------------------|
- * | clusterId! | r!runid1:version1 |
- * | inv Top of | |
- * | Day! | r!runid2:version7 |
- * | userName! | |
- * | flowName | |
- * |-------------------------------------------|
- * </pre>
- */
-public class FlowActivityTable extends BaseTable<FlowActivityTable> {
- /** flow activity table prefix. */
- private static final String PREFIX =
- YarnConfiguration.TIMELINE_SERVICE_PREFIX + ".flowactivity";
-
- /** config param name that specifies the flowactivity table name. */
- public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
-
- /** default value for flowactivity table name. */
- public static final String DEFAULT_TABLE_NAME =
- "timelineservice.flowactivity";
-
- private static final Logger LOG =
- LoggerFactory.getLogger(FlowActivityTable.class);
-
- /** default max number of versions. */
- public static final int DEFAULT_METRICS_MAX_VERSIONS = Integer.MAX_VALUE;
-
- public FlowActivityTable() {
- super(TABLE_NAME_CONF_NAME, DEFAULT_TABLE_NAME);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.BaseTable#createTable
- * (org.apache.hadoop.hbase.client.Admin,
- * org.apache.hadoop.conf.Configuration)
- */
- public void createTable(Admin admin, Configuration hbaseConf)
- throws IOException {
-
- TableName table = getTableName(hbaseConf);
- if (admin.tableExists(table)) {
- // do not disable / delete existing table
- // similar to the approach taken by map-reduce jobs when
- // output directory exists
- throw new IOException("Table " + table.getNameAsString()
- + " already exists.");
- }
-
- HTableDescriptor flowActivityTableDescp = new HTableDescriptor(table);
- HColumnDescriptor infoCF =
- new HColumnDescriptor(FlowActivityColumnFamily.INFO.getBytes());
- infoCF.setBloomFilterType(BloomType.ROWCOL);
- flowActivityTableDescp.addFamily(infoCF);
- infoCF.setMinVersions(1);
- infoCF.setMaxVersions(DEFAULT_METRICS_MAX_VERSIONS);
-
- // TODO: figure the split policy before running in production
- admin.createTable(flowActivityTableDescp);
- LOG.info("Status of table creation for " + table.getNameAsString() + "="
- + admin.tableExists(table));
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java
deleted file mode 100644
index 3797faf..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Column;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.GenericConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
-
-/**
- * Identifies fully qualified columns for the {@link FlowRunTable}.
- */
-public enum FlowRunColumn implements Column<FlowRunTable> {
-
- /**
- * When the flow was started. This is the minimum of currently known
- * application start times.
- */
- MIN_START_TIME(FlowRunColumnFamily.INFO, "min_start_time",
- AggregationOperation.GLOBAL_MIN, new LongConverter()),
-
- /**
- * When the flow ended. This is the maximum of currently known application end
- * times.
- */
- MAX_END_TIME(FlowRunColumnFamily.INFO, "max_end_time",
- AggregationOperation.GLOBAL_MAX, new LongConverter()),
-
- /**
- * The version of the flow that this flow belongs to.
- */
- FLOW_VERSION(FlowRunColumnFamily.INFO, "flow_version", null);
-
- private final ColumnHelper<FlowRunTable> column;
- private final ColumnFamily<FlowRunTable> columnFamily;
- private final String columnQualifier;
- private final byte[] columnQualifierBytes;
- private final AggregationOperation aggOp;
-
- private FlowRunColumn(ColumnFamily<FlowRunTable> columnFamily,
- String columnQualifier, AggregationOperation aggOp) {
- this(columnFamily, columnQualifier, aggOp,
- GenericConverter.getInstance());
- }
-
- private FlowRunColumn(ColumnFamily<FlowRunTable> columnFamily,
- String columnQualifier, AggregationOperation aggOp,
- ValueConverter converter) {
- this.columnFamily = columnFamily;
- this.columnQualifier = columnQualifier;
- this.aggOp = aggOp;
- // Future-proof by ensuring the right column prefix hygiene.
- this.columnQualifierBytes = Bytes.toBytes(Separator.SPACE
- .encode(columnQualifier));
- this.column = new ColumnHelper<FlowRunTable>(columnFamily, converter, true);
- }
-
- /**
- * @return the column name value
- */
- private String getColumnQualifier() {
- return columnQualifier;
- }
-
- @Override
- public byte[] getColumnQualifierBytes() {
- return columnQualifierBytes.clone();
- }
-
- @Override
- public byte[] getColumnFamilyBytes() {
- return columnFamily.getBytes();
- }
-
- public AggregationOperation getAggregationOperation() {
- return aggOp;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.Column#store
- * (byte[], org.apache.hadoop.yarn.server.timelineservice.storage.common.
- * TypedBufferedMutator, java.lang.Long, java.lang.Object,
- * org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute[])
- */
- public void store(byte[] rowKey,
- TypedBufferedMutator<FlowRunTable> tableMutator, Long timestamp,
- Object inputValue, Attribute... attributes) throws IOException {
-
- Attribute[] combinedAttributes =
- HBaseTimelineStorageUtils.combineAttributes(attributes, aggOp);
- column.store(rowKey, tableMutator, columnQualifierBytes, timestamp,
- inputValue, combinedAttributes);
- }
-
- public Object readResult(Result result) throws IOException {
- return column.readResult(result, columnQualifierBytes);
- }
-
- @Override
- public ValueConverter getValueConverter() {
- return column.getValueConverter();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnFamily.java
deleted file mode 100644
index 8faf5f8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnFamily.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
-
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-
-/**
- * Represents the flow run table column families.
- */
-public enum FlowRunColumnFamily implements ColumnFamily<FlowRunTable> {
-
- /**
- * Info column family houses known columns, specifically ones included in
- * columnfamily filters.
- */
- INFO("i");
-
- /**
- * Byte representation of this column family.
- */
- private final byte[] bytes;
-
- /**
- * @param value
- * create a column family with this name. Must be lower case and
- * without spaces.
- */
- private FlowRunColumnFamily(String value) {
- // column families should be lower case and not contain any spaces.
- this.bytes = Bytes.toBytes(Separator.SPACE.encode(value));
- }
-
- public byte[] getBytes() {
- return Bytes.copy(bytes);
- }
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/18] hadoop git commit: YARN-7919. Refactor timelineservice-hbase
module into submodules. Contributed by Haibo Chen.
Posted by ro...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnRWHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnRWHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnRWHelper.java
new file mode 100644
index 0000000..a8e5149
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnRWHelper.java
@@ -0,0 +1,487 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationCompactionDimension;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+
+/**
+ * A set of utility functions that read or read to a column.
+ * This class is meant to be used only by explicit Columns,
+ * and not directly to write by clients.
+ */
+public final class ColumnRWHelper {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ColumnHelper.class);
+
+ private ColumnRWHelper() {
+ }
+
+ /**
+ * Figures out the cell timestamp used in the Put For storing.
+ * Will supplement the timestamp if required. Typically done for flow run
+ * table.If we supplement the timestamp, we left shift the timestamp and
+ * supplement it with the AppId id so that there are no collisions in the flow
+ * run table's cells.
+ */
+ private static long getPutTimestamp(
+ Long timestamp, boolean supplementTs, Attribute[] attributes) {
+ if (timestamp == null) {
+ timestamp = System.currentTimeMillis();
+ }
+ if (!supplementTs) {
+ return timestamp;
+ } else {
+ String appId = getAppIdFromAttributes(attributes);
+ long supplementedTS = TimestampGenerator.getSupplementedTimestamp(
+ timestamp, appId);
+ return supplementedTS;
+ }
+ }
+
+ private static String getAppIdFromAttributes(Attribute[] attributes) {
+ if (attributes == null) {
+ return null;
+ }
+ String appId = null;
+ for (Attribute attribute : attributes) {
+ if (AggregationCompactionDimension.APPLICATION_ID.toString().equals(
+ attribute.getName())) {
+ appId = Bytes.toString(attribute.getValue());
+ }
+ }
+ return appId;
+ }
+
+ /**
+ * Sends a Mutation to the table. The mutations will be buffered and sent over
+ * the wire as part of a batch.
+ *
+ * @param rowKey
+ * identifying the row to write. Nothing gets written when null.
+ * @param tableMutator
+ * used to modify the underlying HBase table
+ * @param column the column that is to be modified
+ * @param timestamp
+ * version timestamp. When null the current timestamp multiplied with
+ * TimestampGenerator.TS_MULTIPLIER and added with last 3 digits of
+ * app id will be used
+ * @param inputValue
+ * the value to write to the rowKey and column qualifier. Nothing
+ * gets written when null.
+ * @param attributes Attributes to be set for HBase Put.
+ * @throws IOException if any problem occurs during store operation(sending
+ * mutation to table).
+ */
+ public static void store(byte[] rowKey, TypedBufferedMutator<?> tableMutator,
+ Column<?> column, Long timestamp,
+ Object inputValue, Attribute... attributes)
+ throws IOException {
+ store(rowKey, tableMutator, column.getColumnFamilyBytes(),
+ column.getColumnQualifierBytes(), timestamp,
+ column.supplementCellTimestamp(), inputValue,
+ column.getValueConverter(),
+ column.getCombinedAttrsWithAggr(attributes));
+ }
+
+ /**
+ * Sends a Mutation to the table. The mutations will be buffered and sent over
+ * the wire as part of a batch.
+ *
+ * @param rowKey
+ * identifying the row to write. Nothing gets written when null.
+ * @param tableMutator
+ * used to modify the underlying HBase table
+ * @param columnFamilyBytes
+ * @param columnQualifier
+ * column qualifier. Nothing gets written when null.
+ * @param timestamp
+ * version timestamp. When null the current timestamp multiplied with
+ * TimestampGenerator.TS_MULTIPLIER and added with last 3 digits of
+ * app id will be used
+ * @param inputValue
+ * the value to write to the rowKey and column qualifier. Nothing
+ * gets written when null.
+ * @param converter
+ * @param attributes Attributes to be set for HBase Put.
+ * @throws IOException if any problem occurs during store operation(sending
+ * mutation to table).
+ */
+ public static void store(byte[] rowKey, TypedBufferedMutator<?> tableMutator,
+ byte[] columnFamilyBytes, byte[] columnQualifier, Long timestamp,
+ boolean supplementTs, Object inputValue, ValueConverter converter,
+ Attribute... attributes) throws IOException {
+ if ((rowKey == null) || (columnQualifier == null) || (inputValue == null)) {
+ return;
+ }
+ Put p = new Put(rowKey);
+ timestamp = getPutTimestamp(timestamp, supplementTs, attributes);
+ p.addColumn(columnFamilyBytes, columnQualifier, timestamp,
+ converter.encodeValue(inputValue));
+ if ((attributes != null) && (attributes.length > 0)) {
+ for (Attribute attribute : attributes) {
+ p.setAttribute(attribute.getName(), attribute.getValue());
+ }
+ }
+ tableMutator.mutate(p);
+ }
+
+ /**
+ * Get the latest version of this specified column. Note: this call clones the
+ * value content of the hosting {@link org.apache.hadoop.hbase.Cell Cell}.
+ *
+ * @param result from which to read the value. Cannot be null
+ * @param columnFamilyBytes
+ * @param columnQualifierBytes referring to the column to be read.
+ * @param converter
+ * @return latest version of the specified column of whichever object was
+ * written.
+ * @throws IOException if any problem occurs while reading result.
+ */
+ public static Object readResult(Result result, byte[] columnFamilyBytes,
+ byte[] columnQualifierBytes, ValueConverter converter)
+ throws IOException {
+ if (result == null || columnQualifierBytes == null) {
+ return null;
+ }
+
+ // Would have preferred to be able to use getValueAsByteBuffer and get a
+ // ByteBuffer to avoid copy, but GenericObjectMapper doesn't seem to like
+ // that.
+ byte[] value = result.getValue(columnFamilyBytes, columnQualifierBytes);
+ return converter.decodeValue(value);
+ }
+
+ /**
+ * Get the latest version of this specified column. Note: this call clones the
+ * value content of the hosting {@link org.apache.hadoop.hbase.Cell Cell}.
+ *
+ * @param result from which to read the value. Cannot be null
+ * @param column the column that the result can be parsed to
+ * @return latest version of the specified column of whichever object was
+ * written.
+ * @throws IOException if any problem occurs while reading result.
+ */
+ public static Object readResult(Result result, Column<?> column)
+ throws IOException {
+ return readResult(result, column.getColumnFamilyBytes(),
+ column.getColumnQualifierBytes(), column.getValueConverter());
+ }
+
+ /**
+ * Get the latest version of this specified column. Note: this call clones the
+ * value content of the hosting {@link org.apache.hadoop.hbase.Cell Cell}.
+ *
+ * @param result Cannot be null
+ * @param columnPrefix column prefix to read from
+ * @param qualifier column qualifier. Nothing gets read when null.
+ * @return result object (can be cast to whatever object was written to) or
+ * null when specified column qualifier for this prefix doesn't exist
+ * in the result.
+ * @throws IOException if there is any exception encountered while reading
+ * result.
+ */
+ public static Object readResult(Result result, ColumnPrefix<?> columnPrefix,
+ String qualifier) throws IOException {
+ byte[] columnQualifier = ColumnHelper.getColumnQualifier(
+ columnPrefix.getColumnPrefixInBytes(), qualifier);
+
+ return readResult(
+ result, columnPrefix.getColumnFamilyBytes(),
+ columnQualifier, columnPrefix.getValueConverter());
+ }
+
+ /**
+ *
+ * @param <K> identifies the type of key converter.
+ * @param result from which to read columns.
+ * @param keyConverter used to convert column bytes to the appropriate key
+ * type
+ * @return the latest values of columns in the column family with this prefix
+ * (or all of them if the prefix value is null).
+ * @throws IOException if there is any exception encountered while reading
+ * results.
+ */
+ public static <K> Map<K, Object> readResults(Result result,
+ ColumnPrefix<?> columnPrefix, KeyConverter<K> keyConverter)
+ throws IOException {
+ return readResults(result,
+ columnPrefix.getColumnFamilyBytes(),
+ columnPrefix.getColumnPrefixInBytes(),
+ keyConverter, columnPrefix.getValueConverter());
+ }
+
+ /**
+ * @param result from which to reads data with timestamps.
+ * @param <K> identifies the type of key converter.
+ * @param <V> the type of the values. The values will be cast into that type.
+ * @param keyConverter used to convert column bytes to the appropriate key
+ * type.
+ * @return the cell values at each respective time in for form
+ * {@literal {idA={timestamp1->value1}, idA={timestamp2->value2},
+ * idB={timestamp3->value3}, idC={timestamp1->value4}}}
+ * @throws IOException if there is any exception encountered while reading
+ * result.
+ */
+ public static <K, V> NavigableMap<K, NavigableMap<Long, V>>
+ readResultsWithTimestamps(Result result, ColumnPrefix<?> columnPrefix,
+ KeyConverter<K> keyConverter) throws IOException {
+ return readResultsWithTimestamps(result,
+ columnPrefix.getColumnFamilyBytes(),
+ columnPrefix.getColumnPrefixInBytes(),
+ keyConverter, columnPrefix.getValueConverter(),
+ columnPrefix.supplementCellTimeStamp());
+ }
+
+ /**
+ * @param result from which to reads data with timestamps
+ * @param columnPrefixBytes optional prefix to limit columns. If null all
+ * columns are returned.
+ * @param <K> identifies the type of column name(indicated by type of key
+ * converter).
+ * @param <V> the type of the values. The values will be cast into that type.
+ * @param keyConverter used to convert column bytes to the appropriate key
+ * type.
+ * @return the cell values at each respective time in for form
+ * {@literal {idA={timestamp1->value1}, idA={timestamp2->value2},
+ * idB={timestamp3->value3}, idC={timestamp1->value4}}}
+ * @throws IOException if any problem occurs while reading results.
+ */
+ @SuppressWarnings("unchecked")
+ public static <K, V> NavigableMap<K, NavigableMap<Long, V>>
+ readResultsWithTimestamps(Result result, byte[] columnFamilyBytes,
+ byte[] columnPrefixBytes, KeyConverter<K> keyConverter,
+ ValueConverter valueConverter, boolean supplementTs)
+ throws IOException {
+
+ NavigableMap<K, NavigableMap<Long, V>> results = new TreeMap<>();
+
+ if (result != null) {
+ NavigableMap<
+ byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> resultMap =
+ result.getMap();
+
+ NavigableMap<byte[], NavigableMap<Long, byte[]>> columnCellMap =
+ resultMap.get(columnFamilyBytes);
+ // could be that there is no such column family.
+ if (columnCellMap != null) {
+ for (Map.Entry<byte[], NavigableMap<Long, byte[]>> entry : columnCellMap
+ .entrySet()) {
+ K converterColumnKey = null;
+ if (columnPrefixBytes == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("null prefix was specified; returning all columns");
+ }
+ try {
+ converterColumnKey = keyConverter.decode(entry.getKey());
+ } catch (IllegalArgumentException iae) {
+ LOG.error("Illegal column found, skipping this column.", iae);
+ continue;
+ }
+ } else {
+ // A non-null prefix means columns are actually of the form
+ // prefix!columnNameRemainder
+ byte[][] columnNameParts =
+ Separator.QUALIFIERS.split(entry.getKey(), 2);
+ byte[] actualColumnPrefixBytes = columnNameParts[0];
+ if (Bytes.equals(columnPrefixBytes, actualColumnPrefixBytes)
+ && columnNameParts.length == 2) {
+ try {
+ // This is the prefix that we want
+ converterColumnKey = keyConverter.decode(columnNameParts[1]);
+ } catch (IllegalArgumentException iae) {
+ LOG.error("Illegal column found, skipping this column.", iae);
+ continue;
+ }
+ }
+ }
+
+ // If this column has the prefix we want
+ if (converterColumnKey != null) {
+ NavigableMap<Long, V> cellResults =
+ new TreeMap<Long, V>();
+ NavigableMap<Long, byte[]> cells = entry.getValue();
+ if (cells != null) {
+ for (Map.Entry<Long, byte[]> cell : cells.entrySet()) {
+ V value =
+ (V) valueConverter.decodeValue(cell.getValue());
+ Long ts = supplementTs ? TimestampGenerator.
+ getTruncatedTimestamp(cell.getKey()) : cell.getKey();
+ cellResults.put(ts, value);
+ }
+ }
+ results.put(converterColumnKey, cellResults);
+ }
+ } // for entry : columnCellMap
+ } // if columnCellMap != null
+ } // if result != null
+ return results;
+ }
+
+ /**
+ * @param <K> identifies the type of column name(indicated by type of key
+ * converter).
+ * @param result from which to read columns
+ * @param columnPrefixBytes optional prefix to limit columns. If null all
+ * columns are returned.
+ * @param keyConverter used to convert column bytes to the appropriate key
+ * type.
+ * @return the latest values of columns in the column family. If the column
+ * prefix is null, the column qualifier is returned as Strings. For a
+ * non-null column prefix bytes, the column qualifier is returned as
+ * a list of parts, each part a byte[]. This is to facilitate
+ * returning byte arrays of values that were not Strings.
+ * @throws IOException if any problem occurs while reading results.
+ */
+ public static <K> Map<K, Object> readResults(Result result,
+ byte[] columnFamilyBytes, byte[] columnPrefixBytes,
+ KeyConverter<K> keyConverter, ValueConverter valueConverter)
+ throws IOException {
+ Map<K, Object> results = new HashMap<K, Object>();
+
+ if (result != null) {
+ Map<byte[], byte[]> columns = result.getFamilyMap(columnFamilyBytes);
+ for (Map.Entry<byte[], byte[]> entry : columns.entrySet()) {
+ byte[] columnKey = entry.getKey();
+ if (columnKey != null && columnKey.length > 0) {
+
+ K converterColumnKey = null;
+ if (columnPrefixBytes == null) {
+ try {
+ converterColumnKey = keyConverter.decode(columnKey);
+ } catch (IllegalArgumentException iae) {
+ LOG.error("Illegal column found, skipping this column.", iae);
+ continue;
+ }
+ } else {
+ // A non-null prefix means columns are actually of the form
+ // prefix!columnNameRemainder
+ byte[][] columnNameParts = Separator.QUALIFIERS.split(columnKey, 2);
+ if (columnNameParts.length > 0) {
+ byte[] actualColumnPrefixBytes = columnNameParts[0];
+ // If this is the prefix that we want
+ if (Bytes.equals(columnPrefixBytes, actualColumnPrefixBytes)
+ && columnNameParts.length == 2) {
+ try {
+ converterColumnKey = keyConverter.decode(columnNameParts[1]);
+ } catch (IllegalArgumentException iae) {
+ LOG.error("Illegal column found, skipping this column.", iae);
+ continue;
+ }
+ }
+ }
+ } // if-else
+
+ // If the columnPrefix is null (we want all columns), or the actual
+ // prefix matches the given prefix we want this column
+ if (converterColumnKey != null) {
+ Object value = valueConverter.decodeValue(entry.getValue());
+ // we return the columnQualifier in parts since we don't know
+ // which part is of which data type.
+ results.put(converterColumnKey, value);
+ }
+ }
+ } // for entry
+ }
+ return results;
+ }
+
+ /**
+ * Sends a Mutation to the table. The mutations will be buffered and sent over
+ * the wire as part of a batch.
+ *
+ * @param rowKey identifying the row to write. Nothing gets written when null.
+ * @param tableMutator used to modify the underlying HBase table. Caller is
+ * responsible to pass a mutator for the table that actually has this
+ * column.
+ * @param qualifier column qualifier. Nothing gets written when null.
+ * @param timestamp version timestamp. When null the server timestamp will be
+ * used.
+ * @param attributes attributes for the mutation that are used by the
+ * coprocessor to set/read the cell tags.
+ * @param inputValue the value to write to the rowKey and column qualifier.
+ * Nothing gets written when null.
+ * @throws IOException if there is any exception encountered while doing
+ * store operation(sending mutation to the table).
+ */
+ public static void store(byte[] rowKey, TypedBufferedMutator<?> tableMutator,
+ ColumnPrefix<?> columnPrefix, byte[] qualifier, Long timestamp,
+ Object inputValue, Attribute... attributes) throws IOException {
+ // Null check
+ if (qualifier == null) {
+ throw new IOException("Cannot store column with null qualifier in "
+ +tableMutator.getName().getNameAsString());
+ }
+
+ byte[] columnQualifier = columnPrefix.getColumnPrefixBytes(qualifier);
+ Attribute[] combinedAttributes =
+ columnPrefix.getCombinedAttrsWithAggr(attributes);
+
+ store(rowKey, tableMutator, columnPrefix.getColumnFamilyBytes(),
+ columnQualifier, timestamp, columnPrefix.supplementCellTimeStamp(),
+ inputValue, columnPrefix.getValueConverter(), combinedAttributes);
+ }
+
+ /**
+ * Sends a Mutation to the table. The mutations will be buffered and sent over
+ * the wire as part of a batch.
+ *
+ * @param rowKey identifying the row to write. Nothing gets written when null.
+ * @param tableMutator used to modify the underlying HBase table. Caller is
+ * responsible to pass a mutator for the table that actually has this
+ * column.
+ * @param qualifier column qualifier. Nothing gets written when null.
+ * @param timestamp version timestamp. When null the server timestamp will be
+ * used.
+ * @param attributes attributes for the mutation that are used by the
+ * coprocessor to set/read the cell tags.
+ * @param inputValue the value to write to the rowKey and column qualifier.
+ * Nothing gets written when null.
+ * @throws IOException if there is any exception encountered while doing
+ * store operation(sending mutation to the table).
+ */
+ public static void store(byte[] rowKey, TypedBufferedMutator<?> tableMutator,
+ ColumnPrefix<?> columnPrefix, String qualifier, Long timestamp,
+ Object inputValue, Attribute... attributes) throws IOException {
+ // Null check
+ if (qualifier == null) {
+ throw new IOException("Cannot store column with null qualifier in "
+ + tableMutator.getName().getNameAsString());
+ }
+
+ byte[] columnQualifier = columnPrefix.getColumnPrefixBytes(qualifier);
+ Attribute[] combinedAttributes =
+ columnPrefix.getCombinedAttrsWithAggr(attributes);
+
+ store(rowKey, tableMutator, columnPrefix.getColumnFamilyBytes(),
+ columnQualifier, timestamp, columnPrefix.supplementCellTimeStamp(),
+ inputValue, columnPrefix.getValueConverter(), combinedAttributes);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java
new file mode 100644
index 0000000..f4cd6fb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Arrays;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.client.Query;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * A bunch of utility functions used in HBase TimelineService backend.
+ */
+public final class HBaseTimelineStorageUtils {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(HBaseTimelineStorageUtils.class);
+
+ private HBaseTimelineStorageUtils() {
+ }
+
+
+ /**
+ * @param conf YARN configuration. Used to see if there is an explicit config
+ * pointing to the HBase config file to read. It should not be null
+ * or a NullPointerException will be thrown.
+ * @return a configuration with the HBase configuration from the classpath,
+ * optionally overwritten by the timeline service configuration URL if
+ * specified.
+ * @throws MalformedURLException if a timeline service HBase configuration URL
+ * is specified but is a malformed URL.
+ */
+ public static Configuration getTimelineServiceHBaseConf(Configuration conf)
+ throws MalformedURLException {
+ if (conf == null) {
+ throw new NullPointerException();
+ }
+
+ Configuration hbaseConf;
+ String timelineServiceHBaseConfFileURL =
+ conf.get(YarnConfiguration.TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE);
+ if (timelineServiceHBaseConfFileURL != null
+ && timelineServiceHBaseConfFileURL.length() > 0) {
+ LOG.info("Using hbase configuration at " +
+ timelineServiceHBaseConfFileURL);
+ // create a clone so that we don't mess with out input one
+ hbaseConf = new Configuration(conf);
+ Configuration plainHBaseConf = new Configuration(false);
+ URL hbaseSiteXML = new URL(timelineServiceHBaseConfFileURL);
+ plainHBaseConf.addResource(hbaseSiteXML);
+ HBaseConfiguration.merge(hbaseConf, plainHBaseConf);
+ } else {
+ // default to what is on the classpath
+ hbaseConf = HBaseConfiguration.create(conf);
+ }
+ return hbaseConf;
+ }
+
+ /**
+ * Given a row key prefix stored in a byte array, return a byte array for its
+ * immediate next row key.
+ *
+ * @param rowKeyPrefix The provided row key prefix, represented in an array.
+ * @return the closest next row key of the provided row key.
+ */
+ public static byte[] calculateTheClosestNextRowKeyForPrefix(
+ byte[] rowKeyPrefix) {
+ // Essentially we are treating it like an 'unsigned very very long' and
+ // doing +1 manually.
+ // Search for the place where the trailing 0xFFs start
+ int offset = rowKeyPrefix.length;
+ while (offset > 0) {
+ if (rowKeyPrefix[offset - 1] != (byte) 0xFF) {
+ break;
+ }
+ offset--;
+ }
+
+ if (offset == 0) {
+ // We got an 0xFFFF... (only FFs) stopRow value which is
+ // the last possible prefix before the end of the table.
+ // So set it to stop at the 'end of the table'
+ return HConstants.EMPTY_END_ROW;
+ }
+
+ // Copy the right length of the original
+ byte[] newStopRow = Arrays.copyOfRange(rowKeyPrefix, 0, offset);
+ // And increment the last one
+ newStopRow[newStopRow.length - 1]++;
+ return newStopRow;
+ }
+
+ public static void setMetricsTimeRange(Query query, byte[] metricsCf,
+ long tsBegin, long tsEnd) {
+ if (tsBegin != 0 || tsEnd != Long.MAX_VALUE) {
+ query.setColumnFamilyTimeRange(metricsCf,
+ tsBegin, ((tsEnd == Long.MAX_VALUE) ? Long.MAX_VALUE : (tsEnd + 1)));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineHBaseSchemaConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineHBaseSchemaConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineHBaseSchemaConstants.java
new file mode 100644
index 0000000..8e6c259
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineHBaseSchemaConstants.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * contains the constants used in the context of schema accesses for
+ * {@link org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity}
+ * information.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public final class TimelineHBaseSchemaConstants {
+ private TimelineHBaseSchemaConstants() {
+ }
+
+ /**
+ * Used to create a pre-split for tables starting with a username in the
+ * prefix. TODO: this may have to become a config variable (string with
+ * separators) so that different installations can presplit based on their own
+ * commonly occurring names.
+ */
+ private final static byte[][] USERNAME_SPLITS = {
+ Bytes.toBytes("a"), Bytes.toBytes("ad"), Bytes.toBytes("an"),
+ Bytes.toBytes("b"), Bytes.toBytes("ca"), Bytes.toBytes("cl"),
+ Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
+ Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i"),
+ Bytes.toBytes("j"), Bytes.toBytes("k"), Bytes.toBytes("l"),
+ Bytes.toBytes("m"), Bytes.toBytes("n"), Bytes.toBytes("o"),
+ Bytes.toBytes("q"), Bytes.toBytes("r"), Bytes.toBytes("s"),
+ Bytes.toBytes("se"), Bytes.toBytes("t"), Bytes.toBytes("u"),
+ Bytes.toBytes("v"), Bytes.toBytes("w"), Bytes.toBytes("x"),
+ Bytes.toBytes("y"), Bytes.toBytes("z")
+ };
+
+ /**
+ * The length at which keys auto-split.
+ */
+ public static final String USERNAME_SPLIT_KEY_PREFIX_LENGTH = "4";
+
+ /**
+ * @return splits for splits where a user is a prefix.
+ */
+ public static byte[][] getUsernameSplits() {
+ byte[][] kloon = USERNAME_SPLITS.clone();
+ // Deep copy.
+ for (int row = 0; row < USERNAME_SPLITS.length; row++) {
+ kloon[row] = Bytes.copy(USERNAME_SPLITS[row]);
+ }
+ return kloon;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TypedBufferedMutator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TypedBufferedMutator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TypedBufferedMutator.java
new file mode 100644
index 0000000..29a07e4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TypedBufferedMutator.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.Mutation;
+
+/**
+ * To be used to wrap an actual {@link BufferedMutator} in a type safe manner.
+ *
+ * @param <T> The class referring to the table to be written to.
+ */
+public class TypedBufferedMutator<T extends BaseTable<T>> {
+
+ private final BufferedMutator bufferedMutator;
+
+ /**
+ * @param bufferedMutator the mutator to be wrapped for delegation. Shall not
+ * be null.
+ */
+ public TypedBufferedMutator(BufferedMutator bufferedMutator) {
+ this.bufferedMutator = bufferedMutator;
+ }
+
+ public TableName getName() {
+ return bufferedMutator.getName();
+ }
+
+ public Configuration getConfiguration() {
+ return bufferedMutator.getConfiguration();
+ }
+
+ public void mutate(Mutation mutation) throws IOException {
+ bufferedMutator.mutate(mutation);
+ }
+
+ public void mutate(List<? extends Mutation> mutations) throws IOException {
+ bufferedMutator.mutate(mutations);
+ }
+
+ public void close() throws IOException {
+ bufferedMutator.close();
+ }
+
+ public void flush() throws IOException {
+ bufferedMutator.flush();
+ }
+
+ public long getWriteBufferSize() {
+ return bufferedMutator.getWriteBufferSize();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java
new file mode 100644
index 0000000..0df5b8a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.storage.common contains
+ * a set of utility classes used across backend storage reader and writer.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTableRW.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTableRW.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTableRW.java
new file mode 100644
index 0000000..111ae71
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTableRW.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTableRW;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBaseSchemaConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Create, read and write to the Entity Table.
+ */
+public class EntityTableRW extends BaseTableRW<EntityTable> {
+ /** entity prefix. */
+ private static final String PREFIX =
+ YarnConfiguration.TIMELINE_SERVICE_PREFIX + "entity";
+
+ /** config param name that specifies the entity table name. */
+ public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
+
+ /**
+ * config param name that specifies the TTL for metrics column family in
+ * entity table.
+ */
+ private static final String METRICS_TTL_CONF_NAME = PREFIX
+ + ".table.metrics.ttl";
+
+ /**
+ * config param name that specifies max-versions for metrics column family in
+ * entity table.
+ */
+ private static final String METRICS_MAX_VERSIONS =
+ PREFIX + ".table.metrics.max-versions";
+
+ /** default value for entity table name. */
+ public static final String DEFAULT_TABLE_NAME = "timelineservice.entity";
+
+ /** default TTL is 30 days for metrics timeseries. */
+ private static final int DEFAULT_METRICS_TTL = 2592000;
+
+ /** default max number of versions. */
+ private static final int DEFAULT_METRICS_MAX_VERSIONS = 10000;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(EntityTableRW.class);
+
+ public EntityTableRW() {
+ super(TABLE_NAME_CONF_NAME, DEFAULT_TABLE_NAME);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.BaseTableRW#
+ * createTable(org.apache.hadoop.hbase.client.Admin,
+ * org.apache.hadoop.conf.Configuration)
+ */
+ public void createTable(Admin admin, Configuration hbaseConf)
+ throws IOException {
+
+ TableName table = getTableName(hbaseConf);
+ if (admin.tableExists(table)) {
+ // do not disable / delete existing table
+ // similar to the approach taken by map-reduce jobs when
+ // output directory exists
+ throw new IOException("Table " + table.getNameAsString()
+ + " already exists.");
+ }
+
+ HTableDescriptor entityTableDescp = new HTableDescriptor(table);
+ HColumnDescriptor infoCF =
+ new HColumnDescriptor(EntityColumnFamily.INFO.getBytes());
+ infoCF.setBloomFilterType(BloomType.ROWCOL);
+ entityTableDescp.addFamily(infoCF);
+
+ HColumnDescriptor configCF =
+ new HColumnDescriptor(EntityColumnFamily.CONFIGS.getBytes());
+ configCF.setBloomFilterType(BloomType.ROWCOL);
+ configCF.setBlockCacheEnabled(true);
+ entityTableDescp.addFamily(configCF);
+
+ HColumnDescriptor metricsCF =
+ new HColumnDescriptor(EntityColumnFamily.METRICS.getBytes());
+ entityTableDescp.addFamily(metricsCF);
+ metricsCF.setBlockCacheEnabled(true);
+ // always keep 1 version (the latest)
+ metricsCF.setMinVersions(1);
+ metricsCF.setMaxVersions(
+ hbaseConf.getInt(METRICS_MAX_VERSIONS, DEFAULT_METRICS_MAX_VERSIONS));
+ metricsCF.setTimeToLive(hbaseConf.getInt(METRICS_TTL_CONF_NAME,
+ DEFAULT_METRICS_TTL));
+ entityTableDescp.setRegionSplitPolicyClassName(
+ "org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy");
+ entityTableDescp.setValue("KeyPrefixRegionSplitPolicy.prefix_length",
+ TimelineHBaseSchemaConstants.USERNAME_SPLIT_KEY_PREFIX_LENGTH);
+ admin.createTable(entityTableDescp,
+ TimelineHBaseSchemaConstants.getUsernameSplits());
+ LOG.info("Status of table creation for " + table.getNameAsString() + "="
+ + admin.tableExists(table));
+ }
+
+ /**
+ * @param metricsTTL time to live parameter for the metricss in this table.
+ * @param hbaseConf configururation in which to set the metrics TTL config
+ * variable.
+ */
+ public void setMetricsTTL(int metricsTTL, Configuration hbaseConf) {
+ hbaseConf.setInt(METRICS_TTL_CONF_NAME, metricsTTL);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/package-info.java
new file mode 100644
index 0000000..bb0e331
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.storage.entity
+ * contains classes related to implementation for entity table.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityTableRW.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityTableRW.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityTableRW.java
new file mode 100644
index 0000000..5b9fe13
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityTableRW.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTableRW;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Create, read and write to the FlowActivity Table.
+ */
+public class FlowActivityTableRW extends BaseTableRW<FlowActivityTable> {
+ /** flow activity table prefix. */
+ private static final String PREFIX =
+ YarnConfiguration.TIMELINE_SERVICE_PREFIX + ".flowactivity";
+
+ /** config param name that specifies the flowactivity table name. */
+ public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
+
+ /** default value for flowactivity table name. */
+ public static final String DEFAULT_TABLE_NAME =
+ "timelineservice.flowactivity";
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(FlowActivityTableRW.class);
+
+ /** default max number of versions. */
+ public static final int DEFAULT_METRICS_MAX_VERSIONS = Integer.MAX_VALUE;
+
+ public FlowActivityTableRW() {
+ super(TABLE_NAME_CONF_NAME, DEFAULT_TABLE_NAME);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.BaseTableRW#
+ * createTable(org.apache.hadoop.hbase.client.Admin,
+ * org.apache.hadoop.conf.Configuration)
+ */
+ public void createTable(Admin admin, Configuration hbaseConf)
+ throws IOException {
+
+ TableName table = getTableName(hbaseConf);
+ if (admin.tableExists(table)) {
+ // do not disable / delete existing table
+ // similar to the approach taken by map-reduce jobs when
+ // output directory exists
+ throw new IOException("Table " + table.getNameAsString()
+ + " already exists.");
+ }
+
+ HTableDescriptor flowActivityTableDescp = new HTableDescriptor(table);
+ HColumnDescriptor infoCF =
+ new HColumnDescriptor(FlowActivityColumnFamily.INFO.getBytes());
+ infoCF.setBloomFilterType(BloomType.ROWCOL);
+ flowActivityTableDescp.addFamily(infoCF);
+ infoCF.setMinVersions(1);
+ infoCF.setMaxVersions(DEFAULT_METRICS_MAX_VERSIONS);
+
+ // TODO: figure the split policy before running in production
+ admin.createTable(flowActivityTableDescp);
+ LOG.info("Status of table creation for " + table.getNameAsString() + "="
+ + admin.tableExists(table));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTableRW.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTableRW.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTableRW.java
new file mode 100644
index 0000000..61c0734
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTableRW.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTableRW;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Coprocessor;
+
+/**
+ * Create, read and write to the FlowRun table.
+ */
+public class FlowRunTableRW extends BaseTableRW<FlowRunTable> {
+ /** entity prefix. */
+ private static final String PREFIX =
+ YarnConfiguration.TIMELINE_SERVICE_PREFIX + ".flowrun";
+
+ /** config param name that specifies the flowrun table name. */
+ public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
+
+ /** default value for flowrun table name. */
+ public static final String DEFAULT_TABLE_NAME = "timelineservice.flowrun";
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(FlowRunTableRW.class);
+
+ /** default max number of versions. */
+ public static final int DEFAULT_METRICS_MAX_VERSIONS = Integer.MAX_VALUE;
+
+ public FlowRunTableRW() {
+ super(TABLE_NAME_CONF_NAME, DEFAULT_TABLE_NAME);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.BaseTableRW#
+ * createTable(org.apache.hadoop.hbase.client.Admin,
+ * org.apache.hadoop.conf.Configuration)
+ */
+ public void createTable(Admin admin, Configuration hbaseConf)
+ throws IOException {
+
+ TableName table = getTableName(hbaseConf);
+ if (admin.tableExists(table)) {
+ // do not disable / delete existing table
+ // similar to the approach taken by map-reduce jobs when
+ // output directory exists
+ throw new IOException("Table " + table.getNameAsString()
+ + " already exists.");
+ }
+
+ HTableDescriptor flowRunTableDescp = new HTableDescriptor(table);
+ HColumnDescriptor infoCF =
+ new HColumnDescriptor(FlowRunColumnFamily.INFO.getBytes());
+ infoCF.setBloomFilterType(BloomType.ROWCOL);
+ flowRunTableDescp.addFamily(infoCF);
+ infoCF.setMinVersions(1);
+ infoCF.setMaxVersions(DEFAULT_METRICS_MAX_VERSIONS);
+
+ // TODO: figure the split policy
+ String coprocessorJarPathStr = hbaseConf.get(
+ YarnConfiguration.FLOW_RUN_COPROCESSOR_JAR_HDFS_LOCATION,
+ YarnConfiguration.DEFAULT_HDFS_LOCATION_FLOW_RUN_COPROCESSOR_JAR);
+
+ Path coprocessorJarPath = new Path(coprocessorJarPathStr);
+ LOG.info("CoprocessorJarPath=" + coprocessorJarPath.toString());
+ flowRunTableDescp.addCoprocessor(
+ "org.apache.hadoop.yarn.server.timelineservice.storage." +
+ "flow.FlowRunCoprocessor", coprocessorJarPath,
+ Coprocessor.PRIORITY_USER, null);
+ admin.createTable(flowRunTableDescp);
+ LOG.info("Status of table creation for " + table.getNameAsString() + "="
+ + admin.tableExists(table));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/package-info.java
new file mode 100644
index 0000000..04963f3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/package-info.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.storage.flow
+ * contains classes related to implementation for flow related tables, viz. flow
+ * run table and flow activity table.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java
new file mode 100644
index 0000000..e78db2a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.storage contains
+ * classes which define and implement reading and writing to backend storage.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/AbstractTimelineStorageReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/AbstractTimelineStorageReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/AbstractTimelineStorageReader.java
new file mode 100644
index 0000000..0956f1e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/AbstractTimelineStorageReader.java
@@ -0,0 +1,159 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTableRW;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnRWHelper;
+import org.apache.hadoop.yarn.webapp.NotFoundException;
+
+/**
+ * The base class for reading timeline data from the HBase storage. This class
+ * provides basic support to validate and augment reader context.
+ */
+public abstract class AbstractTimelineStorageReader {
+
+ private final TimelineReaderContext context;
+ /**
+ * Used to look up the flow context.
+ */
+ private final AppToFlowTableRW appToFlowTable = new AppToFlowTableRW();
+
+ public AbstractTimelineStorageReader(TimelineReaderContext ctxt) {
+ context = ctxt;
+ }
+
+ protected TimelineReaderContext getContext() {
+ return context;
+ }
+
+ /**
+ * Looks up flow context from AppToFlow table.
+ *
+ * @param appToFlowRowKey to identify Cluster and App Ids.
+ * @param clusterId the cluster id.
+ * @param hbaseConf HBase configuration.
+ * @param conn HBase Connection.
+ * @return flow context information.
+ * @throws IOException if any problem occurs while fetching flow information.
+ */
+ protected FlowContext lookupFlowContext(AppToFlowRowKey appToFlowRowKey,
+ String clusterId, Configuration hbaseConf, Connection conn)
+ throws IOException {
+ byte[] rowKey = appToFlowRowKey.getRowKey();
+ Get get = new Get(rowKey);
+ Result result = appToFlowTable.getResult(hbaseConf, conn, get);
+ if (result != null && !result.isEmpty()) {
+ Object flowName = ColumnRWHelper.readResult(
+ result, AppToFlowColumnPrefix.FLOW_NAME, clusterId);
+ Object flowRunId = ColumnRWHelper.readResult(
+ result, AppToFlowColumnPrefix.FLOW_RUN_ID, clusterId);
+ Object userId = ColumnRWHelper.readResult(
+ result, AppToFlowColumnPrefix.USER_ID, clusterId);
+ if (flowName == null || userId == null || flowRunId == null) {
+ throw new NotFoundException(
+ "Unable to find the context flow name, and flow run id, "
+ + "and user id for clusterId=" + clusterId
+ + ", appId=" + appToFlowRowKey.getAppId());
+ }
+ return new FlowContext((String)userId, (String)flowName,
+ ((Number)flowRunId).longValue());
+ } else {
+ throw new NotFoundException(
+ "Unable to find the context flow name, and flow run id, "
+ + "and user id for clusterId=" + clusterId
+ + ", appId=" + appToFlowRowKey.getAppId());
+ }
+ }
+
+ /**
+ * Sets certain parameters to defaults if the values are not provided.
+ *
+ * @param hbaseConf HBase Configuration.
+ * @param conn HBase Connection.
+ * @throws IOException if any exception is encountered while setting params.
+ */
+ protected void augmentParams(Configuration hbaseConf, Connection conn)
+ throws IOException {
+ defaultAugmentParams(hbaseConf, conn);
+ }
+
+ /**
+ * Default behavior for all timeline readers to augment parameters.
+ *
+ * @param hbaseConf HBase Configuration.
+ * @param conn HBase Connection.
+ * @throws IOException if any exception is encountered while setting params.
+ */
+ final protected void defaultAugmentParams(Configuration hbaseConf,
+ Connection conn) throws IOException {
+ // In reality all three should be null or neither should be null
+ if (context.getFlowName() == null || context.getFlowRunId() == null
+ || context.getUserId() == null) {
+ // Get flow context information from AppToFlow table.
+ AppToFlowRowKey appToFlowRowKey =
+ new AppToFlowRowKey(context.getAppId());
+ FlowContext flowContext =
+ lookupFlowContext(appToFlowRowKey, context.getClusterId(), hbaseConf,
+ conn);
+ context.setFlowName(flowContext.flowName);
+ context.setFlowRunId(flowContext.flowRunId);
+ context.setUserId(flowContext.userId);
+ }
+ }
+
+ /**
+ * Validates the required parameters to read the entities.
+ */
+ protected abstract void validateParams();
+
+ /**
+ * Encapsulates flow context information.
+ */
+ protected static class FlowContext {
+ private final String userId;
+ private final String flowName;
+ private final Long flowRunId;
+
+ public FlowContext(String user, String flowName, Long flowRunId) {
+ this.userId = user;
+ this.flowName = flowName;
+ this.flowRunId = flowRunId;
+ }
+
+ protected String getUserId() {
+ return userId;
+ }
+
+ protected String getFlowName() {
+ return flowName;
+ }
+
+ protected Long getFlowRunId() {
+ return flowRunId;
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/18] hadoop git commit: YARN-7919. Refactor timelineservice-hbase
module into submodules. Contributed by Haibo Chen.
Posted by ro...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/package-info.java
new file mode 100644
index 0000000..bb0e331
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.storage.entity
+ * contains classes related to implementation for entity table.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationCompactionDimension.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationCompactionDimension.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationCompactionDimension.java
new file mode 100644
index 0000000..4e2cf2d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationCompactionDimension.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Identifies the compaction dimensions for the data in the {@link FlowRunTable}
+ * .
+ */
+public enum AggregationCompactionDimension {
+
+ /**
+ * the application id.
+ */
+ APPLICATION_ID((byte) 101);
+
+ private byte tagType;
+ private byte[] inBytes;
+
+ private AggregationCompactionDimension(byte tagType) {
+ this.tagType = tagType;
+ this.inBytes = Bytes.toBytes(this.name());
+ }
+
+ public Attribute getAttribute(String attributeValue) {
+ return new Attribute(this.name(), Bytes.toBytes(attributeValue));
+ }
+
+ public byte getTagType() {
+ return tagType;
+ }
+
+ public byte[] getInBytes() {
+ return this.inBytes.clone();
+ }
+
+ public static AggregationCompactionDimension
+ getAggregationCompactionDimension(String aggCompactDimStr) {
+ for (AggregationCompactionDimension aggDim : AggregationCompactionDimension
+ .values()) {
+ if (aggDim.name().equals(aggCompactDimStr)) {
+ return aggDim;
+ }
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationOperation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationOperation.java
new file mode 100644
index 0000000..40cdd2c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationOperation.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Identifies the attributes to be set for puts into the {@link FlowRunTable}.
+ * The numbers used for tagType are prime numbers.
+ */
+public enum AggregationOperation {
+
+ /**
+ * When the flow was started.
+ */
+ GLOBAL_MIN((byte) 71),
+
+ /**
+ * When it ended.
+ */
+ GLOBAL_MAX((byte) 73),
+
+ /**
+ * The metrics of the flow.
+ */
+ SUM((byte) 79),
+
+ /**
+ * application running.
+ */
+ SUM_FINAL((byte) 83),
+
+ /**
+ * Min value as per the latest timestamp
+ * seen for a given app.
+ */
+ LATEST_MIN((byte) 89),
+
+ /**
+ * Max value as per the latest timestamp
+ * seen for a given app.
+ */
+ LATEST_MAX((byte) 97);
+
+ private byte tagType;
+ private byte[] inBytes;
+
+ private AggregationOperation(byte tagType) {
+ this.tagType = tagType;
+ this.inBytes = Bytes.toBytes(this.name());
+ }
+
+ public Attribute getAttribute() {
+ return new Attribute(this.name(), this.inBytes);
+ }
+
+ public byte getTagType() {
+ return tagType;
+ }
+
+ public byte[] getInBytes() {
+ return this.inBytes.clone();
+ }
+
+ /**
+ * returns the AggregationOperation enum that represents that string.
+ * @param aggOpStr Aggregation operation.
+ * @return the AggregationOperation enum that represents that string
+ */
+ public static AggregationOperation getAggregationOperation(String aggOpStr) {
+ for (AggregationOperation aggOp : AggregationOperation.values()) {
+ if (aggOp.name().equals(aggOpStr)) {
+ return aggOp;
+ }
+ }
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/Attribute.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/Attribute.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/Attribute.java
new file mode 100644
index 0000000..d3de518
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/Attribute.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+/**
+ * Defines the attribute tuple to be set for puts into the {@link FlowRunTable}.
+ */
+public class Attribute {
+ private final String name;
+ private final byte[] value;
+
+ public Attribute(String name, byte[] value) {
+ this.name = name;
+ this.value = value.clone();
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public byte[] getValue() {
+ return value.clone();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnFamily.java
new file mode 100644
index 0000000..f9eb5b4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnFamily.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+
+/**
+ * Represents the flow run table column families.
+ */
+public enum FlowActivityColumnFamily
+ implements ColumnFamily<FlowActivityTable> {
+
+ /**
+ * Info column family houses known columns, specifically ones included in
+ * columnfamily filters.
+ */
+ INFO("i");
+
+ /**
+ * Byte representation of this column family.
+ */
+ private final byte[] bytes;
+
+ /**
+ * @param value
+ * create a column family with this name. Must be lower case and
+ * without spaces.
+ */
+ private FlowActivityColumnFamily(String value) {
+ // column families should be lower case and not contain any spaces.
+ this.bytes = Bytes.toBytes(Separator.SPACE.encode(value));
+ }
+
+ public byte[] getBytes() {
+ return Bytes.copy(bytes);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java
new file mode 100644
index 0000000..f468f0b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.GenericConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineSchemaUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
+
+/**
+ * Identifies partially qualified columns for the {@link FlowActivityTable}.
+ */
+public enum FlowActivityColumnPrefix
+ implements ColumnPrefix<FlowActivityTable> {
+
+ /**
+ * To store run ids of the flows.
+ */
+ RUN_ID(FlowActivityColumnFamily.INFO, "r", null);
+
+ private final ColumnFamily<FlowActivityTable> columnFamily;
+ private final ValueConverter valueConverter;
+
+ /**
+ * Can be null for those cases where the provided column qualifier is the
+ * entire column name.
+ */
+ private final String columnPrefix;
+ private final byte[] columnPrefixBytes;
+
+ private final AggregationOperation aggOp;
+
+ /**
+ * Private constructor, meant to be used by the enum definition.
+ *
+ * @param columnFamily
+ * that this column is stored in.
+ * @param columnPrefix
+ * for this column.
+ */
+ private FlowActivityColumnPrefix(
+ ColumnFamily<FlowActivityTable> columnFamily, String columnPrefix,
+ AggregationOperation aggOp) {
+ this(columnFamily, columnPrefix, aggOp, false);
+ }
+
+ private FlowActivityColumnPrefix(
+ ColumnFamily<FlowActivityTable> columnFamily, String columnPrefix,
+ AggregationOperation aggOp, boolean compoundColQual) {
+ this.valueConverter = GenericConverter.getInstance();
+ this.columnFamily = columnFamily;
+ this.columnPrefix = columnPrefix;
+ if (columnPrefix == null) {
+ this.columnPrefixBytes = null;
+ } else {
+ // Future-proof by ensuring the right column prefix hygiene.
+ this.columnPrefixBytes = Bytes.toBytes(Separator.SPACE
+ .encode(columnPrefix));
+ }
+ this.aggOp = aggOp;
+ }
+
+ /**
+ * @return the column name value
+ */
+ public String getColumnPrefix() {
+ return columnPrefix;
+ }
+
+ @Override
+ public byte[] getColumnPrefixBytes(byte[] qualifierPrefix) {
+ return ColumnHelper.getColumnQualifier(
+ this.columnPrefixBytes, qualifierPrefix);
+ }
+
+ @Override
+ public byte[] getColumnPrefixBytes(String qualifierPrefix) {
+ return ColumnHelper.getColumnQualifier(
+ this.columnPrefixBytes, qualifierPrefix);
+ }
+
+ public byte[] getColumnPrefixBytes() {
+ return columnPrefixBytes.clone();
+ }
+
+ @Override
+ public byte[] getColumnFamilyBytes() {
+ return columnFamily.getBytes();
+ }
+
+ @Override
+ public byte[] getColumnPrefixInBytes() {
+ return columnPrefixBytes != null ? columnPrefixBytes.clone() : null;
+ }
+
+ @Override
+ public ValueConverter getValueConverter() {
+ return valueConverter;
+ }
+
+ @Override
+ public Attribute[] getCombinedAttrsWithAggr(Attribute... attributes) {
+ return HBaseTimelineSchemaUtils.combineAttributes(attributes, aggOp);
+ }
+
+ @Override
+ public boolean supplementCellTimeStamp() {
+ return false;
+ }
+
+ public AggregationOperation getAttribute() {
+ return aggOp;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
new file mode 100644
index 0000000..747f6ab
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
@@ -0,0 +1,247 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import java.util.List;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineSchemaUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverterToString;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+
+/**
+ * Represents a rowkey for the flow activity table.
+ */
+public class FlowActivityRowKey {
+
+ private final String clusterId;
+ private final Long dayTs;
+ private final String userId;
+ private final String flowName;
+ private final FlowActivityRowKeyConverter
+ flowActivityRowKeyConverter = new FlowActivityRowKeyConverter();
+
+ /**
+ * @param clusterId identifying the cluster
+ * @param dayTs to be converted to the top of the day timestamp
+ * @param userId identifying user
+ * @param flowName identifying the flow
+ */
+ public FlowActivityRowKey(String clusterId, Long dayTs, String userId,
+ String flowName) {
+ this(clusterId, dayTs, userId, flowName, true);
+ }
+
+ /**
+ * @param clusterId identifying the cluster
+ * @param timestamp when the flow activity happened. May be converted to the
+ * top of the day depending on the convertDayTsToTopOfDay argument.
+ * @param userId identifying user
+ * @param flowName identifying the flow
+ * @param convertDayTsToTopOfDay if true and timestamp isn't null, then
+ * timestamp will be converted to the top-of-the day timestamp
+ */
+ protected FlowActivityRowKey(String clusterId, Long timestamp, String userId,
+ String flowName, boolean convertDayTsToTopOfDay) {
+ this.clusterId = clusterId;
+ if (convertDayTsToTopOfDay && (timestamp != null)) {
+ this.dayTs = HBaseTimelineSchemaUtils.getTopOfTheDayTimestamp(timestamp);
+ } else {
+ this.dayTs = timestamp;
+ }
+ this.userId = userId;
+ this.flowName = flowName;
+ }
+
+ public String getClusterId() {
+ return clusterId;
+ }
+
+ public Long getDayTimestamp() {
+ return dayTs;
+ }
+
+ public String getUserId() {
+ return userId;
+ }
+
+ public String getFlowName() {
+ return flowName;
+ }
+
+ /**
+ * Constructs a row key for the flow activity table as follows:
+ * {@code clusterId!dayTimestamp!user!flowName}.
+ *
+ * @return byte array for the row key
+ */
+ public byte[] getRowKey() {
+ return flowActivityRowKeyConverter.encode(this);
+ }
+
+ /**
+ * Given the raw row key as bytes, returns the row key as an object.
+ *
+ * @param rowKey Byte representation of row key.
+ * @return A <cite>FlowActivityRowKey</cite> object.
+ */
+ public static FlowActivityRowKey parseRowKey(byte[] rowKey) {
+ return new FlowActivityRowKeyConverter().decode(rowKey);
+ }
+
+ /**
+ * Constructs a row key for the flow activity table as follows:
+ * {@code clusterId!dayTimestamp!user!flowName}.
+ * @return String representation of row key
+ */
+ public String getRowKeyAsString() {
+ return flowActivityRowKeyConverter.encodeAsString(this);
+ }
+
+ /**
+ * Given the raw row key as string, returns the row key as an object.
+ * @param encodedRowKey String representation of row key.
+ * @return A <cite>FlowActivityRowKey</cite> object.
+ */
+ public static FlowActivityRowKey parseRowKeyFromString(String encodedRowKey) {
+ return new FlowActivityRowKeyConverter().decodeFromString(encodedRowKey);
+ }
+
+ /**
+ * Encodes and decodes row key for flow activity table. The row key is of the
+ * form : clusterId!dayTimestamp!user!flowName. dayTimestamp(top of the day
+ * timestamp) is a long and rest are strings.
+ * <p>
+ */
+ final private static class FlowActivityRowKeyConverter
+ implements KeyConverter<FlowActivityRowKey>,
+ KeyConverterToString<FlowActivityRowKey> {
+
+ private FlowActivityRowKeyConverter() {
+ }
+
+ /**
+ * The flow activity row key is of the form
+ * clusterId!dayTimestamp!user!flowName with each segment separated by !.
+ * The sizes below indicate sizes of each one of these segements in
+ * sequence. clusterId, user and flowName are strings. Top of the day
+ * timestamp is a long hence 8 bytes in size. Strings are variable in size
+ * (i.e. they end whenever separator is encountered). This is used while
+ * decoding and helps in determining where to split.
+ */
+ private static final int[] SEGMENT_SIZES = {Separator.VARIABLE_SIZE,
+ Bytes.SIZEOF_LONG, Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE };
+
+ /*
+ * (non-Javadoc)
+ *
+ * Encodes FlowActivityRowKey object into a byte array with each
+ * component/field in FlowActivityRowKey separated by Separator#QUALIFIERS.
+ * This leads to an flow activity table row key of the form
+ * clusterId!dayTimestamp!user!flowName. If dayTimestamp in passed
+ * FlowActivityRowKey object is null and clusterId is not null, then this
+ * returns a row key prefix as clusterId! and if userId in
+ * FlowActivityRowKey is null (and the fields preceding it i.e. clusterId
+ * and dayTimestamp are not null), this returns a row key prefix as
+ * clusterId!dayTimeStamp! dayTimestamp is inverted while encoding as it
+ * helps maintain a descending order for row keys in flow activity table.
+ *
+ * @see org.apache.hadoop.yarn.server.timelineservice.storage.common
+ * .KeyConverter#encode(java.lang.Object)
+ */
+ @Override
+ public byte[] encode(FlowActivityRowKey rowKey) {
+ if (rowKey.getDayTimestamp() == null) {
+ return Separator.QUALIFIERS.join(Separator.encode(
+ rowKey.getClusterId(), Separator.SPACE, Separator.TAB,
+ Separator.QUALIFIERS), Separator.EMPTY_BYTES);
+ }
+ if (rowKey.getUserId() == null) {
+ return Separator.QUALIFIERS.join(Separator.encode(
+ rowKey.getClusterId(), Separator.SPACE, Separator.TAB,
+ Separator.QUALIFIERS), Bytes.toBytes(LongConverter
+ .invertLong(rowKey.getDayTimestamp())), Separator.EMPTY_BYTES);
+ }
+ return Separator.QUALIFIERS.join(Separator.encode(rowKey.getClusterId(),
+ Separator.SPACE, Separator.TAB, Separator.QUALIFIERS), Bytes
+ .toBytes(LongConverter.invertLong(rowKey.getDayTimestamp())),
+ Separator.encode(rowKey.getUserId(), Separator.SPACE, Separator.TAB,
+ Separator.QUALIFIERS), Separator.encode(rowKey.getFlowName(),
+ Separator.SPACE, Separator.TAB, Separator.QUALIFIERS));
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common
+ * .KeyConverter#decode(byte[])
+ */
+ @Override
+ public FlowActivityRowKey decode(byte[] rowKey) {
+ byte[][] rowKeyComponents =
+ Separator.QUALIFIERS.split(rowKey, SEGMENT_SIZES);
+ if (rowKeyComponents.length != 4) {
+ throw new IllegalArgumentException("the row key is not valid for "
+ + "a flow activity");
+ }
+ String clusterId =
+ Separator.decode(Bytes.toString(rowKeyComponents[0]),
+ Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
+ Long dayTs = LongConverter.invertLong(Bytes.toLong(rowKeyComponents[1]));
+ String userId =
+ Separator.decode(Bytes.toString(rowKeyComponents[2]),
+ Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
+ String flowName =
+ Separator.decode(Bytes.toString(rowKeyComponents[3]),
+ Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
+ return new FlowActivityRowKey(clusterId, dayTs, userId, flowName);
+ }
+
+ @Override
+ public String encodeAsString(FlowActivityRowKey key) {
+ if (key.getDayTimestamp() == null) {
+ return TimelineReaderUtils
+ .joinAndEscapeStrings(new String[] {key.clusterId});
+ } else if (key.getUserId() == null) {
+ return TimelineReaderUtils.joinAndEscapeStrings(
+ new String[] {key.clusterId, key.dayTs.toString()});
+ } else if (key.getFlowName() == null) {
+ return TimelineReaderUtils.joinAndEscapeStrings(
+ new String[] {key.clusterId, key.dayTs.toString(), key.userId});
+ }
+ return TimelineReaderUtils.joinAndEscapeStrings(new String[] {
+ key.clusterId, key.dayTs.toString(), key.userId, key.flowName});
+ }
+
+ @Override
+ public FlowActivityRowKey decodeFromString(String encodedRowKey) {
+ List<String> split = TimelineReaderUtils.split(encodedRowKey);
+ if (split == null || split.size() != 4) {
+ throw new IllegalArgumentException(
+ "Invalid row key for flow activity.");
+ }
+ Long dayTs = Long.valueOf(split.get(1));
+ return new FlowActivityRowKey(split.get(0), dayTs, split.get(2),
+ split.get(3));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKeyPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKeyPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKeyPrefix.java
new file mode 100644
index 0000000..eb88e54
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKeyPrefix.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.RowKeyPrefix;
+
+/**
+ * A prefix partial rowkey for flow activities.
+ */
+public class FlowActivityRowKeyPrefix extends FlowActivityRowKey implements
+ RowKeyPrefix<FlowActivityRowKey> {
+
+ /**
+ * Constructs a row key prefix for the flow activity table as follows:
+ * {@code clusterId!dayTimestamp!}.
+ *
+ * @param clusterId Cluster Id.
+ * @param dayTs Start of the day timestamp.
+ */
+ public FlowActivityRowKeyPrefix(String clusterId, Long dayTs) {
+ super(clusterId, dayTs, null, null, false);
+ }
+
+ /**
+ * Constructs a row key prefix for the flow activity table as follows:
+ * {@code clusterId!}.
+ *
+ * @param clusterId identifying the cluster
+ */
+ public FlowActivityRowKeyPrefix(String clusterId) {
+ super(clusterId, null, null, null, false);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.application.
+ * RowKeyPrefix#getRowKeyPrefix()
+ */
+ public byte[] getRowKeyPrefix() {
+ return super.getRowKey();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityTable.java
new file mode 100644
index 0000000..e88a2fc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityTable.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+
+/**
+ * The flow activity table has column family info
+ * Stores the daily activity record for flows
+ * Useful as a quick lookup of what flows were
+ * running on a given day
+ *
+ * Example flow activity table record:
+ *
+ * <pre>
+ * |-------------------------------------------|
+ * | Row key | Column Family |
+ * | | info |
+ * |-------------------------------------------|
+ * | clusterId! | r!runid1:version1 |
+ * | inv Top of | |
+ * | Day! | r!runid2:version7 |
+ * | userName! | |
+ * | flowName | |
+ * |-------------------------------------------|
+ * </pre>
+ */
+public final class FlowActivityTable extends BaseTable<FlowActivityTable> {
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java
new file mode 100644
index 0000000..2132d04
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Column;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.GenericConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineSchemaUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
+
+/**
+ * Identifies fully qualified columns for the {@link FlowRunTable}.
+ */
+public enum FlowRunColumn implements Column<FlowRunTable> {
+
+ /**
+ * When the flow was started. This is the minimum of currently known
+ * application start times.
+ */
+ MIN_START_TIME(FlowRunColumnFamily.INFO, "min_start_time",
+ AggregationOperation.GLOBAL_MIN, new LongConverter()),
+
+ /**
+ * When the flow ended. This is the maximum of currently known application end
+ * times.
+ */
+ MAX_END_TIME(FlowRunColumnFamily.INFO, "max_end_time",
+ AggregationOperation.GLOBAL_MAX, new LongConverter()),
+
+ /**
+ * The version of the flow that this flow belongs to.
+ */
+ FLOW_VERSION(FlowRunColumnFamily.INFO, "flow_version", null);
+
+ private final ColumnFamily<FlowRunTable> columnFamily;
+ private final String columnQualifier;
+ private final byte[] columnQualifierBytes;
+ private final AggregationOperation aggOp;
+ private final ValueConverter valueConverter;
+
+ private FlowRunColumn(ColumnFamily<FlowRunTable> columnFamily,
+ String columnQualifier, AggregationOperation aggOp) {
+ this(columnFamily, columnQualifier, aggOp,
+ GenericConverter.getInstance());
+ }
+
+ private FlowRunColumn(ColumnFamily<FlowRunTable> columnFamily,
+ String columnQualifier, AggregationOperation aggOp,
+ ValueConverter converter) {
+ this.columnFamily = columnFamily;
+ this.columnQualifier = columnQualifier;
+ this.aggOp = aggOp;
+ // Future-proof by ensuring the right column prefix hygiene.
+ this.columnQualifierBytes = Bytes.toBytes(Separator.SPACE
+ .encode(columnQualifier));
+ this.valueConverter = converter;
+ }
+
+ /**
+ * @return the column name value
+ */
+ private String getColumnQualifier() {
+ return columnQualifier;
+ }
+
+ @Override
+ public byte[] getColumnQualifierBytes() {
+ return columnQualifierBytes.clone();
+ }
+
+ @Override
+ public byte[] getColumnFamilyBytes() {
+ return columnFamily.getBytes();
+ }
+
+ public AggregationOperation getAggregationOperation() {
+ return aggOp;
+ }
+
+ @Override
+ public ValueConverter getValueConverter() {
+ return valueConverter;
+ }
+
+ @Override
+ public Attribute[] getCombinedAttrsWithAggr(Attribute... attributes) {
+ return HBaseTimelineSchemaUtils.combineAttributes(attributes, aggOp);
+ }
+
+ @Override
+ public boolean supplementCellTimestamp() {
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnFamily.java
new file mode 100644
index 0000000..8faf5f8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnFamily.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+
+/**
+ * Represents the flow run table column families.
+ */
+public enum FlowRunColumnFamily implements ColumnFamily<FlowRunTable> {
+
+ /**
+ * Info column family houses known columns, specifically ones included in
+ * columnfamily filters.
+ */
+ INFO("i");
+
+ /**
+ * Byte representation of this column family.
+ */
+ private final byte[] bytes;
+
+ /**
+ * @param value
+ * create a column family with this name. Must be lower case and
+ * without spaces.
+ */
+ private FlowRunColumnFamily(String value) {
+ // column families should be lower case and not contain any spaces.
+ this.bytes = Bytes.toBytes(Separator.SPACE.encode(value));
+ }
+
+ public byte[] getBytes() {
+ return Bytes.copy(bytes);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java
new file mode 100644
index 0000000..cc06bb4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineSchemaUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
+
+/**
+ * Identifies partially qualified columns for the {@link FlowRunTable}.
+ */
+public enum FlowRunColumnPrefix implements ColumnPrefix<FlowRunTable> {
+
+ /**
+ * To store flow run info values.
+ */
+ METRIC(FlowRunColumnFamily.INFO, "m", null, new LongConverter());
+
+ private final ColumnFamily<FlowRunTable> columnFamily;
+
+ /**
+ * Can be null for those cases where the provided column qualifier is the
+ * entire column name.
+ */
+ private final String columnPrefix;
+ private final byte[] columnPrefixBytes;
+ private final ValueConverter valueConverter;
+
+ private final AggregationOperation aggOp;
+
+ /**
+ * Private constructor, meant to be used by the enum definition.
+ *
+ * @param columnFamily that this column is stored in.
+ * @param columnPrefix for this column.
+ */
+ private FlowRunColumnPrefix(ColumnFamily<FlowRunTable> columnFamily,
+ String columnPrefix, AggregationOperation fra, ValueConverter converter) {
+ this(columnFamily, columnPrefix, fra, converter, false);
+ }
+
+ private FlowRunColumnPrefix(ColumnFamily<FlowRunTable> columnFamily,
+ String columnPrefix, AggregationOperation fra, ValueConverter converter,
+ boolean compoundColQual) {
+ this.valueConverter = converter;
+ this.columnFamily = columnFamily;
+ this.columnPrefix = columnPrefix;
+ if (columnPrefix == null) {
+ this.columnPrefixBytes = null;
+ } else {
+ // Future-proof by ensuring the right column prefix hygiene.
+ this.columnPrefixBytes =
+ Bytes.toBytes(Separator.SPACE.encode(columnPrefix));
+ }
+ this.aggOp = fra;
+ }
+
+ /**
+ * @return the column name value
+ */
+ public String getColumnPrefix() {
+ return columnPrefix;
+ }
+
+ public byte[] getColumnPrefixBytes() {
+ return columnPrefixBytes.clone();
+ }
+
+ @Override
+ public byte[] getColumnPrefixBytes(byte[] qualifierPrefix) {
+ return ColumnHelper.getColumnQualifier(this.columnPrefixBytes,
+ qualifierPrefix);
+ }
+
+ @Override
+ public byte[] getColumnPrefixBytes(String qualifierPrefix) {
+ return ColumnHelper.getColumnQualifier(this.columnPrefixBytes,
+ qualifierPrefix);
+ }
+
+ @Override
+ public byte[] getColumnFamilyBytes() {
+ return columnFamily.getBytes();
+ }
+
+ @Override
+ public byte[] getColumnPrefixInBytes() {
+ return columnPrefixBytes != null ? columnPrefixBytes.clone() : null;
+ }
+
+ @Override
+ public Attribute[] getCombinedAttrsWithAggr(Attribute... attributes) {
+ return HBaseTimelineSchemaUtils.combineAttributes(attributes, aggOp);
+ }
+
+ @Override
+ public boolean supplementCellTimeStamp() {
+ return true;
+ }
+
+ public AggregationOperation getAttribute() {
+ return aggOp;
+ }
+
+ @Override
+ public ValueConverter getValueConverter() {
+ return valueConverter;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
new file mode 100644
index 0000000..7ce91cf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
@@ -0,0 +1,233 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import java.util.List;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverterToString;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+
+/**
+ * Represents a rowkey for the flow run table.
+ */
+public class FlowRunRowKey {
+ private final String clusterId;
+ private final String userId;
+ private final String flowName;
+ private final Long flowRunId;
+ private final FlowRunRowKeyConverter flowRunRowKeyConverter =
+ new FlowRunRowKeyConverter();
+
+ public FlowRunRowKey(String clusterId, String userId, String flowName,
+ Long flowRunId) {
+ this.clusterId = clusterId;
+ this.userId = userId;
+ this.flowName = flowName;
+ this.flowRunId = flowRunId;
+ }
+
+ public String getClusterId() {
+ return clusterId;
+ }
+
+ public String getUserId() {
+ return userId;
+ }
+
+ public String getFlowName() {
+ return flowName;
+ }
+
+ public Long getFlowRunId() {
+ return flowRunId;
+ }
+
+ /**
+ * Constructs a row key for the entity table as follows: {
+ * clusterId!userId!flowName!Inverted Flow Run Id}.
+ *
+ * @return byte array with the row key
+ */
+ public byte[] getRowKey() {
+ return flowRunRowKeyConverter.encode(this);
+ }
+
+
+ /**
+ * Given the raw row key as bytes, returns the row key as an object.
+ * @param rowKey Byte representation of row key.
+ * @return A <cite>FlowRunRowKey</cite> object.
+ */
+ public static FlowRunRowKey parseRowKey(byte[] rowKey) {
+ return new FlowRunRowKeyConverter().decode(rowKey);
+ }
+
+ /**
+ * Constructs a row key for the flow run table as follows:
+ * {@code clusterId!userId!flowName!Flow Run Id}.
+ * @return String representation of row key
+ */
+ public String getRowKeyAsString() {
+ return flowRunRowKeyConverter.encodeAsString(this);
+ }
+
+ /**
+ * Given the encoded row key as string, returns the row key as an object.
+ * @param encodedRowKey String representation of row key.
+ * @return A <cite>FlowRunRowKey</cite> object.
+ */
+ public static FlowRunRowKey parseRowKeyFromString(String encodedRowKey) {
+ return new FlowRunRowKeyConverter().decodeFromString(encodedRowKey);
+ }
+
+ /**
+ * returns the Flow Key as a verbose String output.
+ * @return String
+ */
+ @Override
+ public String toString() {
+ StringBuilder flowKeyStr = new StringBuilder();
+ flowKeyStr.append("{clusterId=" + clusterId);
+ flowKeyStr.append(" userId=" + userId);
+ flowKeyStr.append(" flowName=" + flowName);
+ flowKeyStr.append(" flowRunId=");
+ flowKeyStr.append(flowRunId);
+ flowKeyStr.append("}");
+ return flowKeyStr.toString();
+ }
+
+ /**
+ * Encodes and decodes row key for flow run table.
+ * The row key is of the form : clusterId!userId!flowName!flowrunId.
+ * flowrunId is a long and rest are strings.
+ * <p>
+ */
+ final private static class FlowRunRowKeyConverter implements
+ KeyConverter<FlowRunRowKey>, KeyConverterToString<FlowRunRowKey> {
+
+ private FlowRunRowKeyConverter() {
+ }
+
+ /**
+ * The flow run row key is of the form clusterId!userId!flowName!flowrunId
+ * with each segment separated by !. The sizes below indicate sizes of each
+ * one of these segments in sequence. clusterId, userId and flowName are
+ * strings. flowrunId is a long hence 8 bytes in size. Strings are variable
+ * in size (i.e. end whenever separator is encountered). This is used while
+ * decoding and helps in determining where to split.
+ */
+ private static final int[] SEGMENT_SIZES = {Separator.VARIABLE_SIZE,
+ Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG };
+
+ /*
+ * (non-Javadoc)
+ *
+ * Encodes FlowRunRowKey object into a byte array with each component/field
+ * in FlowRunRowKey separated by Separator#QUALIFIERS. This leads to an flow
+ * run row key of the form clusterId!userId!flowName!flowrunId If flowRunId
+ * in passed FlowRunRowKey object is null (and the fields preceding it i.e.
+ * clusterId, userId and flowName are not null), this returns a row key
+ * prefix of the form clusterId!userName!flowName! flowRunId is inverted
+ * while encoding as it helps maintain a descending order for flow keys in
+ * flow run table.
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common
+ * .KeyConverter#encode(java.lang.Object)
+ */
+ @Override
+ public byte[] encode(FlowRunRowKey rowKey) {
+ byte[] first =
+ Separator.QUALIFIERS.join(Separator.encode(rowKey.getClusterId(),
+ Separator.SPACE, Separator.TAB, Separator.QUALIFIERS), Separator
+ .encode(rowKey.getUserId(), Separator.SPACE, Separator.TAB,
+ Separator.QUALIFIERS), Separator.encode(rowKey.getFlowName(),
+ Separator.SPACE, Separator.TAB, Separator.QUALIFIERS));
+ if (rowKey.getFlowRunId() == null) {
+ return Separator.QUALIFIERS.join(first, Separator.EMPTY_BYTES);
+ } else {
+ // Note that flowRunId is a long, so we can't encode them all at the
+ // same
+ // time.
+ byte[] second =
+ Bytes.toBytes(LongConverter.invertLong(rowKey.getFlowRunId()));
+ return Separator.QUALIFIERS.join(first, second);
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * Decodes an flow run row key of the form
+ * clusterId!userId!flowName!flowrunId represented in byte format and
+ * converts it into an FlowRunRowKey object. flowRunId is inverted while
+ * decoding as it was inverted while encoding.
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common
+ * .KeyConverter#decode(byte[])
+ */
+ @Override
+ public FlowRunRowKey decode(byte[] rowKey) {
+ byte[][] rowKeyComponents =
+ Separator.QUALIFIERS.split(rowKey, SEGMENT_SIZES);
+ if (rowKeyComponents.length != 4) {
+ throw new IllegalArgumentException("the row key is not valid for "
+ + "a flow run");
+ }
+ String clusterId =
+ Separator.decode(Bytes.toString(rowKeyComponents[0]),
+ Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
+ String userId =
+ Separator.decode(Bytes.toString(rowKeyComponents[1]),
+ Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
+ String flowName =
+ Separator.decode(Bytes.toString(rowKeyComponents[2]),
+ Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
+ Long flowRunId =
+ LongConverter.invertLong(Bytes.toLong(rowKeyComponents[3]));
+ return new FlowRunRowKey(clusterId, userId, flowName, flowRunId);
+ }
+
+ @Override
+ public String encodeAsString(FlowRunRowKey key) {
+ if (key.clusterId == null || key.userId == null || key.flowName == null
+ || key.flowRunId == null) {
+ throw new IllegalArgumentException();
+ }
+ return TimelineReaderUtils.joinAndEscapeStrings(new String[] {
+ key.clusterId, key.userId, key.flowName, key.flowRunId.toString()});
+ }
+
+ @Override
+ public FlowRunRowKey decodeFromString(String encodedRowKey) {
+ List<String> split = TimelineReaderUtils.split(encodedRowKey);
+ if (split == null || split.size() != 4) {
+ throw new IllegalArgumentException(
+ "Invalid row key for flow run table.");
+ }
+ Long flowRunId = Long.valueOf(split.get(3));
+ return new FlowRunRowKey(split.get(0), split.get(1), split.get(2),
+ flowRunId);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKeyPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKeyPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKeyPrefix.java
new file mode 100644
index 0000000..23ebc66
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKeyPrefix.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.RowKeyPrefix;
+
+/**
+ * Represents a partial rowkey (without the flowRunId) for the flow run table.
+ */
+public class FlowRunRowKeyPrefix extends FlowRunRowKey implements
+ RowKeyPrefix<FlowRunRowKey> {
+
+ /**
+ * Constructs a row key prefix for the flow run table as follows:
+ * {@code clusterId!userI!flowName!}.
+ *
+ * @param clusterId identifying the cluster
+ * @param userId identifying the user
+ * @param flowName identifying the flow
+ */
+ public FlowRunRowKeyPrefix(String clusterId, String userId,
+ String flowName) {
+ super(clusterId, userId, flowName, null);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.application.
+ * RowKeyPrefix#getRowKeyPrefix()
+ */
+ public byte[] getRowKeyPrefix() {
+ // We know we're a FlowRunRowKey with null florRunId, so we can simply
+ // delegate
+ return super.getRowKey();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java
new file mode 100644
index 0000000..643a102
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+
+/**
+ * The flow run table has column family info
+ * Stores per flow run information
+ * aggregated across applications.
+ *
+ * Metrics are also stored in the info column family.
+ *
+ * Example flow run table record:
+ *
+ * <pre>
+ * flow_run table
+ * |-------------------------------------------|
+ * | Row key | Column Family |
+ * | | info |
+ * |-------------------------------------------|
+ * | clusterId! | flow_version:version7 |
+ * | userName! | |
+ * | flowName! | running_apps:1 |
+ * | flowRunId | |
+ * | | min_start_time:1392995080000 |
+ * | | #0:"" |
+ * | | |
+ * | | min_start_time:1392995081012 |
+ * | | #0:appId2 |
+ * | | |
+ * | | min_start_time:1392993083210 |
+ * | | #0:appId3 |
+ * | | |
+ * | | |
+ * | | max_end_time:1392993084018 |
+ * | | #0:"" |
+ * | | |
+ * | | |
+ * | | m!mapInputRecords:127 |
+ * | | #0:"" |
+ * | | |
+ * | | m!mapInputRecords:31 |
+ * | | #2:appId2 |
+ * | | |
+ * | | m!mapInputRecords:37 |
+ * | | #1:appId3 |
+ * | | |
+ * | | |
+ * | | m!mapOutputRecords:181 |
+ * | | #0:"" |
+ * | | |
+ * | | m!mapOutputRecords:37 |
+ * | | #1:appId3 |
+ * | | |
+ * | | |
+ * |-------------------------------------------|
+ * </pre>
+ */
+public final class FlowRunTable extends BaseTable<FlowRunTable> {
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/package-info.java
new file mode 100644
index 0000000..04963f3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/package-info.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.storage.flow
+ * contains classes related to implementation for flow related tables, viz. flow
+ * run table and flow activity table.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java
new file mode 100644
index 0000000..e78db2a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.storage contains
+ * classes which define and implement reading and writing to backend storage.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumn.java
new file mode 100644
index 0000000..a011a3f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumn.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Column;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.GenericConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
+
+/**
+ * Identifies fully qualified columns for the {@link SubApplicationTable}.
+ */
+public enum SubApplicationColumn implements Column<SubApplicationTable> {
+
+ /**
+ * Identifier for the sub application.
+ */
+ ID(SubApplicationColumnFamily.INFO, "id"),
+
+ /**
+ * The type of sub application.
+ */
+ TYPE(SubApplicationColumnFamily.INFO, "type"),
+
+ /**
+ * When the sub application was created.
+ */
+ CREATED_TIME(SubApplicationColumnFamily.INFO, "created_time",
+ new LongConverter()),
+
+ /**
+ * The version of the flow that this sub application belongs to.
+ */
+ FLOW_VERSION(SubApplicationColumnFamily.INFO, "flow_version");
+
+ private final ColumnFamily<SubApplicationTable> columnFamily;
+ private final String columnQualifier;
+ private final byte[] columnQualifierBytes;
+ private final ValueConverter valueConverter;
+
+ SubApplicationColumn(ColumnFamily<SubApplicationTable> columnFamily,
+ String columnQualifier) {
+ this(columnFamily, columnQualifier, GenericConverter.getInstance());
+ }
+
+ SubApplicationColumn(ColumnFamily<SubApplicationTable> columnFamily,
+ String columnQualifier, ValueConverter converter) {
+ this.columnFamily = columnFamily;
+ this.columnQualifier = columnQualifier;
+ // Future-proof by ensuring the right column prefix hygiene.
+ this.columnQualifierBytes =
+ Bytes.toBytes(Separator.SPACE.encode(columnQualifier));
+ this.valueConverter = converter;
+ }
+
+ @Override
+ public byte[] getColumnQualifierBytes() {
+ return columnQualifierBytes.clone();
+ }
+
+ @Override
+ public byte[] getColumnFamilyBytes() {
+ return columnFamily.getBytes();
+ }
+
+ @Override
+ public ValueConverter getValueConverter() {
+ return valueConverter;
+ }
+
+ @Override
+ public Attribute[] getCombinedAttrsWithAggr(Attribute... attributes) {
+ return attributes;
+ }
+
+ @Override
+ public boolean supplementCellTimestamp() {
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumnFamily.java
new file mode 100644
index 0000000..1d7f8fd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumnFamily.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+
+/**
+ * Represents the sub application table column families.
+ */
+public enum SubApplicationColumnFamily
+ implements ColumnFamily<SubApplicationTable> {
+
+ /**
+ * Info column family houses known columns, specifically ones included in
+ * columnfamily filters.
+ */
+ INFO("i"),
+
+ /**
+ * Configurations are in a separate column family for two reasons:
+ * a) the size of the config values can be very large and
+ * b) we expect that config values
+ * are often separately accessed from other metrics and info columns.
+ */
+ CONFIGS("c"),
+
+ /**
+ * Metrics have a separate column family, because they have a separate TTL.
+ */
+ METRICS("m");
+
+ /**
+ * Byte representation of this column family.
+ */
+ private final byte[] bytes;
+
+ /**
+ * @param value
+ * create a column family with this name. Must be lower case and
+ * without spaces.
+ */
+ SubApplicationColumnFamily(String value) {
+ // column families should be lower case and not contain any spaces.
+ this.bytes = Bytes.toBytes(Separator.SPACE.encode(value));
+ }
+
+ public byte[] getBytes() {
+ return Bytes.copy(bytes);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumnPrefix.java
new file mode 100644
index 0000000..1106e37
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationColumnPrefix.java
@@ -0,0 +1,163 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.GenericConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
+
+/**
+ * Identifies partially qualified columns for the sub app table.
+ */
+public enum SubApplicationColumnPrefix
+ implements ColumnPrefix<SubApplicationTable> {
+
+ /**
+ * To store TimelineEntity getIsRelatedToEntities values.
+ */
+ IS_RELATED_TO(SubApplicationColumnFamily.INFO, "s"),
+
+ /**
+ * To store TimelineEntity getRelatesToEntities values.
+ */
+ RELATES_TO(SubApplicationColumnFamily.INFO, "r"),
+
+ /**
+ * To store TimelineEntity info values.
+ */
+ INFO(SubApplicationColumnFamily.INFO, "i"),
+
+ /**
+ * Lifecycle events for an entity.
+ */
+ EVENT(SubApplicationColumnFamily.INFO, "e", true),
+
+ /**
+ * Config column stores configuration with config key as the column name.
+ */
+ CONFIG(SubApplicationColumnFamily.CONFIGS, null),
+
+ /**
+ * Metrics are stored with the metric name as the column name.
+ */
+ METRIC(SubApplicationColumnFamily.METRICS, null, new LongConverter());
+
+ private final ColumnFamily<SubApplicationTable> columnFamily;
+
+ /**
+ * Can be null for those cases where the provided column qualifier is the
+ * entire column name.
+ */
+ private final String columnPrefix;
+ private final byte[] columnPrefixBytes;
+ private final ValueConverter valueConverter;
+
+ /**
+ * Private constructor, meant to be used by the enum definition.
+ *
+ * @param columnFamily that this column is stored in.
+ * @param columnPrefix for this column.
+ */
+ SubApplicationColumnPrefix(ColumnFamily<SubApplicationTable> columnFamily,
+ String columnPrefix) {
+ this(columnFamily, columnPrefix, false, GenericConverter.getInstance());
+ }
+
+ SubApplicationColumnPrefix(ColumnFamily<SubApplicationTable> columnFamily,
+ String columnPrefix, boolean compondColQual) {
+ this(columnFamily, columnPrefix, compondColQual,
+ GenericConverter.getInstance());
+ }
+
+ SubApplicationColumnPrefix(ColumnFamily<SubApplicationTable> columnFamily,
+ String columnPrefix, ValueConverter converter) {
+ this(columnFamily, columnPrefix, false, converter);
+ }
+
+ /**
+ * Private constructor, meant to be used by the enum definition.
+ *
+ * @param columnFamily that this column is stored in.
+ * @param columnPrefix for this column.
+ * @param converter used to encode/decode values to be stored in HBase for
+ * this column prefix.
+ */
+ SubApplicationColumnPrefix(ColumnFamily<SubApplicationTable> columnFamily,
+ String columnPrefix, boolean compondColQual, ValueConverter converter) {
+ this.valueConverter = converter;
+ this.columnFamily = columnFamily;
+ this.columnPrefix = columnPrefix;
+ if (columnPrefix == null) {
+ this.columnPrefixBytes = null;
+ } else {
+ // Future-proof by ensuring the right column prefix hygiene.
+ this.columnPrefixBytes =
+ Bytes.toBytes(Separator.SPACE.encode(columnPrefix));
+ }
+ }
+
+ /**
+ * @return the column name value
+ */
+ public String getColumnPrefix() {
+ return columnPrefix;
+ }
+
+ @Override
+ public byte[] getColumnPrefixBytes(byte[] qualifierPrefix) {
+ return ColumnHelper.getColumnQualifier(
+ this.columnPrefixBytes, qualifierPrefix);
+ }
+
+ @Override
+ public byte[] getColumnPrefixBytes(String qualifierPrefix) {
+ return ColumnHelper.getColumnQualifier(
+ this.columnPrefixBytes, qualifierPrefix);
+ }
+
+ @Override
+ public byte[] getColumnPrefixInBytes() {
+ return columnPrefixBytes != null ? columnPrefixBytes.clone() : null;
+ }
+
+ @Override
+ public byte[] getColumnFamilyBytes() {
+ return columnFamily.getBytes();
+ }
+
+ @Override
+ public ValueConverter getValueConverter() {
+ return valueConverter;
+ }
+
+ @Override
+ public Attribute[] getCombinedAttrsWithAggr(Attribute... attributes) {
+ return attributes;
+ }
+
+ @Override
+ public boolean supplementCellTimeStamp() {
+ return false;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/18] hadoop git commit: YARN-7919. Refactor timelineservice-hbase
module into submodules. Contributed by Haibo Chen.
Posted by ro...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
new file mode 100644
index 0000000..41a371b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
@@ -0,0 +1,277 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.Tag;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineServerUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimestampGenerator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Coprocessor for flow run table.
+ */
+public class FlowRunCoprocessor extends BaseRegionObserver {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(FlowRunCoprocessor.class);
+
+ private Region region;
+ /**
+ * generate a timestamp that is unique per row in a region this is per region.
+ */
+ private final TimestampGenerator timestampGenerator =
+ new TimestampGenerator();
+
+ @Override
+ public void start(CoprocessorEnvironment e) throws IOException {
+ if (e instanceof RegionCoprocessorEnvironment) {
+ RegionCoprocessorEnvironment env = (RegionCoprocessorEnvironment) e;
+ this.region = env.getRegion();
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * This method adds the tags onto the cells in the Put. It is presumed that
+ * all the cells in one Put have the same set of Tags. The existing cell
+ * timestamp is overwritten for non-metric cells and each such cell gets a new
+ * unique timestamp generated by {@link TimestampGenerator}
+ *
+ * @see
+ * org.apache.hadoop.hbase.coprocessor.BaseRegionObserver#prePut(org.apache
+ * .hadoop.hbase.coprocessor.ObserverContext,
+ * org.apache.hadoop.hbase.client.Put,
+ * org.apache.hadoop.hbase.regionserver.wal.WALEdit,
+ * org.apache.hadoop.hbase.client.Durability)
+ */
+ @Override
+ public void prePut(ObserverContext<RegionCoprocessorEnvironment> e, Put put,
+ WALEdit edit, Durability durability) throws IOException {
+ Map<String, byte[]> attributes = put.getAttributesMap();
+ // Assumption is that all the cells in a put are the same operation.
+ List<Tag> tags = new ArrayList<>();
+ if ((attributes != null) && (attributes.size() > 0)) {
+ for (Map.Entry<String, byte[]> attribute : attributes.entrySet()) {
+ Tag t = HBaseTimelineServerUtils.getTagFromAttribute(attribute);
+ if (t != null) {
+ tags.add(t);
+ }
+ }
+ byte[] tagByteArray = Tag.fromList(tags);
+ NavigableMap<byte[], List<Cell>> newFamilyMap = new TreeMap<>(
+ Bytes.BYTES_COMPARATOR);
+ for (Map.Entry<byte[], List<Cell>> entry : put.getFamilyCellMap()
+ .entrySet()) {
+ List<Cell> newCells = new ArrayList<>(entry.getValue().size());
+ for (Cell cell : entry.getValue()) {
+ // for each cell in the put add the tags
+ // Assumption is that all the cells in
+ // one put are the same operation
+ // also, get a unique cell timestamp for non-metric cells
+ // this way we don't inadvertently overwrite cell versions
+ long cellTimestamp = getCellTimestamp(cell.getTimestamp(), tags);
+ newCells.add(CellUtil.createCell(CellUtil.cloneRow(cell),
+ CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell),
+ cellTimestamp, KeyValue.Type.Put, CellUtil.cloneValue(cell),
+ tagByteArray));
+ }
+ newFamilyMap.put(entry.getKey(), newCells);
+ } // for each entry
+ // Update the family map for the Put
+ put.setFamilyCellMap(newFamilyMap);
+ }
+ }
+
+ /**
+ * Determines if the current cell's timestamp is to be used or a new unique
+ * cell timestamp is to be used. The reason this is done is to inadvertently
+ * overwrite cells when writes come in very fast. But for metric cells, the
+ * cell timestamp signifies the metric timestamp. Hence we don't want to
+ * overwrite it.
+ *
+ * @param timestamp
+ * @param tags
+ * @return cell timestamp
+ */
+ private long getCellTimestamp(long timestamp, List<Tag> tags) {
+ // if ts not set (hbase sets to HConstants.LATEST_TIMESTAMP by default)
+ // then use the generator
+ if (timestamp == HConstants.LATEST_TIMESTAMP) {
+ return timestampGenerator.getUniqueTimestamp();
+ } else {
+ return timestamp;
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * Creates a {@link FlowScanner} Scan so that it can correctly process the
+ * contents of {@link FlowRunTable}.
+ *
+ * @see
+ * org.apache.hadoop.hbase.coprocessor.BaseRegionObserver#preGetOp(org.apache
+ * .hadoop.hbase.coprocessor.ObserverContext,
+ * org.apache.hadoop.hbase.client.Get, java.util.List)
+ */
+ @Override
+ public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> e,
+ Get get, List<Cell> results) throws IOException {
+ Scan scan = new Scan(get);
+ scan.setMaxVersions();
+ RegionScanner scanner = null;
+ try {
+ scanner = new FlowScanner(e.getEnvironment(), scan,
+ region.getScanner(scan), FlowScannerOperation.READ);
+ scanner.next(results);
+ e.bypass();
+ } finally {
+ if (scanner != null) {
+ scanner.close();
+ }
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * Ensures that max versions are set for the Scan so that metrics can be
+ * correctly aggregated and min/max can be correctly determined.
+ *
+ * @see
+ * org.apache.hadoop.hbase.coprocessor.BaseRegionObserver#preScannerOpen(org
+ * .apache.hadoop.hbase.coprocessor.ObserverContext,
+ * org.apache.hadoop.hbase.client.Scan,
+ * org.apache.hadoop.hbase.regionserver.RegionScanner)
+ */
+ @Override
+ public RegionScanner preScannerOpen(
+ ObserverContext<RegionCoprocessorEnvironment> e, Scan scan,
+ RegionScanner scanner) throws IOException {
+ // set max versions for scan to see all
+ // versions to aggregate for metrics
+ scan.setMaxVersions();
+ return scanner;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * Creates a {@link FlowScanner} Scan so that it can correctly process the
+ * contents of {@link FlowRunTable}.
+ *
+ * @see
+ * org.apache.hadoop.hbase.coprocessor.BaseRegionObserver#postScannerOpen(
+ * org.apache.hadoop.hbase.coprocessor.ObserverContext,
+ * org.apache.hadoop.hbase.client.Scan,
+ * org.apache.hadoop.hbase.regionserver.RegionScanner)
+ */
+ @Override
+ public RegionScanner postScannerOpen(
+ ObserverContext<RegionCoprocessorEnvironment> e, Scan scan,
+ RegionScanner scanner) throws IOException {
+ return new FlowScanner(e.getEnvironment(), scan,
+ scanner, FlowScannerOperation.READ);
+ }
+
+ @Override
+ public InternalScanner preFlush(
+ ObserverContext<RegionCoprocessorEnvironment> c, Store store,
+ InternalScanner scanner) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ if (store != null) {
+ LOG.debug("preFlush store = " + store.getColumnFamilyName()
+ + " flushableSize=" + store.getFlushableSize()
+ + " flushedCellsCount=" + store.getFlushedCellsCount()
+ + " compactedCellsCount=" + store.getCompactedCellsCount()
+ + " majorCompactedCellsCount="
+ + store.getMajorCompactedCellsCount() + " memstoreFlushSize="
+ + store.getMemstoreFlushSize() + " memstoreSize="
+ + store.getMemStoreSize() + " size=" + store.getSize()
+ + " storeFilesCount=" + store.getStorefilesCount());
+ }
+ }
+ return new FlowScanner(c.getEnvironment(), scanner,
+ FlowScannerOperation.FLUSH);
+ }
+
+ @Override
+ public void postFlush(ObserverContext<RegionCoprocessorEnvironment> c,
+ Store store, StoreFile resultFile) {
+ if (LOG.isDebugEnabled()) {
+ if (store != null) {
+ LOG.debug("postFlush store = " + store.getColumnFamilyName()
+ + " flushableSize=" + store.getFlushableSize()
+ + " flushedCellsCount=" + store.getFlushedCellsCount()
+ + " compactedCellsCount=" + store.getCompactedCellsCount()
+ + " majorCompactedCellsCount="
+ + store.getMajorCompactedCellsCount() + " memstoreFlushSize="
+ + store.getMemstoreFlushSize() + " memstoreSize="
+ + store.getMemStoreSize() + " size=" + store.getSize()
+ + " storeFilesCount=" + store.getStorefilesCount());
+ }
+ }
+ }
+
+ @Override
+ public InternalScanner preCompact(
+ ObserverContext<RegionCoprocessorEnvironment> e, Store store,
+ InternalScanner scanner, ScanType scanType, CompactionRequest request)
+ throws IOException {
+
+ FlowScannerOperation requestOp = FlowScannerOperation.MINOR_COMPACTION;
+ if (request != null) {
+ requestOp = (request.isMajor() ? FlowScannerOperation.MAJOR_COMPACTION
+ : FlowScannerOperation.MINOR_COMPACTION);
+ LOG.info("Compactionrequest= " + request.toString() + " "
+ + requestOp.toString() + " RegionName=" + e.getEnvironment()
+ .getRegion().getRegionInfo().getRegionNameAsString());
+ }
+ return new FlowScanner(e.getEnvironment(), scanner, requestOp);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
new file mode 100644
index 0000000..7f09e51
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
@@ -0,0 +1,723 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.Tag;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScannerContext;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.GenericConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineServerUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.NumericValueConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimestampGenerator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Invoked via the coprocessor when a Get or a Scan is issued for flow run
+ * table. Looks through the list of cells per row, checks their tags and does
+ * operation on those cells as per the cell tags. Transforms reads of the stored
+ * metrics into calculated sums for each column Also, finds the min and max for
+ * start and end times in a flow run.
+ */
+class FlowScanner implements RegionScanner, Closeable {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(FlowScanner.class);
+
+ /**
+ * use a special application id to represent the flow id this is needed since
+ * TimestampGenerator parses the app id to generate a cell timestamp.
+ */
+ private static final String FLOW_APP_ID = "application_00000000000_0000";
+
+ private final Region region;
+ private final InternalScanner flowRunScanner;
+ private final int batchSize;
+ private final long appFinalValueRetentionThreshold;
+ private RegionScanner regionScanner;
+ private boolean hasMore;
+ private byte[] currentRow;
+ private List<Cell> availableCells = new ArrayList<>();
+ private int currentIndex;
+ private FlowScannerOperation action = FlowScannerOperation.READ;
+
+ FlowScanner(RegionCoprocessorEnvironment env, InternalScanner internalScanner,
+ FlowScannerOperation action) {
+ this(env, null, internalScanner, action);
+ }
+
+ FlowScanner(RegionCoprocessorEnvironment env, Scan incomingScan,
+ InternalScanner internalScanner, FlowScannerOperation action) {
+ this.batchSize = incomingScan == null ? -1 : incomingScan.getBatch();
+ // TODO initialize other scan attributes like Scan#maxResultSize
+ this.flowRunScanner = internalScanner;
+ if (internalScanner instanceof RegionScanner) {
+ this.regionScanner = (RegionScanner) internalScanner;
+ }
+ this.action = action;
+ if (env == null) {
+ this.appFinalValueRetentionThreshold =
+ YarnConfiguration.DEFAULT_APP_FINAL_VALUE_RETENTION_THRESHOLD;
+ this.region = null;
+ } else {
+ this.region = env.getRegion();
+ Configuration hbaseConf = env.getConfiguration();
+ this.appFinalValueRetentionThreshold = hbaseConf.getLong(
+ YarnConfiguration.APP_FINAL_VALUE_RETENTION_THRESHOLD,
+ YarnConfiguration.DEFAULT_APP_FINAL_VALUE_RETENTION_THRESHOLD);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(" batch size=" + batchSize);
+ }
+ }
+
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.hadoop.hbase.regionserver.RegionScanner#getRegionInfo()
+ */
+ @Override
+ public HRegionInfo getRegionInfo() {
+ return region.getRegionInfo();
+ }
+
+ @Override
+ public boolean nextRaw(List<Cell> cells) throws IOException {
+ return nextRaw(cells, ScannerContext.newBuilder().build());
+ }
+
+ @Override
+ public boolean nextRaw(List<Cell> cells, ScannerContext scannerContext)
+ throws IOException {
+ return nextInternal(cells, scannerContext);
+ }
+
+ @Override
+ public boolean next(List<Cell> cells) throws IOException {
+ return next(cells, ScannerContext.newBuilder().build());
+ }
+
+ @Override
+ public boolean next(List<Cell> cells, ScannerContext scannerContext)
+ throws IOException {
+ return nextInternal(cells, scannerContext);
+ }
+
+ /**
+ * Get value converter associated with a column or a column prefix. If nothing
+ * matches, generic converter is returned.
+ * @param colQualifierBytes
+ * @return value converter implementation.
+ */
+ private static ValueConverter getValueConverter(byte[] colQualifierBytes) {
+ // Iterate over all the column prefixes for flow run table and get the
+ // appropriate converter for the column qualifier passed if prefix matches.
+ for (FlowRunColumnPrefix colPrefix : FlowRunColumnPrefix.values()) {
+ byte[] colPrefixBytes = colPrefix.getColumnPrefixBytes("");
+ if (Bytes.compareTo(colPrefixBytes, 0, colPrefixBytes.length,
+ colQualifierBytes, 0, colPrefixBytes.length) == 0) {
+ return colPrefix.getValueConverter();
+ }
+ }
+ // Iterate over all the columns for flow run table and get the
+ // appropriate converter for the column qualifier passed if match occurs.
+ for (FlowRunColumn column : FlowRunColumn.values()) {
+ if (Bytes.compareTo(
+ column.getColumnQualifierBytes(), colQualifierBytes) == 0) {
+ return column.getValueConverter();
+ }
+ }
+ // Return generic converter if nothing matches.
+ return GenericConverter.getInstance();
+ }
+
+ /**
+ * This method loops through the cells in a given row of the
+ * {@link FlowRunTable}. It looks at the tags of each cell to figure out how
+ * to process the contents. It then calculates the sum or min or max for each
+ * column or returns the cell as is.
+ *
+ * @param cells
+ * @param scannerContext
+ * @return true if next row is available for the scanner, false otherwise
+ * @throws IOException
+ */
+ private boolean nextInternal(List<Cell> cells, ScannerContext scannerContext)
+ throws IOException {
+ Cell cell = null;
+ startNext();
+ // Loop through all the cells in this row
+ // For min/max/metrics we do need to scan the entire set of cells to get the
+ // right one
+ // But with flush/compaction, the number of cells being scanned will go down
+ // cells are grouped per column qualifier then sorted by cell timestamp
+ // (latest to oldest) per column qualifier
+ // So all cells in one qualifier come one after the other before we see the
+ // next column qualifier
+ ByteArrayComparator comp = new ByteArrayComparator();
+ byte[] previousColumnQualifier = Separator.EMPTY_BYTES;
+ AggregationOperation currentAggOp = null;
+ SortedSet<Cell> currentColumnCells = new TreeSet<>(KeyValue.COMPARATOR);
+ Set<String> alreadySeenAggDim = new HashSet<>();
+ int addedCnt = 0;
+ long currentTimestamp = System.currentTimeMillis();
+ ValueConverter converter = null;
+ int limit = batchSize;
+
+ while (limit <= 0 || addedCnt < limit) {
+ cell = peekAtNextCell(scannerContext);
+ if (cell == null) {
+ break;
+ }
+ byte[] currentColumnQualifier = CellUtil.cloneQualifier(cell);
+ if (previousColumnQualifier == null) {
+ // first time in loop
+ previousColumnQualifier = currentColumnQualifier;
+ }
+
+ converter = getValueConverter(currentColumnQualifier);
+ if (comp.compare(previousColumnQualifier, currentColumnQualifier) != 0) {
+ addedCnt += emitCells(cells, currentColumnCells, currentAggOp,
+ converter, currentTimestamp);
+ resetState(currentColumnCells, alreadySeenAggDim);
+ previousColumnQualifier = currentColumnQualifier;
+ currentAggOp = getCurrentAggOp(cell);
+ converter = getValueConverter(currentColumnQualifier);
+ }
+ collectCells(currentColumnCells, currentAggOp, cell, alreadySeenAggDim,
+ converter, scannerContext);
+ nextCell(scannerContext);
+ }
+ if ((!currentColumnCells.isEmpty()) && ((limit <= 0 || addedCnt < limit))) {
+ addedCnt += emitCells(cells, currentColumnCells, currentAggOp, converter,
+ currentTimestamp);
+ if (LOG.isDebugEnabled()) {
+ if (addedCnt > 0) {
+ LOG.debug("emitted cells. " + addedCnt + " for " + this.action
+ + " rowKey="
+ + FlowRunRowKey.parseRowKey(CellUtil.cloneRow(cells.get(0))));
+ } else {
+ LOG.debug("emitted no cells for " + this.action);
+ }
+ }
+ }
+ return hasMore();
+ }
+
+ private AggregationOperation getCurrentAggOp(Cell cell) {
+ List<Tag> tags = Tag.asList(cell.getTagsArray(), cell.getTagsOffset(),
+ cell.getTagsLength());
+ // We assume that all the operations for a particular column are the same
+ return HBaseTimelineServerUtils.getAggregationOperationFromTagsList(tags);
+ }
+
+ /**
+ * resets the parameters to an initialized state for next loop iteration.
+ */
+ private void resetState(SortedSet<Cell> currentColumnCells,
+ Set<String> alreadySeenAggDim) {
+ currentColumnCells.clear();
+ alreadySeenAggDim.clear();
+ }
+
+ private void collectCells(SortedSet<Cell> currentColumnCells,
+ AggregationOperation currentAggOp, Cell cell,
+ Set<String> alreadySeenAggDim, ValueConverter converter,
+ ScannerContext scannerContext) throws IOException {
+
+ if (currentAggOp == null) {
+ // not a min/max/metric cell, so just return it as is
+ currentColumnCells.add(cell);
+ return;
+ }
+
+ switch (currentAggOp) {
+ case GLOBAL_MIN:
+ if (currentColumnCells.size() == 0) {
+ currentColumnCells.add(cell);
+ } else {
+ Cell currentMinCell = currentColumnCells.first();
+ Cell newMinCell = compareCellValues(currentMinCell, cell, currentAggOp,
+ (NumericValueConverter) converter);
+ if (!currentMinCell.equals(newMinCell)) {
+ currentColumnCells.remove(currentMinCell);
+ currentColumnCells.add(newMinCell);
+ }
+ }
+ break;
+ case GLOBAL_MAX:
+ if (currentColumnCells.size() == 0) {
+ currentColumnCells.add(cell);
+ } else {
+ Cell currentMaxCell = currentColumnCells.first();
+ Cell newMaxCell = compareCellValues(currentMaxCell, cell, currentAggOp,
+ (NumericValueConverter) converter);
+ if (!currentMaxCell.equals(newMaxCell)) {
+ currentColumnCells.remove(currentMaxCell);
+ currentColumnCells.add(newMaxCell);
+ }
+ }
+ break;
+ case SUM:
+ case SUM_FINAL:
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("In collect cells "
+ + " FlowSannerOperation="
+ + this.action
+ + " currentAggOp="
+ + currentAggOp
+ + " cell qualifier="
+ + Bytes.toString(CellUtil.cloneQualifier(cell))
+ + " cell value= "
+ + converter.decodeValue(CellUtil.cloneValue(cell))
+ + " timestamp=" + cell.getTimestamp());
+ }
+
+ // only if this app has not been seen yet, add to current column cells
+ List<Tag> tags = Tag.asList(cell.getTagsArray(), cell.getTagsOffset(),
+ cell.getTagsLength());
+ String aggDim = HBaseTimelineServerUtils
+ .getAggregationCompactionDimension(tags);
+ if (!alreadySeenAggDim.contains(aggDim)) {
+ // if this agg dimension has already been seen,
+ // since they show up in sorted order
+ // we drop the rest which are older
+ // in other words, this cell is older than previously seen cells
+ // for that agg dim
+ // but when this agg dim is not seen,
+ // consider this cell in our working set
+ currentColumnCells.add(cell);
+ alreadySeenAggDim.add(aggDim);
+ }
+ break;
+ default:
+ break;
+ } // end of switch case
+ }
+
+ /*
+ * Processes the cells in input param currentColumnCells and populates
+ * List<Cell> cells as the output based on the input AggregationOperation
+ * parameter.
+ */
+ private int emitCells(List<Cell> cells, SortedSet<Cell> currentColumnCells,
+ AggregationOperation currentAggOp, ValueConverter converter,
+ long currentTimestamp) throws IOException {
+ if ((currentColumnCells == null) || (currentColumnCells.size() == 0)) {
+ return 0;
+ }
+ if (currentAggOp == null) {
+ cells.addAll(currentColumnCells);
+ return currentColumnCells.size();
+ }
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("In emitCells " + this.action + " currentColumnCells size= "
+ + currentColumnCells.size() + " currentAggOp" + currentAggOp);
+ }
+
+ switch (currentAggOp) {
+ case GLOBAL_MIN:
+ case GLOBAL_MAX:
+ cells.addAll(currentColumnCells);
+ return currentColumnCells.size();
+ case SUM:
+ case SUM_FINAL:
+ switch (action) {
+ case FLUSH:
+ case MINOR_COMPACTION:
+ cells.addAll(currentColumnCells);
+ return currentColumnCells.size();
+ case READ:
+ Cell sumCell = processSummation(currentColumnCells,
+ (NumericValueConverter) converter);
+ cells.add(sumCell);
+ return 1;
+ case MAJOR_COMPACTION:
+ List<Cell> finalCells = processSummationMajorCompaction(
+ currentColumnCells, (NumericValueConverter) converter,
+ currentTimestamp);
+ cells.addAll(finalCells);
+ return finalCells.size();
+ default:
+ cells.addAll(currentColumnCells);
+ return currentColumnCells.size();
+ }
+ default:
+ cells.addAll(currentColumnCells);
+ return currentColumnCells.size();
+ }
+ }
+
+ /*
+ * Returns a cell whose value is the sum of all cell values in the input set.
+ * The new cell created has the timestamp of the most recent metric cell. The
+ * sum of a metric for a flow run is the summation at the point of the last
+ * metric update in that flow till that time.
+ */
+ private Cell processSummation(SortedSet<Cell> currentColumnCells,
+ NumericValueConverter converter) throws IOException {
+ Number sum = 0;
+ Number currentValue = 0;
+ long ts = 0L;
+ long mostCurrentTimestamp = 0L;
+ Cell mostRecentCell = null;
+ for (Cell cell : currentColumnCells) {
+ currentValue = (Number) converter.decodeValue(CellUtil.cloneValue(cell));
+ ts = cell.getTimestamp();
+ if (mostCurrentTimestamp < ts) {
+ mostCurrentTimestamp = ts;
+ mostRecentCell = cell;
+ }
+ sum = converter.add(sum, currentValue);
+ }
+ byte[] sumBytes = converter.encodeValue(sum);
+ Cell sumCell =
+ HBaseTimelineServerUtils.createNewCell(mostRecentCell, sumBytes);
+ return sumCell;
+ }
+
+
+ /**
+ * Returns a list of cells that contains
+ *
+ * A) the latest cells for applications that haven't finished yet
+ * B) summation
+ * for the flow, based on applications that have completed and are older than
+ * a certain time
+ *
+ * The new cell created has the timestamp of the most recent metric cell. The
+ * sum of a metric for a flow run is the summation at the point of the last
+ * metric update in that flow till that time.
+ */
+ @VisibleForTesting
+ List<Cell> processSummationMajorCompaction(
+ SortedSet<Cell> currentColumnCells, NumericValueConverter converter,
+ long currentTimestamp)
+ throws IOException {
+ Number sum = 0;
+ Number currentValue = 0;
+ long ts = 0L;
+ boolean summationDone = false;
+ List<Cell> finalCells = new ArrayList<Cell>();
+ if (currentColumnCells == null) {
+ return finalCells;
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("In processSummationMajorCompaction,"
+ + " will drop cells older than " + currentTimestamp
+ + " CurrentColumnCells size=" + currentColumnCells.size());
+ }
+
+ for (Cell cell : currentColumnCells) {
+ AggregationOperation cellAggOp = getCurrentAggOp(cell);
+ // if this is the existing flow sum cell
+ List<Tag> tags = Tag.asList(cell.getTagsArray(), cell.getTagsOffset(),
+ cell.getTagsLength());
+ String appId = HBaseTimelineServerUtils
+ .getAggregationCompactionDimension(tags);
+ if (appId == FLOW_APP_ID) {
+ sum = converter.add(sum, currentValue);
+ summationDone = true;
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("reading flow app id sum=" + sum);
+ }
+ } else {
+ currentValue = (Number) converter.decodeValue(CellUtil
+ .cloneValue(cell));
+ // read the timestamp truncated by the generator
+ ts = TimestampGenerator.getTruncatedTimestamp(cell.getTimestamp());
+ if ((cellAggOp == AggregationOperation.SUM_FINAL)
+ && ((ts + this.appFinalValueRetentionThreshold)
+ < currentTimestamp)) {
+ sum = converter.add(sum, currentValue);
+ summationDone = true;
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("MAJOR COMPACTION loop sum= " + sum
+ + " discarding now: " + " qualifier="
+ + Bytes.toString(CellUtil.cloneQualifier(cell)) + " value="
+ + converter.decodeValue(CellUtil.cloneValue(cell))
+ + " timestamp=" + cell.getTimestamp() + " " + this.action);
+ }
+ } else {
+ // not a final value but it's the latest cell for this app
+ // so include this cell in the list of cells to write back
+ finalCells.add(cell);
+ }
+ }
+ }
+ if (summationDone) {
+ Cell anyCell = currentColumnCells.first();
+ List<Tag> tags = new ArrayList<Tag>();
+ Tag t = new Tag(AggregationOperation.SUM_FINAL.getTagType(),
+ Bytes.toBytes(FLOW_APP_ID));
+ tags.add(t);
+ t = new Tag(AggregationCompactionDimension.APPLICATION_ID.getTagType(),
+ Bytes.toBytes(FLOW_APP_ID));
+ tags.add(t);
+ byte[] tagByteArray = Tag.fromList(tags);
+ Cell sumCell = HBaseTimelineServerUtils.createNewCell(
+ CellUtil.cloneRow(anyCell),
+ CellUtil.cloneFamily(anyCell),
+ CellUtil.cloneQualifier(anyCell),
+ TimestampGenerator.getSupplementedTimestamp(
+ System.currentTimeMillis(), FLOW_APP_ID),
+ converter.encodeValue(sum), tagByteArray);
+ finalCells.add(sumCell);
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("MAJOR COMPACTION final sum= " + sum + " for "
+ + Bytes.toString(CellUtil.cloneQualifier(sumCell))
+ + " " + this.action);
+ }
+ LOG.info("After major compaction for qualifier="
+ + Bytes.toString(CellUtil.cloneQualifier(sumCell))
+ + " with currentColumnCells.size="
+ + currentColumnCells.size()
+ + " returning finalCells.size=" + finalCells.size()
+ + " with sum=" + sum.longValue()
+ + " with cell timestamp " + sumCell.getTimestamp());
+ } else {
+ String qualifier = "";
+ LOG.info("After major compaction for qualifier=" + qualifier
+ + " with currentColumnCells.size="
+ + currentColumnCells.size()
+ + " returning finalCells.size=" + finalCells.size()
+ + " with zero sum="
+ + sum.longValue());
+ }
+ return finalCells;
+ }
+
+ /**
+ * Determines which cell is to be returned based on the values in each cell
+ * and the comparison operation MIN or MAX.
+ *
+ * @param previouslyChosenCell
+ * @param currentCell
+ * @param currentAggOp
+ * @return the cell which is the min (or max) cell
+ * @throws IOException
+ */
+ private Cell compareCellValues(Cell previouslyChosenCell, Cell currentCell,
+ AggregationOperation currentAggOp, NumericValueConverter converter)
+ throws IOException {
+ if (previouslyChosenCell == null) {
+ return currentCell;
+ }
+ try {
+ Number previouslyChosenCellValue = (Number)converter.decodeValue(
+ CellUtil.cloneValue(previouslyChosenCell));
+ Number currentCellValue = (Number) converter.decodeValue(CellUtil
+ .cloneValue(currentCell));
+ switch (currentAggOp) {
+ case GLOBAL_MIN:
+ if (converter.compare(
+ currentCellValue, previouslyChosenCellValue) < 0) {
+ // new value is minimum, hence return this cell
+ return currentCell;
+ } else {
+ // previously chosen value is miniumum, hence return previous min cell
+ return previouslyChosenCell;
+ }
+ case GLOBAL_MAX:
+ if (converter.compare(
+ currentCellValue, previouslyChosenCellValue) > 0) {
+ // new value is max, hence return this cell
+ return currentCell;
+ } else {
+ // previously chosen value is max, hence return previous max cell
+ return previouslyChosenCell;
+ }
+ default:
+ return currentCell;
+ }
+ } catch (IllegalArgumentException iae) {
+ LOG.error("caught iae during conversion to long ", iae);
+ return currentCell;
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (flowRunScanner != null) {
+ flowRunScanner.close();
+ } else {
+ LOG.warn("scanner close called but scanner is null");
+ }
+ }
+
+ /**
+ * Called to signal the start of the next() call by the scanner.
+ */
+ public void startNext() {
+ currentRow = null;
+ }
+
+ /**
+ * Returns whether or not the underlying scanner has more rows.
+ */
+ public boolean hasMore() {
+ return currentIndex < availableCells.size() ? true : hasMore;
+ }
+
+ /**
+ * Returns the next available cell for the current row and advances the
+ * pointer to the next cell. This method can be called multiple times in a row
+ * to advance through all the available cells.
+ *
+ * @param scannerContext
+ * context information for the batch of cells under consideration
+ * @return the next available cell or null if no more cells are available for
+ * the current row
+ * @throws IOException
+ */
+ public Cell nextCell(ScannerContext scannerContext) throws IOException {
+ Cell cell = peekAtNextCell(scannerContext);
+ if (cell != null) {
+ currentIndex++;
+ }
+ return cell;
+ }
+
+ /**
+ * Returns the next available cell for the current row, without advancing the
+ * pointer. Calling this method multiple times in a row will continue to
+ * return the same cell.
+ *
+ * @param scannerContext
+ * context information for the batch of cells under consideration
+ * @return the next available cell or null if no more cells are available for
+ * the current row
+ * @throws IOException if any problem is encountered while grabbing the next
+ * cell.
+ */
+ public Cell peekAtNextCell(ScannerContext scannerContext) throws IOException {
+ if (currentIndex >= availableCells.size()) {
+ // done with current batch
+ availableCells.clear();
+ currentIndex = 0;
+ hasMore = flowRunScanner.next(availableCells, scannerContext);
+ }
+ Cell cell = null;
+ if (currentIndex < availableCells.size()) {
+ cell = availableCells.get(currentIndex);
+ if (currentRow == null) {
+ currentRow = CellUtil.cloneRow(cell);
+ } else if (!CellUtil.matchingRow(cell, currentRow)) {
+ // moved on to the next row
+ // don't use the current cell
+ // also signal no more cells for this row
+ return null;
+ }
+ }
+ return cell;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.hadoop.hbase.regionserver.RegionScanner#getMaxResultSize()
+ */
+ @Override
+ public long getMaxResultSize() {
+ if (regionScanner == null) {
+ throw new IllegalStateException(
+ "RegionScanner.isFilterDone() called when the flow "
+ + "scanner's scanner is not a RegionScanner");
+ }
+ return regionScanner.getMaxResultSize();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.hadoop.hbase.regionserver.RegionScanner#getMvccReadPoint()
+ */
+ @Override
+ public long getMvccReadPoint() {
+ if (regionScanner == null) {
+ throw new IllegalStateException(
+ "RegionScanner.isFilterDone() called when the flow "
+ + "scanner's internal scanner is not a RegionScanner");
+ }
+ return regionScanner.getMvccReadPoint();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.hadoop.hbase.regionserver.RegionScanner#isFilterDone()
+ */
+ @Override
+ public boolean isFilterDone() throws IOException {
+ if (regionScanner == null) {
+ throw new IllegalStateException(
+ "RegionScanner.isFilterDone() called when the flow "
+ + "scanner's internal scanner is not a RegionScanner");
+ }
+ return regionScanner.isFilterDone();
+
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.hadoop.hbase.regionserver.RegionScanner#reseek(byte[])
+ */
+ @Override
+ public boolean reseek(byte[] bytes) throws IOException {
+ if (regionScanner == null) {
+ throw new IllegalStateException(
+ "RegionScanner.reseek() called when the flow "
+ + "scanner's internal scanner is not a RegionScanner");
+ }
+ return regionScanner.reseek(bytes);
+ }
+
+ @Override
+ public int getBatch() {
+ return batchSize;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScannerOperation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScannerOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScannerOperation.java
new file mode 100644
index 0000000..73c666f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScannerOperation.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+
+/**
+ * Identifies the scanner operation on the {@link FlowRunTable}.
+ */
+public enum FlowScannerOperation {
+
+ /**
+ * If the scanner is opened for reading
+ * during preGet or preScan.
+ */
+ READ,
+
+ /**
+ * If the scanner is opened during preFlush.
+ */
+ FLUSH,
+
+ /**
+ * If the scanner is opened during minor Compaction.
+ */
+ MINOR_COMPACTION,
+
+ /**
+ * If the scanner is opened during major Compaction.
+ */
+ MAJOR_COMPACTION
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/package-info.java
new file mode 100644
index 0000000..04963f3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/package-info.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.storage.flow
+ * contains classes related to implementation for flow related tables, viz. flow
+ * run table and flow activity table.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java
new file mode 100644
index 0000000..e78db2a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.storage contains
+ * classes which define and implement reading and writing to backend storage.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml
index 6369864..7e5a803 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml
@@ -18,197 +18,24 @@
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
- http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
<version>3.2.0-SNAPSHOT</version>
</parent>
+
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-server-timelineservice-hbase</artifactId>
+ <version>3.2.0-SNAPSHOT</version>
<name>Apache Hadoop YARN TimelineService HBase Backend</name>
+ <packaging>pom</packaging>
- <properties>
- <!-- Needed for generating FindBugs warnings using parent pom -->
- <yarn.basedir>${project.parent.parent.basedir}</yarn.basedir>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </dependency>
-
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </dependency>
-
- <dependency>
- <groupId>commons-cli</groupId>
- <artifactId>commons-cli</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-annotations</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-api</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-common</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-applicationhistoryservice</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-timelineservice</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-common</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-server</artifactId>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs-client</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty-sslengine</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
+ <modules>
+ <module>hadoop-yarn-server-timelineservice-hbase-client</module>
+ <module>hadoop-yarn-server-timelineservice-hbase-common</module>
+ <module>hadoop-yarn-server-timelineservice-hbase-server</module>
+ </modules>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- <phase>test-compile</phase>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <configuration>
- <additionnalDependencies>
- <additionnalDependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.11</version>
- </additionnalDependency>
- </additionnalDependencies>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>copy-dependencies</goal>
- </goals>
- <configuration>
- <includeScope>runtime</includeScope>
- <excludeGroupIds>org.slf4j,org.apache.hadoop,com.github.stephenc.findbugs</excludeGroupIds>
- <outputDirectory>${project.build.directory}/lib</outputDirectory>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterUtils.java
deleted file mode 100644
index 8b46d32..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterUtils.java
+++ /dev/null
@@ -1,308 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.reader.filter;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.hadoop.hbase.filter.BinaryComparator;
-import org.apache.hadoop.hbase.filter.BinaryPrefixComparator;
-import org.apache.hadoop.hbase.filter.FamilyFilter;
-import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterList;
-import org.apache.hadoop.hbase.filter.FilterList.Operator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Column;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
-import org.apache.hadoop.hbase.filter.QualifierFilter;
-import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Set of utility methods used by timeline filter classes.
- */
-public final class TimelineFilterUtils {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(TimelineFilterUtils.class);
-
- private TimelineFilterUtils() {
- }
-
- /**
- * Returns the equivalent HBase filter list's {@link Operator}.
- *
- * @param op timeline filter list operator.
- * @return HBase filter list's Operator.
- */
- private static Operator getHBaseOperator(TimelineFilterList.Operator op) {
- switch (op) {
- case AND:
- return Operator.MUST_PASS_ALL;
- case OR:
- return Operator.MUST_PASS_ONE;
- default:
- throw new IllegalArgumentException("Invalid operator");
- }
- }
-
- /**
- * Returns the equivalent HBase compare filter's {@link CompareOp}.
- *
- * @param op timeline compare op.
- * @return HBase compare filter's CompareOp.
- */
- private static CompareOp getHBaseCompareOp(
- TimelineCompareOp op) {
- switch (op) {
- case LESS_THAN:
- return CompareOp.LESS;
- case LESS_OR_EQUAL:
- return CompareOp.LESS_OR_EQUAL;
- case EQUAL:
- return CompareOp.EQUAL;
- case NOT_EQUAL:
- return CompareOp.NOT_EQUAL;
- case GREATER_OR_EQUAL:
- return CompareOp.GREATER_OR_EQUAL;
- case GREATER_THAN:
- return CompareOp.GREATER;
- default:
- throw new IllegalArgumentException("Invalid compare operator");
- }
- }
-
- /**
- * Converts a {@link TimelinePrefixFilter} to an equivalent HBase
- * {@link QualifierFilter}.
- * @param colPrefix
- * @param filter
- * @return a {@link QualifierFilter} object
- */
- private static <T> Filter createHBaseColQualPrefixFilter(
- ColumnPrefix<T> colPrefix, TimelinePrefixFilter filter) {
- return new QualifierFilter(getHBaseCompareOp(filter.getCompareOp()),
- new BinaryPrefixComparator(
- colPrefix.getColumnPrefixBytes(filter.getPrefix())));
- }
-
- /**
- * Create a HBase {@link QualifierFilter} for the passed column prefix and
- * compare op.
- *
- * @param <T> Describes the type of column prefix.
- * @param compareOp compare op.
- * @param columnPrefix column prefix.
- * @return a column qualifier filter.
- */
- public static <T> Filter createHBaseQualifierFilter(CompareOp compareOp,
- ColumnPrefix<T> columnPrefix) {
- return new QualifierFilter(compareOp,
- new BinaryPrefixComparator(
- columnPrefix.getColumnPrefixBytes("")));
- }
-
- /**
- * Create filters for confs or metrics to retrieve. This list includes a
- * configs/metrics family filter and relevant filters for confs/metrics to
- * retrieve, if present.
- *
- * @param <T> Describes the type of column prefix.
- * @param confsOrMetricToRetrieve configs/metrics to retrieve.
- * @param columnFamily config or metric column family.
- * @param columnPrefix config or metric column prefix.
- * @return a filter list.
- * @throws IOException if any problem occurs while creating the filters.
- */
- public static <T> Filter createFilterForConfsOrMetricsToRetrieve(
- TimelineFilterList confsOrMetricToRetrieve, ColumnFamily<T> columnFamily,
- ColumnPrefix<T> columnPrefix) throws IOException {
- Filter familyFilter = new FamilyFilter(CompareOp.EQUAL,
- new BinaryComparator(columnFamily.getBytes()));
- if (confsOrMetricToRetrieve != null &&
- !confsOrMetricToRetrieve.getFilterList().isEmpty()) {
- // If confsOrMetricsToRetrive are specified, create a filter list based
- // on it and family filter.
- FilterList filter = new FilterList(familyFilter);
- filter.addFilter(
- createHBaseFilterList(columnPrefix, confsOrMetricToRetrieve));
- return filter;
- } else {
- // Only the family filter needs to be added.
- return familyFilter;
- }
- }
-
- /**
- * Create 2 HBase {@link SingleColumnValueFilter} filters for the specified
- * value range represented by start and end value and wraps them inside a
- * filter list. Start and end value should not be null.
- *
- * @param <T> Describes the type of column prefix.
- * @param column Column for which single column value filter is to be created.
- * @param startValue Start value.
- * @param endValue End value.
- * @return 2 single column value filters wrapped in a filter list.
- * @throws IOException if any problem is encountered while encoding value.
- */
- public static <T> FilterList createSingleColValueFiltersByRange(
- Column<T> column, Object startValue, Object endValue) throws IOException {
- FilterList list = new FilterList();
- Filter singleColValFilterStart = createHBaseSingleColValueFilter(
- column.getColumnFamilyBytes(), column.getColumnQualifierBytes(),
- column.getValueConverter().encodeValue(startValue),
- CompareOp.GREATER_OR_EQUAL, true);
- list.addFilter(singleColValFilterStart);
-
- Filter singleColValFilterEnd = createHBaseSingleColValueFilter(
- column.getColumnFamilyBytes(), column.getColumnQualifierBytes(),
- column.getValueConverter().encodeValue(endValue),
- CompareOp.LESS_OR_EQUAL, true);
- list.addFilter(singleColValFilterEnd);
- return list;
- }
-
- /**
- * Creates a HBase {@link SingleColumnValueFilter} with specified column.
- * @param <T> Describes the type of column prefix.
- * @param column Column which value to be filtered.
- * @param value Value to be filtered.
- * @param op Compare operator
- * @return a SingleColumnValue Filter
- * @throws IOException if any exception.
- */
- public static <T> Filter createHBaseSingleColValueFilter(Column<T> column,
- Object value, CompareOp op) throws IOException {
- Filter singleColValFilter = createHBaseSingleColValueFilter(
- column.getColumnFamilyBytes(), column.getColumnQualifierBytes(),
- column.getValueConverter().encodeValue(value), op, true);
- return singleColValFilter;
- }
-
- /**
- * Creates a HBase {@link SingleColumnValueFilter}.
- *
- * @param columnFamily Column Family represented as bytes.
- * @param columnQualifier Column Qualifier represented as bytes.
- * @param value Value.
- * @param compareOp Compare operator.
- * @param filterIfMissing This flag decides if we should filter the row if the
- * specified column is missing. This is based on the filter's keyMustExist
- * field.
- * @return a {@link SingleColumnValueFilter} object
- * @throws IOException
- */
- private static SingleColumnValueFilter createHBaseSingleColValueFilter(
- byte[] columnFamily, byte[] columnQualifier, byte[] value,
- CompareOp compareOp, boolean filterIfMissing) throws IOException {
- SingleColumnValueFilter singleColValFilter =
- new SingleColumnValueFilter(columnFamily, columnQualifier, compareOp,
- new BinaryComparator(value));
- singleColValFilter.setLatestVersionOnly(true);
- singleColValFilter.setFilterIfMissing(filterIfMissing);
- return singleColValFilter;
- }
-
- /**
- * Fetch columns from filter list containing exists and multivalue equality
- * filters. This is done to fetch only required columns from back-end and
- * then match event filters or relationships in reader.
- *
- * @param filterList filter list.
- * @return set of columns.
- */
- public static Set<String> fetchColumnsFromFilterList(
- TimelineFilterList filterList) {
- Set<String> strSet = new HashSet<String>();
- for (TimelineFilter filter : filterList.getFilterList()) {
- switch(filter.getFilterType()) {
- case LIST:
- strSet.addAll(fetchColumnsFromFilterList((TimelineFilterList)filter));
- break;
- case KEY_VALUES:
- strSet.add(((TimelineKeyValuesFilter)filter).getKey());
- break;
- case EXISTS:
- strSet.add(((TimelineExistsFilter)filter).getValue());
- break;
- default:
- LOG.info("Unexpected filter type " + filter.getFilterType());
- break;
- }
- }
- return strSet;
- }
-
- /**
- * Creates equivalent HBase {@link FilterList} from {@link TimelineFilterList}
- * while converting different timeline filters(of type {@link TimelineFilter})
- * into their equivalent HBase filters.
- *
- * @param <T> Describes the type of column prefix.
- * @param colPrefix column prefix which will be used for conversion.
- * @param filterList timeline filter list which has to be converted.
- * @return A {@link FilterList} object.
- * @throws IOException if any problem occurs while creating the filter list.
- */
- public static <T> FilterList createHBaseFilterList(ColumnPrefix<T> colPrefix,
- TimelineFilterList filterList) throws IOException {
- FilterList list =
- new FilterList(getHBaseOperator(filterList.getOperator()));
- for (TimelineFilter filter : filterList.getFilterList()) {
- switch(filter.getFilterType()) {
- case LIST:
- list.addFilter(createHBaseFilterList(colPrefix,
- (TimelineFilterList)filter));
- break;
- case PREFIX:
- list.addFilter(createHBaseColQualPrefixFilter(colPrefix,
- (TimelinePrefixFilter)filter));
- break;
- case COMPARE:
- TimelineCompareFilter compareFilter = (TimelineCompareFilter)filter;
- list.addFilter(
- createHBaseSingleColValueFilter(
- colPrefix.getColumnFamilyBytes(),
- colPrefix.getColumnPrefixBytes(compareFilter.getKey()),
- colPrefix.getValueConverter().
- encodeValue(compareFilter.getValue()),
- getHBaseCompareOp(compareFilter.getCompareOp()),
- compareFilter.getKeyMustExist()));
- break;
- case KEY_VALUE:
- TimelineKeyValueFilter kvFilter = (TimelineKeyValueFilter)filter;
- list.addFilter(
- createHBaseSingleColValueFilter(
- colPrefix.getColumnFamilyBytes(),
- colPrefix.getColumnPrefixBytes(kvFilter.getKey()),
- colPrefix.getValueConverter().encodeValue(kvFilter.getValue()),
- getHBaseCompareOp(kvFilter.getCompareOp()),
- kvFilter.getKeyMustExist()));
- break;
- default:
- LOG.info("Unexpected filter type " + filter.getFilterType());
- break;
- }
- }
- return list;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/package-info.java
deleted file mode 100644
index f7c0705..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/package-info.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Package org.apache.hadoop.server.timelineservice.reader.filter stores
- * timeline filter implementations.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-package org.apache.hadoop.yarn.server.timelineservice.reader.filter;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
deleted file mode 100644
index 1ebfab2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-
-import java.io.IOException;
-import java.util.Set;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.reader.EntityTypeReader;
-import org.apache.hadoop.yarn.server.timelineservice.storage.reader.TimelineEntityReader;
-import org.apache.hadoop.yarn.server.timelineservice.storage.reader.TimelineEntityReaderFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * HBase based implementation for {@link TimelineReader}.
- */
-public class HBaseTimelineReaderImpl
- extends AbstractService implements TimelineReader {
-
- private static final Logger LOG = LoggerFactory
- .getLogger(HBaseTimelineReaderImpl.class);
-
- private Configuration hbaseConf = null;
- private Connection conn;
-
- public HBaseTimelineReaderImpl() {
- super(HBaseTimelineReaderImpl.class.getName());
- }
-
- @Override
- public void serviceInit(Configuration conf) throws Exception {
- super.serviceInit(conf);
- hbaseConf = HBaseTimelineStorageUtils.getTimelineServiceHBaseConf(conf);
- conn = ConnectionFactory.createConnection(hbaseConf);
- }
-
- @Override
- protected void serviceStop() throws Exception {
- if (conn != null) {
- LOG.info("closing the hbase Connection");
- conn.close();
- }
- super.serviceStop();
- }
-
- @Override
- public TimelineEntity getEntity(TimelineReaderContext context,
- TimelineDataToRetrieve dataToRetrieve) throws IOException {
- TimelineEntityReader reader =
- TimelineEntityReaderFactory.createSingleEntityReader(context,
- dataToRetrieve);
- return reader.readEntity(hbaseConf, conn);
- }
-
- @Override
- public Set<TimelineEntity> getEntities(TimelineReaderContext context,
- TimelineEntityFilters filters, TimelineDataToRetrieve dataToRetrieve)
- throws IOException {
- TimelineEntityReader reader =
- TimelineEntityReaderFactory.createMultipleEntitiesReader(context,
- filters, dataToRetrieve);
- return reader.readEntities(hbaseConf, conn);
- }
-
- @Override
- public Set<String> getEntityTypes(TimelineReaderContext context)
- throws IOException {
- EntityTypeReader reader = new EntityTypeReader(context);
- return reader.readEntityTypes(hbaseConf, conn);
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/18] hadoop git commit: YARN-7919. Refactor timelineservice-hbase
module into submodules. Contributed by Haibo Chen.
Posted by ro...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
deleted file mode 100644
index f938185..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ /dev/null
@@ -1,593 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
-import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
-import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.EventColumnName;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongKeyConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.StringKeyConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationCompactionDimension;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationOperation;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumn;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationColumn;
-import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationTable;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This implements a hbase based backend for storing the timeline entity
- * information.
- * It writes to multiple tables at the backend
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class HBaseTimelineWriterImpl extends AbstractService implements
- TimelineWriter {
-
- private static final Logger LOG = LoggerFactory
- .getLogger(HBaseTimelineWriterImpl.class);
-
- private Connection conn;
- private TypedBufferedMutator<EntityTable> entityTable;
- private TypedBufferedMutator<AppToFlowTable> appToFlowTable;
- private TypedBufferedMutator<ApplicationTable> applicationTable;
- private TypedBufferedMutator<FlowActivityTable> flowActivityTable;
- private TypedBufferedMutator<FlowRunTable> flowRunTable;
- private TypedBufferedMutator<SubApplicationTable> subApplicationTable;
-
- /**
- * Used to convert strings key components to and from storage format.
- */
- private final KeyConverter<String> stringKeyConverter =
- new StringKeyConverter();
-
- /**
- * Used to convert Long key components to and from storage format.
- */
- private final KeyConverter<Long> longKeyConverter = new LongKeyConverter();
-
- private enum Tables {
- APPLICATION_TABLE, ENTITY_TABLE, SUBAPPLICATION_TABLE
- };
-
- public HBaseTimelineWriterImpl() {
- super(HBaseTimelineWriterImpl.class.getName());
- }
-
- /**
- * initializes the hbase connection to write to the entity table.
- */
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
- super.serviceInit(conf);
- Configuration hbaseConf =
- HBaseTimelineStorageUtils.getTimelineServiceHBaseConf(conf);
- conn = ConnectionFactory.createConnection(hbaseConf);
- entityTable = new EntityTable().getTableMutator(hbaseConf, conn);
- appToFlowTable = new AppToFlowTable().getTableMutator(hbaseConf, conn);
- applicationTable = new ApplicationTable().getTableMutator(hbaseConf, conn);
- flowRunTable = new FlowRunTable().getTableMutator(hbaseConf, conn);
- flowActivityTable =
- new FlowActivityTable().getTableMutator(hbaseConf, conn);
- subApplicationTable =
- new SubApplicationTable().getTableMutator(hbaseConf, conn);
-
- UserGroupInformation ugi = UserGroupInformation.isSecurityEnabled() ?
- UserGroupInformation.getLoginUser() :
- UserGroupInformation.getCurrentUser();
- LOG.info("Initialized HBaseTimelineWriterImpl UGI to " + ugi);
- }
-
- /**
- * Stores the entire information in TimelineEntities to the timeline store.
- */
- @Override
- public TimelineWriteResponse write(TimelineCollectorContext context,
- TimelineEntities data, UserGroupInformation callerUgi)
- throws IOException {
-
- TimelineWriteResponse putStatus = new TimelineWriteResponse();
-
- String clusterId = context.getClusterId();
- String userId = context.getUserId();
- String flowName = context.getFlowName();
- String flowVersion = context.getFlowVersion();
- long flowRunId = context.getFlowRunId();
- String appId = context.getAppId();
- String subApplicationUser = callerUgi.getShortUserName();
-
- // defensive coding to avoid NPE during row key construction
- if ((flowName == null) || (appId == null) || (clusterId == null)
- || (userId == null)) {
- LOG.warn("Found null for one of: flowName=" + flowName + " appId=" + appId
- + " userId=" + userId + " clusterId=" + clusterId
- + " . Not proceeding with writing to hbase");
- return putStatus;
- }
-
- for (TimelineEntity te : data.getEntities()) {
-
- // a set can have at most 1 null
- if (te == null) {
- continue;
- }
-
- // if the entity is the application, the destination is the application
- // table
- boolean isApplication = ApplicationEntity.isApplicationEntity(te);
- byte[] rowKey;
- if (isApplication) {
- ApplicationRowKey applicationRowKey =
- new ApplicationRowKey(clusterId, userId, flowName, flowRunId,
- appId);
- rowKey = applicationRowKey.getRowKey();
- store(rowKey, te, flowVersion, Tables.APPLICATION_TABLE);
- } else {
- EntityRowKey entityRowKey =
- new EntityRowKey(clusterId, userId, flowName, flowRunId, appId,
- te.getType(), te.getIdPrefix(), te.getId());
- rowKey = entityRowKey.getRowKey();
- store(rowKey, te, flowVersion, Tables.ENTITY_TABLE);
- }
-
- if (!isApplication && !userId.equals(subApplicationUser)) {
- SubApplicationRowKey subApplicationRowKey =
- new SubApplicationRowKey(subApplicationUser, clusterId,
- te.getType(), te.getIdPrefix(), te.getId(), userId);
- rowKey = subApplicationRowKey.getRowKey();
- store(rowKey, te, flowVersion, Tables.SUBAPPLICATION_TABLE);
- }
-
- if (isApplication) {
- TimelineEvent event =
- ApplicationEntity.getApplicationEvent(te,
- ApplicationMetricsConstants.CREATED_EVENT_TYPE);
- FlowRunRowKey flowRunRowKey =
- new FlowRunRowKey(clusterId, userId, flowName, flowRunId);
- if (event != null) {
- onApplicationCreated(flowRunRowKey, clusterId, appId, userId,
- flowVersion, te, event.getTimestamp());
- }
- // if it's an application entity, store metrics
- storeFlowMetricsAppRunning(flowRunRowKey, appId, te);
- // if application has finished, store it's finish time and write final
- // values of all metrics
- event = ApplicationEntity.getApplicationEvent(te,
- ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
- if (event != null) {
- onApplicationFinished(flowRunRowKey, flowVersion, appId, te,
- event.getTimestamp());
- }
- }
- }
- return putStatus;
- }
-
- private void onApplicationCreated(FlowRunRowKey flowRunRowKey,
- String clusterId, String appId, String userId, String flowVersion,
- TimelineEntity te, long appCreatedTimeStamp)
- throws IOException {
-
- String flowName = flowRunRowKey.getFlowName();
- Long flowRunId = flowRunRowKey.getFlowRunId();
-
- // store in App to flow table
- AppToFlowRowKey appToFlowRowKey = new AppToFlowRowKey(appId);
- byte[] rowKey = appToFlowRowKey.getRowKey();
- AppToFlowColumnPrefix.FLOW_NAME.store(rowKey, appToFlowTable, clusterId,
- null, flowName);
- AppToFlowColumnPrefix.FLOW_RUN_ID.store(rowKey, appToFlowTable, clusterId,
- null, flowRunId);
- AppToFlowColumnPrefix.USER_ID.store(rowKey, appToFlowTable, clusterId, null,
- userId);
-
- // store in flow run table
- storeAppCreatedInFlowRunTable(flowRunRowKey, appId, te);
-
- // store in flow activity table
- byte[] flowActivityRowKeyBytes =
- new FlowActivityRowKey(flowRunRowKey.getClusterId(),
- appCreatedTimeStamp, flowRunRowKey.getUserId(), flowName)
- .getRowKey();
- byte[] qualifier = longKeyConverter.encode(flowRunRowKey.getFlowRunId());
- FlowActivityColumnPrefix.RUN_ID.store(flowActivityRowKeyBytes,
- flowActivityTable, qualifier, null, flowVersion,
- AggregationCompactionDimension.APPLICATION_ID.getAttribute(appId));
- }
-
- /*
- * updates the {@link FlowRunTable} with Application Created information
- */
- private void storeAppCreatedInFlowRunTable(FlowRunRowKey flowRunRowKey,
- String appId, TimelineEntity te) throws IOException {
- byte[] rowKey = flowRunRowKey.getRowKey();
- FlowRunColumn.MIN_START_TIME.store(rowKey, flowRunTable, null,
- te.getCreatedTime(),
- AggregationCompactionDimension.APPLICATION_ID.getAttribute(appId));
- }
-
-
- /*
- * updates the {@link FlowRunTable} and {@link FlowActivityTable} when an
- * application has finished
- */
- private void onApplicationFinished(FlowRunRowKey flowRunRowKey,
- String flowVersion, String appId, TimelineEntity te,
- long appFinishedTimeStamp) throws IOException {
- // store in flow run table
- storeAppFinishedInFlowRunTable(flowRunRowKey, appId, te,
- appFinishedTimeStamp);
-
- // indicate in the flow activity table that the app has finished
- byte[] rowKey =
- new FlowActivityRowKey(flowRunRowKey.getClusterId(),
- appFinishedTimeStamp, flowRunRowKey.getUserId(),
- flowRunRowKey.getFlowName()).getRowKey();
- byte[] qualifier = longKeyConverter.encode(flowRunRowKey.getFlowRunId());
- FlowActivityColumnPrefix.RUN_ID.store(rowKey, flowActivityTable, qualifier,
- null, flowVersion,
- AggregationCompactionDimension.APPLICATION_ID.getAttribute(appId));
- }
-
- /*
- * Update the {@link FlowRunTable} with Application Finished information
- */
- private void storeAppFinishedInFlowRunTable(FlowRunRowKey flowRunRowKey,
- String appId, TimelineEntity te, long appFinishedTimeStamp)
- throws IOException {
- byte[] rowKey = flowRunRowKey.getRowKey();
- Attribute attributeAppId =
- AggregationCompactionDimension.APPLICATION_ID.getAttribute(appId);
- FlowRunColumn.MAX_END_TIME.store(rowKey, flowRunTable, null,
- appFinishedTimeStamp, attributeAppId);
-
- // store the final value of metrics since application has finished
- Set<TimelineMetric> metrics = te.getMetrics();
- if (metrics != null) {
- storeFlowMetrics(rowKey, metrics, attributeAppId,
- AggregationOperation.SUM_FINAL.getAttribute());
- }
- }
-
- /*
- * Updates the {@link FlowRunTable} with Application Metrics
- */
- private void storeFlowMetricsAppRunning(FlowRunRowKey flowRunRowKey,
- String appId, TimelineEntity te) throws IOException {
- Set<TimelineMetric> metrics = te.getMetrics();
- if (metrics != null) {
- byte[] rowKey = flowRunRowKey.getRowKey();
- storeFlowMetrics(rowKey, metrics,
- AggregationCompactionDimension.APPLICATION_ID.getAttribute(appId),
- AggregationOperation.SUM.getAttribute());
- }
- }
-
- private void storeFlowMetrics(byte[] rowKey, Set<TimelineMetric> metrics,
- Attribute... attributes) throws IOException {
- for (TimelineMetric metric : metrics) {
- byte[] metricColumnQualifier = stringKeyConverter.encode(metric.getId());
- Map<Long, Number> timeseries = metric.getValues();
- for (Map.Entry<Long, Number> timeseriesEntry : timeseries.entrySet()) {
- Long timestamp = timeseriesEntry.getKey();
- FlowRunColumnPrefix.METRIC.store(rowKey, flowRunTable,
- metricColumnQualifier, timestamp, timeseriesEntry.getValue(),
- attributes);
- }
- }
- }
-
- /**
- * Stores the Relations from the {@linkplain TimelineEntity} object.
- */
- private <T> void storeRelations(byte[] rowKey,
- Map<String, Set<String>> connectedEntities, ColumnPrefix<T> columnPrefix,
- TypedBufferedMutator<T> table) throws IOException {
- if (connectedEntities != null) {
- for (Map.Entry<String, Set<String>> connectedEntity : connectedEntities
- .entrySet()) {
- // id3?id4?id5
- String compoundValue =
- Separator.VALUES.joinEncoded(connectedEntity.getValue());
- columnPrefix.store(rowKey, table,
- stringKeyConverter.encode(connectedEntity.getKey()), null,
- compoundValue);
- }
- }
- }
-
- /**
- * Stores information from the {@linkplain TimelineEntity} object.
- */
- private void store(byte[] rowKey, TimelineEntity te,
- String flowVersion,
- Tables table) throws IOException {
- switch (table) {
- case APPLICATION_TABLE:
- ApplicationColumn.ID.store(rowKey, applicationTable, null, te.getId());
- ApplicationColumn.CREATED_TIME.store(rowKey, applicationTable, null,
- te.getCreatedTime());
- ApplicationColumn.FLOW_VERSION.store(rowKey, applicationTable, null,
- flowVersion);
- storeInfo(rowKey, te.getInfo(), flowVersion, ApplicationColumnPrefix.INFO,
- applicationTable);
- storeMetrics(rowKey, te.getMetrics(), ApplicationColumnPrefix.METRIC,
- applicationTable);
- storeEvents(rowKey, te.getEvents(), ApplicationColumnPrefix.EVENT,
- applicationTable);
- storeConfig(rowKey, te.getConfigs(), ApplicationColumnPrefix.CONFIG,
- applicationTable);
- storeRelations(rowKey, te.getIsRelatedToEntities(),
- ApplicationColumnPrefix.IS_RELATED_TO, applicationTable);
- storeRelations(rowKey, te.getRelatesToEntities(),
- ApplicationColumnPrefix.RELATES_TO, applicationTable);
- break;
- case ENTITY_TABLE:
- EntityColumn.ID.store(rowKey, entityTable, null, te.getId());
- EntityColumn.TYPE.store(rowKey, entityTable, null, te.getType());
- EntityColumn.CREATED_TIME.store(rowKey, entityTable, null,
- te.getCreatedTime());
- EntityColumn.FLOW_VERSION.store(rowKey, entityTable, null, flowVersion);
- storeInfo(rowKey, te.getInfo(), flowVersion, EntityColumnPrefix.INFO,
- entityTable);
- storeMetrics(rowKey, te.getMetrics(), EntityColumnPrefix.METRIC,
- entityTable);
- storeEvents(rowKey, te.getEvents(), EntityColumnPrefix.EVENT,
- entityTable);
- storeConfig(rowKey, te.getConfigs(), EntityColumnPrefix.CONFIG,
- entityTable);
- storeRelations(rowKey, te.getIsRelatedToEntities(),
- EntityColumnPrefix.IS_RELATED_TO, entityTable);
- storeRelations(rowKey, te.getRelatesToEntities(),
- EntityColumnPrefix.RELATES_TO, entityTable);
- break;
- case SUBAPPLICATION_TABLE:
- SubApplicationColumn.ID.store(rowKey, subApplicationTable, null,
- te.getId());
- SubApplicationColumn.TYPE.store(rowKey, subApplicationTable, null,
- te.getType());
- SubApplicationColumn.CREATED_TIME.store(rowKey, subApplicationTable, null,
- te.getCreatedTime());
- SubApplicationColumn.FLOW_VERSION.store(rowKey, subApplicationTable, null,
- flowVersion);
- storeInfo(rowKey, te.getInfo(), flowVersion,
- SubApplicationColumnPrefix.INFO, subApplicationTable);
- storeMetrics(rowKey, te.getMetrics(), SubApplicationColumnPrefix.METRIC,
- subApplicationTable);
- storeEvents(rowKey, te.getEvents(), SubApplicationColumnPrefix.EVENT,
- subApplicationTable);
- storeConfig(rowKey, te.getConfigs(), SubApplicationColumnPrefix.CONFIG,
- subApplicationTable);
- storeRelations(rowKey, te.getIsRelatedToEntities(),
- SubApplicationColumnPrefix.IS_RELATED_TO, subApplicationTable);
- storeRelations(rowKey, te.getRelatesToEntities(),
- SubApplicationColumnPrefix.RELATES_TO, subApplicationTable);
- break;
- default:
- LOG.info("Invalid table name provided.");
- break;
- }
- }
-
- /**
- * stores the info information from {@linkplain TimelineEntity}.
- */
- private <T> void storeInfo(byte[] rowKey, Map<String, Object> info,
- String flowVersion, ColumnPrefix<T> columnPrefix,
- TypedBufferedMutator<T> table) throws IOException {
- if (info != null) {
- for (Map.Entry<String, Object> entry : info.entrySet()) {
- columnPrefix.store(rowKey, table,
- stringKeyConverter.encode(entry.getKey()), null, entry.getValue());
- }
- }
- }
-
- /**
- * stores the config information from {@linkplain TimelineEntity}.
- */
- private <T> void storeConfig(byte[] rowKey, Map<String, String> config,
- ColumnPrefix<T> columnPrefix, TypedBufferedMutator<T> table)
- throws IOException {
- if (config != null) {
- for (Map.Entry<String, String> entry : config.entrySet()) {
- byte[] configKey = stringKeyConverter.encode(entry.getKey());
- columnPrefix.store(rowKey, table, configKey, null, entry.getValue());
- }
- }
- }
-
- /**
- * stores the {@linkplain TimelineMetric} information from the
- * {@linkplain TimelineEvent} object.
- */
- private <T> void storeMetrics(byte[] rowKey, Set<TimelineMetric> metrics,
- ColumnPrefix<T> columnPrefix, TypedBufferedMutator<T> table)
- throws IOException {
- if (metrics != null) {
- for (TimelineMetric metric : metrics) {
- byte[] metricColumnQualifier =
- stringKeyConverter.encode(metric.getId());
- Map<Long, Number> timeseries = metric.getValues();
- for (Map.Entry<Long, Number> timeseriesEntry : timeseries.entrySet()) {
- Long timestamp = timeseriesEntry.getKey();
- columnPrefix.store(rowKey, table, metricColumnQualifier, timestamp,
- timeseriesEntry.getValue());
- }
- }
- }
- }
-
- /**
- * Stores the events from the {@linkplain TimelineEvent} object.
- */
- private <T> void storeEvents(byte[] rowKey, Set<TimelineEvent> events,
- ColumnPrefix<T> columnPrefix, TypedBufferedMutator<T> table)
- throws IOException {
- if (events != null) {
- for (TimelineEvent event : events) {
- if (event != null) {
- String eventId = event.getId();
- if (eventId != null) {
- long eventTimestamp = event.getTimestamp();
- // if the timestamp is not set, use the current timestamp
- if (eventTimestamp == TimelineEvent.INVALID_TIMESTAMP) {
- LOG.warn("timestamp is not set for event " + eventId +
- "! Using the current timestamp");
- eventTimestamp = System.currentTimeMillis();
- }
- Map<String, Object> eventInfo = event.getInfo();
- if ((eventInfo == null) || (eventInfo.size() == 0)) {
- byte[] columnQualifierBytes =
- new EventColumnName(eventId, eventTimestamp, null)
- .getColumnQualifier();
- columnPrefix.store(rowKey, table, columnQualifierBytes, null,
- Separator.EMPTY_BYTES);
- } else {
- for (Map.Entry<String, Object> info : eventInfo.entrySet()) {
- // eventId=infoKey
- byte[] columnQualifierBytes =
- new EventColumnName(eventId, eventTimestamp, info.getKey())
- .getColumnQualifier();
- columnPrefix.store(rowKey, table, columnQualifierBytes, null,
- info.getValue());
- } // for info: eventInfo
- }
- }
- }
- } // event : events
- }
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage
- * .TimelineWriter#aggregate
- * (org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity,
- * org.apache
- * .hadoop.yarn.server.timelineservice.storage.TimelineAggregationTrack)
- */
- @Override
- public TimelineWriteResponse aggregate(TimelineEntity data,
- TimelineAggregationTrack track) throws IOException {
- return null;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter#flush
- * ()
- */
- @Override
- public void flush() throws IOException {
- // flush all buffered mutators
- entityTable.flush();
- appToFlowTable.flush();
- applicationTable.flush();
- flowRunTable.flush();
- flowActivityTable.flush();
- subApplicationTable.flush();
- }
-
- /**
- * close the hbase connections The close APIs perform flushing and release any
- * resources held.
- */
- @Override
- protected void serviceStop() throws Exception {
- if (entityTable != null) {
- LOG.info("closing the entity table");
- // The close API performs flushing and releases any resources held
- entityTable.close();
- }
- if (appToFlowTable != null) {
- LOG.info("closing the app_flow table");
- // The close API performs flushing and releases any resources held
- appToFlowTable.close();
- }
- if (applicationTable != null) {
- LOG.info("closing the application table");
- applicationTable.close();
- }
- if (flowRunTable != null) {
- LOG.info("closing the flow run table");
- // The close API performs flushing and releases any resources held
- flowRunTable.close();
- }
- if (flowActivityTable != null) {
- LOG.info("closing the flowActivityTable table");
- // The close API performs flushing and releases any resources held
- flowActivityTable.close();
- }
- if (subApplicationTable != null) {
- subApplicationTable.close();
- }
- if (conn != null) {
- LOG.info("closing the hbase Connection");
- conn.close();
- }
- super.serviceStop();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
deleted file mode 100644
index c9f7cec..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
+++ /dev/null
@@ -1,367 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.cli.PosixParser;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationTable;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This creates the schema for a hbase based backend for storing application
- * timeline information.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public final class TimelineSchemaCreator {
- private TimelineSchemaCreator() {
- }
-
- final static String NAME = TimelineSchemaCreator.class.getSimpleName();
- private static final Logger LOG =
- LoggerFactory.getLogger(TimelineSchemaCreator.class);
- private static final String SKIP_EXISTING_TABLE_OPTION_SHORT = "s";
- private static final String APP_METRICS_TTL_OPTION_SHORT = "ma";
- private static final String SUB_APP_METRICS_TTL_OPTION_SHORT = "msa";
- private static final String APP_TABLE_NAME_SHORT = "a";
- private static final String SUB_APP_TABLE_NAME_SHORT = "sa";
- private static final String APP_TO_FLOW_TABLE_NAME_SHORT = "a2f";
- private static final String ENTITY_METRICS_TTL_OPTION_SHORT = "me";
- private static final String ENTITY_TABLE_NAME_SHORT = "e";
- private static final String HELP_SHORT = "h";
- private static final String CREATE_TABLES_SHORT = "c";
-
- public static void main(String[] args) throws Exception {
-
- LOG.info("Starting the schema creation");
- Configuration hbaseConf =
- HBaseTimelineStorageUtils.getTimelineServiceHBaseConf(
- new YarnConfiguration());
- // Grab input args and allow for -Dxyz style arguments
- String[] otherArgs = new GenericOptionsParser(hbaseConf, args)
- .getRemainingArgs();
-
- // Grab the arguments we're looking for.
- CommandLine commandLine = parseArgs(otherArgs);
-
- if (commandLine.hasOption(HELP_SHORT)) {
- // -help option has the highest precedence
- printUsage();
- } else if (commandLine.hasOption(CREATE_TABLES_SHORT)) {
- // Grab the entityTableName argument
- String entityTableName = commandLine.getOptionValue(
- ENTITY_TABLE_NAME_SHORT);
- if (StringUtils.isNotBlank(entityTableName)) {
- hbaseConf.set(EntityTable.TABLE_NAME_CONF_NAME, entityTableName);
- }
- // Grab the entity metrics TTL
- String entityTableMetricsTTL = commandLine.getOptionValue(
- ENTITY_METRICS_TTL_OPTION_SHORT);
- if (StringUtils.isNotBlank(entityTableMetricsTTL)) {
- int entityMetricsTTL = Integer.parseInt(entityTableMetricsTTL);
- new EntityTable().setMetricsTTL(entityMetricsTTL, hbaseConf);
- }
- // Grab the appToflowTableName argument
- String appToflowTableName = commandLine.getOptionValue(
- APP_TO_FLOW_TABLE_NAME_SHORT);
- if (StringUtils.isNotBlank(appToflowTableName)) {
- hbaseConf.set(AppToFlowTable.TABLE_NAME_CONF_NAME, appToflowTableName);
- }
- // Grab the applicationTableName argument
- String applicationTableName = commandLine.getOptionValue(
- APP_TABLE_NAME_SHORT);
- if (StringUtils.isNotBlank(applicationTableName)) {
- hbaseConf.set(ApplicationTable.TABLE_NAME_CONF_NAME,
- applicationTableName);
- }
- // Grab the application metrics TTL
- String applicationTableMetricsTTL = commandLine.getOptionValue(
- APP_METRICS_TTL_OPTION_SHORT);
- if (StringUtils.isNotBlank(applicationTableMetricsTTL)) {
- int appMetricsTTL = Integer.parseInt(applicationTableMetricsTTL);
- new ApplicationTable().setMetricsTTL(appMetricsTTL, hbaseConf);
- }
-
- // Grab the subApplicationTableName argument
- String subApplicationTableName = commandLine.getOptionValue(
- SUB_APP_TABLE_NAME_SHORT);
- if (StringUtils.isNotBlank(subApplicationTableName)) {
- hbaseConf.set(SubApplicationTable.TABLE_NAME_CONF_NAME,
- subApplicationTableName);
- }
- // Grab the subApplication metrics TTL
- String subApplicationTableMetricsTTL = commandLine
- .getOptionValue(SUB_APP_METRICS_TTL_OPTION_SHORT);
- if (StringUtils.isNotBlank(subApplicationTableMetricsTTL)) {
- int subAppMetricsTTL = Integer.parseInt(subApplicationTableMetricsTTL);
- new SubApplicationTable().setMetricsTTL(subAppMetricsTTL, hbaseConf);
- }
-
- // create all table schemas in hbase
- final boolean skipExisting = commandLine.hasOption(
- SKIP_EXISTING_TABLE_OPTION_SHORT);
- createAllSchemas(hbaseConf, skipExisting);
- } else {
- // print usage information if -create is not specified
- printUsage();
- }
- }
-
- /**
- * Parse command-line arguments.
- *
- * @param args
- * command line arguments passed to program.
- * @return parsed command line.
- * @throws ParseException
- */
- private static CommandLine parseArgs(String[] args) throws ParseException {
- Options options = new Options();
-
- // Input
- Option o = new Option(HELP_SHORT, "help", false, "print help information");
- o.setRequired(false);
- options.addOption(o);
-
- o = new Option(CREATE_TABLES_SHORT, "create", false,
- "a mandatory option to create hbase tables");
- o.setRequired(false);
- options.addOption(o);
-
- o = new Option(ENTITY_TABLE_NAME_SHORT, "entityTableName", true,
- "entity table name");
- o.setArgName("entityTableName");
- o.setRequired(false);
- options.addOption(o);
-
- o = new Option(ENTITY_METRICS_TTL_OPTION_SHORT, "entityMetricsTTL", true,
- "TTL for metrics column family");
- o.setArgName("entityMetricsTTL");
- o.setRequired(false);
- options.addOption(o);
-
- o = new Option(APP_TO_FLOW_TABLE_NAME_SHORT, "appToflowTableName", true,
- "app to flow table name");
- o.setArgName("appToflowTableName");
- o.setRequired(false);
- options.addOption(o);
-
- o = new Option(APP_TABLE_NAME_SHORT, "applicationTableName", true,
- "application table name");
- o.setArgName("applicationTableName");
- o.setRequired(false);
- options.addOption(o);
-
- o = new Option(APP_METRICS_TTL_OPTION_SHORT, "applicationMetricsTTL", true,
- "TTL for metrics column family");
- o.setArgName("applicationMetricsTTL");
- o.setRequired(false);
- options.addOption(o);
-
- o = new Option(SUB_APP_TABLE_NAME_SHORT, "subApplicationTableName", true,
- "subApplication table name");
- o.setArgName("subApplicationTableName");
- o.setRequired(false);
- options.addOption(o);
-
- o = new Option(SUB_APP_METRICS_TTL_OPTION_SHORT, "subApplicationMetricsTTL",
- true, "TTL for metrics column family");
- o.setArgName("subApplicationMetricsTTL");
- o.setRequired(false);
- options.addOption(o);
-
- // Options without an argument
- // No need to set arg name since we do not need an argument here
- o = new Option(SKIP_EXISTING_TABLE_OPTION_SHORT, "skipExistingTable",
- false, "skip existing Hbase tables and continue to create new tables");
- o.setRequired(false);
- options.addOption(o);
-
- CommandLineParser parser = new PosixParser();
- CommandLine commandLine = null;
- try {
- commandLine = parser.parse(options, args);
- } catch (Exception e) {
- LOG.error("ERROR: " + e.getMessage() + "\n");
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp(NAME + " ", options, true);
- System.exit(-1);
- }
-
- return commandLine;
- }
-
- private static void printUsage() {
- StringBuilder usage = new StringBuilder("Command Usage: \n");
- usage.append("TimelineSchemaCreator [-help] Display help info" +
- " for all commands. Or\n");
- usage.append("TimelineSchemaCreator -create [OPTIONAL_OPTIONS]" +
- " Create hbase tables.\n\n");
- usage.append("The Optional options for creating tables include: \n");
- usage.append("[-entityTableName <Entity Table Name>] " +
- "The name of the Entity table\n");
- usage.append("[-entityMetricsTTL <Entity Table Metrics TTL>]" +
- " TTL for metrics in the Entity table\n");
- usage.append("[-appToflowTableName <AppToflow Table Name>]" +
- " The name of the AppToFlow table\n");
- usage.append("[-applicationTableName <Application Table Name>]" +
- " The name of the Application table\n");
- usage.append("[-applicationMetricsTTL <Application Table Metrics TTL>]" +
- " TTL for metrics in the Application table\n");
- usage.append("[-subApplicationTableName <SubApplication Table Name>]" +
- " The name of the SubApplication table\n");
- usage.append("[-subApplicationMetricsTTL " +
- " <SubApplication Table Metrics TTL>]" +
- " TTL for metrics in the SubApplication table\n");
- usage.append("[-skipExistingTable] Whether to skip existing" +
- " hbase tables\n");
- System.out.println(usage.toString());
- }
-
- /**
- * Create all table schemas and log success or exception if failed.
- * @param hbaseConf the hbase configuration to create tables with
- * @param skipExisting whether to skip existing hbase tables
- */
- private static void createAllSchemas(Configuration hbaseConf,
- boolean skipExisting) {
- List<Exception> exceptions = new ArrayList<>();
- try {
- if (skipExisting) {
- LOG.info("Will skip existing tables and continue on htable creation "
- + "exceptions!");
- }
- createAllTables(hbaseConf, skipExisting);
- LOG.info("Successfully created HBase schema. ");
- } catch (IOException e) {
- LOG.error("Error in creating hbase tables: ", e);
- exceptions.add(e);
- }
-
- if (exceptions.size() > 0) {
- LOG.warn("Schema creation finished with the following exceptions");
- for (Exception e : exceptions) {
- LOG.warn(e.getMessage());
- }
- System.exit(-1);
- } else {
- LOG.info("Schema creation finished successfully");
- }
- }
-
- @VisibleForTesting
- public static void createAllTables(Configuration hbaseConf,
- boolean skipExisting) throws IOException {
-
- Connection conn = null;
- try {
- conn = ConnectionFactory.createConnection(hbaseConf);
- Admin admin = conn.getAdmin();
- if (admin == null) {
- throw new IOException("Cannot create table since admin is null");
- }
- try {
- new EntityTable().createTable(admin, hbaseConf);
- } catch (IOException e) {
- if (skipExisting) {
- LOG.warn("Skip and continue on: " + e.getMessage());
- } else {
- throw e;
- }
- }
- try {
- new AppToFlowTable().createTable(admin, hbaseConf);
- } catch (IOException e) {
- if (skipExisting) {
- LOG.warn("Skip and continue on: " + e.getMessage());
- } else {
- throw e;
- }
- }
- try {
- new ApplicationTable().createTable(admin, hbaseConf);
- } catch (IOException e) {
- if (skipExisting) {
- LOG.warn("Skip and continue on: " + e.getMessage());
- } else {
- throw e;
- }
- }
- try {
- new FlowRunTable().createTable(admin, hbaseConf);
- } catch (IOException e) {
- if (skipExisting) {
- LOG.warn("Skip and continue on: " + e.getMessage());
- } else {
- throw e;
- }
- }
- try {
- new FlowActivityTable().createTable(admin, hbaseConf);
- } catch (IOException e) {
- if (skipExisting) {
- LOG.warn("Skip and continue on: " + e.getMessage());
- } else {
- throw e;
- }
- }
- try {
- new SubApplicationTable().createTable(admin, hbaseConf);
- } catch (IOException e) {
- if (skipExisting) {
- LOG.warn("Skip and continue on: " + e.getMessage());
- } else {
- throw e;
- }
- }
- } finally {
- if (conn != null) {
- conn.close();
- }
- }
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
deleted file mode 100644
index 00eaa7e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.application;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Column;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.GenericConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
-
-/**
- * Identifies fully qualified columns for the {@link ApplicationTable}.
- */
-public enum ApplicationColumn implements Column<ApplicationTable> {
-
- /**
- * App id.
- */
- ID(ApplicationColumnFamily.INFO, "id"),
-
- /**
- * When the application was created.
- */
- CREATED_TIME(ApplicationColumnFamily.INFO, "created_time",
- new LongConverter()),
-
- /**
- * The version of the flow that this app belongs to.
- */
- FLOW_VERSION(ApplicationColumnFamily.INFO, "flow_version");
-
- private final ColumnHelper<ApplicationTable> column;
- private final ColumnFamily<ApplicationTable> columnFamily;
- private final String columnQualifier;
- private final byte[] columnQualifierBytes;
-
- private ApplicationColumn(ColumnFamily<ApplicationTable> columnFamily,
- String columnQualifier) {
- this(columnFamily, columnQualifier, GenericConverter.getInstance());
- }
-
- private ApplicationColumn(ColumnFamily<ApplicationTable> columnFamily,
- String columnQualifier, ValueConverter converter) {
- this.columnFamily = columnFamily;
- this.columnQualifier = columnQualifier;
- // Future-proof by ensuring the right column prefix hygiene.
- this.columnQualifierBytes =
- Bytes.toBytes(Separator.SPACE.encode(columnQualifier));
- this.column = new ColumnHelper<ApplicationTable>(columnFamily, converter);
- }
-
- /**
- * @return the column name value
- */
- private String getColumnQualifier() {
- return columnQualifier;
- }
-
- public void store(byte[] rowKey,
- TypedBufferedMutator<ApplicationTable> tableMutator, Long timestamp,
- Object inputValue, Attribute... attributes) throws IOException {
- column.store(rowKey, tableMutator, columnQualifierBytes, timestamp,
- inputValue, attributes);
- }
-
- public Object readResult(Result result) throws IOException {
- return column.readResult(result, columnQualifierBytes);
- }
-
- @Override
- public byte[] getColumnQualifierBytes() {
- return columnQualifierBytes.clone();
- }
-
- @Override
- public byte[] getColumnFamilyBytes() {
- return columnFamily.getBytes();
- }
-
- @Override
- public ValueConverter getValueConverter() {
- return column.getValueConverter();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnFamily.java
deleted file mode 100644
index 97e5f7b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnFamily.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.application;
-
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-
-/**
- * Represents the application table column families.
- */
-public enum ApplicationColumnFamily implements ColumnFamily<ApplicationTable> {
-
- /**
- * Info column family houses known columns, specifically ones included in
- * columnfamily filters.
- */
- INFO("i"),
-
- /**
- * Configurations are in a separate column family for two reasons: a) the size
- * of the config values can be very large and b) we expect that config values
- * are often separately accessed from other metrics and info columns.
- */
- CONFIGS("c"),
-
- /**
- * Metrics have a separate column family, because they have a separate TTL.
- */
- METRICS("m");
-
- /**
- * Byte representation of this column family.
- */
- private final byte[] bytes;
-
- /**
- * @param value create a column family with this name. Must be lower case and
- * without spaces.
- */
- private ApplicationColumnFamily(String value) {
- // column families should be lower case and not contain any spaces.
- this.bytes = Bytes.toBytes(Separator.SPACE.encode(value));
- }
-
- public byte[] getBytes() {
- return Bytes.copy(bytes);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
deleted file mode 100644
index 8297dc5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
+++ /dev/null
@@ -1,236 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.application;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.NavigableMap;
-
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.GenericConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
-
-/**
- * Identifies partially qualified columns for the application table.
- */
-public enum ApplicationColumnPrefix implements ColumnPrefix<ApplicationTable> {
-
- /**
- * To store TimelineEntity getIsRelatedToEntities values.
- */
- IS_RELATED_TO(ApplicationColumnFamily.INFO, "s"),
-
- /**
- * To store TimelineEntity getRelatesToEntities values.
- */
- RELATES_TO(ApplicationColumnFamily.INFO, "r"),
-
- /**
- * To store TimelineEntity info values.
- */
- INFO(ApplicationColumnFamily.INFO, "i"),
-
- /**
- * Lifecycle events for an application.
- */
- EVENT(ApplicationColumnFamily.INFO, "e"),
-
- /**
- * Config column stores configuration with config key as the column name.
- */
- CONFIG(ApplicationColumnFamily.CONFIGS, null),
-
- /**
- * Metrics are stored with the metric name as the column name.
- */
- METRIC(ApplicationColumnFamily.METRICS, null, new LongConverter());
-
- private final ColumnHelper<ApplicationTable> column;
- private final ColumnFamily<ApplicationTable> columnFamily;
-
- /**
- * Can be null for those cases where the provided column qualifier is the
- * entire column name.
- */
- private final String columnPrefix;
- private final byte[] columnPrefixBytes;
-
- /**
- * Private constructor, meant to be used by the enum definition.
- *
- * @param columnFamily that this column is stored in.
- * @param columnPrefix for this column.
- */
- private ApplicationColumnPrefix(ColumnFamily<ApplicationTable> columnFamily,
- String columnPrefix) {
- this(columnFamily, columnPrefix, GenericConverter.getInstance());
- }
-
- /**
- * Private constructor, meant to be used by the enum definition.
- *
- * @param columnFamily that this column is stored in.
- * @param columnPrefix for this column.
- * @param converter used to encode/decode values to be stored in HBase for
- * this column prefix.
- */
- private ApplicationColumnPrefix(ColumnFamily<ApplicationTable> columnFamily,
- String columnPrefix, ValueConverter converter) {
- column = new ColumnHelper<ApplicationTable>(columnFamily, converter);
- this.columnFamily = columnFamily;
- this.columnPrefix = columnPrefix;
- if (columnPrefix == null) {
- this.columnPrefixBytes = null;
- } else {
- // Future-proof by ensuring the right column prefix hygiene.
- this.columnPrefixBytes =
- Bytes.toBytes(Separator.SPACE.encode(columnPrefix));
- }
- }
-
- /**
- * @return the column name value
- */
- private String getColumnPrefix() {
- return columnPrefix;
- }
-
- @Override
- public byte[] getColumnPrefixBytes(byte[] qualifierPrefix) {
- return ColumnHelper.getColumnQualifier(
- this.columnPrefixBytes, qualifierPrefix);
- }
-
- @Override
- public byte[] getColumnPrefixBytes(String qualifierPrefix) {
- return ColumnHelper.getColumnQualifier(
- this.columnPrefixBytes, qualifierPrefix);
- }
-
- @Override
- public byte[] getColumnFamilyBytes() {
- return columnFamily.getBytes();
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
- * #store(byte[],
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.
- * TypedBufferedMutator, java.lang.String, java.lang.Long, java.lang.Object)
- */
- public void store(byte[] rowKey,
- TypedBufferedMutator<ApplicationTable> tableMutator, byte[] qualifier,
- Long timestamp, Object inputValue, Attribute... attributes)
- throws IOException {
-
- // Null check
- if (qualifier == null) {
- throw new IOException("Cannot store column with null qualifier in "
- + tableMutator.getName().getNameAsString());
- }
-
- byte[] columnQualifier = getColumnPrefixBytes(qualifier);
-
- column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue,
- attributes);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
- * #store(byte[],
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.
- * TypedBufferedMutator, java.lang.String, java.lang.Long, java.lang.Object)
- */
- public void store(byte[] rowKey,
- TypedBufferedMutator<ApplicationTable> tableMutator, String qualifier,
- Long timestamp, Object inputValue, Attribute...attributes)
- throws IOException {
-
- // Null check
- if (qualifier == null) {
- throw new IOException("Cannot store column with null qualifier in "
- + tableMutator.getName().getNameAsString());
- }
-
- byte[] columnQualifier = getColumnPrefixBytes(qualifier);
-
- column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue,
- attributes);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
- * #readResult(org.apache.hadoop.hbase.client.Result, java.lang.String)
- */
- public Object readResult(Result result, String qualifier) throws IOException {
- byte[] columnQualifier =
- ColumnHelper.getColumnQualifier(this.columnPrefixBytes, qualifier);
- return column.readResult(result, columnQualifier);
- }
-
- public ValueConverter getValueConverter() {
- return column.getValueConverter();
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
- * #readResults(org.apache.hadoop.hbase.client.Result,
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter)
- */
- public <K> Map<K, Object> readResults(Result result,
- KeyConverter<K> keyConverter) throws IOException {
- return column.readResults(result, columnPrefixBytes, keyConverter);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
- * #readResultsWithTimestamps(org.apache.hadoop.hbase.client.Result,
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter)
- */
- public <K, V> NavigableMap<K, NavigableMap<Long, V>>
- readResultsWithTimestamps(Result result, KeyConverter<K> keyConverter)
- throws IOException {
- return column.readResultsWithTimestamps(result, columnPrefixBytes,
- keyConverter);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
deleted file mode 100644
index e89a6a7..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
+++ /dev/null
@@ -1,251 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.storage.application;
-
-import java.util.List;
-
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.AppIdKeyConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverterToString;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-
-/**
- * Represents a rowkey for the application table.
- */
-public class ApplicationRowKey {
- private final String clusterId;
- private final String userId;
- private final String flowName;
- private final Long flowRunId;
- private final String appId;
- private final ApplicationRowKeyConverter appRowKeyConverter =
- new ApplicationRowKeyConverter();
-
- public ApplicationRowKey(String clusterId, String userId, String flowName,
- Long flowRunId, String appId) {
- this.clusterId = clusterId;
- this.userId = userId;
- this.flowName = flowName;
- this.flowRunId = flowRunId;
- this.appId = appId;
- }
-
- public String getClusterId() {
- return clusterId;
- }
-
- public String getUserId() {
- return userId;
- }
-
- public String getFlowName() {
- return flowName;
- }
-
- public Long getFlowRunId() {
- return flowRunId;
- }
-
- public String getAppId() {
- return appId;
- }
-
- /**
- * Constructs a row key for the application table as follows:
- * {@code clusterId!userName!flowName!flowRunId!AppId}.
- *
- * @return byte array with the row key
- */
- public byte[] getRowKey() {
- return appRowKeyConverter.encode(this);
- }
-
- /**
- * Given the raw row key as bytes, returns the row key as an object.
- *
- * @param rowKey Byte representation of row key.
- * @return An <cite>ApplicationRowKey</cite> object.
- */
- public static ApplicationRowKey parseRowKey(byte[] rowKey) {
- return new ApplicationRowKeyConverter().decode(rowKey);
- }
-
- /**
- * Constructs a row key for the application table as follows:
- * {@code clusterId!userName!flowName!flowRunId!AppId}.
- * @return String representation of row key.
- */
- public String getRowKeyAsString() {
- return appRowKeyConverter.encodeAsString(this);
- }
-
- /**
- * Given the encoded row key as string, returns the row key as an object.
- * @param encodedRowKey String representation of row key.
- * @return A <cite>ApplicationRowKey</cite> object.
- */
- public static ApplicationRowKey parseRowKeyFromString(String encodedRowKey) {
- return new ApplicationRowKeyConverter().decodeFromString(encodedRowKey);
- }
-
- /**
- * Encodes and decodes row key for application table. The row key is of the
- * form: clusterId!userName!flowName!flowRunId!appId. flowRunId is a long,
- * appId is encoded and decoded using {@link AppIdKeyConverter} and rest are
- * strings.
- * <p>
- */
- final private static class ApplicationRowKeyConverter implements
- KeyConverter<ApplicationRowKey>, KeyConverterToString<ApplicationRowKey> {
-
- private final KeyConverter<String> appIDKeyConverter =
- new AppIdKeyConverter();
-
- /**
- * Intended for use in ApplicationRowKey only.
- */
- private ApplicationRowKeyConverter() {
- }
-
- /**
- * Application row key is of the form
- * clusterId!userName!flowName!flowRunId!appId with each segment separated
- * by !. The sizes below indicate sizes of each one of these segements in
- * sequence. clusterId, userName and flowName are strings. flowrunId is a
- * long hence 8 bytes in size. app id is represented as 12 bytes with
- * cluster timestamp part of appid takes 8 bytes(long) and seq id takes 4
- * bytes(int). Strings are variable in size (i.e. end whenever separator is
- * encountered). This is used while decoding and helps in determining where
- * to split.
- */
- private static final int[] SEGMENT_SIZES = {Separator.VARIABLE_SIZE,
- Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
- AppIdKeyConverter.getKeySize() };
-
- /*
- * (non-Javadoc)
- *
- * Encodes ApplicationRowKey object into a byte array with each
- * component/field in ApplicationRowKey separated by Separator#QUALIFIERS.
- * This leads to an application table row key of the form
- * clusterId!userName!flowName!flowRunId!appId If flowRunId in passed
- * ApplicationRowKey object is null (and the fields preceding it i.e.
- * clusterId, userId and flowName are not null), this returns a row key
- * prefix of the form clusterId!userName!flowName! and if appId in
- * ApplicationRowKey is null (other 4 components all are not null), this
- * returns a row key prefix of the form
- * clusterId!userName!flowName!flowRunId! flowRunId is inverted while
- * encoding as it helps maintain a descending order for row keys in the
- * application table.
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common
- * .KeyConverter#encode(java.lang.Object)
- */
- @Override
- public byte[] encode(ApplicationRowKey rowKey) {
- byte[] cluster =
- Separator.encode(rowKey.getClusterId(), Separator.SPACE,
- Separator.TAB, Separator.QUALIFIERS);
- byte[] user =
- Separator.encode(rowKey.getUserId(), Separator.SPACE, Separator.TAB,
- Separator.QUALIFIERS);
- byte[] flow =
- Separator.encode(rowKey.getFlowName(), Separator.SPACE,
- Separator.TAB, Separator.QUALIFIERS);
- byte[] first = Separator.QUALIFIERS.join(cluster, user, flow);
- // Note that flowRunId is a long, so we can't encode them all at the same
- // time.
- if (rowKey.getFlowRunId() == null) {
- return Separator.QUALIFIERS.join(first, Separator.EMPTY_BYTES);
- }
- byte[] second =
- Bytes.toBytes(LongConverter.invertLong(
- rowKey.getFlowRunId()));
- if (rowKey.getAppId() == null || rowKey.getAppId().isEmpty()) {
- return Separator.QUALIFIERS.join(first, second, Separator.EMPTY_BYTES);
- }
- byte[] third = appIDKeyConverter.encode(rowKey.getAppId());
- return Separator.QUALIFIERS.join(first, second, third);
- }
-
- /*
- * (non-Javadoc)
- *
- * Decodes an application row key of the form
- * clusterId!userName!flowName!flowRunId!appId represented in byte format
- * and converts it into an ApplicationRowKey object.flowRunId is inverted
- * while decoding as it was inverted while encoding.
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common
- * .KeyConverter#decode(byte[])
- */
- @Override
- public ApplicationRowKey decode(byte[] rowKey) {
- byte[][] rowKeyComponents =
- Separator.QUALIFIERS.split(rowKey, SEGMENT_SIZES);
- if (rowKeyComponents.length != 5) {
- throw new IllegalArgumentException("the row key is not valid for "
- + "an application");
- }
- String clusterId =
- Separator.decode(Bytes.toString(rowKeyComponents[0]),
- Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
- String userId =
- Separator.decode(Bytes.toString(rowKeyComponents[1]),
- Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
- String flowName =
- Separator.decode(Bytes.toString(rowKeyComponents[2]),
- Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
- Long flowRunId =
- LongConverter.invertLong(Bytes.toLong(rowKeyComponents[3]));
- String appId = appIDKeyConverter.decode(rowKeyComponents[4]);
- return new ApplicationRowKey(clusterId, userId, flowName, flowRunId,
- appId);
- }
-
- @Override
- public String encodeAsString(ApplicationRowKey key) {
- if (key.clusterId == null || key.userId == null || key.flowName == null
- || key.flowRunId == null || key.appId == null) {
- throw new IllegalArgumentException();
- }
- return TimelineReaderUtils
- .joinAndEscapeStrings(new String[] {key.clusterId, key.userId,
- key.flowName, key.flowRunId.toString(), key.appId});
- }
-
- @Override
- public ApplicationRowKey decodeFromString(String encodedRowKey) {
- List<String> split = TimelineReaderUtils.split(encodedRowKey);
- if (split == null || split.size() != 5) {
- throw new IllegalArgumentException(
- "Invalid row key for application table.");
- }
- Long flowRunId = Long.valueOf(split.get(3));
- return new ApplicationRowKey(split.get(0), split.get(1), split.get(2),
- flowRunId, split.get(4));
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKeyPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKeyPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKeyPrefix.java
deleted file mode 100644
index f61b0e9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKeyPrefix.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.application;
-
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.RowKeyPrefix;
-
-/**
- * Represents a partial rowkey (without flowName or without flowName and
- * flowRunId) for the application table.
- */
-public class ApplicationRowKeyPrefix extends ApplicationRowKey implements
- RowKeyPrefix<ApplicationRowKey> {
-
- /**
- * Creates a prefix which generates the following rowKeyPrefixes for the
- * application table: {@code clusterId!userName!flowName!}.
- *
- * @param clusterId the cluster on which applications ran
- * @param userId the user that ran applications
- * @param flowName the name of the flow that was run by the user on the
- * cluster
- */
- public ApplicationRowKeyPrefix(String clusterId, String userId,
- String flowName) {
- super(clusterId, userId, flowName, null, null);
- }
-
- /**
- * Creates a prefix which generates the following rowKeyPrefixes for the
- * application table: {@code clusterId!userName!flowName!flowRunId!}.
- *
- * @param clusterId identifying the cluster
- * @param userId identifying the user
- * @param flowName identifying the flow
- * @param flowRunId identifying the instance of this flow
- */
- public ApplicationRowKeyPrefix(String clusterId, String userId,
- String flowName, Long flowRunId) {
- super(clusterId, userId, flowName, flowRunId, null);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.application.
- * RowKeyPrefix#getRowKeyPrefix()
- */
- @Override
- public byte[] getRowKeyPrefix() {
- return super.getRowKey();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
deleted file mode 100644
index 4da720e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.application;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.regionserver.BloomType;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBaseSchemaConstants;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * The application table as column families info, config and metrics. Info
- * stores information about a YARN application entity, config stores
- * configuration data of a YARN application, metrics stores the metrics of a
- * YARN application. This table is entirely analogous to the entity table but
- * created for better performance.
- *
- * Example application table record:
- *
- * <pre>
- * |-------------------------------------------------------------------------|
- * | Row | Column Family | Column Family| Column Family|
- * | key | info | metrics | config |
- * |-------------------------------------------------------------------------|
- * | clusterId! | id:appId | metricId1: | configKey1: |
- * | userName! | | metricValue1 | configValue1 |
- * | flowName! | created_time: | @timestamp1 | |
- * | flowRunId! | 1392993084018 | | configKey2: |
- * | AppId | | metriciD1: | configValue2 |
- * | | i!infoKey: | metricValue2 | |
- * | | infoValue | @timestamp2 | |
- * | | | | |
- * | | r!relatesToKey: | metricId2: | |
- * | | id3=id4=id5 | metricValue1 | |
- * | | | @timestamp2 | |
- * | | s!isRelatedToKey: | | |
- * | | id7=id9=id6 | | |
- * | | | | |
- * | | e!eventId=timestamp=infoKey: | | |
- * | | eventInfoValue | | |
- * | | | | |
- * | | flowVersion: | | |
- * | | versionValue | | |
- * |-------------------------------------------------------------------------|
- * </pre>
- */
-public class ApplicationTable extends BaseTable<ApplicationTable> {
- /** application prefix. */
- private static final String PREFIX =
- YarnConfiguration.TIMELINE_SERVICE_PREFIX + "application";
-
- /** config param name that specifies the application table name. */
- public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
-
- /**
- * config param name that specifies the TTL for metrics column family in
- * application table.
- */
- private static final String METRICS_TTL_CONF_NAME = PREFIX
- + ".table.metrics.ttl";
-
- /**
- * config param name that specifies max-versions for metrics column family in
- * entity table.
- */
- private static final String METRICS_MAX_VERSIONS =
- PREFIX + ".table.metrics.max-versions";
-
- /** default value for application table name. */
- private static final String DEFAULT_TABLE_NAME =
- "timelineservice.application";
-
- /** default TTL is 30 days for metrics timeseries. */
- private static final int DEFAULT_METRICS_TTL = 2592000;
-
- /** default max number of versions. */
- private static final int DEFAULT_METRICS_MAX_VERSIONS = 10000;
-
- private static final Logger LOG =
- LoggerFactory.getLogger(ApplicationTable.class);
-
- public ApplicationTable() {
- super(TABLE_NAME_CONF_NAME, DEFAULT_TABLE_NAME);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.BaseTable#createTable
- * (org.apache.hadoop.hbase.client.Admin,
- * org.apache.hadoop.conf.Configuration)
- */
- public void createTable(Admin admin, Configuration hbaseConf)
- throws IOException {
-
- TableName table = getTableName(hbaseConf);
- if (admin.tableExists(table)) {
- // do not disable / delete existing table
- // similar to the approach taken by map-reduce jobs when
- // output directory exists
- throw new IOException("Table " + table.getNameAsString()
- + " already exists.");
- }
-
- HTableDescriptor applicationTableDescp = new HTableDescriptor(table);
- HColumnDescriptor infoCF =
- new HColumnDescriptor(ApplicationColumnFamily.INFO.getBytes());
- infoCF.setBloomFilterType(BloomType.ROWCOL);
- applicationTableDescp.addFamily(infoCF);
-
- HColumnDescriptor configCF =
- new HColumnDescriptor(ApplicationColumnFamily.CONFIGS.getBytes());
- configCF.setBloomFilterType(BloomType.ROWCOL);
- configCF.setBlockCacheEnabled(true);
- applicationTableDescp.addFamily(configCF);
-
- HColumnDescriptor metricsCF =
- new HColumnDescriptor(ApplicationColumnFamily.METRICS.getBytes());
- applicationTableDescp.addFamily(metricsCF);
- metricsCF.setBlockCacheEnabled(true);
- // always keep 1 version (the latest)
- metricsCF.setMinVersions(1);
- metricsCF.setMaxVersions(
- hbaseConf.getInt(METRICS_MAX_VERSIONS, DEFAULT_METRICS_MAX_VERSIONS));
- metricsCF.setTimeToLive(hbaseConf.getInt(METRICS_TTL_CONF_NAME,
- DEFAULT_METRICS_TTL));
- applicationTableDescp.setRegionSplitPolicyClassName(
- "org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy");
- applicationTableDescp.setValue("KeyPrefixRegionSplitPolicy.prefix_length",
- TimelineHBaseSchemaConstants.USERNAME_SPLIT_KEY_PREFIX_LENGTH);
- admin.createTable(applicationTableDescp,
- TimelineHBaseSchemaConstants.getUsernameSplits());
- LOG.info("Status of table creation for " + table.getNameAsString() + "="
- + admin.tableExists(table));
- }
-
- /**
- * @param metricsTTL time to live parameter for the metrics in this table.
- * @param hbaseConf configuration in which to set the metrics TTL config
- * variable.
- */
- public void setMetricsTTL(int metricsTTL, Configuration hbaseConf) {
- hbaseConf.setInt(METRICS_TTL_CONF_NAME, metricsTTL);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/package-info.java
deleted file mode 100644
index 03f508f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/package-info.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Package org.apache.hadoop.yarn.server.timelineservice.storage.application
- * contains classes related to implementation for application table.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-package org.apache.hadoop.yarn.server.timelineservice.storage.application;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java
deleted file mode 100644
index 67497fc..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow;
-
-
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Column;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
-
-import java.io.IOException;
-
-/**
- * Identifies fully qualified columns for the {@link AppToFlowTable}.
- */
-public enum AppToFlowColumn implements Column<AppToFlowTable> {
-
- /**
- * The flow ID.
- */
- FLOW_ID(AppToFlowColumnFamily.MAPPING, "flow_id"),
-
- /**
- * The flow run ID.
- */
- FLOW_RUN_ID(AppToFlowColumnFamily.MAPPING, "flow_run_id"),
-
- /**
- * The user.
- */
- USER_ID(AppToFlowColumnFamily.MAPPING, "user_id");
-
- private final ColumnHelper<AppToFlowTable> column;
- private final ColumnFamily<AppToFlowTable> columnFamily;
- private final String columnQualifier;
- private final byte[] columnQualifierBytes;
-
- AppToFlowColumn(ColumnFamily<AppToFlowTable> columnFamily,
- String columnQualifier) {
- this.columnFamily = columnFamily;
- this.columnQualifier = columnQualifier;
- // Future-proof by ensuring the right column prefix hygiene.
- this.columnQualifierBytes =
- Bytes.toBytes(Separator.SPACE.encode(columnQualifier));
- this.column = new ColumnHelper<AppToFlowTable>(columnFamily);
- }
-
- /**
- * @return the column name value
- */
- private String getColumnQualifier() {
- return columnQualifier;
- }
-
- @Override
- public byte[] getColumnQualifierBytes() {
- return columnQualifierBytes.clone();
- }
-
- public void store(byte[] rowKey,
- TypedBufferedMutator<AppToFlowTable> tableMutator, Long timestamp,
- Object inputValue, Attribute... attributes) throws IOException {
- column.store(rowKey, tableMutator, columnQualifierBytes, timestamp,
- inputValue, attributes);
- }
-
- @Override
- public byte[] getColumnFamilyBytes() {
- return columnFamily.getBytes();
- }
-
- @Override
- public ValueConverter getValueConverter() {
- return column.getValueConverter();
- }
-
- public Object readResult(Result result) throws IOException {
- return column.readResult(result, columnQualifierBytes);
- }
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/18] hadoop git commit: YARN-7919. Refactor timelineservice-hbase
module into submodules. Contributed by Haibo Chen.
Posted by ro...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/SubApplicationEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/SubApplicationEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/SubApplicationEntityReader.java
new file mode 100644
index 0000000..faed348
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/SubApplicationEntityReader.java
@@ -0,0 +1,489 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Query;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.BinaryComparator;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.filter.FamilyFilter;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.FilterList.Operator;
+import org.apache.hadoop.hbase.filter.PageFilter;
+import org.apache.hadoop.hbase.filter.QualifierFilter;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTableRW;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnRWHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.RowKeyPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationRowKeyPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationTableRW;
+import org.apache.hadoop.yarn.webapp.BadRequestException;
+
+import com.google.common.base.Preconditions;
+
+class SubApplicationEntityReader extends GenericEntityReader {
+ private static final SubApplicationTableRW SUB_APPLICATION_TABLE =
+ new SubApplicationTableRW();
+
+ SubApplicationEntityReader(TimelineReaderContext ctxt,
+ TimelineEntityFilters entityFilters, TimelineDataToRetrieve toRetrieve) {
+ super(ctxt, entityFilters, toRetrieve);
+ }
+
+ SubApplicationEntityReader(TimelineReaderContext ctxt,
+ TimelineDataToRetrieve toRetrieve) {
+ super(ctxt, toRetrieve);
+ }
+
+ /**
+ * Uses the {@link SubApplicationTableRW}.
+ */
+ protected BaseTableRW<?> getTable() {
+ return SUB_APPLICATION_TABLE;
+ }
+
+ @Override
+ protected FilterList constructFilterListBasedOnFilters() throws IOException {
+ // Filters here cannot be null for multiple entity reads as they are set in
+ // augmentParams if null.
+ FilterList listBasedOnFilters = new FilterList();
+ TimelineEntityFilters filters = getFilters();
+ // Create filter list based on created time range and add it to
+ // listBasedOnFilters.
+ long createdTimeBegin = filters.getCreatedTimeBegin();
+ long createdTimeEnd = filters.getCreatedTimeEnd();
+ if (createdTimeBegin != 0 || createdTimeEnd != Long.MAX_VALUE) {
+ listBasedOnFilters.addFilter(TimelineFilterUtils
+ .createSingleColValueFiltersByRange(SubApplicationColumn.CREATED_TIME,
+ createdTimeBegin, createdTimeEnd));
+ }
+ // Create filter list based on metric filters and add it to
+ // listBasedOnFilters.
+ TimelineFilterList metricFilters = filters.getMetricFilters();
+ if (metricFilters != null && !metricFilters.getFilterList().isEmpty()) {
+ listBasedOnFilters.addFilter(TimelineFilterUtils.createHBaseFilterList(
+ SubApplicationColumnPrefix.METRIC, metricFilters));
+ }
+ // Create filter list based on config filters and add it to
+ // listBasedOnFilters.
+ TimelineFilterList configFilters = filters.getConfigFilters();
+ if (configFilters != null && !configFilters.getFilterList().isEmpty()) {
+ listBasedOnFilters.addFilter(TimelineFilterUtils.createHBaseFilterList(
+ SubApplicationColumnPrefix.CONFIG, configFilters));
+ }
+ // Create filter list based on info filters and add it to listBasedOnFilters
+ TimelineFilterList infoFilters = filters.getInfoFilters();
+ if (infoFilters != null && !infoFilters.getFilterList().isEmpty()) {
+ listBasedOnFilters.addFilter(TimelineFilterUtils
+ .createHBaseFilterList(SubApplicationColumnPrefix.INFO, infoFilters));
+ }
+ return listBasedOnFilters;
+ }
+
+ /**
+ * Add {@link QualifierFilter} filters to filter list for each column of
+ * entity table.
+ *
+ * @param list filter list to which qualifier filters have to be added.
+ */
+ protected void updateFixedColumns(FilterList list) {
+ for (SubApplicationColumn column : SubApplicationColumn.values()) {
+ list.addFilter(new QualifierFilter(CompareOp.EQUAL,
+ new BinaryComparator(column.getColumnQualifierBytes())));
+ }
+ }
+
+ /**
+ * Creates a filter list which indicates that only some of the column
+ * qualifiers in the info column family will be returned in result.
+ *
+ * @param isApplication If true, it means operations are to be performed for
+ * application table, otherwise for entity table.
+ * @return filter list.
+ * @throws IOException if any problem occurs while creating filter list.
+ */
+ private FilterList createFilterListForColsOfInfoFamily() throws IOException {
+ FilterList infoFamilyColsFilter = new FilterList(Operator.MUST_PASS_ONE);
+ // Add filters for each column in entity table.
+ updateFixedColumns(infoFamilyColsFilter);
+ EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
+ // If INFO field has to be retrieved, add a filter for fetching columns
+ // with INFO column prefix.
+ if (hasField(fieldsToRetrieve, Field.INFO)) {
+ infoFamilyColsFilter.addFilter(
+ TimelineFilterUtils.createHBaseQualifierFilter(CompareOp.EQUAL,
+ SubApplicationColumnPrefix.INFO));
+ }
+ TimelineFilterList relatesTo = getFilters().getRelatesTo();
+ if (hasField(fieldsToRetrieve, Field.RELATES_TO)) {
+ // If RELATES_TO field has to be retrieved, add a filter for fetching
+ // columns with RELATES_TO column prefix.
+ infoFamilyColsFilter.addFilter(
+ TimelineFilterUtils.createHBaseQualifierFilter(CompareOp.EQUAL,
+ SubApplicationColumnPrefix.RELATES_TO));
+ } else if (relatesTo != null && !relatesTo.getFilterList().isEmpty()) {
+ // Even if fields to retrieve does not contain RELATES_TO, we still
+ // need to have a filter to fetch some of the column qualifiers if
+ // relatesTo filters are specified. relatesTo filters will then be
+ // matched after fetching rows from HBase.
+ Set<String> relatesToCols =
+ TimelineFilterUtils.fetchColumnsFromFilterList(relatesTo);
+ infoFamilyColsFilter.addFilter(createFiltersFromColumnQualifiers(
+ SubApplicationColumnPrefix.RELATES_TO, relatesToCols));
+ }
+ TimelineFilterList isRelatedTo = getFilters().getIsRelatedTo();
+ if (hasField(fieldsToRetrieve, Field.IS_RELATED_TO)) {
+ // If IS_RELATED_TO field has to be retrieved, add a filter for fetching
+ // columns with IS_RELATED_TO column prefix.
+ infoFamilyColsFilter.addFilter(
+ TimelineFilterUtils.createHBaseQualifierFilter(CompareOp.EQUAL,
+ SubApplicationColumnPrefix.IS_RELATED_TO));
+ } else if (isRelatedTo != null && !isRelatedTo.getFilterList().isEmpty()) {
+ // Even if fields to retrieve does not contain IS_RELATED_TO, we still
+ // need to have a filter to fetch some of the column qualifiers if
+ // isRelatedTo filters are specified. isRelatedTo filters will then be
+ // matched after fetching rows from HBase.
+ Set<String> isRelatedToCols =
+ TimelineFilterUtils.fetchColumnsFromFilterList(isRelatedTo);
+ infoFamilyColsFilter.addFilter(createFiltersFromColumnQualifiers(
+ SubApplicationColumnPrefix.IS_RELATED_TO, isRelatedToCols));
+ }
+ TimelineFilterList eventFilters = getFilters().getEventFilters();
+ if (hasField(fieldsToRetrieve, Field.EVENTS)) {
+ // If EVENTS field has to be retrieved, add a filter for fetching columns
+ // with EVENT column prefix.
+ infoFamilyColsFilter.addFilter(
+ TimelineFilterUtils.createHBaseQualifierFilter(CompareOp.EQUAL,
+ SubApplicationColumnPrefix.EVENT));
+ } else if (eventFilters != null
+ && !eventFilters.getFilterList().isEmpty()) {
+ // Even if fields to retrieve does not contain EVENTS, we still need to
+ // have a filter to fetch some of the column qualifiers on the basis of
+ // event filters specified. Event filters will then be matched after
+ // fetching rows from HBase.
+ Set<String> eventCols =
+ TimelineFilterUtils.fetchColumnsFromFilterList(eventFilters);
+ infoFamilyColsFilter.addFilter(createFiltersFromColumnQualifiers(
+ SubApplicationColumnPrefix.EVENT, eventCols));
+ }
+ return infoFamilyColsFilter;
+ }
+
+ /**
+ * Exclude column prefixes via filters which are not required(based on fields
+ * to retrieve) from info column family. These filters are added to filter
+ * list which contains a filter for getting info column family.
+ *
+ * @param infoColFamilyList filter list for info column family.
+ */
+ private void excludeFieldsFromInfoColFamily(FilterList infoColFamilyList) {
+ EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
+ // Events not required.
+ if (!hasField(fieldsToRetrieve, Field.EVENTS)) {
+ infoColFamilyList.addFilter(
+ TimelineFilterUtils.createHBaseQualifierFilter(CompareOp.NOT_EQUAL,
+ SubApplicationColumnPrefix.EVENT));
+ }
+ // info not required.
+ if (!hasField(fieldsToRetrieve, Field.INFO)) {
+ infoColFamilyList.addFilter(
+ TimelineFilterUtils.createHBaseQualifierFilter(CompareOp.NOT_EQUAL,
+ SubApplicationColumnPrefix.INFO));
+ }
+ // is related to not required.
+ if (!hasField(fieldsToRetrieve, Field.IS_RELATED_TO)) {
+ infoColFamilyList.addFilter(
+ TimelineFilterUtils.createHBaseQualifierFilter(CompareOp.NOT_EQUAL,
+ SubApplicationColumnPrefix.IS_RELATED_TO));
+ }
+ // relates to not required.
+ if (!hasField(fieldsToRetrieve, Field.RELATES_TO)) {
+ infoColFamilyList.addFilter(
+ TimelineFilterUtils.createHBaseQualifierFilter(CompareOp.NOT_EQUAL,
+ SubApplicationColumnPrefix.RELATES_TO));
+ }
+ }
+
+ /**
+ * Updates filter list based on fields for confs and metrics to retrieve.
+ *
+ * @param listBasedOnFields filter list based on fields.
+ * @throws IOException if any problem occurs while updating filter list.
+ */
+ private void updateFilterForConfsAndMetricsToRetrieve(
+ FilterList listBasedOnFields) throws IOException {
+ TimelineDataToRetrieve dataToRetrieve = getDataToRetrieve();
+ // Please note that if confsToRetrieve is specified, we would have added
+ // CONFS to fields to retrieve in augmentParams() even if not specified.
+ if (dataToRetrieve.getFieldsToRetrieve().contains(Field.CONFIGS)) {
+ // Create a filter list for configs.
+ listBasedOnFields.addFilter(
+ TimelineFilterUtils.createFilterForConfsOrMetricsToRetrieve(
+ dataToRetrieve.getConfsToRetrieve(),
+ SubApplicationColumnFamily.CONFIGS,
+ SubApplicationColumnPrefix.CONFIG));
+ }
+
+ // Please note that if metricsToRetrieve is specified, we would have added
+ // METRICS to fields to retrieve in augmentParams() even if not specified.
+ if (dataToRetrieve.getFieldsToRetrieve().contains(Field.METRICS)) {
+ // Create a filter list for metrics.
+ listBasedOnFields.addFilter(
+ TimelineFilterUtils.createFilterForConfsOrMetricsToRetrieve(
+ dataToRetrieve.getMetricsToRetrieve(),
+ SubApplicationColumnFamily.METRICS,
+ SubApplicationColumnPrefix.METRIC));
+ }
+ }
+
+ @Override
+ protected FilterList constructFilterListBasedOnFields() throws IOException {
+ if (!needCreateFilterListBasedOnFields()) {
+ // Fetch all the columns. No need of a filter.
+ return null;
+ }
+ FilterList listBasedOnFields = new FilterList(Operator.MUST_PASS_ONE);
+ FilterList infoColFamilyList = new FilterList();
+ // By default fetch everything in INFO column family.
+ FamilyFilter infoColumnFamily = new FamilyFilter(CompareOp.EQUAL,
+ new BinaryComparator(SubApplicationColumnFamily.INFO.getBytes()));
+ infoColFamilyList.addFilter(infoColumnFamily);
+ if (fetchPartialColsFromInfoFamily()) {
+ // We can fetch only some of the columns from info family.
+ infoColFamilyList.addFilter(createFilterListForColsOfInfoFamily());
+ } else {
+ // Exclude column prefixes in info column family which are not required
+ // based on fields to retrieve.
+ excludeFieldsFromInfoColFamily(infoColFamilyList);
+ }
+ listBasedOnFields.addFilter(infoColFamilyList);
+ updateFilterForConfsAndMetricsToRetrieve(listBasedOnFields);
+ return listBasedOnFields;
+ }
+
+ @Override
+ protected void validateParams() {
+ Preconditions.checkNotNull(getContext(), "context shouldn't be null");
+ Preconditions.checkNotNull(getDataToRetrieve(),
+ "data to retrieve shouldn't be null");
+ Preconditions.checkNotNull(getContext().getClusterId(),
+ "clusterId shouldn't be null");
+ Preconditions.checkNotNull(getContext().getDoAsUser(),
+ "DoAsUser shouldn't be null");
+ Preconditions.checkNotNull(getContext().getEntityType(),
+ "entityType shouldn't be null");
+ }
+
+ @Override
+ protected void augmentParams(Configuration hbaseConf, Connection conn)
+ throws IOException {
+ getDataToRetrieve().addFieldsBasedOnConfsAndMetricsToRetrieve();
+ createFiltersIfNull();
+ }
+
+ private void setMetricsTimeRange(Query query) {
+ // Set time range for metric values.
+ HBaseTimelineStorageUtils.setMetricsTimeRange(query,
+ SubApplicationColumnFamily.METRICS.getBytes(),
+ getDataToRetrieve().getMetricsTimeBegin(),
+ getDataToRetrieve().getMetricsTimeEnd());
+ }
+
+ @Override
+ protected ResultScanner getResults(Configuration hbaseConf, Connection conn,
+ FilterList filterList) throws IOException {
+
+ // Scan through part of the table to find the entities belong to one app
+ // and one type
+ Scan scan = new Scan();
+ TimelineReaderContext context = getContext();
+ if (context.getDoAsUser() == null) {
+ throw new BadRequestException("Invalid user!");
+ }
+
+ RowKeyPrefix<SubApplicationRowKey> subApplicationRowKeyPrefix = null;
+ // default mode, will always scans from beginning of entity type.
+ if (getFilters() == null || getFilters().getFromId() == null) {
+ subApplicationRowKeyPrefix = new SubApplicationRowKeyPrefix(
+ context.getDoAsUser(), context.getClusterId(),
+ context.getEntityType(), null, null, null);
+ scan.setRowPrefixFilter(subApplicationRowKeyPrefix.getRowKeyPrefix());
+ } else { // pagination mode, will scan from given entityIdPrefix!enitityId
+
+ SubApplicationRowKey entityRowKey = null;
+ try {
+ entityRowKey = SubApplicationRowKey
+ .parseRowKeyFromString(getFilters().getFromId());
+ } catch (IllegalArgumentException e) {
+ throw new BadRequestException("Invalid filter fromid is provided.");
+ }
+ if (!context.getClusterId().equals(entityRowKey.getClusterId())) {
+ throw new BadRequestException(
+ "fromid doesn't belong to clusterId=" + context.getClusterId());
+ }
+
+ // set start row
+ scan.setStartRow(entityRowKey.getRowKey());
+
+ // get the bytes for stop row
+ subApplicationRowKeyPrefix = new SubApplicationRowKeyPrefix(
+ context.getDoAsUser(), context.getClusterId(),
+ context.getEntityType(), null, null, null);
+
+ // set stop row
+ scan.setStopRow(
+ HBaseTimelineStorageUtils.calculateTheClosestNextRowKeyForPrefix(
+ subApplicationRowKeyPrefix.getRowKeyPrefix()));
+
+ // set page filter to limit. This filter has to set only in pagination
+ // mode.
+ filterList.addFilter(new PageFilter(getFilters().getLimit()));
+ }
+ setMetricsTimeRange(scan);
+ scan.setMaxVersions(getDataToRetrieve().getMetricsLimit());
+ if (filterList != null && !filterList.getFilters().isEmpty()) {
+ scan.setFilter(filterList);
+ }
+ return getTable().getResultScanner(hbaseConf, conn, scan);
+ }
+
+ @Override
+ protected Result getResult(Configuration hbaseConf, Connection conn,
+ FilterList filterList) throws IOException {
+ throw new UnsupportedOperationException(
+ "we don't support a single entity query");
+ }
+
+ @Override
+ protected TimelineEntity parseEntity(Result result) throws IOException {
+ if (result == null || result.isEmpty()) {
+ return null;
+ }
+ TimelineEntity entity = new TimelineEntity();
+ SubApplicationRowKey parseRowKey =
+ SubApplicationRowKey.parseRowKey(result.getRow());
+ entity.setType(parseRowKey.getEntityType());
+ entity.setId(parseRowKey.getEntityId());
+ entity.setIdPrefix(parseRowKey.getEntityIdPrefix().longValue());
+
+ TimelineEntityFilters filters = getFilters();
+ // fetch created time
+ Long createdTime = (Long) ColumnRWHelper.readResult(result,
+ SubApplicationColumn.CREATED_TIME);
+ entity.setCreatedTime(createdTime);
+
+ EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
+ // fetch is related to entities and match isRelatedTo filter. If isRelatedTo
+ // filters do not match, entity would be dropped. We have to match filters
+ // locally as relevant HBase filters to filter out rows on the basis of
+ // isRelatedTo are not set in HBase scan.
+ boolean checkIsRelatedTo =
+ filters.getIsRelatedTo() != null
+ && filters.getIsRelatedTo().getFilterList().size() > 0;
+ if (hasField(fieldsToRetrieve, Field.IS_RELATED_TO) || checkIsRelatedTo) {
+ readRelationship(entity, result, SubApplicationColumnPrefix.IS_RELATED_TO,
+ true);
+ if (checkIsRelatedTo && !TimelineStorageUtils.matchIsRelatedTo(entity,
+ filters.getIsRelatedTo())) {
+ return null;
+ }
+ if (!hasField(fieldsToRetrieve, Field.IS_RELATED_TO)) {
+ entity.getIsRelatedToEntities().clear();
+ }
+ }
+
+ // fetch relates to entities and match relatesTo filter. If relatesTo
+ // filters do not match, entity would be dropped. We have to match filters
+ // locally as relevant HBase filters to filter out rows on the basis of
+ // relatesTo are not set in HBase scan.
+ boolean checkRelatesTo =
+ !isSingleEntityRead() && filters.getRelatesTo() != null
+ && filters.getRelatesTo().getFilterList().size() > 0;
+ if (hasField(fieldsToRetrieve, Field.RELATES_TO) || checkRelatesTo) {
+ readRelationship(entity, result, SubApplicationColumnPrefix.RELATES_TO,
+ false);
+ if (checkRelatesTo && !TimelineStorageUtils.matchRelatesTo(entity,
+ filters.getRelatesTo())) {
+ return null;
+ }
+ if (!hasField(fieldsToRetrieve, Field.RELATES_TO)) {
+ entity.getRelatesToEntities().clear();
+ }
+ }
+
+ // fetch info if fieldsToRetrieve contains INFO or ALL.
+ if (hasField(fieldsToRetrieve, Field.INFO)) {
+ readKeyValuePairs(entity, result, SubApplicationColumnPrefix.INFO, false);
+ }
+
+ // fetch configs if fieldsToRetrieve contains CONFIGS or ALL.
+ if (hasField(fieldsToRetrieve, Field.CONFIGS)) {
+ readKeyValuePairs(entity, result, SubApplicationColumnPrefix.CONFIG,
+ true);
+ }
+
+ // fetch events and match event filters if they exist. If event filters do
+ // not match, entity would be dropped. We have to match filters locally
+ // as relevant HBase filters to filter out rows on the basis of events
+ // are not set in HBase scan.
+ boolean checkEvents =
+ !isSingleEntityRead() && filters.getEventFilters() != null
+ && filters.getEventFilters().getFilterList().size() > 0;
+ if (hasField(fieldsToRetrieve, Field.EVENTS) || checkEvents) {
+ readEvents(entity, result, SubApplicationColumnPrefix.EVENT);
+ if (checkEvents && !TimelineStorageUtils.matchEventFilters(entity,
+ filters.getEventFilters())) {
+ return null;
+ }
+ if (!hasField(fieldsToRetrieve, Field.EVENTS)) {
+ entity.getEvents().clear();
+ }
+ }
+
+ // fetch metrics if fieldsToRetrieve contains METRICS or ALL.
+ if (hasField(fieldsToRetrieve, Field.METRICS)) {
+ readMetrics(entity, result, SubApplicationColumnPrefix.METRIC);
+ }
+
+ entity.getInfo().put(TimelineReaderUtils.FROMID_KEY,
+ parseRowKey.getRowKeyAsString());
+ return entity;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
new file mode 100644
index 0000000..3168163
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
@@ -0,0 +1,464 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.filter.BinaryPrefixComparator;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.FilterList.Operator;
+import org.apache.hadoop.hbase.filter.QualifierFilter;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTableRW;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnRWHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.EventColumnName;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.EventColumnNameConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.StringKeyConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The base class for reading and deserializing timeline entities from the
+ * HBase storage. Different types can be defined for different types of the
+ * entities that are being requested.
+ */
+public abstract class TimelineEntityReader extends
+ AbstractTimelineStorageReader {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TimelineEntityReader.class);
+
+ private final boolean singleEntityRead;
+ private TimelineDataToRetrieve dataToRetrieve;
+ // used only for multiple entity read mode
+ private TimelineEntityFilters filters;
+
+ /**
+ * Main table the entity reader uses.
+ */
+ private BaseTableRW<?> table;
+
+ /**
+ * Used to convert strings key components to and from storage format.
+ */
+ private final KeyConverter<String> stringKeyConverter =
+ new StringKeyConverter();
+
+ /**
+ * Instantiates a reader for multiple-entity reads.
+ *
+ * @param ctxt Reader context which defines the scope in which query has to be
+ * made.
+ * @param entityFilters Filters which limit the entities returned.
+ * @param toRetrieve Data to retrieve for each entity.
+ */
+ protected TimelineEntityReader(TimelineReaderContext ctxt,
+ TimelineEntityFilters entityFilters, TimelineDataToRetrieve toRetrieve) {
+ super(ctxt);
+ this.singleEntityRead = false;
+ this.dataToRetrieve = toRetrieve;
+ this.filters = entityFilters;
+
+ this.setTable(getTable());
+ }
+
+ /**
+ * Instantiates a reader for single-entity reads.
+ *
+ * @param ctxt Reader context which defines the scope in which query has to be
+ * made.
+ * @param toRetrieve Data to retrieve for each entity.
+ */
+ protected TimelineEntityReader(TimelineReaderContext ctxt,
+ TimelineDataToRetrieve toRetrieve) {
+ super(ctxt);
+ this.singleEntityRead = true;
+ this.dataToRetrieve = toRetrieve;
+
+ this.setTable(getTable());
+ }
+
+ /**
+ * Creates a {@link FilterList} based on fields, confs and metrics to
+ * retrieve. This filter list will be set in Scan/Get objects to trim down
+ * results fetched from HBase back-end storage. This is called only for
+ * multiple entity reads.
+ *
+ * @return a {@link FilterList} object.
+ * @throws IOException if any problem occurs while creating filter list.
+ */
+ protected abstract FilterList constructFilterListBasedOnFields()
+ throws IOException;
+
+ /**
+ * Creates a {@link FilterList} based on info, config and metric filters. This
+ * filter list will be set in HBase Get to trim down results fetched from
+ * HBase back-end storage.
+ *
+ * @return a {@link FilterList} object.
+ * @throws IOException if any problem occurs while creating filter list.
+ */
+ protected abstract FilterList constructFilterListBasedOnFilters()
+ throws IOException;
+
+ /**
+ * Combines filter lists created based on fields and based on filters.
+ *
+ * @return a {@link FilterList} object if it can be constructed. Returns null,
+ * if filter list cannot be created either on the basis of filters or on the
+ * basis of fields.
+ * @throws IOException if any problem occurs while creating filter list.
+ */
+ private FilterList createFilterList() throws IOException {
+ FilterList listBasedOnFilters = constructFilterListBasedOnFilters();
+ boolean hasListBasedOnFilters = listBasedOnFilters != null &&
+ !listBasedOnFilters.getFilters().isEmpty();
+ FilterList listBasedOnFields = constructFilterListBasedOnFields();
+ boolean hasListBasedOnFields = listBasedOnFields != null &&
+ !listBasedOnFields.getFilters().isEmpty();
+ // If filter lists based on both filters and fields can be created,
+ // combine them in a new filter list and return it.
+ // If either one of them has been created, return that filter list.
+ // Return null, if none of the filter lists can be created. This indicates
+ // that no filter list needs to be added to HBase Scan as filters are not
+ // specified for the query or only the default view of entity needs to be
+ // returned.
+ if (hasListBasedOnFilters && hasListBasedOnFields) {
+ FilterList list = new FilterList();
+ list.addFilter(listBasedOnFilters);
+ list.addFilter(listBasedOnFields);
+ return list;
+ } else if (hasListBasedOnFilters) {
+ return listBasedOnFilters;
+ } else if (hasListBasedOnFields) {
+ return listBasedOnFields;
+ }
+ return null;
+ }
+
+ protected TimelineDataToRetrieve getDataToRetrieve() {
+ return dataToRetrieve;
+ }
+
+ protected TimelineEntityFilters getFilters() {
+ return filters;
+ }
+
+ /**
+ * Create a {@link TimelineEntityFilters} object with default values for
+ * filters.
+ */
+ protected void createFiltersIfNull() {
+ if (filters == null) {
+ filters = new TimelineEntityFilters.Builder().build();
+ }
+ }
+
+ /**
+ * Reads and deserializes a single timeline entity from the HBase storage.
+ *
+ * @param hbaseConf HBase Configuration.
+ * @param conn HBase Connection.
+ * @return A <cite>TimelineEntity</cite> object.
+ * @throws IOException if there is any exception encountered while reading
+ * entity.
+ */
+ public TimelineEntity readEntity(Configuration hbaseConf, Connection conn)
+ throws IOException {
+ validateParams();
+ augmentParams(hbaseConf, conn);
+
+ FilterList filterList = constructFilterListBasedOnFields();
+ if (LOG.isDebugEnabled() && filterList != null) {
+ LOG.debug("FilterList created for get is - " + filterList);
+ }
+ Result result = getResult(hbaseConf, conn, filterList);
+ if (result == null || result.isEmpty()) {
+ // Could not find a matching row.
+ LOG.info("Cannot find matching entity of type " +
+ getContext().getEntityType());
+ return null;
+ }
+ return parseEntity(result);
+ }
+
+ /**
+ * Reads and deserializes a set of timeline entities from the HBase storage.
+ * It goes through all the results available, and returns the number of
+ * entries as specified in the limit in the entity's natural sort order.
+ *
+ * @param hbaseConf HBase Configuration.
+ * @param conn HBase Connection.
+ * @return a set of <cite>TimelineEntity</cite> objects.
+ * @throws IOException if any exception is encountered while reading entities.
+ */
+ public Set<TimelineEntity> readEntities(Configuration hbaseConf,
+ Connection conn) throws IOException {
+ validateParams();
+ augmentParams(hbaseConf, conn);
+
+ Set<TimelineEntity> entities = new LinkedHashSet<>();
+ FilterList filterList = createFilterList();
+ if (LOG.isDebugEnabled() && filterList != null) {
+ LOG.debug("FilterList created for scan is - " + filterList);
+ }
+ ResultScanner results = getResults(hbaseConf, conn, filterList);
+ try {
+ for (Result result : results) {
+ TimelineEntity entity = parseEntity(result);
+ if (entity == null) {
+ continue;
+ }
+ entities.add(entity);
+ if (entities.size() == filters.getLimit()) {
+ break;
+ }
+ }
+ return entities;
+ } finally {
+ results.close();
+ }
+ }
+
+ /**
+ * Returns the main table to be used by the entity reader.
+ *
+ * @return A reference to the table.
+ */
+ protected BaseTableRW<?> getTable() {
+ return table;
+ }
+
+ /**
+ * Fetches a {@link Result} instance for a single-entity read.
+ *
+ * @param hbaseConf HBase Configuration.
+ * @param conn HBase Connection.
+ * @param filterList filter list which will be applied to HBase Get.
+ * @return the {@link Result} instance or null if no such record is found.
+ * @throws IOException if any exception is encountered while getting result.
+ */
+ protected abstract Result getResult(Configuration hbaseConf, Connection conn,
+ FilterList filterList) throws IOException;
+
+ /**
+ * Fetches a {@link ResultScanner} for a multi-entity read.
+ *
+ * @param hbaseConf HBase Configuration.
+ * @param conn HBase Connection.
+ * @param filterList filter list which will be applied to HBase Scan.
+ * @return the {@link ResultScanner} instance.
+ * @throws IOException if any exception is encountered while getting results.
+ */
+ protected abstract ResultScanner getResults(Configuration hbaseConf,
+ Connection conn, FilterList filterList) throws IOException;
+
+ /**
+ * Parses the result retrieved from HBase backend and convert it into a
+ * {@link TimelineEntity} object.
+ *
+ * @param result Single row result of a Get/Scan.
+ * @return the <cite>TimelineEntity</cite> instance or null if the entity is
+ * filtered.
+ * @throws IOException if any exception is encountered while parsing entity.
+ */
+ protected abstract TimelineEntity parseEntity(Result result)
+ throws IOException;
+
+ /**
+ * Helper method for reading and deserializing {@link TimelineMetric} objects
+ * using the specified column prefix. The timeline metrics then are added to
+ * the given timeline entity.
+ *
+ * @param entity {@link TimelineEntity} object.
+ * @param result {@link Result} object retrieved from backend.
+ * @param columnPrefix Metric column prefix
+ * @throws IOException if any exception is encountered while reading metrics.
+ */
+ protected void readMetrics(TimelineEntity entity, Result result,
+ ColumnPrefix<?> columnPrefix) throws IOException {
+ NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
+ ColumnRWHelper.readResultsWithTimestamps(
+ result, columnPrefix, stringKeyConverter);
+ for (Map.Entry<String, NavigableMap<Long, Number>> metricResult:
+ metricsResult.entrySet()) {
+ TimelineMetric metric = new TimelineMetric();
+ metric.setId(metricResult.getKey());
+ // Simply assume that if the value set contains more than 1 elements, the
+ // metric is a TIME_SERIES metric, otherwise, it's a SINGLE_VALUE metric
+ TimelineMetric.Type metricType = metricResult.getValue().size() > 1 ?
+ TimelineMetric.Type.TIME_SERIES : TimelineMetric.Type.SINGLE_VALUE;
+ metric.setType(metricType);
+ metric.addValues(metricResult.getValue());
+ entity.addMetric(metric);
+ }
+ }
+
+ /**
+ * Checks whether the reader has been created to fetch single entity or
+ * multiple entities.
+ *
+ * @return true, if query is for single entity, false otherwise.
+ */
+ public boolean isSingleEntityRead() {
+ return singleEntityRead;
+ }
+
+ protected void setTable(BaseTableRW<?> baseTable) {
+ this.table = baseTable;
+ }
+
+ /**
+ * Check if we have a certain field amongst fields to retrieve. This method
+ * checks against {@link Field#ALL} as well because that would mean field
+ * passed needs to be matched.
+ *
+ * @param fieldsToRetrieve fields to be retrieved.
+ * @param requiredField fields to be checked in fieldsToRetrieve.
+ * @return true if has the required field, false otherwise.
+ */
+ protected boolean hasField(EnumSet<Field> fieldsToRetrieve,
+ Field requiredField) {
+ return fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(requiredField);
+ }
+
+ /**
+ * Create a filter list of qualifier filters based on passed set of columns.
+ *
+ * @param <T> Describes the type of column prefix.
+ * @param colPrefix Column Prefix.
+ * @param columns set of column qualifiers.
+ * @return filter list.
+ */
+ protected <T extends BaseTable<T>> FilterList
+ createFiltersFromColumnQualifiers(
+ ColumnPrefix<T> colPrefix, Set<String> columns) {
+ FilterList list = new FilterList(Operator.MUST_PASS_ONE);
+ for (String column : columns) {
+ // For columns which have compound column qualifiers (eg. events), we need
+ // to include the required separator.
+ byte[] compoundColQual = createColQualifierPrefix(colPrefix, column);
+ list.addFilter(new QualifierFilter(CompareOp.EQUAL,
+ new BinaryPrefixComparator(colPrefix
+ .getColumnPrefixBytes(compoundColQual))));
+ }
+ return list;
+ }
+
+ protected <T extends BaseTable<T>> byte[] createColQualifierPrefix(
+ ColumnPrefix<T> colPrefix, String column) {
+ if (colPrefix == ApplicationColumnPrefix.EVENT
+ || colPrefix == EntityColumnPrefix.EVENT) {
+ return new EventColumnName(column, null, null).getColumnQualifier();
+ } else {
+ return stringKeyConverter.encode(column);
+ }
+ }
+
+ /**
+ * Helper method for reading relationship.
+ *
+ * @param <T> Describes the type of column prefix.
+ * @param entity entity to fill.
+ * @param result result from HBase.
+ * @param prefix column prefix.
+ * @param isRelatedTo if true, means relationship is to be added to
+ * isRelatedTo, otherwise its added to relatesTo.
+ * @throws IOException if any problem is encountered while reading result.
+ */
+ protected <T extends BaseTable<T>> void readRelationship(
+ TimelineEntity entity, Result result,
+ ColumnPrefix<T> prefix, boolean isRelatedTo) throws IOException {
+ // isRelatedTo and relatesTo are of type Map<String, Set<String>>
+ Map<String, Object> columns = ColumnRWHelper.readResults(
+ result, prefix, stringKeyConverter);
+ for (Map.Entry<String, Object> column : columns.entrySet()) {
+ for (String id : Separator.VALUES.splitEncoded(column.getValue()
+ .toString())) {
+ if (isRelatedTo) {
+ entity.addIsRelatedToEntity(column.getKey(), id);
+ } else {
+ entity.addRelatesToEntity(column.getKey(), id);
+ }
+ }
+ }
+ }
+
+ /**
+ * Read events from the entity table or the application table. The column name
+ * is of the form "eventId=timestamp=infoKey" where "infoKey" may be omitted
+ * if there is no info associated with the event.
+ *
+ * @param <T> Describes the type of column prefix.
+ * @param entity entity to fill.
+ * @param result HBase Result.
+ * @param prefix column prefix.
+ * @throws IOException if any problem is encountered while reading result.
+ */
+ protected static <T extends BaseTable<T>> void readEvents(
+ TimelineEntity entity, Result result,
+ ColumnPrefix<T> prefix) throws IOException {
+ Map<String, TimelineEvent> eventsMap = new HashMap<>();
+ Map<EventColumnName, Object> eventsResult = ColumnRWHelper.readResults(
+ result, prefix, new EventColumnNameConverter());
+ for (Map.Entry<EventColumnName, Object>
+ eventResult : eventsResult.entrySet()) {
+ EventColumnName eventColumnName = eventResult.getKey();
+ String key = eventColumnName.getId() +
+ Long.toString(eventColumnName.getTimestamp());
+ // Retrieve previously seen event to add to it
+ TimelineEvent event = eventsMap.get(key);
+ if (event == null) {
+ // First time we're seeing this event, add it to the eventsMap
+ event = new TimelineEvent();
+ event.setId(eventColumnName.getId());
+ event.setTimestamp(eventColumnName.getTimestamp());
+ eventsMap.put(key, event);
+ }
+ if (eventColumnName.getInfoKey() != null) {
+ event.addInfo(eventColumnName.getInfoKey(), eventResult.getValue());
+ }
+ }
+ Set<TimelineEvent> eventsSet = new HashSet<>(eventsMap.values());
+ entity.addEvents(eventsSet);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
new file mode 100644
index 0000000..fa16077
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
+
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
+
+/**
+ * Factory methods for instantiating a timeline entity reader.
+ */
+public final class TimelineEntityReaderFactory {
+ private TimelineEntityReaderFactory() {
+ }
+
+ /**
+ * Creates a timeline entity reader instance for reading a single entity with
+ * the specified input.
+ *
+ * @param context Reader context which defines the scope in which query has to
+ * be made.
+ * @param dataToRetrieve Data to retrieve for each entity.
+ * @return An implementation of <cite>TimelineEntityReader</cite> object
+ * depending on entity type.
+ */
+ public static TimelineEntityReader createSingleEntityReader(
+ TimelineReaderContext context, TimelineDataToRetrieve dataToRetrieve) {
+ // currently the types that are handled separate from the generic entity
+ // table are application, flow run, and flow activity entities
+ if (TimelineEntityType.YARN_APPLICATION.matches(context.getEntityType())) {
+ return new ApplicationEntityReader(context, dataToRetrieve);
+ } else if (TimelineEntityType.
+ YARN_FLOW_RUN.matches(context.getEntityType())) {
+ return new FlowRunEntityReader(context, dataToRetrieve);
+ } else if (TimelineEntityType.
+ YARN_FLOW_ACTIVITY.matches(context.getEntityType())) {
+ return new FlowActivityEntityReader(context, dataToRetrieve);
+ } else {
+ // assume we're dealing with a generic entity read
+ return new GenericEntityReader(context, dataToRetrieve);
+ }
+ }
+
+ /**
+ * Creates a timeline entity reader instance for reading set of entities with
+ * the specified input and predicates.
+ *
+ * @param context Reader context which defines the scope in which query has to
+ * be made.
+ * @param filters Filters which limit the entities returned.
+ * @param dataToRetrieve Data to retrieve for each entity.
+ * @return An implementation of <cite>TimelineEntityReader</cite> object
+ * depending on entity type.
+ */
+ public static TimelineEntityReader createMultipleEntitiesReader(
+ TimelineReaderContext context, TimelineEntityFilters filters,
+ TimelineDataToRetrieve dataToRetrieve) {
+ // currently the types that are handled separate from the generic entity
+ // table are application, flow run, and flow activity entities
+ if (TimelineEntityType.YARN_APPLICATION.matches(context.getEntityType())) {
+ return new ApplicationEntityReader(context, filters, dataToRetrieve);
+ } else if (TimelineEntityType.
+ YARN_FLOW_ACTIVITY.matches(context.getEntityType())) {
+ return new FlowActivityEntityReader(context, filters, dataToRetrieve);
+ } else if (TimelineEntityType.
+ YARN_FLOW_RUN.matches(context.getEntityType())) {
+ return new FlowRunEntityReader(context, filters, dataToRetrieve);
+ } else {
+ if (context.getDoAsUser() != null) {
+ return new SubApplicationEntityReader(context, filters, dataToRetrieve);
+ }
+ // assume we're dealing with a generic entity read
+ return new GenericEntityReader(context, filters, dataToRetrieve);
+ }
+ }
+
+ /**
+ * Creates a timeline entity type reader that will read all available entity
+ * types within the specified context.
+ *
+ * @param context Reader context which defines the scope in which query has to
+ * be made. Limited to application level only.
+ * @return an <cite>EntityTypeReader</cite> object
+ */
+ public static EntityTypeReader createEntityTypeReader(
+ TimelineReaderContext context) {
+ return new EntityTypeReader(context);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/package-info.java
new file mode 100644
index 0000000..9814d6d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.storage.reader
+ * contains classes used to read entities from backend based on query type.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationTableRW.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationTableRW.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationTableRW.java
new file mode 100644
index 0000000..256b24b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationTableRW.java
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTableRW;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBaseSchemaConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Create, read and write to the SubApplication table.
+ */
+public class SubApplicationTableRW extends BaseTableRW<SubApplicationTable> {
+ /** sub app prefix. */
+ private static final String PREFIX =
+ YarnConfiguration.TIMELINE_SERVICE_PREFIX + "subapplication";
+
+ /** config param name that specifies the subapplication table name. */
+ public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
+
+ /**
+ * config param name that specifies the TTL for metrics column family in
+ * subapplication table.
+ */
+ private static final String METRICS_TTL_CONF_NAME = PREFIX
+ + ".table.metrics.ttl";
+
+ /**
+ * config param name that specifies max-versions for
+ * metrics column family in subapplication table.
+ */
+ private static final String METRICS_MAX_VERSIONS =
+ PREFIX + ".table.metrics.max-versions";
+
+ /** default value for subapplication table name. */
+ public static final String DEFAULT_TABLE_NAME =
+ "timelineservice.subapplication";
+
+ /** default TTL is 30 days for metrics timeseries. */
+ private static final int DEFAULT_METRICS_TTL = 2592000;
+
+ /** default max number of versions. */
+ private static final int DEFAULT_METRICS_MAX_VERSIONS = 10000;
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ SubApplicationTableRW.class);
+
+ public SubApplicationTableRW() {
+ super(TABLE_NAME_CONF_NAME, DEFAULT_TABLE_NAME);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.BaseTableRW#
+ * createTable(org.apache.hadoop.hbase.client.Admin,
+ * org.apache.hadoop.conf.Configuration)
+ */
+ public void createTable(Admin admin, Configuration hbaseConf)
+ throws IOException {
+
+ TableName table = getTableName(hbaseConf);
+ if (admin.tableExists(table)) {
+ // do not disable / delete existing table
+ // similar to the approach taken by map-reduce jobs when
+ // output directory exists
+ throw new IOException("Table " + table.getNameAsString()
+ + " already exists.");
+ }
+
+ HTableDescriptor subAppTableDescp = new HTableDescriptor(table);
+ HColumnDescriptor infoCF =
+ new HColumnDescriptor(SubApplicationColumnFamily.INFO.getBytes());
+ infoCF.setBloomFilterType(BloomType.ROWCOL);
+ subAppTableDescp.addFamily(infoCF);
+
+ HColumnDescriptor configCF =
+ new HColumnDescriptor(SubApplicationColumnFamily.CONFIGS.getBytes());
+ configCF.setBloomFilterType(BloomType.ROWCOL);
+ configCF.setBlockCacheEnabled(true);
+ subAppTableDescp.addFamily(configCF);
+
+ HColumnDescriptor metricsCF =
+ new HColumnDescriptor(SubApplicationColumnFamily.METRICS.getBytes());
+ subAppTableDescp.addFamily(metricsCF);
+ metricsCF.setBlockCacheEnabled(true);
+ // always keep 1 version (the latest)
+ metricsCF.setMinVersions(1);
+ metricsCF.setMaxVersions(
+ hbaseConf.getInt(METRICS_MAX_VERSIONS, DEFAULT_METRICS_MAX_VERSIONS));
+ metricsCF.setTimeToLive(hbaseConf.getInt(METRICS_TTL_CONF_NAME,
+ DEFAULT_METRICS_TTL));
+ subAppTableDescp.setRegionSplitPolicyClassName(
+ "org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy");
+ subAppTableDescp.setValue("KeyPrefixRegionSplitPolicy.prefix_length",
+ TimelineHBaseSchemaConstants.USERNAME_SPLIT_KEY_PREFIX_LENGTH);
+ admin.createTable(subAppTableDescp,
+ TimelineHBaseSchemaConstants.getUsernameSplits());
+ LOG.info("Status of table creation for " + table.getNameAsString() + "="
+ + admin.tableExists(table));
+ }
+
+ /**
+ * @param metricsTTL time to live parameter for the metricss in this table.
+ * @param hbaseConf configururation in which to set the metrics TTL config
+ * variable.
+ */
+ public void setMetricsTTL(int metricsTTL, Configuration hbaseConf) {
+ hbaseConf.setInt(METRICS_TTL_CONF_NAME, metricsTTL);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/package-info.java
new file mode 100644
index 0000000..52cc399
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication
+ * contains classes related to implementation for subapplication table.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java
new file mode 100644
index 0000000..402a89b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.junit.Test;
+
+/**
+ * Unit tests for HBaseTimelineStorageUtils static methos.
+ */
+public class TestHBaseTimelineStorageUtils {
+
+ @Test(expected=NullPointerException.class)
+ public void testGetTimelineServiceHBaseConfNullArgument() throws Exception {
+ HBaseTimelineStorageUtils.getTimelineServiceHBaseConf(null);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml
new file mode 100644
index 0000000..cb0d6e2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml
@@ -0,0 +1,132 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+ http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>hadoop-yarn-server-timelineservice-hbase</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <version>3.2.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>hadoop-yarn-server-timelineservice-hbase-common</artifactId>
+ <name>Apache Hadoop YARN TimelineService HBase Common</name>
+ <version>3.2.0-SNAPSHOT</version>
+
+ <properties>
+ <!-- Needed for generating FindBugs warnings using parent pom -->
+ <yarn.basedir>${project.parent.parent.parent.basedir}</yarn.basedir>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-timelineservice</artifactId>
+ </dependency>
+
+ <!-- This is needed for GenericObjectMapper in GenericConverter -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-applicationhistoryservice</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- 'mvn dependency:analyze' fails to detect use of this direct
+ dependency -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ <phase>test-compile</phase>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <includeScope>runtime</includeScope>
+ <excludeGroupIds>org.slf4j,org.apache.hadoop,com.github.stephenc.findbugs</excludeGroupIds>
+ <outputDirectory>${project.build.directory}/lib</outputDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
new file mode 100644
index 0000000..c3d6a52
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.application;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Column;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.GenericConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
+
+/**
+ * Identifies fully qualified columns for the {@link ApplicationTable}.
+ */
+public enum ApplicationColumn implements Column<ApplicationTable> {
+
+ /**
+ * App id.
+ */
+ ID(ApplicationColumnFamily.INFO, "id"),
+
+ /**
+ * When the application was created.
+ */
+ CREATED_TIME(ApplicationColumnFamily.INFO, "created_time",
+ new LongConverter()),
+
+ /**
+ * The version of the flow that this app belongs to.
+ */
+ FLOW_VERSION(ApplicationColumnFamily.INFO, "flow_version");
+
+ private final ColumnFamily<ApplicationTable> columnFamily;
+ private final String columnQualifier;
+ private final byte[] columnQualifierBytes;
+ private final ValueConverter valueConverter;
+
+ private ApplicationColumn(ColumnFamily<ApplicationTable> columnFamily,
+ String columnQualifier) {
+ this(columnFamily, columnQualifier, GenericConverter.getInstance());
+ }
+
+ private ApplicationColumn(ColumnFamily<ApplicationTable> columnFamily,
+ String columnQualifier, ValueConverter converter) {
+ this.columnFamily = columnFamily;
+ this.columnQualifier = columnQualifier;
+ // Future-proof by ensuring the right column prefix hygiene.
+ this.columnQualifierBytes =
+ Bytes.toBytes(Separator.SPACE.encode(columnQualifier));
+ this.valueConverter = converter;
+ }
+
+ /**
+ * @return the column name value
+ */
+ private String getColumnQualifier() {
+ return columnQualifier;
+ }
+
+ @Override
+ public byte[] getColumnQualifierBytes() {
+ return columnQualifierBytes.clone();
+ }
+
+ @Override
+ public byte[] getColumnFamilyBytes() {
+ return columnFamily.getBytes();
+ }
+
+ @Override
+ public ValueConverter getValueConverter() {
+ return valueConverter;
+ }
+
+ @Override
+ public Attribute[] getCombinedAttrsWithAggr(Attribute... attributes) {
+ return attributes;
+ }
+
+ @Override
+ public boolean supplementCellTimestamp() {
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnFamily.java
new file mode 100644
index 0000000..97e5f7b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnFamily.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.application;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+
+/**
+ * Represents the application table column families.
+ */
+public enum ApplicationColumnFamily implements ColumnFamily<ApplicationTable> {
+
+ /**
+ * Info column family houses known columns, specifically ones included in
+ * columnfamily filters.
+ */
+ INFO("i"),
+
+ /**
+ * Configurations are in a separate column family for two reasons: a) the size
+ * of the config values can be very large and b) we expect that config values
+ * are often separately accessed from other metrics and info columns.
+ */
+ CONFIGS("c"),
+
+ /**
+ * Metrics have a separate column family, because they have a separate TTL.
+ */
+ METRICS("m");
+
+ /**
+ * Byte representation of this column family.
+ */
+ private final byte[] bytes;
+
+ /**
+ * @param value create a column family with this name. Must be lower case and
+ * without spaces.
+ */
+ private ApplicationColumnFamily(String value) {
+ // column families should be lower case and not contain any spaces.
+ this.bytes = Bytes.toBytes(Separator.SPACE.encode(value));
+ }
+
+ public byte[] getBytes() {
+ return Bytes.copy(bytes);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
new file mode 100644
index 0000000..89412f4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.application;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.GenericConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
+
+/**
+ * Identifies partially qualified columns for the application table.
+ */
+public enum ApplicationColumnPrefix implements ColumnPrefix<ApplicationTable> {
+
+ /**
+ * To store TimelineEntity getIsRelatedToEntities values.
+ */
+ IS_RELATED_TO(ApplicationColumnFamily.INFO, "s"),
+
+ /**
+ * To store TimelineEntity getRelatesToEntities values.
+ */
+ RELATES_TO(ApplicationColumnFamily.INFO, "r"),
+
+ /**
+ * To store TimelineEntity info values.
+ */
+ INFO(ApplicationColumnFamily.INFO, "i"),
+
+ /**
+ * Lifecycle events for an application.
+ */
+ EVENT(ApplicationColumnFamily.INFO, "e"),
+
+ /**
+ * Config column stores configuration with config key as the column name.
+ */
+ CONFIG(ApplicationColumnFamily.CONFIGS, null),
+
+ /**
+ * Metrics are stored with the metric name as the column name.
+ */
+ METRIC(ApplicationColumnFamily.METRICS, null, new LongConverter());
+
+ private final ColumnFamily<ApplicationTable> columnFamily;
+
+ /**
+ * Can be null for those cases where the provided column qualifier is the
+ * entire column name.
+ */
+ private final String columnPrefix;
+ private final byte[] columnPrefixBytes;
+ private final ValueConverter valueConverter;
+
+ /**
+ * Private constructor, meant to be used by the enum definition.
+ *
+ * @param columnFamily that this column is stored in.
+ * @param columnPrefix for this column.
+ */
+ private ApplicationColumnPrefix(ColumnFamily<ApplicationTable> columnFamily,
+ String columnPrefix) {
+ this(columnFamily, columnPrefix, GenericConverter.getInstance());
+ }
+
+ /**
+ * Private constructor, meant to be used by the enum definition.
+ *
+ * @param columnFamily that this column is stored in.
+ * @param columnPrefix for this column.
+ * @param converter used to encode/decode values to be stored in HBase for
+ * this column prefix.
+ */
+ private ApplicationColumnPrefix(ColumnFamily<ApplicationTable> columnFamily,
+ String columnPrefix, ValueConverter converter) {
+ this.valueConverter = converter;
+ this.columnFamily = columnFamily;
+ this.columnPrefix = columnPrefix;
+ if (columnPrefix == null) {
+ this.columnPrefixBytes = null;
+ } else {
+ // Future-proof by ensuring the right column prefix hygiene.
+ this.columnPrefixBytes =
+ Bytes.toBytes(Separator.SPACE.encode(columnPrefix));
+ }
+ }
+
+ /**
+ * @return the column name value
+ */
+ private String getColumnPrefix() {
+ return columnPrefix;
+ }
+
+ @Override
+ public byte[] getColumnPrefixBytes(byte[] qualifierPrefix) {
+ return ColumnHelper.getColumnQualifier(
+ this.columnPrefixBytes, qualifierPrefix);
+ }
+
+ @Override
+ public byte[] getColumnPrefixBytes(String qualifierPrefix) {
+ return ColumnHelper.getColumnQualifier(
+ this.columnPrefixBytes, qualifierPrefix);
+ }
+
+ @Override
+ public byte[] getColumnFamilyBytes() {
+ return columnFamily.getBytes();
+ }
+
+ @Override
+ public byte[] getColumnPrefixInBytes() {
+ return columnPrefixBytes != null ? columnPrefixBytes.clone() : null;
+ }
+
+ @Override
+ public Attribute[] getCombinedAttrsWithAggr(Attribute... attributes) {
+ return attributes;
+ }
+
+ @Override
+ public boolean supplementCellTimeStamp() {
+ return false;
+ }
+
+ public ValueConverter getValueConverter() {
+ return valueConverter;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/18] hadoop git commit: YARN-7919. Refactor timelineservice-hbase
module into submodules. Contributed by Haibo Chen.
Posted by ro...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
new file mode 100644
index 0000000..7440316
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
@@ -0,0 +1,523 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Query;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.BinaryComparator;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.filter.FamilyFilter;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.FilterList.Operator;
+import org.apache.hadoop.hbase.filter.PageFilter;
+import org.apache.hadoop.hbase.filter.QualifierFilter;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKeyPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTableRW;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTableRW;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnRWHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.RowKeyPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
+import org.apache.hadoop.yarn.webapp.BadRequestException;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Timeline entity reader for application entities that are stored in the
+ * application table.
+ */
+class ApplicationEntityReader extends GenericEntityReader {
+ private static final ApplicationTableRW APPLICATION_TABLE =
+ new ApplicationTableRW();
+
+ ApplicationEntityReader(TimelineReaderContext ctxt,
+ TimelineEntityFilters entityFilters, TimelineDataToRetrieve toRetrieve) {
+ super(ctxt, entityFilters, toRetrieve);
+ }
+
+ ApplicationEntityReader(TimelineReaderContext ctxt,
+ TimelineDataToRetrieve toRetrieve) {
+ super(ctxt, toRetrieve);
+ }
+
+ /**
+ * Uses the {@link ApplicationTableRW}.
+ */
+ protected BaseTableRW<?> getTable() {
+ return APPLICATION_TABLE;
+ }
+
+ /**
+ * This method is called only for multiple entity reads.
+ */
+ @Override
+ protected FilterList constructFilterListBasedOnFilters() throws IOException {
+ // Filters here cannot be null for multiple entity reads as they are set in
+ // augmentParams if null.
+ TimelineEntityFilters filters = getFilters();
+ FilterList listBasedOnFilters = new FilterList();
+ // Create filter list based on created time range and add it to
+ // listBasedOnFilters.
+ long createdTimeBegin = filters.getCreatedTimeBegin();
+ long createdTimeEnd = filters.getCreatedTimeEnd();
+ if (createdTimeBegin != 0 || createdTimeEnd != Long.MAX_VALUE) {
+ listBasedOnFilters.addFilter(
+ TimelineFilterUtils.createSingleColValueFiltersByRange(
+ ApplicationColumn.CREATED_TIME, createdTimeBegin, createdTimeEnd));
+ }
+ // Create filter list based on metric filters and add it to
+ // listBasedOnFilters.
+ TimelineFilterList metricFilters = filters.getMetricFilters();
+ if (metricFilters != null && !metricFilters.getFilterList().isEmpty()) {
+ listBasedOnFilters.addFilter(
+ TimelineFilterUtils.createHBaseFilterList(
+ ApplicationColumnPrefix.METRIC, metricFilters));
+ }
+ // Create filter list based on config filters and add it to
+ // listBasedOnFilters.
+ TimelineFilterList configFilters = filters.getConfigFilters();
+ if (configFilters != null && !configFilters.getFilterList().isEmpty()) {
+ listBasedOnFilters.addFilter(
+ TimelineFilterUtils.createHBaseFilterList(
+ ApplicationColumnPrefix.CONFIG, configFilters));
+ }
+ // Create filter list based on info filters and add it to listBasedOnFilters
+ TimelineFilterList infoFilters = filters.getInfoFilters();
+ if (infoFilters != null && !infoFilters.getFilterList().isEmpty()) {
+ listBasedOnFilters.addFilter(
+ TimelineFilterUtils.createHBaseFilterList(
+ ApplicationColumnPrefix.INFO, infoFilters));
+ }
+ return listBasedOnFilters;
+ }
+
+ /**
+ * Add {@link QualifierFilter} filters to filter list for each column of
+ * application table.
+ *
+ * @param list filter list to which qualifier filters have to be added.
+ */
+ @Override
+ protected void updateFixedColumns(FilterList list) {
+ for (ApplicationColumn column : ApplicationColumn.values()) {
+ list.addFilter(new QualifierFilter(CompareOp.EQUAL,
+ new BinaryComparator(column.getColumnQualifierBytes())));
+ }
+ }
+
+ /**
+ * Creates a filter list which indicates that only some of the column
+ * qualifiers in the info column family will be returned in result.
+ *
+ * @return filter list.
+ * @throws IOException if any problem occurs while creating filter list.
+ */
+ private FilterList createFilterListForColsOfInfoFamily()
+ throws IOException {
+ FilterList infoFamilyColsFilter = new FilterList(Operator.MUST_PASS_ONE);
+ // Add filters for each column in entity table.
+ updateFixedColumns(infoFamilyColsFilter);
+ EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
+ // If INFO field has to be retrieved, add a filter for fetching columns
+ // with INFO column prefix.
+ if (hasField(fieldsToRetrieve, Field.INFO)) {
+ infoFamilyColsFilter.addFilter(
+ TimelineFilterUtils.createHBaseQualifierFilter(
+ CompareOp.EQUAL, ApplicationColumnPrefix.INFO));
+ }
+ TimelineFilterList relatesTo = getFilters().getRelatesTo();
+ if (hasField(fieldsToRetrieve, Field.RELATES_TO)) {
+ // If RELATES_TO field has to be retrieved, add a filter for fetching
+ // columns with RELATES_TO column prefix.
+ infoFamilyColsFilter.addFilter(
+ TimelineFilterUtils.createHBaseQualifierFilter(
+ CompareOp.EQUAL, ApplicationColumnPrefix.RELATES_TO));
+ } else if (relatesTo != null && !relatesTo.getFilterList().isEmpty()) {
+ // Even if fields to retrieve does not contain RELATES_TO, we still
+ // need to have a filter to fetch some of the column qualifiers if
+ // relatesTo filters are specified. relatesTo filters will then be
+ // matched after fetching rows from HBase.
+ Set<String> relatesToCols =
+ TimelineFilterUtils.fetchColumnsFromFilterList(relatesTo);
+ infoFamilyColsFilter.addFilter(createFiltersFromColumnQualifiers(
+ ApplicationColumnPrefix.RELATES_TO, relatesToCols));
+ }
+ TimelineFilterList isRelatedTo = getFilters().getIsRelatedTo();
+ if (hasField(fieldsToRetrieve, Field.IS_RELATED_TO)) {
+ // If IS_RELATED_TO field has to be retrieved, add a filter for fetching
+ // columns with IS_RELATED_TO column prefix.
+ infoFamilyColsFilter.addFilter(
+ TimelineFilterUtils.createHBaseQualifierFilter(
+ CompareOp.EQUAL, ApplicationColumnPrefix.IS_RELATED_TO));
+ } else if (isRelatedTo != null && !isRelatedTo.getFilterList().isEmpty()) {
+ // Even if fields to retrieve does not contain IS_RELATED_TO, we still
+ // need to have a filter to fetch some of the column qualifiers if
+ // isRelatedTo filters are specified. isRelatedTo filters will then be
+ // matched after fetching rows from HBase.
+ Set<String> isRelatedToCols =
+ TimelineFilterUtils.fetchColumnsFromFilterList(isRelatedTo);
+ infoFamilyColsFilter.addFilter(createFiltersFromColumnQualifiers(
+ ApplicationColumnPrefix.IS_RELATED_TO, isRelatedToCols));
+ }
+ TimelineFilterList eventFilters = getFilters().getEventFilters();
+ if (hasField(fieldsToRetrieve, Field.EVENTS)) {
+ // If EVENTS field has to be retrieved, add a filter for fetching columns
+ // with EVENT column prefix.
+ infoFamilyColsFilter.addFilter(
+ TimelineFilterUtils.createHBaseQualifierFilter(
+ CompareOp.EQUAL, ApplicationColumnPrefix.EVENT));
+ } else if (eventFilters != null && !eventFilters.getFilterList().isEmpty()){
+ // Even if fields to retrieve does not contain EVENTS, we still need to
+ // have a filter to fetch some of the column qualifiers on the basis of
+ // event filters specified. Event filters will then be matched after
+ // fetching rows from HBase.
+ Set<String> eventCols =
+ TimelineFilterUtils.fetchColumnsFromFilterList(eventFilters);
+ infoFamilyColsFilter.addFilter(createFiltersFromColumnQualifiers(
+ ApplicationColumnPrefix.EVENT, eventCols));
+ }
+ return infoFamilyColsFilter;
+ }
+
+ /**
+ * Exclude column prefixes via filters which are not required(based on fields
+ * to retrieve) from info column family. These filters are added to filter
+ * list which contains a filter for getting info column family.
+ *
+ * @param infoColFamilyList filter list for info column family.
+ */
+ private void excludeFieldsFromInfoColFamily(FilterList infoColFamilyList) {
+ EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
+ // Events not required.
+ if (!hasField(fieldsToRetrieve, Field.EVENTS)) {
+ infoColFamilyList.addFilter(
+ TimelineFilterUtils.createHBaseQualifierFilter(
+ CompareOp.NOT_EQUAL, ApplicationColumnPrefix.EVENT));
+ }
+ // info not required.
+ if (!hasField(fieldsToRetrieve, Field.INFO)) {
+ infoColFamilyList.addFilter(
+ TimelineFilterUtils.createHBaseQualifierFilter(
+ CompareOp.NOT_EQUAL, ApplicationColumnPrefix.INFO));
+ }
+ // is related to not required.
+ if (!hasField(fieldsToRetrieve, Field.IS_RELATED_TO)) {
+ infoColFamilyList.addFilter(
+ TimelineFilterUtils.createHBaseQualifierFilter(
+ CompareOp.NOT_EQUAL, ApplicationColumnPrefix.IS_RELATED_TO));
+ }
+ // relates to not required.
+ if (!hasField(fieldsToRetrieve, Field.RELATES_TO)) {
+ infoColFamilyList.addFilter(
+ TimelineFilterUtils.createHBaseQualifierFilter(
+ CompareOp.NOT_EQUAL, ApplicationColumnPrefix.RELATES_TO));
+ }
+ }
+
+ /**
+ * Updates filter list based on fields for confs and metrics to retrieve.
+ *
+ * @param listBasedOnFields filter list based on fields.
+ * @throws IOException if any problem occurs while updating filter list.
+ */
+ private void updateFilterForConfsAndMetricsToRetrieve(
+ FilterList listBasedOnFields) throws IOException {
+ TimelineDataToRetrieve dataToRetrieve = getDataToRetrieve();
+ // Please note that if confsToRetrieve is specified, we would have added
+ // CONFS to fields to retrieve in augmentParams() even if not specified.
+ if (dataToRetrieve.getFieldsToRetrieve().contains(Field.CONFIGS)) {
+ // Create a filter list for configs.
+ listBasedOnFields.addFilter(TimelineFilterUtils.
+ createFilterForConfsOrMetricsToRetrieve(
+ dataToRetrieve.getConfsToRetrieve(),
+ ApplicationColumnFamily.CONFIGS, ApplicationColumnPrefix.CONFIG));
+ }
+
+ // Please note that if metricsToRetrieve is specified, we would have added
+ // METRICS to fields to retrieve in augmentParams() even if not specified.
+ if (dataToRetrieve.getFieldsToRetrieve().contains(Field.METRICS)) {
+ // Create a filter list for metrics.
+ listBasedOnFields.addFilter(TimelineFilterUtils.
+ createFilterForConfsOrMetricsToRetrieve(
+ dataToRetrieve.getMetricsToRetrieve(),
+ ApplicationColumnFamily.METRICS, ApplicationColumnPrefix.METRIC));
+ }
+ }
+
+ @Override
+ protected FilterList constructFilterListBasedOnFields() throws IOException {
+ if (!needCreateFilterListBasedOnFields()) {
+ // Fetch all the columns. No need of a filter.
+ return null;
+ }
+ FilterList listBasedOnFields = new FilterList(Operator.MUST_PASS_ONE);
+ FilterList infoColFamilyList = new FilterList();
+ // By default fetch everything in INFO column family.
+ FamilyFilter infoColumnFamily =
+ new FamilyFilter(CompareOp.EQUAL,
+ new BinaryComparator(ApplicationColumnFamily.INFO.getBytes()));
+ infoColFamilyList.addFilter(infoColumnFamily);
+ if (!isSingleEntityRead() && fetchPartialColsFromInfoFamily()) {
+ // We can fetch only some of the columns from info family.
+ infoColFamilyList.addFilter(createFilterListForColsOfInfoFamily());
+ } else {
+ // Exclude column prefixes in info column family which are not required
+ // based on fields to retrieve.
+ excludeFieldsFromInfoColFamily(infoColFamilyList);
+ }
+ listBasedOnFields.addFilter(infoColFamilyList);
+
+ updateFilterForConfsAndMetricsToRetrieve(listBasedOnFields);
+ return listBasedOnFields;
+ }
+
+ @Override
+ protected Result getResult(Configuration hbaseConf, Connection conn,
+ FilterList filterList) throws IOException {
+ TimelineReaderContext context = getContext();
+ ApplicationRowKey applicationRowKey =
+ new ApplicationRowKey(context.getClusterId(), context.getUserId(),
+ context.getFlowName(), context.getFlowRunId(), context.getAppId());
+ byte[] rowKey = applicationRowKey.getRowKey();
+ Get get = new Get(rowKey);
+ // Set time range for metric values.
+ setMetricsTimeRange(get);
+ get.setMaxVersions(getDataToRetrieve().getMetricsLimit());
+ if (filterList != null && !filterList.getFilters().isEmpty()) {
+ get.setFilter(filterList);
+ }
+ return getTable().getResult(hbaseConf, conn, get);
+ }
+
+ @Override
+ protected void validateParams() {
+ Preconditions.checkNotNull(getContext(), "context shouldn't be null");
+ Preconditions.checkNotNull(
+ getDataToRetrieve(), "data to retrieve shouldn't be null");
+ Preconditions.checkNotNull(getContext().getClusterId(),
+ "clusterId shouldn't be null");
+ Preconditions.checkNotNull(getContext().getEntityType(),
+ "entityType shouldn't be null");
+ if (isSingleEntityRead()) {
+ Preconditions.checkNotNull(getContext().getAppId(),
+ "appId shouldn't be null");
+ } else {
+ Preconditions.checkNotNull(getContext().getUserId(),
+ "userId shouldn't be null");
+ Preconditions.checkNotNull(getContext().getFlowName(),
+ "flowName shouldn't be null");
+ }
+ }
+
+ @Override
+ protected void augmentParams(Configuration hbaseConf, Connection conn)
+ throws IOException {
+ if (isSingleEntityRead()) {
+ // Get flow context information from AppToFlow table.
+ defaultAugmentParams(hbaseConf, conn);
+ }
+ // Add configs/metrics to fields to retrieve if confsToRetrieve and/or
+ // metricsToRetrieve are specified.
+ getDataToRetrieve().addFieldsBasedOnConfsAndMetricsToRetrieve();
+ if (!isSingleEntityRead()) {
+ createFiltersIfNull();
+ }
+ }
+
+ private void setMetricsTimeRange(Query query) {
+ // Set time range for metric values.
+ HBaseTimelineStorageUtils.setMetricsTimeRange(
+ query, ApplicationColumnFamily.METRICS.getBytes(),
+ getDataToRetrieve().getMetricsTimeBegin(),
+ getDataToRetrieve().getMetricsTimeEnd());
+ }
+
+ @Override
+ protected ResultScanner getResults(Configuration hbaseConf,
+ Connection conn, FilterList filterList) throws IOException {
+ Scan scan = new Scan();
+ TimelineReaderContext context = getContext();
+ RowKeyPrefix<ApplicationRowKey> applicationRowKeyPrefix = null;
+
+ // Whether or not flowRunID is null doesn't matter, the
+ // ApplicationRowKeyPrefix will do the right thing.
+ // default mode, will always scans from beginning of entity type.
+ if (getFilters().getFromId() == null) {
+ applicationRowKeyPrefix = new ApplicationRowKeyPrefix(
+ context.getClusterId(), context.getUserId(), context.getFlowName(),
+ context.getFlowRunId());
+ scan.setRowPrefixFilter(applicationRowKeyPrefix.getRowKeyPrefix());
+ } else {
+ ApplicationRowKey applicationRowKey = null;
+ try {
+ applicationRowKey =
+ ApplicationRowKey.parseRowKeyFromString(getFilters().getFromId());
+ } catch (IllegalArgumentException e) {
+ throw new BadRequestException("Invalid filter fromid is provided.");
+ }
+ if (!context.getClusterId().equals(applicationRowKey.getClusterId())) {
+ throw new BadRequestException(
+ "fromid doesn't belong to clusterId=" + context.getClusterId());
+ }
+
+ // set start row
+ scan.setStartRow(applicationRowKey.getRowKey());
+
+ // get the bytes for stop row
+ applicationRowKeyPrefix = new ApplicationRowKeyPrefix(
+ context.getClusterId(), context.getUserId(), context.getFlowName(),
+ context.getFlowRunId());
+
+ // set stop row
+ scan.setStopRow(
+ HBaseTimelineStorageUtils.calculateTheClosestNextRowKeyForPrefix(
+ applicationRowKeyPrefix.getRowKeyPrefix()));
+ }
+
+ FilterList newList = new FilterList();
+ newList.addFilter(new PageFilter(getFilters().getLimit()));
+ if (filterList != null && !filterList.getFilters().isEmpty()) {
+ newList.addFilter(filterList);
+ }
+ scan.setFilter(newList);
+
+ // Set time range for metric values.
+ setMetricsTimeRange(scan);
+ scan.setMaxVersions(getDataToRetrieve().getMetricsLimit());
+ return getTable().getResultScanner(hbaseConf, conn, scan);
+ }
+
+ @Override
+ protected TimelineEntity parseEntity(Result result) throws IOException {
+ if (result == null || result.isEmpty()) {
+ return null;
+ }
+ TimelineEntity entity = new TimelineEntity();
+ entity.setType(TimelineEntityType.YARN_APPLICATION.toString());
+ String entityId =
+ ColumnRWHelper.readResult(result, ApplicationColumn.ID).toString();
+ entity.setId(entityId);
+
+ TimelineEntityFilters filters = getFilters();
+ // fetch created time
+ Long createdTime = (Long) ColumnRWHelper.readResult(result,
+ ApplicationColumn.CREATED_TIME);
+ entity.setCreatedTime(createdTime);
+
+ EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
+ // fetch is related to entities and match isRelatedTo filter. If isRelatedTo
+ // filters do not match, entity would be dropped. We have to match filters
+ // locally as relevant HBase filters to filter out rows on the basis of
+ // isRelatedTo are not set in HBase scan.
+ boolean checkIsRelatedTo =
+ !isSingleEntityRead() && filters.getIsRelatedTo() != null &&
+ filters.getIsRelatedTo().getFilterList().size() > 0;
+ if (hasField(fieldsToRetrieve, Field.IS_RELATED_TO) || checkIsRelatedTo) {
+ readRelationship(entity, result, ApplicationColumnPrefix.IS_RELATED_TO,
+ true);
+ if (checkIsRelatedTo && !TimelineStorageUtils.matchIsRelatedTo(entity,
+ filters.getIsRelatedTo())) {
+ return null;
+ }
+ if (!hasField(fieldsToRetrieve,
+ Field.IS_RELATED_TO)) {
+ entity.getIsRelatedToEntities().clear();
+ }
+ }
+
+ // fetch relates to entities and match relatesTo filter. If relatesTo
+ // filters do not match, entity would be dropped. We have to match filters
+ // locally as relevant HBase filters to filter out rows on the basis of
+ // relatesTo are not set in HBase scan.
+ boolean checkRelatesTo =
+ !isSingleEntityRead() && filters.getRelatesTo() != null &&
+ filters.getRelatesTo().getFilterList().size() > 0;
+ if (hasField(fieldsToRetrieve, Field.RELATES_TO) ||
+ checkRelatesTo) {
+ readRelationship(entity, result, ApplicationColumnPrefix.RELATES_TO,
+ false);
+ if (checkRelatesTo && !TimelineStorageUtils.matchRelatesTo(entity,
+ filters.getRelatesTo())) {
+ return null;
+ }
+ if (!hasField(fieldsToRetrieve, Field.RELATES_TO)) {
+ entity.getRelatesToEntities().clear();
+ }
+ }
+
+ // fetch info if fieldsToRetrieve contains INFO or ALL.
+ if (hasField(fieldsToRetrieve, Field.INFO)) {
+ readKeyValuePairs(entity, result, ApplicationColumnPrefix.INFO, false);
+ }
+
+ // fetch configs if fieldsToRetrieve contains CONFIGS or ALL.
+ if (hasField(fieldsToRetrieve, Field.CONFIGS)) {
+ readKeyValuePairs(entity, result, ApplicationColumnPrefix.CONFIG, true);
+ }
+
+ // fetch events and match event filters if they exist. If event filters do
+ // not match, entity would be dropped. We have to match filters locally
+ // as relevant HBase filters to filter out rows on the basis of events
+ // are not set in HBase scan.
+ boolean checkEvents =
+ !isSingleEntityRead() && filters.getEventFilters() != null &&
+ filters.getEventFilters().getFilterList().size() > 0;
+ if (hasField(fieldsToRetrieve, Field.EVENTS) || checkEvents) {
+ readEvents(entity, result, ApplicationColumnPrefix.EVENT);
+ if (checkEvents && !TimelineStorageUtils.matchEventFilters(entity,
+ filters.getEventFilters())) {
+ return null;
+ }
+ if (!hasField(fieldsToRetrieve, Field.EVENTS)) {
+ entity.getEvents().clear();
+ }
+ }
+
+ // fetch metrics if fieldsToRetrieve contains METRICS or ALL.
+ if (hasField(fieldsToRetrieve, Field.METRICS)) {
+ readMetrics(entity, result, ApplicationColumnPrefix.METRIC);
+ }
+
+ ApplicationRowKey rowKey = ApplicationRowKey.parseRowKey(result.getRow());
+ entity.getInfo().put(TimelineReaderUtils.FROMID_KEY,
+ rowKey.getRowKeyAsString());
+ return entity;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/EntityTypeReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/EntityTypeReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/EntityTypeReader.java
new file mode 100644
index 0000000..ebe21a4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/EntityTypeReader.java
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
+import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
+import org.apache.hadoop.hbase.filter.PageFilter;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKeyPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTableRW;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * Timeline entity reader for listing all available entity types given one
+ * reader context. Right now only supports listing all entity types within one
+ * YARN application.
+ */
+public final class EntityTypeReader extends AbstractTimelineStorageReader {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(EntityTypeReader.class);
+ private static final EntityTableRW ENTITY_TABLE = new EntityTableRW();
+
+ public EntityTypeReader(TimelineReaderContext context) {
+ super(context);
+ }
+
+ /**
+ * Reads a set of timeline entity types from the HBase storage for the given
+ * context.
+ *
+ * @param hbaseConf HBase Configuration.
+ * @param conn HBase Connection.
+ * @return a set of <cite>TimelineEntity</cite> objects, with only type field
+ * set.
+ * @throws IOException if any exception is encountered while reading entities.
+ */
+ public Set<String> readEntityTypes(Configuration hbaseConf,
+ Connection conn) throws IOException {
+
+ validateParams();
+ augmentParams(hbaseConf, conn);
+
+ Set<String> types = new TreeSet<>();
+ TimelineReaderContext context = getContext();
+ EntityRowKeyPrefix prefix = new EntityRowKeyPrefix(context.getClusterId(),
+ context.getUserId(), context.getFlowName(), context.getFlowRunId(),
+ context.getAppId());
+ byte[] currRowKey = prefix.getRowKeyPrefix();
+ byte[] nextRowKey = prefix.getRowKeyPrefix();
+ nextRowKey[nextRowKey.length - 1]++;
+
+ FilterList typeFilterList = new FilterList();
+ typeFilterList.addFilter(new FirstKeyOnlyFilter());
+ typeFilterList.addFilter(new KeyOnlyFilter());
+ typeFilterList.addFilter(new PageFilter(1));
+ LOG.debug("FilterList created for scan is - {}", typeFilterList);
+
+ int counter = 0;
+ while (true) {
+ try (ResultScanner results =
+ getResult(hbaseConf, conn, typeFilterList, currRowKey, nextRowKey)) {
+ TimelineEntity entity = parseEntityForType(results.next());
+ if (entity == null) {
+ break;
+ }
+ ++counter;
+ if (!types.add(entity.getType())) {
+ LOG.warn("Failed to add type " + entity.getType()
+ + " to the result set because there is a duplicated copy. ");
+ }
+ String currType = entity.getType();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Current row key: " + Arrays.toString(currRowKey));
+ LOG.debug("New entity type discovered: " + currType);
+ }
+ currRowKey = getNextRowKey(prefix.getRowKeyPrefix(), currType);
+ }
+ }
+ LOG.debug("Scanned {} records for {} types", counter, types.size());
+ return types;
+ }
+
+ @Override
+ protected void validateParams() {
+ Preconditions.checkNotNull(getContext(), "context shouldn't be null");
+ Preconditions.checkNotNull(getContext().getClusterId(),
+ "clusterId shouldn't be null");
+ Preconditions.checkNotNull(getContext().getAppId(),
+ "appId shouldn't be null");
+ }
+
+ /**
+ * Gets the possibly next row key prefix given current prefix and type.
+ *
+ * @param currRowKeyPrefix The current prefix that contains user, cluster,
+ * flow, run, and application id.
+ * @param entityType Current entity type.
+ * @return A new prefix for the possibly immediately next row key.
+ */
+ private static byte[] getNextRowKey(byte[] currRowKeyPrefix,
+ String entityType) {
+ if (currRowKeyPrefix == null || entityType == null) {
+ return null;
+ }
+
+ byte[] entityTypeEncoded = Separator.QUALIFIERS.join(
+ Separator.encode(entityType, Separator.SPACE, Separator.TAB,
+ Separator.QUALIFIERS),
+ Separator.EMPTY_BYTES);
+
+ byte[] currRowKey
+ = new byte[currRowKeyPrefix.length + entityTypeEncoded.length];
+ System.arraycopy(currRowKeyPrefix, 0, currRowKey, 0,
+ currRowKeyPrefix.length);
+ System.arraycopy(entityTypeEncoded, 0, currRowKey, currRowKeyPrefix.length,
+ entityTypeEncoded.length);
+
+ return HBaseTimelineStorageUtils.calculateTheClosestNextRowKeyForPrefix(
+ currRowKey);
+ }
+
+ private ResultScanner getResult(Configuration hbaseConf, Connection conn,
+ FilterList filterList, byte[] startPrefix, byte[] endPrefix)
+ throws IOException {
+ Scan scan = new Scan(startPrefix, endPrefix);
+ scan.setFilter(filterList);
+ scan.setSmall(true);
+ return ENTITY_TABLE.getResultScanner(hbaseConf, conn, scan);
+ }
+
+ private TimelineEntity parseEntityForType(Result result)
+ throws IOException {
+ if (result == null || result.isEmpty()) {
+ return null;
+ }
+ TimelineEntity entity = new TimelineEntity();
+ EntityRowKey newRowKey = EntityRowKey.parseRowKey(result.getRow());
+ entity.setType(newRowKey.getEntityType());
+ return entity;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
new file mode 100644
index 0000000..d0a0f3b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
@@ -0,0 +1,186 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.PageFilter;
+import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTableRW;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnRWHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongKeyConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKeyPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityTableRW;
+import org.apache.hadoop.yarn.webapp.BadRequestException;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Timeline entity reader for flow activity entities that are stored in the
+ * flow activity table.
+ */
+class FlowActivityEntityReader extends TimelineEntityReader {
+ private static final FlowActivityTableRW FLOW_ACTIVITY_TABLE =
+ new FlowActivityTableRW();
+
+ /**
+ * Used to convert Long key components to and from storage format.
+ */
+ private final KeyConverter<Long> longKeyConverter = new LongKeyConverter();
+
+
+ FlowActivityEntityReader(TimelineReaderContext ctxt,
+ TimelineEntityFilters entityFilters, TimelineDataToRetrieve toRetrieve) {
+ super(ctxt, entityFilters, toRetrieve);
+ }
+
+ FlowActivityEntityReader(TimelineReaderContext ctxt,
+ TimelineDataToRetrieve toRetrieve) {
+ super(ctxt, toRetrieve);
+ }
+
+ /**
+ * Uses the {@link FlowActivityTableRW}.
+ */
+ @Override
+ protected BaseTableRW<?> getTable() {
+ return FLOW_ACTIVITY_TABLE;
+ }
+
+ @Override
+ protected void validateParams() {
+ Preconditions.checkNotNull(getContext().getClusterId(),
+ "clusterId shouldn't be null");
+ }
+
+ @Override
+ protected void augmentParams(Configuration hbaseConf, Connection conn)
+ throws IOException {
+ createFiltersIfNull();
+ }
+
+ @Override
+ protected FilterList constructFilterListBasedOnFilters() throws IOException {
+ return null;
+ }
+
+ @Override
+ protected FilterList constructFilterListBasedOnFields() {
+ return null;
+ }
+
+ @Override
+ protected Result getResult(Configuration hbaseConf, Connection conn,
+ FilterList filterList) throws IOException {
+ throw new UnsupportedOperationException(
+ "we don't support a single entity query");
+ }
+
+ @Override
+ protected ResultScanner getResults(Configuration hbaseConf,
+ Connection conn, FilterList filterList) throws IOException {
+ Scan scan = new Scan();
+ String clusterId = getContext().getClusterId();
+ if (getFilters().getFromId() == null
+ && getFilters().getCreatedTimeBegin() == 0L
+ && getFilters().getCreatedTimeEnd() == Long.MAX_VALUE) {
+ // All records have to be chosen.
+ scan.setRowPrefixFilter(new FlowActivityRowKeyPrefix(clusterId)
+ .getRowKeyPrefix());
+ } else if (getFilters().getFromId() != null) {
+ FlowActivityRowKey key = null;
+ try {
+ key =
+ FlowActivityRowKey.parseRowKeyFromString(getFilters().getFromId());
+ } catch (IllegalArgumentException e) {
+ throw new BadRequestException("Invalid filter fromid is provided.");
+ }
+ if (!clusterId.equals(key.getClusterId())) {
+ throw new BadRequestException(
+ "fromid doesn't belong to clusterId=" + clusterId);
+ }
+ scan.setStartRow(key.getRowKey());
+ scan.setStopRow(
+ new FlowActivityRowKeyPrefix(clusterId,
+ (getFilters().getCreatedTimeBegin() <= 0 ? 0
+ : (getFilters().getCreatedTimeBegin() - 1)))
+ .getRowKeyPrefix());
+ } else {
+ scan.setStartRow(new FlowActivityRowKeyPrefix(clusterId, getFilters()
+ .getCreatedTimeEnd()).getRowKeyPrefix());
+ scan.setStopRow(new FlowActivityRowKeyPrefix(clusterId, (getFilters()
+ .getCreatedTimeBegin() <= 0 ? 0
+ : (getFilters().getCreatedTimeBegin() - 1))).getRowKeyPrefix());
+ }
+ // use the page filter to limit the result to the page size
+ // the scanner may still return more than the limit; therefore we need to
+ // read the right number as we iterate
+ scan.setFilter(new PageFilter(getFilters().getLimit()));
+ return getTable().getResultScanner(hbaseConf, conn, scan);
+ }
+
+ @Override
+ protected TimelineEntity parseEntity(Result result) throws IOException {
+ FlowActivityRowKey rowKey = FlowActivityRowKey.parseRowKey(result.getRow());
+
+ Long time = rowKey.getDayTimestamp();
+ String user = rowKey.getUserId();
+ String flowName = rowKey.getFlowName();
+
+ FlowActivityEntity flowActivity = new FlowActivityEntity(
+ getContext().getClusterId(), time, user, flowName);
+ // set the id
+ flowActivity.setId(flowActivity.getId());
+ // get the list of run ids along with the version that are associated with
+ // this flow on this day
+ Map<Long, Object> runIdsMap = ColumnRWHelper.readResults(result,
+ FlowActivityColumnPrefix.RUN_ID, longKeyConverter);
+ for (Map.Entry<Long, Object> e : runIdsMap.entrySet()) {
+ Long runId = e.getKey();
+ String version = (String)e.getValue();
+ FlowRunEntity flowRun = new FlowRunEntity();
+ flowRun.setUser(user);
+ flowRun.setName(flowName);
+ flowRun.setRunId(runId);
+ flowRun.setVersion(version);
+ // set the id
+ flowRun.setId(flowRun.getId());
+ flowActivity.addFlowRun(flowRun);
+ }
+ flowActivity.getInfo().put(TimelineReaderUtils.FROMID_KEY,
+ rowKey.getRowKeyAsString());
+ return flowActivity;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
new file mode 100644
index 0000000..33a2cf6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
@@ -0,0 +1,298 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
+
+import java.io.IOException;
+import java.util.EnumSet;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.BinaryComparator;
+import org.apache.hadoop.hbase.filter.BinaryPrefixComparator;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.filter.FamilyFilter;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.FilterList.Operator;
+import org.apache.hadoop.hbase.filter.PageFilter;
+import org.apache.hadoop.hbase.filter.QualifierFilter;
+import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTableRW;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnRWHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.RowKeyPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunRowKeyPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTableRW;
+import org.apache.hadoop.yarn.webapp.BadRequestException;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Timeline entity reader for flow run entities that are stored in the flow run
+ * table.
+ */
+class FlowRunEntityReader extends TimelineEntityReader {
+ private static final FlowRunTableRW FLOW_RUN_TABLE = new FlowRunTableRW();
+
+ FlowRunEntityReader(TimelineReaderContext ctxt,
+ TimelineEntityFilters entityFilters, TimelineDataToRetrieve toRetrieve) {
+ super(ctxt, entityFilters, toRetrieve);
+ }
+
+ FlowRunEntityReader(TimelineReaderContext ctxt,
+ TimelineDataToRetrieve toRetrieve) {
+ super(ctxt, toRetrieve);
+ }
+
+ /**
+ * Uses the {@link FlowRunTableRW}.
+ */
+ @Override
+ protected BaseTableRW<?> getTable() {
+ return FLOW_RUN_TABLE;
+ }
+
+ @Override
+ protected void validateParams() {
+ Preconditions.checkNotNull(getContext(), "context shouldn't be null");
+ Preconditions.checkNotNull(getDataToRetrieve(),
+ "data to retrieve shouldn't be null");
+ Preconditions.checkNotNull(getContext().getClusterId(),
+ "clusterId shouldn't be null");
+ Preconditions.checkNotNull(getContext().getUserId(),
+ "userId shouldn't be null");
+ Preconditions.checkNotNull(getContext().getFlowName(),
+ "flowName shouldn't be null");
+ if (isSingleEntityRead()) {
+ Preconditions.checkNotNull(getContext().getFlowRunId(),
+ "flowRunId shouldn't be null");
+ }
+ EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
+ if (!isSingleEntityRead() && fieldsToRetrieve != null) {
+ for (Field field : fieldsToRetrieve) {
+ if (field != Field.ALL && field != Field.METRICS) {
+ throw new BadRequestException("Invalid field " + field
+ + " specified while querying flow runs.");
+ }
+ }
+ }
+ }
+
+ @Override
+ protected void augmentParams(Configuration hbaseConf, Connection conn) {
+ // Add metrics to fields to retrieve if metricsToRetrieve is specified.
+ getDataToRetrieve().addFieldsBasedOnConfsAndMetricsToRetrieve();
+ if (!isSingleEntityRead()) {
+ createFiltersIfNull();
+ }
+ }
+
+ protected FilterList constructFilterListBasedOnFilters() throws IOException {
+ FilterList listBasedOnFilters = new FilterList();
+ // Filter based on created time range.
+ Long createdTimeBegin = getFilters().getCreatedTimeBegin();
+ Long createdTimeEnd = getFilters().getCreatedTimeEnd();
+ if (createdTimeBegin != 0 || createdTimeEnd != Long.MAX_VALUE) {
+ listBasedOnFilters.addFilter(TimelineFilterUtils
+ .createSingleColValueFiltersByRange(FlowRunColumn.MIN_START_TIME,
+ createdTimeBegin, createdTimeEnd));
+ }
+ // Filter based on metric filters.
+ TimelineFilterList metricFilters = getFilters().getMetricFilters();
+ if (metricFilters != null && !metricFilters.getFilterList().isEmpty()) {
+ listBasedOnFilters.addFilter(TimelineFilterUtils.createHBaseFilterList(
+ FlowRunColumnPrefix.METRIC, metricFilters));
+ }
+ return listBasedOnFilters;
+ }
+
+ /**
+ * Add {@link QualifierFilter} filters to filter list for each column of flow
+ * run table.
+ *
+ * @return filter list to which qualifier filters have been added.
+ */
+ private FilterList updateFixedColumns() {
+ FilterList columnsList = new FilterList(Operator.MUST_PASS_ONE);
+ for (FlowRunColumn column : FlowRunColumn.values()) {
+ columnsList.addFilter(new QualifierFilter(CompareOp.EQUAL,
+ new BinaryComparator(column.getColumnQualifierBytes())));
+ }
+ return columnsList;
+ }
+
+ @Override
+ protected FilterList constructFilterListBasedOnFields() throws IOException {
+ FilterList list = new FilterList(Operator.MUST_PASS_ONE);
+ // By default fetch everything in INFO column family.
+ FamilyFilter infoColumnFamily =
+ new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(
+ FlowRunColumnFamily.INFO.getBytes()));
+ TimelineDataToRetrieve dataToRetrieve = getDataToRetrieve();
+ // If multiple entities have to be retrieved, check if metrics have to be
+ // retrieved and if not, add a filter so that metrics can be excluded.
+ // Metrics are always returned if we are reading a single entity.
+ if (!isSingleEntityRead()
+ && !hasField(dataToRetrieve.getFieldsToRetrieve(), Field.METRICS)) {
+ FilterList infoColFamilyList = new FilterList(Operator.MUST_PASS_ONE);
+ infoColFamilyList.addFilter(infoColumnFamily);
+ infoColFamilyList.addFilter(new QualifierFilter(CompareOp.NOT_EQUAL,
+ new BinaryPrefixComparator(FlowRunColumnPrefix.METRIC
+ .getColumnPrefixBytes(""))));
+ list.addFilter(infoColFamilyList);
+ } else {
+ // Check if metricsToRetrieve are specified and if they are, create a
+ // filter list for info column family by adding flow run tables columns
+ // and a list for metrics to retrieve. Pls note that fieldsToRetrieve
+ // will have METRICS added to it if metricsToRetrieve are specified
+ // (in augmentParams()).
+ TimelineFilterList metricsToRetrieve =
+ dataToRetrieve.getMetricsToRetrieve();
+ if (metricsToRetrieve != null
+ && !metricsToRetrieve.getFilterList().isEmpty()) {
+ FilterList infoColFamilyList = new FilterList();
+ infoColFamilyList.addFilter(infoColumnFamily);
+ FilterList columnsList = updateFixedColumns();
+ columnsList.addFilter(TimelineFilterUtils.createHBaseFilterList(
+ FlowRunColumnPrefix.METRIC, metricsToRetrieve));
+ infoColFamilyList.addFilter(columnsList);
+ list.addFilter(infoColFamilyList);
+ }
+ }
+ return list;
+ }
+
+ @Override
+ protected Result getResult(Configuration hbaseConf, Connection conn,
+ FilterList filterList) throws IOException {
+ TimelineReaderContext context = getContext();
+ FlowRunRowKey flowRunRowKey =
+ new FlowRunRowKey(context.getClusterId(), context.getUserId(),
+ context.getFlowName(), context.getFlowRunId());
+ byte[] rowKey = flowRunRowKey.getRowKey();
+ Get get = new Get(rowKey);
+ get.setMaxVersions(Integer.MAX_VALUE);
+ if (filterList != null && !filterList.getFilters().isEmpty()) {
+ get.setFilter(filterList);
+ }
+ return getTable().getResult(hbaseConf, conn, get);
+ }
+
+ @Override
+ protected ResultScanner getResults(Configuration hbaseConf, Connection conn,
+ FilterList filterList) throws IOException {
+ Scan scan = new Scan();
+ TimelineReaderContext context = getContext();
+ RowKeyPrefix<FlowRunRowKey> flowRunRowKeyPrefix = null;
+ if (getFilters().getFromId() == null) {
+ flowRunRowKeyPrefix = new FlowRunRowKeyPrefix(context.getClusterId(),
+ context.getUserId(), context.getFlowName());
+ scan.setRowPrefixFilter(flowRunRowKeyPrefix.getRowKeyPrefix());
+ } else {
+ FlowRunRowKey flowRunRowKey = null;
+ try {
+ flowRunRowKey =
+ FlowRunRowKey.parseRowKeyFromString(getFilters().getFromId());
+ } catch (IllegalArgumentException e) {
+ throw new BadRequestException("Invalid filter fromid is provided.");
+ }
+ if (!context.getClusterId().equals(flowRunRowKey.getClusterId())) {
+ throw new BadRequestException(
+ "fromid doesn't belong to clusterId=" + context.getClusterId());
+ }
+ // set start row
+ scan.setStartRow(flowRunRowKey.getRowKey());
+
+ // get the bytes for stop row
+ flowRunRowKeyPrefix = new FlowRunRowKeyPrefix(context.getClusterId(),
+ context.getUserId(), context.getFlowName());
+
+ // set stop row
+ scan.setStopRow(
+ HBaseTimelineStorageUtils.calculateTheClosestNextRowKeyForPrefix(
+ flowRunRowKeyPrefix.getRowKeyPrefix()));
+ }
+
+ FilterList newList = new FilterList();
+ newList.addFilter(new PageFilter(getFilters().getLimit()));
+ if (filterList != null && !filterList.getFilters().isEmpty()) {
+ newList.addFilter(filterList);
+ }
+ scan.setFilter(newList);
+ scan.setMaxVersions(Integer.MAX_VALUE);
+ return getTable().getResultScanner(hbaseConf, conn, scan);
+ }
+
+ @Override
+ protected TimelineEntity parseEntity(Result result) throws IOException {
+ FlowRunEntity flowRun = new FlowRunEntity();
+ FlowRunRowKey rowKey = FlowRunRowKey.parseRowKey(result.getRow());
+ flowRun.setRunId(rowKey.getFlowRunId());
+ flowRun.setUser(rowKey.getUserId());
+ flowRun.setName(rowKey.getFlowName());
+
+ // read the start time
+ Long startTime = (Long) ColumnRWHelper.readResult(result,
+ FlowRunColumn.MIN_START_TIME);
+ if (startTime != null) {
+ flowRun.setStartTime(startTime.longValue());
+ }
+
+ // read the end time if available
+ Long endTime = (Long) ColumnRWHelper.readResult(result,
+ FlowRunColumn.MAX_END_TIME);
+ if (endTime != null) {
+ flowRun.setMaxEndTime(endTime.longValue());
+ }
+
+ // read the flow version
+ String version = (String) ColumnRWHelper.readResult(result,
+ FlowRunColumn.FLOW_VERSION);
+ if (version != null) {
+ flowRun.setVersion(version);
+ }
+
+ // read metrics if its a single entity query or if METRICS are part of
+ // fieldsToRetrieve.
+ if (isSingleEntityRead()
+ || hasField(getDataToRetrieve().getFieldsToRetrieve(), Field.METRICS)) {
+ readMetrics(flowRun, result, FlowRunColumnPrefix.METRIC);
+ }
+
+ // set the id
+ flowRun.setId(flowRun.getId());
+ flowRun.getInfo().put(TimelineReaderUtils.FROMID_KEY,
+ rowKey.getRowKeyAsString());
+ return flowRun;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
new file mode 100644
index 0000000..02eca84
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
@@ -0,0 +1,655 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Query;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.BinaryComparator;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.filter.FamilyFilter;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.PageFilter;
+import org.apache.hadoop.hbase.filter.FilterList.Operator;
+import org.apache.hadoop.hbase.filter.QualifierFilter;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTableRW;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnRWHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.RowKeyPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.StringKeyConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKeyPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTableRW;
+import org.apache.hadoop.yarn.webapp.BadRequestException;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Timeline entity reader for generic entities that are stored in the entity
+ * table.
+ */
+class GenericEntityReader extends TimelineEntityReader {
+ private static final EntityTableRW ENTITY_TABLE = new EntityTableRW();
+
+ /**
+ * Used to convert strings key components to and from storage format.
+ */
+ private final KeyConverter<String> stringKeyConverter =
+ new StringKeyConverter();
+
+ GenericEntityReader(TimelineReaderContext ctxt,
+ TimelineEntityFilters entityFilters, TimelineDataToRetrieve toRetrieve) {
+ super(ctxt, entityFilters, toRetrieve);
+ }
+
+ GenericEntityReader(TimelineReaderContext ctxt,
+ TimelineDataToRetrieve toRetrieve) {
+ super(ctxt, toRetrieve);
+ }
+
+ /**
+ * Uses the {@link EntityTableRW}.
+ */
+ protected BaseTableRW<?> getTable() {
+ return ENTITY_TABLE;
+ }
+
+ @Override
+ protected FilterList constructFilterListBasedOnFilters() throws IOException {
+ // Filters here cannot be null for multiple entity reads as they are set in
+ // augmentParams if null.
+ FilterList listBasedOnFilters = new FilterList();
+ TimelineEntityFilters filters = getFilters();
+ // Create filter list based on created time range and add it to
+ // listBasedOnFilters.
+ long createdTimeBegin = filters.getCreatedTimeBegin();
+ long createdTimeEnd = filters.getCreatedTimeEnd();
+ if (createdTimeBegin != 0 || createdTimeEnd != Long.MAX_VALUE) {
+ listBasedOnFilters.addFilter(TimelineFilterUtils
+ .createSingleColValueFiltersByRange(EntityColumn.CREATED_TIME,
+ createdTimeBegin, createdTimeEnd));
+ }
+ // Create filter list based on metric filters and add it to
+ // listBasedOnFilters.
+ TimelineFilterList metricFilters = filters.getMetricFilters();
+ if (metricFilters != null && !metricFilters.getFilterList().isEmpty()) {
+ listBasedOnFilters.addFilter(TimelineFilterUtils.createHBaseFilterList(
+ EntityColumnPrefix.METRIC, metricFilters));
+ }
+ // Create filter list based on config filters and add it to
+ // listBasedOnFilters.
+ TimelineFilterList configFilters = filters.getConfigFilters();
+ if (configFilters != null && !configFilters.getFilterList().isEmpty()) {
+ listBasedOnFilters.addFilter(TimelineFilterUtils.createHBaseFilterList(
+ EntityColumnPrefix.CONFIG, configFilters));
+ }
+ // Create filter list based on info filters and add it to listBasedOnFilters
+ TimelineFilterList infoFilters = filters.getInfoFilters();
+ if (infoFilters != null && !infoFilters.getFilterList().isEmpty()) {
+ listBasedOnFilters.addFilter(TimelineFilterUtils.createHBaseFilterList(
+ EntityColumnPrefix.INFO, infoFilters));
+ }
+ return listBasedOnFilters;
+ }
+
+ /**
+ * Check if we need to fetch only some of the event columns.
+ *
+ * @return true if we need to fetch some of the columns, false otherwise.
+ */
+ protected boolean fetchPartialEventCols(TimelineFilterList eventFilters,
+ EnumSet<Field> fieldsToRetrieve) {
+ return (eventFilters != null && !eventFilters.getFilterList().isEmpty() &&
+ !hasField(fieldsToRetrieve, Field.EVENTS));
+ }
+
+ /**
+ * Check if we need to fetch only some of the relates_to columns.
+ *
+ * @return true if we need to fetch some of the columns, false otherwise.
+ */
+ protected boolean fetchPartialRelatesToCols(TimelineFilterList relatesTo,
+ EnumSet<Field> fieldsToRetrieve) {
+ return (relatesTo != null && !relatesTo.getFilterList().isEmpty() &&
+ !hasField(fieldsToRetrieve, Field.RELATES_TO));
+ }
+
+ /**
+ * Check if we need to fetch only some of the is_related_to columns.
+ *
+ * @return true if we need to fetch some of the columns, false otherwise.
+ */
+ private boolean fetchPartialIsRelatedToCols(TimelineFilterList isRelatedTo,
+ EnumSet<Field> fieldsToRetrieve) {
+ return (isRelatedTo != null && !isRelatedTo.getFilterList().isEmpty() &&
+ !hasField(fieldsToRetrieve, Field.IS_RELATED_TO));
+ }
+
+ /**
+ * Check if we need to fetch only some of the columns based on event filters,
+ * relatesto and isrelatedto from info family.
+ *
+ * @return true, if we need to fetch only some of the columns, false if we
+ * need to fetch all the columns under info column family.
+ */
+ protected boolean fetchPartialColsFromInfoFamily() {
+ EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
+ TimelineEntityFilters filters = getFilters();
+ return fetchPartialEventCols(filters.getEventFilters(), fieldsToRetrieve)
+ || fetchPartialRelatesToCols(filters.getRelatesTo(), fieldsToRetrieve)
+ || fetchPartialIsRelatedToCols(filters.getIsRelatedTo(),
+ fieldsToRetrieve);
+ }
+
+ /**
+ * Check if we need to create filter list based on fields. We need to create a
+ * filter list iff all fields need not be retrieved or we have some specific
+ * fields or metrics to retrieve. We also need to create a filter list if we
+ * have relationships(relatesTo/isRelatedTo) and event filters specified for
+ * the query.
+ *
+ * @return true if we need to create the filter list, false otherwise.
+ */
+ protected boolean needCreateFilterListBasedOnFields() {
+ TimelineDataToRetrieve dataToRetrieve = getDataToRetrieve();
+ // Check if all fields are to be retrieved or not. If all fields have to
+ // be retrieved, also check if we have some metrics or configs to
+ // retrieve specified for the query because then a filter list will have
+ // to be created.
+ boolean flag =
+ !dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL)
+ || (dataToRetrieve.getConfsToRetrieve() != null && !dataToRetrieve
+ .getConfsToRetrieve().getFilterList().isEmpty())
+ || (dataToRetrieve.getMetricsToRetrieve() != null && !dataToRetrieve
+ .getMetricsToRetrieve().getFilterList().isEmpty());
+ // Filters need to be checked only if we are reading multiple entities. If
+ // condition above is false, we check if there are relationships(relatesTo/
+ // isRelatedTo) and event filters specified for the query.
+ if (!flag && !isSingleEntityRead()) {
+ TimelineEntityFilters filters = getFilters();
+ flag =
+ (filters.getEventFilters() != null && !filters.getEventFilters()
+ .getFilterList().isEmpty())
+ || (filters.getIsRelatedTo() != null && !filters.getIsRelatedTo()
+ .getFilterList().isEmpty())
+ || (filters.getRelatesTo() != null && !filters.getRelatesTo()
+ .getFilterList().isEmpty());
+ }
+ return flag;
+ }
+
+ /**
+ * Add {@link QualifierFilter} filters to filter list for each column of
+ * entity table.
+ *
+ * @param list filter list to which qualifier filters have to be added.
+ */
+ protected void updateFixedColumns(FilterList list) {
+ for (EntityColumn column : EntityColumn.values()) {
+ list.addFilter(new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(
+ column.getColumnQualifierBytes())));
+ }
+ }
+
+ /**
+ * Creates a filter list which indicates that only some of the column
+ * qualifiers in the info column family will be returned in result.
+ *
+ * @param isApplication If true, it means operations are to be performed for
+ * application table, otherwise for entity table.
+ * @return filter list.
+ * @throws IOException if any problem occurs while creating filter list.
+ */
+ private FilterList createFilterListForColsOfInfoFamily() throws IOException {
+ FilterList infoFamilyColsFilter = new FilterList(Operator.MUST_PASS_ONE);
+ // Add filters for each column in entity table.
+ updateFixedColumns(infoFamilyColsFilter);
+ EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
+ // If INFO field has to be retrieved, add a filter for fetching columns
+ // with INFO column prefix.
+ if (hasField(fieldsToRetrieve, Field.INFO)) {
+ infoFamilyColsFilter
+ .addFilter(TimelineFilterUtils.createHBaseQualifierFilter(
+ CompareOp.EQUAL, EntityColumnPrefix.INFO));
+ }
+ TimelineFilterList relatesTo = getFilters().getRelatesTo();
+ if (hasField(fieldsToRetrieve, Field.RELATES_TO)) {
+ // If RELATES_TO field has to be retrieved, add a filter for fetching
+ // columns with RELATES_TO column prefix.
+ infoFamilyColsFilter.addFilter(TimelineFilterUtils
+ .createHBaseQualifierFilter(CompareOp.EQUAL,
+ EntityColumnPrefix.RELATES_TO));
+ } else if (relatesTo != null && !relatesTo.getFilterList().isEmpty()) {
+ // Even if fields to retrieve does not contain RELATES_TO, we still
+ // need to have a filter to fetch some of the column qualifiers if
+ // relatesTo filters are specified. relatesTo filters will then be
+ // matched after fetching rows from HBase.
+ Set<String> relatesToCols =
+ TimelineFilterUtils.fetchColumnsFromFilterList(relatesTo);
+ infoFamilyColsFilter.addFilter(createFiltersFromColumnQualifiers(
+ EntityColumnPrefix.RELATES_TO, relatesToCols));
+ }
+ TimelineFilterList isRelatedTo = getFilters().getIsRelatedTo();
+ if (hasField(fieldsToRetrieve, Field.IS_RELATED_TO)) {
+ // If IS_RELATED_TO field has to be retrieved, add a filter for fetching
+ // columns with IS_RELATED_TO column prefix.
+ infoFamilyColsFilter.addFilter(TimelineFilterUtils
+ .createHBaseQualifierFilter(CompareOp.EQUAL,
+ EntityColumnPrefix.IS_RELATED_TO));
+ } else if (isRelatedTo != null && !isRelatedTo.getFilterList().isEmpty()) {
+ // Even if fields to retrieve does not contain IS_RELATED_TO, we still
+ // need to have a filter to fetch some of the column qualifiers if
+ // isRelatedTo filters are specified. isRelatedTo filters will then be
+ // matched after fetching rows from HBase.
+ Set<String> isRelatedToCols =
+ TimelineFilterUtils.fetchColumnsFromFilterList(isRelatedTo);
+ infoFamilyColsFilter.addFilter(createFiltersFromColumnQualifiers(
+ EntityColumnPrefix.IS_RELATED_TO, isRelatedToCols));
+ }
+ TimelineFilterList eventFilters = getFilters().getEventFilters();
+ if (hasField(fieldsToRetrieve, Field.EVENTS)) {
+ // If EVENTS field has to be retrieved, add a filter for fetching columns
+ // with EVENT column prefix.
+ infoFamilyColsFilter
+ .addFilter(TimelineFilterUtils.createHBaseQualifierFilter(
+ CompareOp.EQUAL, EntityColumnPrefix.EVENT));
+ } else if (eventFilters != null &&
+ !eventFilters.getFilterList().isEmpty()) {
+ // Even if fields to retrieve does not contain EVENTS, we still need to
+ // have a filter to fetch some of the column qualifiers on the basis of
+ // event filters specified. Event filters will then be matched after
+ // fetching rows from HBase.
+ Set<String> eventCols =
+ TimelineFilterUtils.fetchColumnsFromFilterList(eventFilters);
+ infoFamilyColsFilter.addFilter(createFiltersFromColumnQualifiers(
+ EntityColumnPrefix.EVENT, eventCols));
+ }
+ return infoFamilyColsFilter;
+ }
+
+ /**
+ * Exclude column prefixes via filters which are not required(based on fields
+ * to retrieve) from info column family. These filters are added to filter
+ * list which contains a filter for getting info column family.
+ *
+ * @param infoColFamilyList filter list for info column family.
+ */
+ private void excludeFieldsFromInfoColFamily(FilterList infoColFamilyList) {
+ EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
+ // Events not required.
+ if (!hasField(fieldsToRetrieve, Field.EVENTS)) {
+ infoColFamilyList.addFilter(TimelineFilterUtils
+ .createHBaseQualifierFilter(CompareOp.NOT_EQUAL,
+ EntityColumnPrefix.EVENT));
+ }
+ // info not required.
+ if (!hasField(fieldsToRetrieve, Field.INFO)) {
+ infoColFamilyList.addFilter(TimelineFilterUtils
+ .createHBaseQualifierFilter(CompareOp.NOT_EQUAL,
+ EntityColumnPrefix.INFO));
+ }
+ // is related to not required.
+ if (!hasField(fieldsToRetrieve, Field.IS_RELATED_TO)) {
+ infoColFamilyList.addFilter(TimelineFilterUtils
+ .createHBaseQualifierFilter(CompareOp.NOT_EQUAL,
+ EntityColumnPrefix.IS_RELATED_TO));
+ }
+ // relates to not required.
+ if (!hasField(fieldsToRetrieve, Field.RELATES_TO)) {
+ infoColFamilyList.addFilter(TimelineFilterUtils
+ .createHBaseQualifierFilter(CompareOp.NOT_EQUAL,
+ EntityColumnPrefix.RELATES_TO));
+ }
+ }
+
+ /**
+ * Updates filter list based on fields for confs and metrics to retrieve.
+ *
+ * @param listBasedOnFields filter list based on fields.
+ * @throws IOException if any problem occurs while updating filter list.
+ */
+ private void updateFilterForConfsAndMetricsToRetrieve(
+ FilterList listBasedOnFields) throws IOException {
+ TimelineDataToRetrieve dataToRetrieve = getDataToRetrieve();
+ // Please note that if confsToRetrieve is specified, we would have added
+ // CONFS to fields to retrieve in augmentParams() even if not specified.
+ if (dataToRetrieve.getFieldsToRetrieve().contains(Field.CONFIGS)) {
+ // Create a filter list for configs.
+ listBasedOnFields.addFilter(TimelineFilterUtils
+ .createFilterForConfsOrMetricsToRetrieve(
+ dataToRetrieve.getConfsToRetrieve(), EntityColumnFamily.CONFIGS,
+ EntityColumnPrefix.CONFIG));
+ }
+
+ // Please note that if metricsToRetrieve is specified, we would have added
+ // METRICS to fields to retrieve in augmentParams() even if not specified.
+ if (dataToRetrieve.getFieldsToRetrieve().contains(Field.METRICS)) {
+ // Create a filter list for metrics.
+ listBasedOnFields.addFilter(TimelineFilterUtils
+ .createFilterForConfsOrMetricsToRetrieve(
+ dataToRetrieve.getMetricsToRetrieve(),
+ EntityColumnFamily.METRICS, EntityColumnPrefix.METRIC));
+ }
+ }
+
+ @Override
+ protected FilterList constructFilterListBasedOnFields() throws IOException {
+ if (!needCreateFilterListBasedOnFields()) {
+ // Fetch all the columns. No need of a filter.
+ return null;
+ }
+ FilterList listBasedOnFields = new FilterList(Operator.MUST_PASS_ONE);
+ FilterList infoColFamilyList = new FilterList();
+ // By default fetch everything in INFO column family.
+ FamilyFilter infoColumnFamily =
+ new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(
+ EntityColumnFamily.INFO.getBytes()));
+ infoColFamilyList.addFilter(infoColumnFamily);
+ if (!isSingleEntityRead() && fetchPartialColsFromInfoFamily()) {
+ // We can fetch only some of the columns from info family.
+ infoColFamilyList.addFilter(createFilterListForColsOfInfoFamily());
+ } else {
+ // Exclude column prefixes in info column family which are not required
+ // based on fields to retrieve.
+ excludeFieldsFromInfoColFamily(infoColFamilyList);
+ }
+ listBasedOnFields.addFilter(infoColFamilyList);
+ updateFilterForConfsAndMetricsToRetrieve(listBasedOnFields);
+ return listBasedOnFields;
+ }
+
+ @Override
+ protected void validateParams() {
+ Preconditions.checkNotNull(getContext(), "context shouldn't be null");
+ Preconditions.checkNotNull(getDataToRetrieve(),
+ "data to retrieve shouldn't be null");
+ Preconditions.checkNotNull(getContext().getClusterId(),
+ "clusterId shouldn't be null");
+ Preconditions.checkNotNull(getContext().getAppId(),
+ "appId shouldn't be null");
+ Preconditions.checkNotNull(getContext().getEntityType(),
+ "entityType shouldn't be null");
+ if (isSingleEntityRead()) {
+ Preconditions.checkNotNull(getContext().getEntityId(),
+ "entityId shouldn't be null");
+ }
+ }
+
+ @Override
+ protected void augmentParams(Configuration hbaseConf, Connection conn)
+ throws IOException {
+ defaultAugmentParams(hbaseConf, conn);
+ // Add configs/metrics to fields to retrieve if confsToRetrieve and/or
+ // metricsToRetrieve are specified.
+ getDataToRetrieve().addFieldsBasedOnConfsAndMetricsToRetrieve();
+ if (!isSingleEntityRead()) {
+ createFiltersIfNull();
+ }
+ }
+
+ @Override
+ protected Result getResult(Configuration hbaseConf, Connection conn,
+ FilterList filterList) throws IOException {
+ TimelineReaderContext context = getContext();
+ Result result = null;
+ if (context.getEntityIdPrefix() != null) {
+ byte[] rowKey = new EntityRowKey(context.getClusterId(),
+ context.getUserId(), context.getFlowName(), context.getFlowRunId(),
+ context.getAppId(), context.getEntityType(),
+ context.getEntityIdPrefix(), context.getEntityId()).getRowKey();
+ Get get = new Get(rowKey);
+ setMetricsTimeRange(get);
+ get.setMaxVersions(getDataToRetrieve().getMetricsLimit());
+ if (filterList != null && !filterList.getFilters().isEmpty()) {
+ get.setFilter(filterList);
+ }
+ result = getTable().getResult(hbaseConf, conn, get);
+
+ } else {
+ // Prepare for range scan
+ // create single SingleColumnValueFilter and add to existing filters.
+ FilterList filter = new FilterList(Operator.MUST_PASS_ALL);
+ if (filterList != null && !filterList.getFilters().isEmpty()) {
+ filter.addFilter(filterList);
+ }
+ FilterList newFilter = new FilterList();
+ newFilter.addFilter(TimelineFilterUtils.createHBaseSingleColValueFilter(
+ EntityColumn.ID, context.getEntityId(), CompareOp.EQUAL));
+ newFilter.addFilter(new PageFilter(1));
+ filter.addFilter(newFilter);
+
+ ResultScanner results = getResults(hbaseConf, conn, filter);
+ try {
+ Iterator<Result> iterator = results.iterator();
+ if (iterator.hasNext()) {
+ result = iterator.next();
+ }
+ } finally {
+ results.close();
+ }
+ }
+ return result;
+ }
+
+ private void setMetricsTimeRange(Query query) {
+ // Set time range for metric values.
+ HBaseTimelineStorageUtils.setMetricsTimeRange(
+ query, EntityColumnFamily.METRICS.getBytes(),
+ getDataToRetrieve().getMetricsTimeBegin(),
+ getDataToRetrieve().getMetricsTimeEnd());
+ }
+
+ @Override
+ protected ResultScanner getResults(Configuration hbaseConf, Connection conn,
+ FilterList filterList) throws IOException {
+ // Scan through part of the table to find the entities belong to one app
+ // and one type
+ Scan scan = new Scan();
+ TimelineReaderContext context = getContext();
+ RowKeyPrefix<EntityRowKey> entityRowKeyPrefix = null;
+ // default mode, will always scans from beginning of entity type.
+ if (getFilters() == null || getFilters().getFromId() == null) {
+ entityRowKeyPrefix = new EntityRowKeyPrefix(context.getClusterId(),
+ context.getUserId(), context.getFlowName(), context.getFlowRunId(),
+ context.getAppId(), context.getEntityType(), null, null);
+ scan.setRowPrefixFilter(entityRowKeyPrefix.getRowKeyPrefix());
+ } else { // pagination mode, will scan from given entityIdPrefix!enitityId
+
+ EntityRowKey entityRowKey = null;
+ try {
+ entityRowKey =
+ EntityRowKey.parseRowKeyFromString(getFilters().getFromId());
+ } catch (IllegalArgumentException e) {
+ throw new BadRequestException("Invalid filter fromid is provided.");
+ }
+ if (!context.getClusterId().equals(entityRowKey.getClusterId())) {
+ throw new BadRequestException(
+ "fromid doesn't belong to clusterId=" + context.getClusterId());
+ }
+
+ // set start row
+ scan.setStartRow(entityRowKey.getRowKey());
+
+ // get the bytes for stop row
+ entityRowKeyPrefix = new EntityRowKeyPrefix(context.getClusterId(),
+ context.getUserId(), context.getFlowName(), context.getFlowRunId(),
+ context.getAppId(), context.getEntityType(), null, null);
+
+ // set stop row
+ scan.setStopRow(
+ HBaseTimelineStorageUtils.calculateTheClosestNextRowKeyForPrefix(
+ entityRowKeyPrefix.getRowKeyPrefix()));
+
+ // set page filter to limit. This filter has to set only in pagination
+ // mode.
+ filterList.addFilter(new PageFilter(getFilters().getLimit()));
+ }
+ setMetricsTimeRange(scan);
+ scan.setMaxVersions(getDataToRetrieve().getMetricsLimit());
+ if (filterList != null && !filterList.getFilters().isEmpty()) {
+ scan.setFilter(filterList);
+ }
+ return getTable().getResultScanner(hbaseConf, conn, scan);
+ }
+
+ @Override
+ protected TimelineEntity parseEntity(Result result) throws IOException {
+ if (result == null || result.isEmpty()) {
+ return null;
+ }
+ TimelineEntity entity = new TimelineEntity();
+ EntityRowKey parseRowKey = EntityRowKey.parseRowKey(result.getRow());
+ entity.setType(parseRowKey.getEntityType());
+ entity.setId(parseRowKey.getEntityId());
+ entity.setIdPrefix(parseRowKey.getEntityIdPrefix().longValue());
+
+ TimelineEntityFilters filters = getFilters();
+ // fetch created time
+ Long createdTime = (Long) ColumnRWHelper.readResult(result,
+ EntityColumn.CREATED_TIME);
+ entity.setCreatedTime(createdTime);
+
+ EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
+ // fetch is related to entities and match isRelatedTo filter. If isRelatedTo
+ // filters do not match, entity would be dropped. We have to match filters
+ // locally as relevant HBase filters to filter out rows on the basis of
+ // isRelatedTo are not set in HBase scan.
+ boolean checkIsRelatedTo =
+ !isSingleEntityRead() && filters.getIsRelatedTo() != null
+ && filters.getIsRelatedTo().getFilterList().size() > 0;
+ if (hasField(fieldsToRetrieve, Field.IS_RELATED_TO) || checkIsRelatedTo) {
+ readRelationship(entity, result, EntityColumnPrefix.IS_RELATED_TO, true);
+ if (checkIsRelatedTo
+ && !TimelineStorageUtils.matchIsRelatedTo(entity,
+ filters.getIsRelatedTo())) {
+ return null;
+ }
+ if (!hasField(fieldsToRetrieve, Field.IS_RELATED_TO)) {
+ entity.getIsRelatedToEntities().clear();
+ }
+ }
+
+ // fetch relates to entities and match relatesTo filter. If relatesTo
+ // filters do not match, entity would be dropped. We have to match filters
+ // locally as relevant HBase filters to filter out rows on the basis of
+ // relatesTo are not set in HBase scan.
+ boolean checkRelatesTo =
+ !isSingleEntityRead() && filters.getRelatesTo() != null
+ && filters.getRelatesTo().getFilterList().size() > 0;
+ if (hasField(fieldsToRetrieve, Field.RELATES_TO)
+ || checkRelatesTo) {
+ readRelationship(entity, result, EntityColumnPrefix.RELATES_TO, false);
+ if (checkRelatesTo
+ && !TimelineStorageUtils.matchRelatesTo(entity,
+ filters.getRelatesTo())) {
+ return null;
+ }
+ if (!hasField(fieldsToRetrieve, Field.RELATES_TO)) {
+ entity.getRelatesToEntities().clear();
+ }
+ }
+
+ // fetch info if fieldsToRetrieve contains INFO or ALL.
+ if (hasField(fieldsToRetrieve, Field.INFO)) {
+ readKeyValuePairs(entity, result, EntityColumnPrefix.INFO, false);
+ }
+
+ // fetch configs if fieldsToRetrieve contains CONFIGS or ALL.
+ if (hasField(fieldsToRetrieve, Field.CONFIGS)) {
+ readKeyValuePairs(entity, result, EntityColumnPrefix.CONFIG, true);
+ }
+
+ // fetch events and match event filters if they exist. If event filters do
+ // not match, entity would be dropped. We have to match filters locally
+ // as relevant HBase filters to filter out rows on the basis of events
+ // are not set in HBase scan.
+ boolean checkEvents =
+ !isSingleEntityRead() && filters.getEventFilters() != null
+ && filters.getEventFilters().getFilterList().size() > 0;
+ if (hasField(fieldsToRetrieve, Field.EVENTS) || checkEvents) {
+ readEvents(entity, result, EntityColumnPrefix.EVENT);
+ if (checkEvents
+ && !TimelineStorageUtils.matchEventFilters(entity,
+ filters.getEventFilters())) {
+ return null;
+ }
+ if (!hasField(fieldsToRetrieve, Field.EVENTS)) {
+ entity.getEvents().clear();
+ }
+ }
+
+ // fetch metrics if fieldsToRetrieve contains METRICS or ALL.
+ if (hasField(fieldsToRetrieve, Field.METRICS)) {
+ readMetrics(entity, result, EntityColumnPrefix.METRIC);
+ }
+
+ entity.getInfo().put(TimelineReaderUtils.FROMID_KEY,
+ parseRowKey.getRowKeyAsString());
+ return entity;
+ }
+
+ /**
+ * Helper method for reading key-value pairs for either info or config.
+ *
+ * @param <T> Describes the type of column prefix.
+ * @param entity entity to fill.
+ * @param result result from HBase.
+ * @param prefix column prefix.
+ * @param isConfig if true, means we are reading configs, otherwise info.
+ * @throws IOException if any problem is encountered while reading result.
+ */
+ protected <T extends BaseTable<T>> void readKeyValuePairs(
+ TimelineEntity entity, Result result,
+ ColumnPrefix<T> prefix, boolean isConfig) throws IOException {
+ // info and configuration are of type Map<String, Object or String>
+ Map<String, Object> columns =
+ ColumnRWHelper.readResults(result, prefix, stringKeyConverter);
+ if (isConfig) {
+ for (Map.Entry<String, Object> column : columns.entrySet()) {
+ entity.addConfig(column.getKey(), column.getValue().toString());
+ }
+ } else {
+ entity.addInfo(columns);
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/18] hadoop git commit: YARN-7919. Refactor timelineservice-hbase
module into submodules. Contributed by Haibo Chen.
Posted by ro...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongConverter.java
new file mode 100644
index 0000000..0857980
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongConverter.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import java.io.IOException;
+import java.io.Serializable;
+
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Encodes a value by interpreting it as a Long and converting it to bytes and
+ * decodes a set of bytes as a Long.
+ */
+public final class LongConverter implements NumericValueConverter,
+ Serializable {
+
+ /**
+ * Added because we implement Comparator<Number>.
+ */
+ private static final long serialVersionUID = 1L;
+
+ public LongConverter() {
+ }
+
+ @Override
+ public byte[] encodeValue(Object value) throws IOException {
+ if (!HBaseTimelineSchemaUtils.isIntegralValue(value)) {
+ throw new IOException("Expected integral value");
+ }
+ return Bytes.toBytes(((Number)value).longValue());
+ }
+
+ @Override
+ public Object decodeValue(byte[] bytes) throws IOException {
+ if (bytes == null) {
+ return null;
+ }
+ return Bytes.toLong(bytes);
+ }
+
+ /**
+ * Compares two numbers as longs. If either number is null, it will be taken
+ * as 0.
+ *
+ * @param num1 the first {@code Long} to compare.
+ * @param num2 the second {@code Long} to compare.
+ * @return -1 if num1 is less than num2, 0 if num1 is equal to num2 and 1 if
+ * num1 is greater than num2.
+ */
+ @Override
+ public int compare(Number num1, Number num2) {
+ return Long.compare((num1 == null) ? 0L : num1.longValue(),
+ (num2 == null) ? 0L : num2.longValue());
+ }
+
+ @Override
+ public Number add(Number num1, Number num2, Number...numbers) {
+ long sum = ((num1 == null) ? 0L : num1.longValue()) +
+ ((num2 == null) ? 0L : num2.longValue());
+ for (Number num : numbers) {
+ sum = sum + ((num == null) ? 0L : num.longValue());
+ }
+ return sum;
+ }
+
+ /**
+ * Converts a timestamp into it's inverse timestamp to be used in (row) keys
+ * where we want to have the most recent timestamp in the top of the table
+ * (scans start at the most recent timestamp first).
+ *
+ * @param key value to be inverted so that the latest version will be first in
+ * a scan.
+ * @return inverted long
+ */
+ public static long invertLong(long key) {
+ return Long.MAX_VALUE - key;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongKeyConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongKeyConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongKeyConverter.java
new file mode 100644
index 0000000..4a724d6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongKeyConverter.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import java.io.IOException;
+/**
+ * Encodes and decodes column names / row keys which are long.
+ */
+public final class LongKeyConverter implements KeyConverter<Long> {
+
+ /**
+ * To delegate the actual work to.
+ */
+ private final LongConverter longConverter = new LongConverter();
+
+ public LongKeyConverter() {
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter
+ * #encode(java.lang.Object)
+ */
+ @Override
+ public byte[] encode(Long key) {
+ try {
+ // IOException will not be thrown here as we are explicitly passing
+ // Long.
+ return longConverter.encodeValue(key);
+ } catch (IOException e) {
+ return null;
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter
+ * #decode(byte[])
+ */
+ @Override
+ public Long decode(byte[] bytes) {
+ try {
+ return (Long) longConverter.decodeValue(bytes);
+ } catch (IOException e) {
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/NumericValueConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/NumericValueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/NumericValueConverter.java
new file mode 100644
index 0000000..8fb6536
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/NumericValueConverter.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import java.util.Comparator;
+
+/**
+ * Extends ValueConverter interface for numeric converters to support numerical
+ * operations such as comparison, addition, etc.
+ */
+public interface NumericValueConverter extends ValueConverter,
+ Comparator<Number> {
+ /**
+ * Adds two or more numbers. If either of the numbers are null, it is taken as
+ * 0.
+ *
+ * @param num1 the first number to add.
+ * @param num2 the second number to add.
+ * @param numbers Rest of the numbers to be added.
+ * @return result after adding up the numbers.
+ */
+ Number add(Number num1, Number num2, Number...numbers);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Range.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Range.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Range.java
new file mode 100644
index 0000000..8a2e01a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Range.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Encapsulates a range with start and end indices.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class Range {
+ private final int startIdx;
+ private final int endIdx;
+
+ /**
+ * Defines a range from start index (inclusive) to end index (exclusive).
+ *
+ * @param start
+ * Starting index position
+ * @param end
+ * Ending index position (exclusive)
+ */
+ public Range(int start, int end) {
+ if (start < 0 || end < start) {
+ throw new IllegalArgumentException(
+ "Invalid range, required that: 0 <= start <= end; start=" + start
+ + ", end=" + end);
+ }
+
+ this.startIdx = start;
+ this.endIdx = end;
+ }
+
+ public int start() {
+ return startIdx;
+ }
+
+ public int end() {
+ return endIdx;
+ }
+
+ public int length() {
+ return endIdx - startIdx;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/RowKeyPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/RowKeyPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/RowKeyPrefix.java
new file mode 100644
index 0000000..6159dc7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/RowKeyPrefix.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+/**
+ * In queries where a single result is needed, an exact rowkey can be used
+ * through the corresponding rowkey#getRowKey() method. For queries that need to
+ * scan over a range of rowkeys, a partial (the initial part) of rowkeys are
+ * used. Classes implementing RowKeyPrefix indicate that they are the initial
+ * part of rowkeys, with different constructors with fewer number of argument to
+ * form a partial rowkey, a prefix.
+ *
+ * @param <R> indicating the type of rowkey that a particular implementation is
+ * a prefix for.
+ */
+public interface RowKeyPrefix<R> {
+
+ /**
+ * Create a row key prefix, meaning a partial rowkey that can be used in range
+ * scans. Which fields are included in the prefix will depend on the
+ * constructor of the specific instance that was used. Output depends on which
+ * constructor was used.
+ * @return a prefix of the following form {@code fist!second!...!last!}
+ */
+ byte[] getRowKeyPrefix();
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
new file mode 100644
index 0000000..5090b4d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
@@ -0,0 +1,575 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Used to separate row qualifiers, column qualifiers and compound fields.
+ */
+public enum Separator {
+
+ /**
+ * separator in key or column qualifier fields.
+ */
+ QUALIFIERS("!", "%0$"),
+
+ /**
+ * separator in values, and/or compound key/column qualifier fields.
+ */
+ VALUES("=", "%1$"),
+
+ /**
+ * separator in values, often used to avoid having these in qualifiers and
+ * names. Note that if we use HTML form encoding through URLEncoder, we end up
+ * getting a + for a space, which may already occur in strings, so we don't
+ * want that.
+ */
+ SPACE(" ", "%2$"),
+
+ /**
+ * separator in values, often used to avoid having these in qualifiers and
+ * names.
+ */
+ TAB("\t", "%3$");
+
+ // a reserved character that starts each of the encoded values and is encoded
+ // first in order to escape naturally occurring instances of encoded values
+ // although it can be expressed as an enum instance, we define them as private
+ // variables to hide it from callers
+ private static final String PERCENT = "%";
+ private static final String PERCENT_ENCODED = "%9$";
+
+ private static final Pattern PERCENT_PATTERN =
+ Pattern.compile(PERCENT, Pattern.LITERAL);
+ private static final String PERCENT_REPLACEMENT =
+ Matcher.quoteReplacement(PERCENT);
+
+ private static final Pattern PERCENT_ENCODED_PATTERN =
+ Pattern.compile(PERCENT_ENCODED, Pattern.LITERAL);
+ private static final String PERCENT_ENCODED_REPLACEMENT =
+ Matcher.quoteReplacement(PERCENT_ENCODED);
+
+ /**
+ * The string value of this separator.
+ */
+ private final String value;
+
+ /**
+ * The bye representation of value.
+ */
+ private final byte[] bytes;
+
+ // pre-compiled patterns and quoted replacements for optimization
+ private final Pattern valuePattern;
+ private final String valueReplacement;
+
+ private final Pattern encodedValuePattern;
+ private final String encodedValueReplacement;
+
+ /**
+ * Indicator for variable size of an individual segment in a split. The
+ * segment ends wherever separator is encountered.
+ * Typically used for string.
+ * Also used to indicate that there is no fixed number of splits which need to
+ * be returned. If split limit is specified as this, all possible splits are
+ * returned.
+ */
+ public static final int VARIABLE_SIZE = 0;
+
+
+ /** empty string. */
+ public static final String EMPTY_STRING = "";
+
+ /** empty bytes. */
+ public static final byte[] EMPTY_BYTES = new byte[0];
+
+ /**
+ * @param value of the separator to use. Cannot be null or empty string.
+ * @param encodedValue choose something that isn't likely to occur in the data
+ * itself. Cannot be null or empty string.
+ */
+ private Separator(String value, String encodedValue) {
+ this.value = value;
+
+ // validation
+ if (value == null || value.length() == 0 || encodedValue == null
+ || encodedValue.length() == 0) {
+ throw new IllegalArgumentException(
+ "Cannot create separator from null or empty string.");
+ }
+
+ this.bytes = Bytes.toBytes(value);
+ this.valuePattern = Pattern.compile(value, Pattern.LITERAL);
+ this.valueReplacement = Matcher.quoteReplacement(value);
+
+ this.encodedValuePattern = Pattern.compile(encodedValue, Pattern.LITERAL);
+ this.encodedValueReplacement = Matcher.quoteReplacement(encodedValue);
+ }
+
+ /**
+ * @return the original value of the separator
+ */
+ public String getValue() {
+ return value;
+ }
+
+ /**
+ * Used to make token safe to be used with this separator without collisions.
+ * It <em>must</em> be paired with {@link #decode(String)} for it to be
+ * decoded correctly.
+ * <p>
+ * If you need to encode a given string for multiple separators,
+ * {@link #encode(String, Separator...)} should be used over successive
+ * invocations of this method. It will result in a more compact version of the
+ * encoded value.
+ *
+ * @param token Token to be encoded.
+ * @return the token with any occurrences of this separator URLEncoded.
+ */
+ public String encode(String token) {
+ if (token == null || token.length() == 0) {
+ // Nothing to replace
+ return token;
+ }
+ // first encode the percent to escape naturally occurring encoded values
+ String escaped = encodePercent(token);
+ return encodeSingle(escaped, this);
+ }
+
+ private static String replace(String token, Pattern pattern,
+ String replacement) {
+ return pattern.matcher(token).replaceAll(replacement);
+ }
+
+ private static String encodeSingle(String token, Separator separator) {
+ return replace(token, separator.valuePattern,
+ separator.encodedValueReplacement);
+ }
+
+ private static String encodePercent(String token) {
+ return replace(token, PERCENT_PATTERN, PERCENT_ENCODED_REPLACEMENT);
+ }
+
+ /**
+ * Decode the token encoded using {@link #encode(String)}. It <em>must</em> be
+ * used for the result encoded with {@link #encode(String)} to be able to
+ * recover the original.
+ *
+ * @param token Token to be decoded.
+ * @return the token with any occurrences of the encoded separator replaced by
+ * the separator itself.
+ */
+ public String decode(String token) {
+ if (token == null || token.length() == 0) {
+ // Nothing to replace
+ return token;
+ }
+ String escaped = decodeSingle(token, this);
+ // decode percent to de-escape
+ return decodePercent(escaped);
+ }
+
+ private static String decodeSingle(String token, Separator separator) {
+ return replace(token, separator.encodedValuePattern,
+ separator.valueReplacement);
+ }
+
+ private static String decodePercent(String token) {
+ return replace(token, PERCENT_ENCODED_PATTERN, PERCENT_REPLACEMENT);
+ }
+
+ /**
+ * Encode the given separators in the token with their encoding equivalents.
+ * It <em>must</em> be paired with {@link #decode(byte[], Separator...)} or
+ * {@link #decode(String, Separator...)} with the same separators for it to be
+ * decoded correctly.
+ * <p>
+ * If you need to encode a given string for multiple separators, this form of
+ * encoding should be used over successive invocations of
+ * {@link #encode(String)}. It will result in a more compact version of the
+ * encoded value.
+ *
+ * @param token containing possible separators that need to be encoded.
+ * @param separators to be encoded in the token with their URLEncoding
+ * equivalent.
+ * @return non-null byte representation of the token with occurrences of the
+ * separators encoded.
+ */
+ public static byte[] encode(String token, Separator... separators) {
+ if (token == null || token.length() == 0) {
+ return EMPTY_BYTES;
+ }
+ String result = token;
+ // first encode the percent to escape naturally occurring encoded values
+ result = encodePercent(token);
+ for (Separator separator : separators) {
+ if (separator != null) {
+ result = encodeSingle(result, separator);
+ }
+ }
+ return Bytes.toBytes(result);
+ }
+
+ /**
+ * Decode the given separators in the token with their decoding equivalents.
+ * It <em>must</em> be used for the result encoded with
+ * {@link #encode(String, Separator...)} with the same separators to be able
+ * to recover the original.
+ *
+ * @param token containing possible separators that need to be encoded.
+ * @param separators to be encoded in the token with their URLEncoding
+ * equivalent.
+ * @return String representation of the token with occurrences of the URL
+ * encoded separators decoded.
+ */
+ public static String decode(byte[] token, Separator... separators) {
+ if (token == null) {
+ return null;
+ }
+ return decode(Bytes.toString(token), separators);
+ }
+
+ /**
+ * Decode the given separators in the token with their decoding equivalents.
+ * It <em>must</em> be used for the result encoded with
+ * {@link #encode(String, Separator...)} with the same separators to be able
+ * to recover the original.
+ *
+ * @param token containing possible separators that need to be encoded.
+ * @param separators to be encoded in the token with their URLEncoding
+ * equivalent.
+ * @return String representation of the token with occurrences of the URL
+ * encoded separators decoded.
+ */
+ public static String decode(String token, Separator... separators) {
+ if (token == null) {
+ return null;
+ }
+ String result = token;
+ for (Separator separator : separators) {
+ if (separator != null) {
+ result = decodeSingle(result, separator);
+ }
+ }
+ // decode percent to de-escape
+ return decodePercent(result);
+ }
+
+ /**
+ * Returns a single byte array containing all of the individual arrays
+ * components separated by this separator.
+ *
+ * @param components Byte array components to be joined together.
+ * @return byte array after joining the components
+ */
+ public byte[] join(byte[]... components) {
+ if (components == null || components.length == 0) {
+ return EMPTY_BYTES;
+ }
+
+ int finalSize = 0;
+ finalSize = this.value.length() * (components.length - 1);
+ for (byte[] comp : components) {
+ if (comp != null) {
+ finalSize += comp.length;
+ }
+ }
+
+ byte[] buf = new byte[finalSize];
+ int offset = 0;
+ for (int i = 0; i < components.length; i++) {
+ if (components[i] != null) {
+ System.arraycopy(components[i], 0, buf, offset, components[i].length);
+ offset += components[i].length;
+ }
+ if (i < (components.length - 1)) {
+ System.arraycopy(this.bytes, 0, buf, offset, this.value.length());
+ offset += this.value.length();
+ }
+ }
+ return buf;
+ }
+
+ /**
+ * Concatenates items (as String), using this separator.
+ *
+ * @param items Items join, {@code toString()} will be called in each item.
+ * Any occurrence of the separator in the individual strings will be
+ * first encoded. Cannot be null.
+ * @return non-null joined result. Note that when separator is {@literal null}
+ * the result is simply all items concatenated and the process is not
+ * reversible through {@link #splitEncoded(String)}
+ */
+ public String joinEncoded(String... items) {
+ if (items == null || items.length == 0) {
+ return "";
+ }
+
+ StringBuilder sb = new StringBuilder(encode(items[0].toString()));
+ // Start at 1, we've already grabbed the first value at index 0
+ for (int i = 1; i < items.length; i++) {
+ sb.append(this.value);
+ sb.append(encode(items[i].toString()));
+ }
+
+ return sb.toString();
+ }
+
+ /**
+ * Concatenates items (as String), using this separator.
+ *
+ * @param items Items join, {@code toString()} will be called in each item.
+ * Any occurrence of the separator in the individual strings will be
+ * first encoded. Cannot be null.
+ * @return non-null joined result. Note that when separator is {@literal null}
+ * the result is simply all items concatenated and the process is not
+ * reversible through {@link #splitEncoded(String)}
+ */
+ public String joinEncoded(Iterable<?> items) {
+ if (items == null) {
+ return "";
+ }
+ Iterator<?> i = items.iterator();
+ if (!i.hasNext()) {
+ return "";
+ }
+
+ StringBuilder sb = new StringBuilder(encode(i.next().toString()));
+ while (i.hasNext()) {
+ sb.append(this.value);
+ sb.append(encode(i.next().toString()));
+ }
+
+ return sb.toString();
+ }
+
+ /**
+ * @param compoundValue containing individual values separated by this
+ * separator, which have that separator encoded.
+ * @return non-null set of values from the compoundValue with the separator
+ * decoded.
+ */
+ public Collection<String> splitEncoded(String compoundValue) {
+ List<String> result = new ArrayList<String>();
+ if (compoundValue != null) {
+ for (String val : valuePattern.split(compoundValue)) {
+ result.add(decode(val));
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Splits the source array into multiple array segments using this separator,
+ * up to a maximum of count items. This will naturally produce copied byte
+ * arrays for each of the split segments.
+ *
+ * @param source to be split
+ * @param limit on how many segments are supposed to be returned. A
+ * non-positive value indicates no limit on number of segments.
+ * @return source split by this separator.
+ */
+ public byte[][] split(byte[] source, int limit) {
+ return split(source, this.bytes, limit);
+ }
+
+ /**
+ * Splits the source array into multiple array segments using this separator.
+ * The sizes indicate the sizes of the relative components/segments.
+ * In case one of the segments contains this separator before the specified
+ * size is reached, the separator will be considered part of that segment and
+ * we will continue till size is reached.
+ * Variable length strings cannot contain this separator and are indiced with
+ * a size of {@value #VARIABLE_SIZE}. Such strings are encoded for this
+ * separator and decoded after the results from split is returned.
+ *
+ * @param source byte array to be split.
+ * @param sizes sizes of relative components/segments.
+ * @return source split by this separator as per the sizes specified..
+ */
+ public byte[][] split(byte[] source, int[] sizes) {
+ return split(source, this.bytes, sizes);
+ }
+
+ /**
+ * Splits the source array into multiple array segments using this separator,
+ * as many times as splits are found. This will naturally produce copied byte
+ * arrays for each of the split segments.
+ *
+ * @param source byte array to be split
+ * @return source split by this separator.
+ */
+ public byte[][] split(byte[] source) {
+ return split(source, this.bytes);
+ }
+
+ /**
+ * Returns a list of ranges identifying [start, end) -- closed, open --
+ * positions within the source byte array that would be split using the
+ * separator byte array.
+ * The sizes indicate the sizes of the relative components/segments.
+ * In case one of the segments contains this separator before the specified
+ * size is reached, the separator will be considered part of that segment and
+ * we will continue till size is reached.
+ * Variable length strings cannot contain this separator and are indiced with
+ * a size of {@value #VARIABLE_SIZE}. Such strings are encoded for this
+ * separator and decoded after the results from split is returned.
+ *
+ * @param source the source data
+ * @param separator the separator pattern to look for
+ * @param sizes indicate the sizes of the relative components/segments.
+ * @return a list of ranges.
+ */
+ private static List<Range> splitRanges(byte[] source, byte[] separator,
+ int[] sizes) {
+ List<Range> segments = new ArrayList<Range>();
+ if (source == null || separator == null) {
+ return segments;
+ }
+ // VARIABLE_SIZE here indicates that there is no limit to number of segments
+ // to return.
+ int limit = VARIABLE_SIZE;
+ if (sizes != null && sizes.length > 0) {
+ limit = sizes.length;
+ }
+ int start = 0;
+ int currentSegment = 0;
+ itersource: for (int i = 0; i < source.length; i++) {
+ for (int j = 0; j < separator.length; j++) {
+ if (source[i + j] != separator[j]) {
+ continue itersource;
+ }
+ }
+ // all separator elements matched
+ if (limit > VARIABLE_SIZE) {
+ if (segments.size() >= (limit - 1)) {
+ // everything else goes in one final segment
+ break;
+ }
+ if (sizes != null) {
+ int currentSegExpectedSize = sizes[currentSegment];
+ if (currentSegExpectedSize > VARIABLE_SIZE) {
+ int currentSegSize = i - start;
+ if (currentSegSize < currentSegExpectedSize) {
+ // Segment not yet complete. More bytes to parse.
+ continue itersource;
+ } else if (currentSegSize > currentSegExpectedSize) {
+ // Segment is not as per size.
+ throw new IllegalArgumentException(
+ "Segments not separated as per expected sizes");
+ }
+ }
+ }
+ }
+ segments.add(new Range(start, i));
+ start = i + separator.length;
+ // i will be incremented again in outer for loop
+ i += separator.length - 1;
+ currentSegment++;
+ }
+ // add in remaining to a final range
+ if (start <= source.length) {
+ if (sizes != null) {
+ // Check if final segment is as per size specified.
+ if (sizes[currentSegment] > VARIABLE_SIZE &&
+ source.length - start > sizes[currentSegment]) {
+ // Segment is not as per size.
+ throw new IllegalArgumentException(
+ "Segments not separated as per expected sizes");
+ }
+ }
+ segments.add(new Range(start, source.length));
+ }
+ return segments;
+ }
+
+ /**
+ * Splits based on segments calculated based on limit/sizes specified for the
+ * separator.
+ *
+ * @param source byte array to be split.
+ * @param segments specifies the range for each segment.
+ * @return a byte[][] split as per the segment ranges.
+ */
+ private static byte[][] split(byte[] source, List<Range> segments) {
+ byte[][] splits = new byte[segments.size()][];
+ for (int i = 0; i < segments.size(); i++) {
+ Range r = segments.get(i);
+ byte[] tmp = new byte[r.length()];
+ if (tmp.length > 0) {
+ System.arraycopy(source, r.start(), tmp, 0, r.length());
+ }
+ splits[i] = tmp;
+ }
+ return splits;
+ }
+
+ /**
+ * Splits the source array into multiple array segments using the given
+ * separator based on the sizes. This will naturally produce copied byte
+ * arrays for each of the split segments.
+ *
+ * @param source source array.
+ * @param separator separator represented as a byte array.
+ * @param sizes sizes of relative components/segments.
+ * @return byte[][] after splitting the source.
+ */
+ private static byte[][] split(byte[] source, byte[] separator, int[] sizes) {
+ List<Range> segments = splitRanges(source, separator, sizes);
+ return split(source, segments);
+ }
+
+ /**
+ * Splits the source array into multiple array segments using the given
+ * separator. This will naturally produce copied byte arrays for each of the
+ * split segments.
+ *
+ * @param source Source array.
+ * @param separator Separator represented as a byte array.
+ * @return byte[][] after splitting the source.
+ */
+ private static byte[][] split(byte[] source, byte[] separator) {
+ return split(source, separator, (int[]) null);
+ }
+
+ /**
+ * Splits the source array into multiple array segments using the given
+ * separator, up to a maximum of count items. This will naturally produce
+ * copied byte arrays for each of the split segments.
+ *
+ * @param source Source array.
+ * @param separator Separator represented as a byte array.
+ * @param limit a non-positive value indicates no limit on number of segments.
+ * @return byte[][] after splitting the input source.
+ */
+ private static byte[][] split(byte[] source, byte[] separator, int limit) {
+ int[] sizes = null;
+ if (limit > VARIABLE_SIZE) {
+ sizes = new int[limit];
+ }
+ List<Range> segments = splitRanges(source, separator, sizes);
+ return split(source, segments);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/StringKeyConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/StringKeyConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/StringKeyConverter.java
new file mode 100644
index 0000000..282848e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/StringKeyConverter.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+/**
+ * Encodes and decodes column names / row keys which are merely strings.
+ * Column prefixes are not part of the column name passed for encoding. It is
+ * added later, if required in the associated ColumnPrefix implementations.
+ */
+public final class StringKeyConverter implements KeyConverter<String> {
+
+ public StringKeyConverter() {
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter
+ * #encode(java.lang.Object)
+ */
+ @Override
+ public byte[] encode(String key) {
+ return Separator.encode(key, Separator.SPACE, Separator.TAB);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter
+ * #decode(byte[])
+ */
+ @Override
+ public String decode(byte[] bytes) {
+ return Separator.decode(bytes, Separator.TAB, Separator.SPACE);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimestampGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimestampGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimestampGenerator.java
new file mode 100644
index 0000000..d03b37d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimestampGenerator.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+/**
+ * Utility class that allows HBase coprocessors to interact with unique
+ * timestamps.
+ */
+public class TimestampGenerator {
+
+ /*
+ * if this is changed, then reading cell timestamps written with older
+ * multiplier value will not work
+ */
+ public static final long TS_MULTIPLIER = 1000000L;
+
+ private final AtomicLong lastTimestamp = new AtomicLong();
+
+ /**
+ * Returns the current wall clock time in milliseconds, multiplied by the
+ * required precision.
+ *
+ * @return current timestamp.
+ */
+ public long currentTime() {
+ // We want to align cell timestamps with current time.
+ // cell timestamps are not be less than
+ // System.currentTimeMillis() * TS_MULTIPLIER.
+ return System.currentTimeMillis() * TS_MULTIPLIER;
+ }
+
+ /**
+ * Returns a timestamp value unique within the scope of this
+ * {@code TimestampGenerator} instance. For usage by HBase
+ * {@code RegionObserver} coprocessors, this normally means unique within a
+ * given region.
+ *
+ * Unlikely scenario of generating a non-unique timestamp: if there is a
+ * sustained rate of more than 1M hbase writes per second AND if region fails
+ * over within that time range of timestamps being generated then there may be
+ * collisions writing to a cell version of the same column.
+ *
+ * @return unique timestamp.
+ */
+ public long getUniqueTimestamp() {
+ long lastTs;
+ long nextTs;
+ do {
+ lastTs = lastTimestamp.get();
+ nextTs = Math.max(lastTs + 1, currentTime());
+ } while (!lastTimestamp.compareAndSet(lastTs, nextTs));
+ return nextTs;
+ }
+
+ /**
+ * Returns a timestamp multiplied with TS_MULTIPLIER and last few digits of
+ * application id.
+ *
+ * Unlikely scenario of generating a timestamp that is a duplicate: If more
+ * than a 1M concurrent apps are running in one flow run AND write to same
+ * column at the same time, then say appId of 1M and 1 will overlap
+ * with appId of 001 and there may be collisions for that flow run's
+ * specific column.
+ *
+ * @param incomingTS Timestamp to be converted.
+ * @param appId Application Id.
+ * @return a timestamp multiplied with TS_MULTIPLIER and last few digits of
+ * application id
+ */
+ public static long getSupplementedTimestamp(long incomingTS, String appId) {
+ long suffix = getAppIdSuffix(appId);
+ long outgoingTS = incomingTS * TS_MULTIPLIER + suffix;
+ return outgoingTS;
+
+ }
+
+ private static long getAppIdSuffix(String appIdStr) {
+ if (appIdStr == null) {
+ return 0L;
+ }
+ ApplicationId appId = ApplicationId.fromString(appIdStr);
+ long id = appId.getId() % TS_MULTIPLIER;
+ return id;
+ }
+
+ /**
+ * truncates the last few digits of the timestamp which were supplemented by
+ * the TimestampGenerator#getSupplementedTimestamp function.
+ *
+ * @param incomingTS Timestamp to be truncated.
+ * @return a truncated timestamp value
+ */
+ public static long getTruncatedTimestamp(long incomingTS) {
+ return incomingTS / TS_MULTIPLIER;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ValueConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ValueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ValueConverter.java
new file mode 100644
index 0000000..757a6d3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ValueConverter.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import java.io.IOException;
+
+/**
+ * Converter used to encode/decode value associated with a column prefix or a
+ * column.
+ */
+public interface ValueConverter {
+
+ /**
+ * Encode an object as a byte array depending on the converter implementation.
+ *
+ * @param value Value to be encoded.
+ * @return a byte array
+ * @throws IOException if any problem is encountered while encoding.
+ */
+ byte[] encodeValue(Object value) throws IOException;
+
+ /**
+ * Decode a byte array and convert it into an object depending on the
+ * converter implementation.
+ *
+ * @param bytes Byte array to be decoded.
+ * @return an object
+ * @throws IOException if any problem is encountered while decoding.
+ */
+ Object decodeValue(byte[] bytes) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java
new file mode 100644
index 0000000..0df5b8a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.storage.common contains
+ * a set of utility classes used across backend storage reader and writer.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
new file mode 100644
index 0000000..81961d3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Column;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.GenericConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
+
+/**
+ * Identifies fully qualified columns for the {@link EntityTable}.
+ */
+public enum EntityColumn implements Column<EntityTable> {
+
+ /**
+ * Identifier for the entity.
+ */
+ ID(EntityColumnFamily.INFO, "id"),
+
+ /**
+ * The type of entity.
+ */
+ TYPE(EntityColumnFamily.INFO, "type"),
+
+ /**
+ * When the entity was created.
+ */
+ CREATED_TIME(EntityColumnFamily.INFO, "created_time", new LongConverter()),
+
+ /**
+ * The version of the flow that this entity belongs to.
+ */
+ FLOW_VERSION(EntityColumnFamily.INFO, "flow_version");
+
+ private final ColumnFamily<EntityTable> columnFamily;
+ private final String columnQualifier;
+ private final byte[] columnQualifierBytes;
+ private final ValueConverter valueConverter;
+
+ EntityColumn(ColumnFamily<EntityTable> columnFamily,
+ String columnQualifier) {
+ this(columnFamily, columnQualifier, GenericConverter.getInstance());
+ }
+
+ EntityColumn(ColumnFamily<EntityTable> columnFamily,
+ String columnQualifier, ValueConverter converter) {
+ this.columnFamily = columnFamily;
+ this.columnQualifier = columnQualifier;
+ // Future-proof by ensuring the right column prefix hygiene.
+ this.columnQualifierBytes =
+ Bytes.toBytes(Separator.SPACE.encode(columnQualifier));
+ this.valueConverter = converter;
+ }
+
+ /**
+ * @return the column name value
+ */
+ private String getColumnQualifier() {
+ return columnQualifier;
+ }
+
+ @Override
+ public byte[] getColumnQualifierBytes() {
+ return columnQualifierBytes.clone();
+ }
+
+ @Override
+ public byte[] getColumnFamilyBytes() {
+ return columnFamily.getBytes();
+ }
+
+ @Override
+ public ValueConverter getValueConverter() {
+ return valueConverter;
+ }
+
+ @Override
+ public Attribute[] getCombinedAttrsWithAggr(Attribute... attributes) {
+ return attributes;
+ }
+
+ @Override
+ public boolean supplementCellTimestamp() {
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java
new file mode 100644
index 0000000..7c63727
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+
+/**
+ * Represents the entity table column families.
+ */
+public enum EntityColumnFamily implements ColumnFamily<EntityTable> {
+
+ /**
+ * Info column family houses known columns, specifically ones included in
+ * columnfamily filters.
+ */
+ INFO("i"),
+
+ /**
+ * Configurations are in a separate column family for two reasons: a) the size
+ * of the config values can be very large and b) we expect that config values
+ * are often separately accessed from other metrics and info columns.
+ */
+ CONFIGS("c"),
+
+ /**
+ * Metrics have a separate column family, because they have a separate TTL.
+ */
+ METRICS("m");
+
+ /**
+ * Byte representation of this column family.
+ */
+ private final byte[] bytes;
+
+ /**
+ * @param value create a column family with this name. Must be lower case and
+ * without spaces.
+ */
+ EntityColumnFamily(String value) {
+ // column families should be lower case and not contain any spaces.
+ this.bytes = Bytes.toBytes(Separator.SPACE.encode(value));
+ }
+
+ public byte[] getBytes() {
+ return Bytes.copy(bytes);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
new file mode 100644
index 0000000..08234d3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.GenericConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
+
+/**
+ * Identifies partially qualified columns for the entity table.
+ */
+public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
+
+ /**
+ * To store TimelineEntity getIsRelatedToEntities values.
+ */
+ IS_RELATED_TO(EntityColumnFamily.INFO, "s"),
+
+ /**
+ * To store TimelineEntity getRelatesToEntities values.
+ */
+ RELATES_TO(EntityColumnFamily.INFO, "r"),
+
+ /**
+ * To store TimelineEntity info values.
+ */
+ INFO(EntityColumnFamily.INFO, "i"),
+
+ /**
+ * Lifecycle events for an entity.
+ */
+ EVENT(EntityColumnFamily.INFO, "e", true),
+
+ /**
+ * Config column stores configuration with config key as the column name.
+ */
+ CONFIG(EntityColumnFamily.CONFIGS, null),
+
+ /**
+ * Metrics are stored with the metric name as the column name.
+ */
+ METRIC(EntityColumnFamily.METRICS, null, new LongConverter());
+
+ private final ColumnFamily<EntityTable> columnFamily;
+
+ /**
+ * Can be null for those cases where the provided column qualifier is the
+ * entire column name.
+ */
+ private final String columnPrefix;
+ private final byte[] columnPrefixBytes;
+ private final ValueConverter valueConverter;
+
+ /**
+ * Private constructor, meant to be used by the enum definition.
+ *
+ * @param columnFamily that this column is stored in.
+ * @param columnPrefix for this column.
+ */
+ EntityColumnPrefix(ColumnFamily<EntityTable> columnFamily,
+ String columnPrefix) {
+ this(columnFamily, columnPrefix, false, GenericConverter.getInstance());
+ }
+
+ EntityColumnPrefix(ColumnFamily<EntityTable> columnFamily,
+ String columnPrefix, boolean compondColQual) {
+ this(columnFamily, columnPrefix, compondColQual,
+ GenericConverter.getInstance());
+ }
+
+ EntityColumnPrefix(ColumnFamily<EntityTable> columnFamily,
+ String columnPrefix, ValueConverter converter) {
+ this(columnFamily, columnPrefix, false, converter);
+ }
+
+ /**
+ * Private constructor, meant to be used by the enum definition.
+ *
+ * @param columnFamily that this column is stored in.
+ * @param columnPrefix for this column.
+ * @param converter used to encode/decode values to be stored in HBase for
+ * this column prefix.
+ */
+ EntityColumnPrefix(ColumnFamily<EntityTable> columnFamily,
+ String columnPrefix, boolean compondColQual, ValueConverter converter) {
+ this.valueConverter = converter;
+ this.columnFamily = columnFamily;
+ this.columnPrefix = columnPrefix;
+ if (columnPrefix == null) {
+ this.columnPrefixBytes = null;
+ } else {
+ // Future-proof by ensuring the right column prefix hygiene.
+ this.columnPrefixBytes =
+ Bytes.toBytes(Separator.SPACE.encode(columnPrefix));
+ }
+ }
+
+ /**
+ * @return the column name value
+ */
+ public String getColumnPrefix() {
+ return columnPrefix;
+ }
+
+ @Override
+ public byte[] getColumnPrefixBytes(byte[] qualifierPrefix) {
+ return ColumnHelper.getColumnQualifier(
+ this.columnPrefixBytes, qualifierPrefix);
+ }
+
+ @Override
+ public byte[] getColumnPrefixBytes(String qualifierPrefix) {
+ return ColumnHelper.getColumnQualifier(
+ this.columnPrefixBytes, qualifierPrefix);
+ }
+
+ @Override
+ public byte[] getColumnPrefixInBytes() {
+ return columnPrefixBytes != null ? columnPrefixBytes.clone() : null;
+ }
+
+ @Override
+ public byte[] getColumnFamilyBytes() {
+ return columnFamily.getBytes();
+ }
+
+ @Override
+ public ValueConverter getValueConverter() {
+ return valueConverter;
+ }
+
+ @Override
+ public Attribute[] getCombinedAttrsWithAggr(Attribute... attributes) {
+ return attributes;
+ }
+
+ @Override
+ public boolean supplementCellTimeStamp() {
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
new file mode 100644
index 0000000..b85a9b0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
@@ -0,0 +1,299 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
+
+import java.util.List;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.AppIdKeyConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverterToString;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+
+/**
+ * Represents a rowkey for the entity table.
+ */
+public class EntityRowKey {
+ private final String clusterId;
+ private final String userId;
+ private final String flowName;
+ private final Long flowRunId;
+ private final String appId;
+ private final String entityType;
+ private final Long entityIdPrefix;
+ private final String entityId;
+ private final EntityRowKeyConverter entityRowKeyConverter =
+ new EntityRowKeyConverter();
+
+ public EntityRowKey(String clusterId, String userId, String flowName,
+ Long flowRunId, String appId, String entityType, Long entityIdPrefix,
+ String entityId) {
+ this.clusterId = clusterId;
+ this.userId = userId;
+ this.flowName = flowName;
+ this.flowRunId = flowRunId;
+ this.appId = appId;
+ this.entityType = entityType;
+ this.entityIdPrefix = entityIdPrefix;
+ this.entityId = entityId;
+ }
+
+ public String getClusterId() {
+ return clusterId;
+ }
+
+ public String getUserId() {
+ return userId;
+ }
+
+ public String getFlowName() {
+ return flowName;
+ }
+
+ public Long getFlowRunId() {
+ return flowRunId;
+ }
+
+ public String getAppId() {
+ return appId;
+ }
+
+ public String getEntityType() {
+ return entityType;
+ }
+
+ public String getEntityId() {
+ return entityId;
+ }
+
+ public Long getEntityIdPrefix() {
+ return entityIdPrefix;
+ }
+
+ /**
+ * Constructs a row key for the entity table as follows:
+ * {@code userName!clusterId!flowName!flowRunId!AppId!entityType!entityId}.
+ * Typically used while querying a specific entity.
+ *
+ * @return byte array with the row key.
+ */
+ public byte[] getRowKey() {
+ return entityRowKeyConverter.encode(this);
+ }
+
+ /**
+ * Given the raw row key as bytes, returns the row key as an object.
+ * @param rowKey byte representation of row key.
+ * @return An <cite>EntityRowKey</cite> object.
+ */
+ public static EntityRowKey parseRowKey(byte[] rowKey) {
+ return new EntityRowKeyConverter().decode(rowKey);
+ }
+
+ /**
+ * Constructs a row key for the entity table as follows:
+ * <p>
+ * {@code userName!clusterId!flowName!flowRunId!AppId!
+ * entityType!entityIdPrefix!entityId}.
+ * </p>
+ * @return String representation of row key.
+ */
+ public String getRowKeyAsString() {
+ return entityRowKeyConverter.encodeAsString(this);
+ }
+
+ /**
+ * Given the encoded row key as string, returns the row key as an object.
+ * @param encodedRowKey String representation of row key.
+ * @return A <cite>EntityRowKey</cite> object.
+ */
+ public static EntityRowKey parseRowKeyFromString(String encodedRowKey) {
+ return new EntityRowKeyConverter().decodeFromString(encodedRowKey);
+ }
+
+ /**
+ * Encodes and decodes row key for entity table. The row key is of the form :
+ * userName!clusterId!flowName!flowRunId!appId!entityType!entityId. flowRunId
+ * is a long, appId is encoded/decoded using {@link AppIdKeyConverter} and
+ * rest are strings.
+ * <p>
+ */
+ final private static class EntityRowKeyConverter implements
+ KeyConverter<EntityRowKey>, KeyConverterToString<EntityRowKey> {
+
+ private final AppIdKeyConverter appIDKeyConverter = new AppIdKeyConverter();
+
+ private EntityRowKeyConverter() {
+ }
+
+ /**
+ * Entity row key is of the form
+ * userName!clusterId!flowName!flowRunId!appId!entityType!entityId w. each
+ * segment separated by !. The sizes below indicate sizes of each one of
+ * these segments in sequence. clusterId, userName, flowName, entityType and
+ * entityId are strings. flowrunId is a long hence 8 bytes in size. app id
+ * is represented as 12 bytes with cluster timestamp part of appid being 8
+ * bytes (long) and seq id being 4 bytes(int). Strings are variable in size
+ * (i.e. end whenever separator is encountered). This is used while decoding
+ * and helps in determining where to split.
+ */
+ private static final int[] SEGMENT_SIZES = {Separator.VARIABLE_SIZE,
+ Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
+ AppIdKeyConverter.getKeySize(), Separator.VARIABLE_SIZE,
+ Bytes.SIZEOF_LONG, Separator.VARIABLE_SIZE };
+
+ /*
+ * (non-Javadoc)
+ *
+ * Encodes EntityRowKey object into a byte array with each component/field
+ * in EntityRowKey separated by Separator#QUALIFIERS. This leads to an
+ * entity table row key of the form
+ * userName!clusterId!flowName!flowRunId!appId!entityType!entityId If
+ * entityType in passed EntityRowKey object is null (and the fields
+ * preceding it i.e. clusterId, userId and flowName, flowRunId and appId
+ * are not null), this returns a row key prefix of the form
+ * userName!clusterId!flowName!flowRunId!appId! and if entityId in
+ * EntityRowKey is null (other 6 components are not null), this returns a
+ * row key prefix of the form
+ * userName!clusterId!flowName!flowRunId!appId!entityType! flowRunId is
+ * inverted while encoding as it helps maintain a descending order for row
+ * keys in entity table.
+ *
+ * @see org.apache.hadoop.yarn.server.timelineservice.storage.common
+ * .KeyConverter#encode(java.lang.Object)
+ */
+ @Override
+ public byte[] encode(EntityRowKey rowKey) {
+ byte[] user =
+ Separator.encode(rowKey.getUserId(), Separator.SPACE, Separator.TAB,
+ Separator.QUALIFIERS);
+ byte[] cluster =
+ Separator.encode(rowKey.getClusterId(), Separator.SPACE,
+ Separator.TAB, Separator.QUALIFIERS);
+ byte[] flow =
+ Separator.encode(rowKey.getFlowName(), Separator.SPACE,
+ Separator.TAB, Separator.QUALIFIERS);
+ byte[] first = Separator.QUALIFIERS.join(user, cluster, flow);
+ // Note that flowRunId is a long, so we can't encode them all at the same
+ // time.
+ byte[] second =
+ Bytes.toBytes(LongConverter.invertLong(rowKey.getFlowRunId()));
+ byte[] third = appIDKeyConverter.encode(rowKey.getAppId());
+ if (rowKey.getEntityType() == null) {
+ return Separator.QUALIFIERS.join(first, second, third,
+ Separator.EMPTY_BYTES);
+ }
+ byte[] entityType =
+ Separator.encode(rowKey.getEntityType(), Separator.SPACE,
+ Separator.TAB, Separator.QUALIFIERS);
+
+ if (rowKey.getEntityIdPrefix() == null) {
+ return Separator.QUALIFIERS.join(first, second, third, entityType,
+ Separator.EMPTY_BYTES);
+ }
+
+ byte[] entityIdPrefix = Bytes.toBytes(rowKey.getEntityIdPrefix());
+
+ if (rowKey.getEntityId() == null) {
+ return Separator.QUALIFIERS.join(first, second, third, entityType,
+ entityIdPrefix, Separator.EMPTY_BYTES);
+ }
+
+ byte[] entityId = Separator.encode(rowKey.getEntityId(), Separator.SPACE,
+ Separator.TAB, Separator.QUALIFIERS);
+
+ byte[] fourth =
+ Separator.QUALIFIERS.join(entityType, entityIdPrefix, entityId);
+
+ return Separator.QUALIFIERS.join(first, second, third, fourth);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * Decodes an application row key of the form
+ * userName!clusterId!flowName!flowRunId!appId!entityType!entityId
+ * represented in byte format and converts it into an EntityRowKey object.
+ * flowRunId is inverted while decoding as it was inverted while encoding.
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common
+ * .KeyConverter#decode(byte[])
+ */
+ @Override
+ public EntityRowKey decode(byte[] rowKey) {
+ byte[][] rowKeyComponents =
+ Separator.QUALIFIERS.split(rowKey, SEGMENT_SIZES);
+ if (rowKeyComponents.length != 8) {
+ throw new IllegalArgumentException("the row key is not valid for "
+ + "an entity");
+ }
+ String userId =
+ Separator.decode(Bytes.toString(rowKeyComponents[0]),
+ Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
+ String clusterId =
+ Separator.decode(Bytes.toString(rowKeyComponents[1]),
+ Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
+ String flowName =
+ Separator.decode(Bytes.toString(rowKeyComponents[2]),
+ Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
+ Long flowRunId =
+ LongConverter.invertLong(Bytes.toLong(rowKeyComponents[3]));
+ String appId = appIDKeyConverter.decode(rowKeyComponents[4]);
+ String entityType =
+ Separator.decode(Bytes.toString(rowKeyComponents[5]),
+ Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
+
+ Long entityPrefixId = Bytes.toLong(rowKeyComponents[6]);
+
+ String entityId =
+ Separator.decode(Bytes.toString(rowKeyComponents[7]),
+ Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
+ return new EntityRowKey(clusterId, userId, flowName, flowRunId, appId,
+ entityType, entityPrefixId, entityId);
+ }
+
+ @Override
+ public String encodeAsString(EntityRowKey key) {
+ if (key.clusterId == null || key.userId == null || key.flowName == null
+ || key.flowRunId == null || key.appId == null
+ || key.entityType == null || key.entityIdPrefix == null
+ || key.entityId == null) {
+ throw new IllegalArgumentException();
+ }
+ return TimelineReaderUtils
+ .joinAndEscapeStrings(new String[] {key.clusterId, key.userId,
+ key.flowName, key.flowRunId.toString(), key.appId, key.entityType,
+ key.entityIdPrefix.toString(), key.entityId});
+ }
+
+ @Override
+ public EntityRowKey decodeFromString(String encodedRowKey) {
+ List<String> split = TimelineReaderUtils.split(encodedRowKey);
+ if (split == null || split.size() != 8) {
+ throw new IllegalArgumentException("Invalid row key for entity table.");
+ }
+ Long flowRunId = Long.valueOf(split.get(3));
+ Long entityIdPrefix = Long.valueOf(split.get(6));
+ return new EntityRowKey(split.get(0), split.get(1), split.get(2),
+ flowRunId, split.get(4), split.get(5), entityIdPrefix, split.get(7));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKeyPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKeyPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKeyPrefix.java
new file mode 100644
index 0000000..47a1789
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKeyPrefix.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
+
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.RowKeyPrefix;
+
+/**
+ * Represents a partial rowkey without the entityId or without entityType and
+ * entityId for the entity table.
+ *
+ */
+public class EntityRowKeyPrefix extends EntityRowKey implements
+ RowKeyPrefix<EntityRowKey> {
+
+ /**
+ * Creates a prefix which generates the following rowKeyPrefixes for the
+ * entity table:
+ * {@code userName!clusterId!flowName!flowRunId!AppId!entityType!}.
+ * @param clusterId identifying the cluster
+ * @param userId identifying the user
+ * @param flowName identifying the flow
+ * @param flowRunId identifying the individual run of this flow
+ * @param appId identifying the application
+ * @param entityType which entity type
+ * @param entityIdPrefix for entityId
+ * @param entityId for an entity
+ */
+ public EntityRowKeyPrefix(String clusterId, String userId, String flowName,
+ Long flowRunId, String appId, String entityType, Long entityIdPrefix,
+ String entityId) {
+ super(clusterId, userId, flowName, flowRunId, appId, entityType,
+ entityIdPrefix, entityId);
+ }
+
+ /**
+ * Creates a prefix which generates the following rowKeyPrefixes for the
+ * entity table:
+ * {@code userName!clusterId!flowName!flowRunId!AppId!entityType!entityId}.
+ *
+ * @param clusterId identifying the cluster
+ * @param userId identifying the user
+ * @param flowName identifying the flow
+ * @param flowRunId identifying the individual run of this flow
+ * @param appId identifying the application
+ */
+ public EntityRowKeyPrefix(String clusterId, String userId, String flowName,
+ Long flowRunId, String appId) {
+ this(clusterId, userId, flowName, flowRunId, appId, null, null, null);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.application.
+ * RowKeyPrefix#getRowKeyPrefix()
+ */
+ public byte[] getRowKeyPrefix() {
+ return super.getRowKey();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
new file mode 100644
index 0000000..dceeb99
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
+
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+
+/**
+ * The entity table as column families info, config and metrics. Info stores
+ * information about a timeline entity object config stores configuration data
+ * of a timeline entity object metrics stores the metrics of a timeline entity
+ * object
+ *
+ * Example entity table record:
+ *
+ * <pre>
+ * |-------------------------------------------------------------------------|
+ * | Row | Column Family | Column Family| Column Family|
+ * | key | info | metrics | config |
+ * |-------------------------------------------------------------------------|
+ * | userName! | id:entityId | metricId1: | configKey1: |
+ * | clusterId! | | metricValue1 | configValue1 |
+ * | flowName! | type:entityType | @timestamp1 | |
+ * | flowRunId! | | | configKey2: |
+ * | AppId! | created_time: | metricId1: | configValue2 |
+ * | entityType!| 1392993084018 | metricValue2 | |
+ * | idPrefix! | | @timestamp2 | |
+ * | entityId | i!infoKey: | | |
+ * | | infoValue | metricId1: | |
+ * | | | metricValue1 | |
+ * | | r!relatesToKey: | @timestamp2 | |
+ * | | id3=id4=id5 | | |
+ * | | | | |
+ * | | s!isRelatedToKey | | |
+ * | | id7=id9=id6 | | |
+ * | | | | |
+ * | | e!eventId=timestamp=infoKey: | | |
+ * | | eventInfoValue | | |
+ * | | | | |
+ * | | flowVersion: | | |
+ * | | versionValue | | |
+ * |-------------------------------------------------------------------------|
+ * </pre>
+ */
+public final class EntityTable extends BaseTable<EntityTable> {
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/18] hadoop git commit: YARN-7919. Refactor timelineservice-hbase
module into submodules. Contributed by Haibo Chen.
Posted by ro...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
deleted file mode 100644
index 0edd6a5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
+++ /dev/null
@@ -1,520 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.Set;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Query;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.filter.BinaryComparator;
-import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
-import org.apache.hadoop.hbase.filter.FamilyFilter;
-import org.apache.hadoop.hbase.filter.FilterList;
-import org.apache.hadoop.hbase.filter.FilterList.Operator;
-import org.apache.hadoop.hbase.filter.PageFilter;
-import org.apache.hadoop.hbase.filter.QualifierFilter;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKeyPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.RowKeyPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
-import org.apache.hadoop.yarn.webapp.BadRequestException;
-
-import com.google.common.base.Preconditions;
-
-/**
- * Timeline entity reader for application entities that are stored in the
- * application table.
- */
-class ApplicationEntityReader extends GenericEntityReader {
- private static final ApplicationTable APPLICATION_TABLE =
- new ApplicationTable();
-
- public ApplicationEntityReader(TimelineReaderContext ctxt,
- TimelineEntityFilters entityFilters, TimelineDataToRetrieve toRetrieve) {
- super(ctxt, entityFilters, toRetrieve);
- }
-
- public ApplicationEntityReader(TimelineReaderContext ctxt,
- TimelineDataToRetrieve toRetrieve) {
- super(ctxt, toRetrieve);
- }
-
- /**
- * Uses the {@link ApplicationTable}.
- */
- protected BaseTable<?> getTable() {
- return APPLICATION_TABLE;
- }
-
- /**
- * This method is called only for multiple entity reads.
- */
- @Override
- protected FilterList constructFilterListBasedOnFilters() throws IOException {
- // Filters here cannot be null for multiple entity reads as they are set in
- // augmentParams if null.
- TimelineEntityFilters filters = getFilters();
- FilterList listBasedOnFilters = new FilterList();
- // Create filter list based on created time range and add it to
- // listBasedOnFilters.
- long createdTimeBegin = filters.getCreatedTimeBegin();
- long createdTimeEnd = filters.getCreatedTimeEnd();
- if (createdTimeBegin != 0 || createdTimeEnd != Long.MAX_VALUE) {
- listBasedOnFilters.addFilter(
- TimelineFilterUtils.createSingleColValueFiltersByRange(
- ApplicationColumn.CREATED_TIME, createdTimeBegin, createdTimeEnd));
- }
- // Create filter list based on metric filters and add it to
- // listBasedOnFilters.
- TimelineFilterList metricFilters = filters.getMetricFilters();
- if (metricFilters != null && !metricFilters.getFilterList().isEmpty()) {
- listBasedOnFilters.addFilter(
- TimelineFilterUtils.createHBaseFilterList(
- ApplicationColumnPrefix.METRIC, metricFilters));
- }
- // Create filter list based on config filters and add it to
- // listBasedOnFilters.
- TimelineFilterList configFilters = filters.getConfigFilters();
- if (configFilters != null && !configFilters.getFilterList().isEmpty()) {
- listBasedOnFilters.addFilter(
- TimelineFilterUtils.createHBaseFilterList(
- ApplicationColumnPrefix.CONFIG, configFilters));
- }
- // Create filter list based on info filters and add it to listBasedOnFilters
- TimelineFilterList infoFilters = filters.getInfoFilters();
- if (infoFilters != null && !infoFilters.getFilterList().isEmpty()) {
- listBasedOnFilters.addFilter(
- TimelineFilterUtils.createHBaseFilterList(
- ApplicationColumnPrefix.INFO, infoFilters));
- }
- return listBasedOnFilters;
- }
-
- /**
- * Add {@link QualifierFilter} filters to filter list for each column of
- * application table.
- *
- * @param list filter list to which qualifier filters have to be added.
- */
- @Override
- protected void updateFixedColumns(FilterList list) {
- for (ApplicationColumn column : ApplicationColumn.values()) {
- list.addFilter(new QualifierFilter(CompareOp.EQUAL,
- new BinaryComparator(column.getColumnQualifierBytes())));
- }
- }
-
- /**
- * Creates a filter list which indicates that only some of the column
- * qualifiers in the info column family will be returned in result.
- *
- * @return filter list.
- * @throws IOException if any problem occurs while creating filter list.
- */
- private FilterList createFilterListForColsOfInfoFamily()
- throws IOException {
- FilterList infoFamilyColsFilter = new FilterList(Operator.MUST_PASS_ONE);
- // Add filters for each column in entity table.
- updateFixedColumns(infoFamilyColsFilter);
- EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
- // If INFO field has to be retrieved, add a filter for fetching columns
- // with INFO column prefix.
- if (hasField(fieldsToRetrieve, Field.INFO)) {
- infoFamilyColsFilter.addFilter(
- TimelineFilterUtils.createHBaseQualifierFilter(
- CompareOp.EQUAL, ApplicationColumnPrefix.INFO));
- }
- TimelineFilterList relatesTo = getFilters().getRelatesTo();
- if (hasField(fieldsToRetrieve, Field.RELATES_TO)) {
- // If RELATES_TO field has to be retrieved, add a filter for fetching
- // columns with RELATES_TO column prefix.
- infoFamilyColsFilter.addFilter(
- TimelineFilterUtils.createHBaseQualifierFilter(
- CompareOp.EQUAL, ApplicationColumnPrefix.RELATES_TO));
- } else if (relatesTo != null && !relatesTo.getFilterList().isEmpty()) {
- // Even if fields to retrieve does not contain RELATES_TO, we still
- // need to have a filter to fetch some of the column qualifiers if
- // relatesTo filters are specified. relatesTo filters will then be
- // matched after fetching rows from HBase.
- Set<String> relatesToCols =
- TimelineFilterUtils.fetchColumnsFromFilterList(relatesTo);
- infoFamilyColsFilter.addFilter(createFiltersFromColumnQualifiers(
- ApplicationColumnPrefix.RELATES_TO, relatesToCols));
- }
- TimelineFilterList isRelatedTo = getFilters().getIsRelatedTo();
- if (hasField(fieldsToRetrieve, Field.IS_RELATED_TO)) {
- // If IS_RELATED_TO field has to be retrieved, add a filter for fetching
- // columns with IS_RELATED_TO column prefix.
- infoFamilyColsFilter.addFilter(
- TimelineFilterUtils.createHBaseQualifierFilter(
- CompareOp.EQUAL, ApplicationColumnPrefix.IS_RELATED_TO));
- } else if (isRelatedTo != null && !isRelatedTo.getFilterList().isEmpty()) {
- // Even if fields to retrieve does not contain IS_RELATED_TO, we still
- // need to have a filter to fetch some of the column qualifiers if
- // isRelatedTo filters are specified. isRelatedTo filters will then be
- // matched after fetching rows from HBase.
- Set<String> isRelatedToCols =
- TimelineFilterUtils.fetchColumnsFromFilterList(isRelatedTo);
- infoFamilyColsFilter.addFilter(createFiltersFromColumnQualifiers(
- ApplicationColumnPrefix.IS_RELATED_TO, isRelatedToCols));
- }
- TimelineFilterList eventFilters = getFilters().getEventFilters();
- if (hasField(fieldsToRetrieve, Field.EVENTS)) {
- // If EVENTS field has to be retrieved, add a filter for fetching columns
- // with EVENT column prefix.
- infoFamilyColsFilter.addFilter(
- TimelineFilterUtils.createHBaseQualifierFilter(
- CompareOp.EQUAL, ApplicationColumnPrefix.EVENT));
- } else if (eventFilters != null && !eventFilters.getFilterList().isEmpty()){
- // Even if fields to retrieve does not contain EVENTS, we still need to
- // have a filter to fetch some of the column qualifiers on the basis of
- // event filters specified. Event filters will then be matched after
- // fetching rows from HBase.
- Set<String> eventCols =
- TimelineFilterUtils.fetchColumnsFromFilterList(eventFilters);
- infoFamilyColsFilter.addFilter(createFiltersFromColumnQualifiers(
- ApplicationColumnPrefix.EVENT, eventCols));
- }
- return infoFamilyColsFilter;
- }
-
- /**
- * Exclude column prefixes via filters which are not required(based on fields
- * to retrieve) from info column family. These filters are added to filter
- * list which contains a filter for getting info column family.
- *
- * @param infoColFamilyList filter list for info column family.
- */
- private void excludeFieldsFromInfoColFamily(FilterList infoColFamilyList) {
- EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
- // Events not required.
- if (!hasField(fieldsToRetrieve, Field.EVENTS)) {
- infoColFamilyList.addFilter(
- TimelineFilterUtils.createHBaseQualifierFilter(
- CompareOp.NOT_EQUAL, ApplicationColumnPrefix.EVENT));
- }
- // info not required.
- if (!hasField(fieldsToRetrieve, Field.INFO)) {
- infoColFamilyList.addFilter(
- TimelineFilterUtils.createHBaseQualifierFilter(
- CompareOp.NOT_EQUAL, ApplicationColumnPrefix.INFO));
- }
- // is related to not required.
- if (!hasField(fieldsToRetrieve, Field.IS_RELATED_TO)) {
- infoColFamilyList.addFilter(
- TimelineFilterUtils.createHBaseQualifierFilter(
- CompareOp.NOT_EQUAL, ApplicationColumnPrefix.IS_RELATED_TO));
- }
- // relates to not required.
- if (!hasField(fieldsToRetrieve, Field.RELATES_TO)) {
- infoColFamilyList.addFilter(
- TimelineFilterUtils.createHBaseQualifierFilter(
- CompareOp.NOT_EQUAL, ApplicationColumnPrefix.RELATES_TO));
- }
- }
-
- /**
- * Updates filter list based on fields for confs and metrics to retrieve.
- *
- * @param listBasedOnFields filter list based on fields.
- * @throws IOException if any problem occurs while updating filter list.
- */
- private void updateFilterForConfsAndMetricsToRetrieve(
- FilterList listBasedOnFields) throws IOException {
- TimelineDataToRetrieve dataToRetrieve = getDataToRetrieve();
- // Please note that if confsToRetrieve is specified, we would have added
- // CONFS to fields to retrieve in augmentParams() even if not specified.
- if (dataToRetrieve.getFieldsToRetrieve().contains(Field.CONFIGS)) {
- // Create a filter list for configs.
- listBasedOnFields.addFilter(TimelineFilterUtils.
- createFilterForConfsOrMetricsToRetrieve(
- dataToRetrieve.getConfsToRetrieve(),
- ApplicationColumnFamily.CONFIGS, ApplicationColumnPrefix.CONFIG));
- }
-
- // Please note that if metricsToRetrieve is specified, we would have added
- // METRICS to fields to retrieve in augmentParams() even if not specified.
- if (dataToRetrieve.getFieldsToRetrieve().contains(Field.METRICS)) {
- // Create a filter list for metrics.
- listBasedOnFields.addFilter(TimelineFilterUtils.
- createFilterForConfsOrMetricsToRetrieve(
- dataToRetrieve.getMetricsToRetrieve(),
- ApplicationColumnFamily.METRICS, ApplicationColumnPrefix.METRIC));
- }
- }
-
- @Override
- protected FilterList constructFilterListBasedOnFields() throws IOException {
- if (!needCreateFilterListBasedOnFields()) {
- // Fetch all the columns. No need of a filter.
- return null;
- }
- FilterList listBasedOnFields = new FilterList(Operator.MUST_PASS_ONE);
- FilterList infoColFamilyList = new FilterList();
- // By default fetch everything in INFO column family.
- FamilyFilter infoColumnFamily =
- new FamilyFilter(CompareOp.EQUAL,
- new BinaryComparator(ApplicationColumnFamily.INFO.getBytes()));
- infoColFamilyList.addFilter(infoColumnFamily);
- if (!isSingleEntityRead() && fetchPartialColsFromInfoFamily()) {
- // We can fetch only some of the columns from info family.
- infoColFamilyList.addFilter(createFilterListForColsOfInfoFamily());
- } else {
- // Exclude column prefixes in info column family which are not required
- // based on fields to retrieve.
- excludeFieldsFromInfoColFamily(infoColFamilyList);
- }
- listBasedOnFields.addFilter(infoColFamilyList);
-
- updateFilterForConfsAndMetricsToRetrieve(listBasedOnFields);
- return listBasedOnFields;
- }
-
- @Override
- protected Result getResult(Configuration hbaseConf, Connection conn,
- FilterList filterList) throws IOException {
- TimelineReaderContext context = getContext();
- ApplicationRowKey applicationRowKey =
- new ApplicationRowKey(context.getClusterId(), context.getUserId(),
- context.getFlowName(), context.getFlowRunId(), context.getAppId());
- byte[] rowKey = applicationRowKey.getRowKey();
- Get get = new Get(rowKey);
- // Set time range for metric values.
- setMetricsTimeRange(get);
- get.setMaxVersions(getDataToRetrieve().getMetricsLimit());
- if (filterList != null && !filterList.getFilters().isEmpty()) {
- get.setFilter(filterList);
- }
- return getTable().getResult(hbaseConf, conn, get);
- }
-
- @Override
- protected void validateParams() {
- Preconditions.checkNotNull(getContext(), "context shouldn't be null");
- Preconditions.checkNotNull(
- getDataToRetrieve(), "data to retrieve shouldn't be null");
- Preconditions.checkNotNull(getContext().getClusterId(),
- "clusterId shouldn't be null");
- Preconditions.checkNotNull(getContext().getEntityType(),
- "entityType shouldn't be null");
- if (isSingleEntityRead()) {
- Preconditions.checkNotNull(getContext().getAppId(),
- "appId shouldn't be null");
- } else {
- Preconditions.checkNotNull(getContext().getUserId(),
- "userId shouldn't be null");
- Preconditions.checkNotNull(getContext().getFlowName(),
- "flowName shouldn't be null");
- }
- }
-
- @Override
- protected void augmentParams(Configuration hbaseConf, Connection conn)
- throws IOException {
- if (isSingleEntityRead()) {
- // Get flow context information from AppToFlow table.
- defaultAugmentParams(hbaseConf, conn);
- }
- // Add configs/metrics to fields to retrieve if confsToRetrieve and/or
- // metricsToRetrieve are specified.
- getDataToRetrieve().addFieldsBasedOnConfsAndMetricsToRetrieve();
- if (!isSingleEntityRead()) {
- createFiltersIfNull();
- }
- }
-
- private void setMetricsTimeRange(Query query) {
- // Set time range for metric values.
- HBaseTimelineStorageUtils.setMetricsTimeRange(
- query, ApplicationColumnFamily.METRICS.getBytes(),
- getDataToRetrieve().getMetricsTimeBegin(),
- getDataToRetrieve().getMetricsTimeEnd());
- }
-
- @Override
- protected ResultScanner getResults(Configuration hbaseConf,
- Connection conn, FilterList filterList) throws IOException {
- Scan scan = new Scan();
- TimelineReaderContext context = getContext();
- RowKeyPrefix<ApplicationRowKey> applicationRowKeyPrefix = null;
-
- // Whether or not flowRunID is null doesn't matter, the
- // ApplicationRowKeyPrefix will do the right thing.
- // default mode, will always scans from beginning of entity type.
- if (getFilters().getFromId() == null) {
- applicationRowKeyPrefix = new ApplicationRowKeyPrefix(
- context.getClusterId(), context.getUserId(), context.getFlowName(),
- context.getFlowRunId());
- scan.setRowPrefixFilter(applicationRowKeyPrefix.getRowKeyPrefix());
- } else {
- ApplicationRowKey applicationRowKey = null;
- try {
- applicationRowKey =
- ApplicationRowKey.parseRowKeyFromString(getFilters().getFromId());
- } catch (IllegalArgumentException e) {
- throw new BadRequestException("Invalid filter fromid is provided.");
- }
- if (!context.getClusterId().equals(applicationRowKey.getClusterId())) {
- throw new BadRequestException(
- "fromid doesn't belong to clusterId=" + context.getClusterId());
- }
-
- // set start row
- scan.setStartRow(applicationRowKey.getRowKey());
-
- // get the bytes for stop row
- applicationRowKeyPrefix = new ApplicationRowKeyPrefix(
- context.getClusterId(), context.getUserId(), context.getFlowName(),
- context.getFlowRunId());
-
- // set stop row
- scan.setStopRow(
- HBaseTimelineStorageUtils.calculateTheClosestNextRowKeyForPrefix(
- applicationRowKeyPrefix.getRowKeyPrefix()));
- }
-
- FilterList newList = new FilterList();
- newList.addFilter(new PageFilter(getFilters().getLimit()));
- if (filterList != null && !filterList.getFilters().isEmpty()) {
- newList.addFilter(filterList);
- }
- scan.setFilter(newList);
-
- // Set time range for metric values.
- setMetricsTimeRange(scan);
- scan.setMaxVersions(getDataToRetrieve().getMetricsLimit());
- return getTable().getResultScanner(hbaseConf, conn, scan);
- }
-
- @Override
- protected TimelineEntity parseEntity(Result result) throws IOException {
- if (result == null || result.isEmpty()) {
- return null;
- }
- TimelineEntity entity = new TimelineEntity();
- entity.setType(TimelineEntityType.YARN_APPLICATION.toString());
- String entityId = ApplicationColumn.ID.readResult(result).toString();
- entity.setId(entityId);
-
- TimelineEntityFilters filters = getFilters();
- // fetch created time
- Long createdTime = (Long) ApplicationColumn.CREATED_TIME.readResult(result);
- entity.setCreatedTime(createdTime);
-
- EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
- // fetch is related to entities and match isRelatedTo filter. If isRelatedTo
- // filters do not match, entity would be dropped. We have to match filters
- // locally as relevant HBase filters to filter out rows on the basis of
- // isRelatedTo are not set in HBase scan.
- boolean checkIsRelatedTo =
- !isSingleEntityRead() && filters.getIsRelatedTo() != null &&
- filters.getIsRelatedTo().getFilterList().size() > 0;
- if (hasField(fieldsToRetrieve, Field.IS_RELATED_TO) || checkIsRelatedTo) {
- readRelationship(entity, result, ApplicationColumnPrefix.IS_RELATED_TO,
- true);
- if (checkIsRelatedTo && !TimelineStorageUtils.matchIsRelatedTo(entity,
- filters.getIsRelatedTo())) {
- return null;
- }
- if (!hasField(fieldsToRetrieve,
- Field.IS_RELATED_TO)) {
- entity.getIsRelatedToEntities().clear();
- }
- }
-
- // fetch relates to entities and match relatesTo filter. If relatesTo
- // filters do not match, entity would be dropped. We have to match filters
- // locally as relevant HBase filters to filter out rows on the basis of
- // relatesTo are not set in HBase scan.
- boolean checkRelatesTo =
- !isSingleEntityRead() && filters.getRelatesTo() != null &&
- filters.getRelatesTo().getFilterList().size() > 0;
- if (hasField(fieldsToRetrieve, Field.RELATES_TO) ||
- checkRelatesTo) {
- readRelationship(entity, result, ApplicationColumnPrefix.RELATES_TO,
- false);
- if (checkRelatesTo && !TimelineStorageUtils.matchRelatesTo(entity,
- filters.getRelatesTo())) {
- return null;
- }
- if (!hasField(fieldsToRetrieve, Field.RELATES_TO)) {
- entity.getRelatesToEntities().clear();
- }
- }
-
- // fetch info if fieldsToRetrieve contains INFO or ALL.
- if (hasField(fieldsToRetrieve, Field.INFO)) {
- readKeyValuePairs(entity, result, ApplicationColumnPrefix.INFO, false);
- }
-
- // fetch configs if fieldsToRetrieve contains CONFIGS or ALL.
- if (hasField(fieldsToRetrieve, Field.CONFIGS)) {
- readKeyValuePairs(entity, result, ApplicationColumnPrefix.CONFIG, true);
- }
-
- // fetch events and match event filters if they exist. If event filters do
- // not match, entity would be dropped. We have to match filters locally
- // as relevant HBase filters to filter out rows on the basis of events
- // are not set in HBase scan.
- boolean checkEvents =
- !isSingleEntityRead() && filters.getEventFilters() != null &&
- filters.getEventFilters().getFilterList().size() > 0;
- if (hasField(fieldsToRetrieve, Field.EVENTS) || checkEvents) {
- readEvents(entity, result, ApplicationColumnPrefix.EVENT);
- if (checkEvents && !TimelineStorageUtils.matchEventFilters(entity,
- filters.getEventFilters())) {
- return null;
- }
- if (!hasField(fieldsToRetrieve, Field.EVENTS)) {
- entity.getEvents().clear();
- }
- }
-
- // fetch metrics if fieldsToRetrieve contains METRICS or ALL.
- if (hasField(fieldsToRetrieve, Field.METRICS)) {
- readMetrics(entity, result, ApplicationColumnPrefix.METRIC);
- }
-
- ApplicationRowKey rowKey = ApplicationRowKey.parseRowKey(result.getRow());
- entity.getInfo().put(TimelineReaderUtils.FROMID_KEY,
- rowKey.getRowKeyAsString());
- return entity;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/EntityTypeReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/EntityTypeReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/EntityTypeReader.java
deleted file mode 100644
index 3f72334..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/EntityTypeReader.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.filter.FilterList;
-import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
-import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
-import org.apache.hadoop.hbase.filter.PageFilter;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKeyPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Set;
-import java.util.TreeSet;
-
-/**
- * Timeline entity reader for listing all available entity types given one
- * reader context. Right now only supports listing all entity types within one
- * YARN application.
- */
-public final class EntityTypeReader extends AbstractTimelineStorageReader {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(EntityTypeReader.class);
- private static final EntityTable ENTITY_TABLE = new EntityTable();
-
- public EntityTypeReader(TimelineReaderContext context) {
- super(context);
- }
-
- /**
- * Reads a set of timeline entity types from the HBase storage for the given
- * context.
- *
- * @param hbaseConf HBase Configuration.
- * @param conn HBase Connection.
- * @return a set of <cite>TimelineEntity</cite> objects, with only type field
- * set.
- * @throws IOException if any exception is encountered while reading entities.
- */
- public Set<String> readEntityTypes(Configuration hbaseConf,
- Connection conn) throws IOException {
-
- validateParams();
- augmentParams(hbaseConf, conn);
-
- Set<String> types = new TreeSet<>();
- TimelineReaderContext context = getContext();
- EntityRowKeyPrefix prefix = new EntityRowKeyPrefix(context.getClusterId(),
- context.getUserId(), context.getFlowName(), context.getFlowRunId(),
- context.getAppId());
- byte[] currRowKey = prefix.getRowKeyPrefix();
- byte[] nextRowKey = prefix.getRowKeyPrefix();
- nextRowKey[nextRowKey.length - 1]++;
-
- FilterList typeFilterList = new FilterList();
- typeFilterList.addFilter(new FirstKeyOnlyFilter());
- typeFilterList.addFilter(new KeyOnlyFilter());
- typeFilterList.addFilter(new PageFilter(1));
- LOG.debug("FilterList created for scan is - {}", typeFilterList);
-
- int counter = 0;
- while (true) {
- try (ResultScanner results =
- getResult(hbaseConf, conn, typeFilterList, currRowKey, nextRowKey)) {
- TimelineEntity entity = parseEntityForType(results.next());
- if (entity == null) {
- break;
- }
- ++counter;
- if (!types.add(entity.getType())) {
- LOG.warn("Failed to add type " + entity.getType()
- + " to the result set because there is a duplicated copy. ");
- }
- String currType = entity.getType();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Current row key: " + Arrays.toString(currRowKey));
- LOG.debug("New entity type discovered: " + currType);
- }
- currRowKey = getNextRowKey(prefix.getRowKeyPrefix(), currType);
- }
- }
- LOG.debug("Scanned {} records for {} types", counter, types.size());
- return types;
- }
-
- @Override
- protected void validateParams() {
- Preconditions.checkNotNull(getContext(), "context shouldn't be null");
- Preconditions.checkNotNull(getContext().getClusterId(),
- "clusterId shouldn't be null");
- Preconditions.checkNotNull(getContext().getAppId(),
- "appId shouldn't be null");
- }
-
- /**
- * Gets the possibly next row key prefix given current prefix and type.
- *
- * @param currRowKeyPrefix The current prefix that contains user, cluster,
- * flow, run, and application id.
- * @param entityType Current entity type.
- * @return A new prefix for the possibly immediately next row key.
- */
- private static byte[] getNextRowKey(byte[] currRowKeyPrefix,
- String entityType) {
- if (currRowKeyPrefix == null || entityType == null) {
- return null;
- }
-
- byte[] entityTypeEncoded = Separator.QUALIFIERS.join(
- Separator.encode(entityType, Separator.SPACE, Separator.TAB,
- Separator.QUALIFIERS),
- Separator.EMPTY_BYTES);
-
- byte[] currRowKey
- = new byte[currRowKeyPrefix.length + entityTypeEncoded.length];
- System.arraycopy(currRowKeyPrefix, 0, currRowKey, 0,
- currRowKeyPrefix.length);
- System.arraycopy(entityTypeEncoded, 0, currRowKey, currRowKeyPrefix.length,
- entityTypeEncoded.length);
-
- return HBaseTimelineStorageUtils.calculateTheClosestNextRowKeyForPrefix(
- currRowKey);
- }
-
- private ResultScanner getResult(Configuration hbaseConf, Connection conn,
- FilterList filterList, byte[] startPrefix, byte[] endPrefix)
- throws IOException {
- Scan scan = new Scan(startPrefix, endPrefix);
- scan.setFilter(filterList);
- scan.setSmall(true);
- return ENTITY_TABLE.getResultScanner(hbaseConf, conn, scan);
- }
-
- private TimelineEntity parseEntityForType(Result result)
- throws IOException {
- if (result == null || result.isEmpty()) {
- return null;
- }
- TimelineEntity entity = new TimelineEntity();
- EntityRowKey newRowKey = EntityRowKey.parseRowKey(result.getRow());
- entity.setType(newRowKey.getEntityType());
- return entity;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
deleted file mode 100644
index a1cdb29..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
-
-import java.io.IOException;
-import java.util.Map;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.filter.FilterList;
-import org.apache.hadoop.hbase.filter.PageFilter;
-import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongKeyConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKeyPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityTable;
-import org.apache.hadoop.yarn.webapp.BadRequestException;
-
-import com.google.common.base.Preconditions;
-
-/**
- * Timeline entity reader for flow activity entities that are stored in the
- * flow activity table.
- */
-class FlowActivityEntityReader extends TimelineEntityReader {
- private static final FlowActivityTable FLOW_ACTIVITY_TABLE =
- new FlowActivityTable();
-
- /**
- * Used to convert Long key components to and from storage format.
- */
- private final KeyConverter<Long> longKeyConverter = new LongKeyConverter();
-
-
- public FlowActivityEntityReader(TimelineReaderContext ctxt,
- TimelineEntityFilters entityFilters, TimelineDataToRetrieve toRetrieve) {
- super(ctxt, entityFilters, toRetrieve);
- }
-
- public FlowActivityEntityReader(TimelineReaderContext ctxt,
- TimelineDataToRetrieve toRetrieve) {
- super(ctxt, toRetrieve);
- }
-
- /**
- * Uses the {@link FlowActivityTable}.
- */
- @Override
- protected BaseTable<?> getTable() {
- return FLOW_ACTIVITY_TABLE;
- }
-
- @Override
- protected void validateParams() {
- Preconditions.checkNotNull(getContext().getClusterId(),
- "clusterId shouldn't be null");
- }
-
- @Override
- protected void augmentParams(Configuration hbaseConf, Connection conn)
- throws IOException {
- createFiltersIfNull();
- }
-
- @Override
- protected FilterList constructFilterListBasedOnFilters() throws IOException {
- return null;
- }
-
- @Override
- protected FilterList constructFilterListBasedOnFields() {
- return null;
- }
-
- @Override
- protected Result getResult(Configuration hbaseConf, Connection conn,
- FilterList filterList) throws IOException {
- throw new UnsupportedOperationException(
- "we don't support a single entity query");
- }
-
- @Override
- protected ResultScanner getResults(Configuration hbaseConf,
- Connection conn, FilterList filterList) throws IOException {
- Scan scan = new Scan();
- String clusterId = getContext().getClusterId();
- if (getFilters().getFromId() == null
- && getFilters().getCreatedTimeBegin() == 0L
- && getFilters().getCreatedTimeEnd() == Long.MAX_VALUE) {
- // All records have to be chosen.
- scan.setRowPrefixFilter(new FlowActivityRowKeyPrefix(clusterId)
- .getRowKeyPrefix());
- } else if (getFilters().getFromId() != null) {
- FlowActivityRowKey key = null;
- try {
- key =
- FlowActivityRowKey.parseRowKeyFromString(getFilters().getFromId());
- } catch (IllegalArgumentException e) {
- throw new BadRequestException("Invalid filter fromid is provided.");
- }
- if (!clusterId.equals(key.getClusterId())) {
- throw new BadRequestException(
- "fromid doesn't belong to clusterId=" + clusterId);
- }
- scan.setStartRow(key.getRowKey());
- scan.setStopRow(
- new FlowActivityRowKeyPrefix(clusterId,
- (getFilters().getCreatedTimeBegin() <= 0 ? 0
- : (getFilters().getCreatedTimeBegin() - 1)))
- .getRowKeyPrefix());
- } else {
- scan.setStartRow(new FlowActivityRowKeyPrefix(clusterId, getFilters()
- .getCreatedTimeEnd()).getRowKeyPrefix());
- scan.setStopRow(new FlowActivityRowKeyPrefix(clusterId, (getFilters()
- .getCreatedTimeBegin() <= 0 ? 0
- : (getFilters().getCreatedTimeBegin() - 1))).getRowKeyPrefix());
- }
- // use the page filter to limit the result to the page size
- // the scanner may still return more than the limit; therefore we need to
- // read the right number as we iterate
- scan.setFilter(new PageFilter(getFilters().getLimit()));
- return getTable().getResultScanner(hbaseConf, conn, scan);
- }
-
- @Override
- protected TimelineEntity parseEntity(Result result) throws IOException {
- FlowActivityRowKey rowKey = FlowActivityRowKey.parseRowKey(result.getRow());
-
- Long time = rowKey.getDayTimestamp();
- String user = rowKey.getUserId();
- String flowName = rowKey.getFlowName();
-
- FlowActivityEntity flowActivity = new FlowActivityEntity(
- getContext().getClusterId(), time, user, flowName);
- // set the id
- flowActivity.setId(flowActivity.getId());
- // get the list of run ids along with the version that are associated with
- // this flow on this day
- Map<Long, Object> runIdsMap =
- FlowActivityColumnPrefix.RUN_ID.readResults(result, longKeyConverter);
- for (Map.Entry<Long, Object> e : runIdsMap.entrySet()) {
- Long runId = e.getKey();
- String version = (String)e.getValue();
- FlowRunEntity flowRun = new FlowRunEntity();
- flowRun.setUser(user);
- flowRun.setName(flowName);
- flowRun.setRunId(runId);
- flowRun.setVersion(version);
- // set the id
- flowRun.setId(flowRun.getId());
- flowActivity.addFlowRun(flowRun);
- }
- flowActivity.getInfo().put(TimelineReaderUtils.FROMID_KEY,
- rowKey.getRowKeyAsString());
- return flowActivity;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
deleted file mode 100644
index af043b3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
+++ /dev/null
@@ -1,294 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
-
-import java.io.IOException;
-import java.util.EnumSet;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.filter.BinaryComparator;
-import org.apache.hadoop.hbase.filter.BinaryPrefixComparator;
-import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
-import org.apache.hadoop.hbase.filter.FamilyFilter;
-import org.apache.hadoop.hbase.filter.FilterList;
-import org.apache.hadoop.hbase.filter.FilterList.Operator;
-import org.apache.hadoop.hbase.filter.PageFilter;
-import org.apache.hadoop.hbase.filter.QualifierFilter;
-import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.RowKeyPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumn;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunRowKeyPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTable;
-import org.apache.hadoop.yarn.webapp.BadRequestException;
-
-import com.google.common.base.Preconditions;
-
-/**
- * Timeline entity reader for flow run entities that are stored in the flow run
- * table.
- */
-class FlowRunEntityReader extends TimelineEntityReader {
- private static final FlowRunTable FLOW_RUN_TABLE = new FlowRunTable();
-
- public FlowRunEntityReader(TimelineReaderContext ctxt,
- TimelineEntityFilters entityFilters, TimelineDataToRetrieve toRetrieve) {
- super(ctxt, entityFilters, toRetrieve);
- }
-
- public FlowRunEntityReader(TimelineReaderContext ctxt,
- TimelineDataToRetrieve toRetrieve) {
- super(ctxt, toRetrieve);
- }
-
- /**
- * Uses the {@link FlowRunTable}.
- */
- @Override
- protected BaseTable<?> getTable() {
- return FLOW_RUN_TABLE;
- }
-
- @Override
- protected void validateParams() {
- Preconditions.checkNotNull(getContext(), "context shouldn't be null");
- Preconditions.checkNotNull(getDataToRetrieve(),
- "data to retrieve shouldn't be null");
- Preconditions.checkNotNull(getContext().getClusterId(),
- "clusterId shouldn't be null");
- Preconditions.checkNotNull(getContext().getUserId(),
- "userId shouldn't be null");
- Preconditions.checkNotNull(getContext().getFlowName(),
- "flowName shouldn't be null");
- if (isSingleEntityRead()) {
- Preconditions.checkNotNull(getContext().getFlowRunId(),
- "flowRunId shouldn't be null");
- }
- EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
- if (!isSingleEntityRead() && fieldsToRetrieve != null) {
- for (Field field : fieldsToRetrieve) {
- if (field != Field.ALL && field != Field.METRICS) {
- throw new BadRequestException("Invalid field " + field
- + " specified while querying flow runs.");
- }
- }
- }
- }
-
- @Override
- protected void augmentParams(Configuration hbaseConf, Connection conn) {
- // Add metrics to fields to retrieve if metricsToRetrieve is specified.
- getDataToRetrieve().addFieldsBasedOnConfsAndMetricsToRetrieve();
- if (!isSingleEntityRead()) {
- createFiltersIfNull();
- }
- }
-
- protected FilterList constructFilterListBasedOnFilters() throws IOException {
- FilterList listBasedOnFilters = new FilterList();
- // Filter based on created time range.
- Long createdTimeBegin = getFilters().getCreatedTimeBegin();
- Long createdTimeEnd = getFilters().getCreatedTimeEnd();
- if (createdTimeBegin != 0 || createdTimeEnd != Long.MAX_VALUE) {
- listBasedOnFilters.addFilter(TimelineFilterUtils
- .createSingleColValueFiltersByRange(FlowRunColumn.MIN_START_TIME,
- createdTimeBegin, createdTimeEnd));
- }
- // Filter based on metric filters.
- TimelineFilterList metricFilters = getFilters().getMetricFilters();
- if (metricFilters != null && !metricFilters.getFilterList().isEmpty()) {
- listBasedOnFilters.addFilter(TimelineFilterUtils.createHBaseFilterList(
- FlowRunColumnPrefix.METRIC, metricFilters));
- }
- return listBasedOnFilters;
- }
-
- /**
- * Add {@link QualifierFilter} filters to filter list for each column of flow
- * run table.
- *
- * @return filter list to which qualifier filters have been added.
- */
- private FilterList updateFixedColumns() {
- FilterList columnsList = new FilterList(Operator.MUST_PASS_ONE);
- for (FlowRunColumn column : FlowRunColumn.values()) {
- columnsList.addFilter(new QualifierFilter(CompareOp.EQUAL,
- new BinaryComparator(column.getColumnQualifierBytes())));
- }
- return columnsList;
- }
-
- @Override
- protected FilterList constructFilterListBasedOnFields() throws IOException {
- FilterList list = new FilterList(Operator.MUST_PASS_ONE);
- // By default fetch everything in INFO column family.
- FamilyFilter infoColumnFamily =
- new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(
- FlowRunColumnFamily.INFO.getBytes()));
- TimelineDataToRetrieve dataToRetrieve = getDataToRetrieve();
- // If multiple entities have to be retrieved, check if metrics have to be
- // retrieved and if not, add a filter so that metrics can be excluded.
- // Metrics are always returned if we are reading a single entity.
- if (!isSingleEntityRead()
- && !hasField(dataToRetrieve.getFieldsToRetrieve(), Field.METRICS)) {
- FilterList infoColFamilyList = new FilterList(Operator.MUST_PASS_ONE);
- infoColFamilyList.addFilter(infoColumnFamily);
- infoColFamilyList.addFilter(new QualifierFilter(CompareOp.NOT_EQUAL,
- new BinaryPrefixComparator(FlowRunColumnPrefix.METRIC
- .getColumnPrefixBytes(""))));
- list.addFilter(infoColFamilyList);
- } else {
- // Check if metricsToRetrieve are specified and if they are, create a
- // filter list for info column family by adding flow run tables columns
- // and a list for metrics to retrieve. Pls note that fieldsToRetrieve
- // will have METRICS added to it if metricsToRetrieve are specified
- // (in augmentParams()).
- TimelineFilterList metricsToRetrieve =
- dataToRetrieve.getMetricsToRetrieve();
- if (metricsToRetrieve != null
- && !metricsToRetrieve.getFilterList().isEmpty()) {
- FilterList infoColFamilyList = new FilterList();
- infoColFamilyList.addFilter(infoColumnFamily);
- FilterList columnsList = updateFixedColumns();
- columnsList.addFilter(TimelineFilterUtils.createHBaseFilterList(
- FlowRunColumnPrefix.METRIC, metricsToRetrieve));
- infoColFamilyList.addFilter(columnsList);
- list.addFilter(infoColFamilyList);
- }
- }
- return list;
- }
-
- @Override
- protected Result getResult(Configuration hbaseConf, Connection conn,
- FilterList filterList) throws IOException {
- TimelineReaderContext context = getContext();
- FlowRunRowKey flowRunRowKey =
- new FlowRunRowKey(context.getClusterId(), context.getUserId(),
- context.getFlowName(), context.getFlowRunId());
- byte[] rowKey = flowRunRowKey.getRowKey();
- Get get = new Get(rowKey);
- get.setMaxVersions(Integer.MAX_VALUE);
- if (filterList != null && !filterList.getFilters().isEmpty()) {
- get.setFilter(filterList);
- }
- return getTable().getResult(hbaseConf, conn, get);
- }
-
- @Override
- protected ResultScanner getResults(Configuration hbaseConf, Connection conn,
- FilterList filterList) throws IOException {
- Scan scan = new Scan();
- TimelineReaderContext context = getContext();
- RowKeyPrefix<FlowRunRowKey> flowRunRowKeyPrefix = null;
- if (getFilters().getFromId() == null) {
- flowRunRowKeyPrefix = new FlowRunRowKeyPrefix(context.getClusterId(),
- context.getUserId(), context.getFlowName());
- scan.setRowPrefixFilter(flowRunRowKeyPrefix.getRowKeyPrefix());
- } else {
- FlowRunRowKey flowRunRowKey = null;
- try {
- flowRunRowKey =
- FlowRunRowKey.parseRowKeyFromString(getFilters().getFromId());
- } catch (IllegalArgumentException e) {
- throw new BadRequestException("Invalid filter fromid is provided.");
- }
- if (!context.getClusterId().equals(flowRunRowKey.getClusterId())) {
- throw new BadRequestException(
- "fromid doesn't belong to clusterId=" + context.getClusterId());
- }
- // set start row
- scan.setStartRow(flowRunRowKey.getRowKey());
-
- // get the bytes for stop row
- flowRunRowKeyPrefix = new FlowRunRowKeyPrefix(context.getClusterId(),
- context.getUserId(), context.getFlowName());
-
- // set stop row
- scan.setStopRow(
- HBaseTimelineStorageUtils.calculateTheClosestNextRowKeyForPrefix(
- flowRunRowKeyPrefix.getRowKeyPrefix()));
- }
-
- FilterList newList = new FilterList();
- newList.addFilter(new PageFilter(getFilters().getLimit()));
- if (filterList != null && !filterList.getFilters().isEmpty()) {
- newList.addFilter(filterList);
- }
- scan.setFilter(newList);
- scan.setMaxVersions(Integer.MAX_VALUE);
- return getTable().getResultScanner(hbaseConf, conn, scan);
- }
-
- @Override
- protected TimelineEntity parseEntity(Result result) throws IOException {
- FlowRunEntity flowRun = new FlowRunEntity();
- FlowRunRowKey rowKey = FlowRunRowKey.parseRowKey(result.getRow());
- flowRun.setRunId(rowKey.getFlowRunId());
- flowRun.setUser(rowKey.getUserId());
- flowRun.setName(rowKey.getFlowName());
-
- // read the start time
- Long startTime = (Long) FlowRunColumn.MIN_START_TIME.readResult(result);
- if (startTime != null) {
- flowRun.setStartTime(startTime.longValue());
- }
-
- // read the end time if available
- Long endTime = (Long) FlowRunColumn.MAX_END_TIME.readResult(result);
- if (endTime != null) {
- flowRun.setMaxEndTime(endTime.longValue());
- }
-
- // read the flow version
- String version = (String) FlowRunColumn.FLOW_VERSION.readResult(result);
- if (version != null) {
- flowRun.setVersion(version);
- }
-
- // read metrics if its a single entity query or if METRICS are part of
- // fieldsToRetrieve.
- if (isSingleEntityRead()
- || hasField(getDataToRetrieve().getFieldsToRetrieve(), Field.METRICS)) {
- readMetrics(flowRun, result, FlowRunColumnPrefix.METRIC);
- }
-
- // set the id
- flowRun.setId(flowRun.getId());
- flowRun.getInfo().put(TimelineReaderUtils.FROMID_KEY,
- rowKey.getRowKeyAsString());
- return flowRun;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
deleted file mode 100644
index 3a44445..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
+++ /dev/null
@@ -1,651 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Query;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.filter.BinaryComparator;
-import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
-import org.apache.hadoop.hbase.filter.FamilyFilter;
-import org.apache.hadoop.hbase.filter.FilterList;
-import org.apache.hadoop.hbase.filter.PageFilter;
-import org.apache.hadoop.hbase.filter.FilterList.Operator;
-import org.apache.hadoop.hbase.filter.QualifierFilter;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.RowKeyPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.StringKeyConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKeyPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
-import org.apache.hadoop.yarn.webapp.BadRequestException;
-
-import com.google.common.base.Preconditions;
-
-/**
- * Timeline entity reader for generic entities that are stored in the entity
- * table.
- */
-class GenericEntityReader extends TimelineEntityReader {
- private static final EntityTable ENTITY_TABLE = new EntityTable();
-
- /**
- * Used to convert strings key components to and from storage format.
- */
- private final KeyConverter<String> stringKeyConverter =
- new StringKeyConverter();
-
- public GenericEntityReader(TimelineReaderContext ctxt,
- TimelineEntityFilters entityFilters, TimelineDataToRetrieve toRetrieve) {
- super(ctxt, entityFilters, toRetrieve);
- }
-
- public GenericEntityReader(TimelineReaderContext ctxt,
- TimelineDataToRetrieve toRetrieve) {
- super(ctxt, toRetrieve);
- }
-
- /**
- * Uses the {@link EntityTable}.
- */
- protected BaseTable<?> getTable() {
- return ENTITY_TABLE;
- }
-
- @Override
- protected FilterList constructFilterListBasedOnFilters() throws IOException {
- // Filters here cannot be null for multiple entity reads as they are set in
- // augmentParams if null.
- FilterList listBasedOnFilters = new FilterList();
- TimelineEntityFilters filters = getFilters();
- // Create filter list based on created time range and add it to
- // listBasedOnFilters.
- long createdTimeBegin = filters.getCreatedTimeBegin();
- long createdTimeEnd = filters.getCreatedTimeEnd();
- if (createdTimeBegin != 0 || createdTimeEnd != Long.MAX_VALUE) {
- listBasedOnFilters.addFilter(TimelineFilterUtils
- .createSingleColValueFiltersByRange(EntityColumn.CREATED_TIME,
- createdTimeBegin, createdTimeEnd));
- }
- // Create filter list based on metric filters and add it to
- // listBasedOnFilters.
- TimelineFilterList metricFilters = filters.getMetricFilters();
- if (metricFilters != null && !metricFilters.getFilterList().isEmpty()) {
- listBasedOnFilters.addFilter(TimelineFilterUtils.createHBaseFilterList(
- EntityColumnPrefix.METRIC, metricFilters));
- }
- // Create filter list based on config filters and add it to
- // listBasedOnFilters.
- TimelineFilterList configFilters = filters.getConfigFilters();
- if (configFilters != null && !configFilters.getFilterList().isEmpty()) {
- listBasedOnFilters.addFilter(TimelineFilterUtils.createHBaseFilterList(
- EntityColumnPrefix.CONFIG, configFilters));
- }
- // Create filter list based on info filters and add it to listBasedOnFilters
- TimelineFilterList infoFilters = filters.getInfoFilters();
- if (infoFilters != null && !infoFilters.getFilterList().isEmpty()) {
- listBasedOnFilters.addFilter(TimelineFilterUtils.createHBaseFilterList(
- EntityColumnPrefix.INFO, infoFilters));
- }
- return listBasedOnFilters;
- }
-
- /**
- * Check if we need to fetch only some of the event columns.
- *
- * @return true if we need to fetch some of the columns, false otherwise.
- */
- protected boolean fetchPartialEventCols(TimelineFilterList eventFilters,
- EnumSet<Field> fieldsToRetrieve) {
- return (eventFilters != null && !eventFilters.getFilterList().isEmpty() &&
- !hasField(fieldsToRetrieve, Field.EVENTS));
- }
-
- /**
- * Check if we need to fetch only some of the relates_to columns.
- *
- * @return true if we need to fetch some of the columns, false otherwise.
- */
- protected boolean fetchPartialRelatesToCols(TimelineFilterList relatesTo,
- EnumSet<Field> fieldsToRetrieve) {
- return (relatesTo != null && !relatesTo.getFilterList().isEmpty() &&
- !hasField(fieldsToRetrieve, Field.RELATES_TO));
- }
-
- /**
- * Check if we need to fetch only some of the is_related_to columns.
- *
- * @return true if we need to fetch some of the columns, false otherwise.
- */
- private boolean fetchPartialIsRelatedToCols(TimelineFilterList isRelatedTo,
- EnumSet<Field> fieldsToRetrieve) {
- return (isRelatedTo != null && !isRelatedTo.getFilterList().isEmpty() &&
- !hasField(fieldsToRetrieve, Field.IS_RELATED_TO));
- }
-
- /**
- * Check if we need to fetch only some of the columns based on event filters,
- * relatesto and isrelatedto from info family.
- *
- * @return true, if we need to fetch only some of the columns, false if we
- * need to fetch all the columns under info column family.
- */
- protected boolean fetchPartialColsFromInfoFamily() {
- EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
- TimelineEntityFilters filters = getFilters();
- return fetchPartialEventCols(filters.getEventFilters(), fieldsToRetrieve)
- || fetchPartialRelatesToCols(filters.getRelatesTo(), fieldsToRetrieve)
- || fetchPartialIsRelatedToCols(filters.getIsRelatedTo(),
- fieldsToRetrieve);
- }
-
- /**
- * Check if we need to create filter list based on fields. We need to create a
- * filter list iff all fields need not be retrieved or we have some specific
- * fields or metrics to retrieve. We also need to create a filter list if we
- * have relationships(relatesTo/isRelatedTo) and event filters specified for
- * the query.
- *
- * @return true if we need to create the filter list, false otherwise.
- */
- protected boolean needCreateFilterListBasedOnFields() {
- TimelineDataToRetrieve dataToRetrieve = getDataToRetrieve();
- // Check if all fields are to be retrieved or not. If all fields have to
- // be retrieved, also check if we have some metrics or configs to
- // retrieve specified for the query because then a filter list will have
- // to be created.
- boolean flag =
- !dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL)
- || (dataToRetrieve.getConfsToRetrieve() != null && !dataToRetrieve
- .getConfsToRetrieve().getFilterList().isEmpty())
- || (dataToRetrieve.getMetricsToRetrieve() != null && !dataToRetrieve
- .getMetricsToRetrieve().getFilterList().isEmpty());
- // Filters need to be checked only if we are reading multiple entities. If
- // condition above is false, we check if there are relationships(relatesTo/
- // isRelatedTo) and event filters specified for the query.
- if (!flag && !isSingleEntityRead()) {
- TimelineEntityFilters filters = getFilters();
- flag =
- (filters.getEventFilters() != null && !filters.getEventFilters()
- .getFilterList().isEmpty())
- || (filters.getIsRelatedTo() != null && !filters.getIsRelatedTo()
- .getFilterList().isEmpty())
- || (filters.getRelatesTo() != null && !filters.getRelatesTo()
- .getFilterList().isEmpty());
- }
- return flag;
- }
-
- /**
- * Add {@link QualifierFilter} filters to filter list for each column of
- * entity table.
- *
- * @param list filter list to which qualifier filters have to be added.
- */
- protected void updateFixedColumns(FilterList list) {
- for (EntityColumn column : EntityColumn.values()) {
- list.addFilter(new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(
- column.getColumnQualifierBytes())));
- }
- }
-
- /**
- * Creates a filter list which indicates that only some of the column
- * qualifiers in the info column family will be returned in result.
- *
- * @param isApplication If true, it means operations are to be performed for
- * application table, otherwise for entity table.
- * @return filter list.
- * @throws IOException if any problem occurs while creating filter list.
- */
- private FilterList createFilterListForColsOfInfoFamily() throws IOException {
- FilterList infoFamilyColsFilter = new FilterList(Operator.MUST_PASS_ONE);
- // Add filters for each column in entity table.
- updateFixedColumns(infoFamilyColsFilter);
- EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
- // If INFO field has to be retrieved, add a filter for fetching columns
- // with INFO column prefix.
- if (hasField(fieldsToRetrieve, Field.INFO)) {
- infoFamilyColsFilter
- .addFilter(TimelineFilterUtils.createHBaseQualifierFilter(
- CompareOp.EQUAL, EntityColumnPrefix.INFO));
- }
- TimelineFilterList relatesTo = getFilters().getRelatesTo();
- if (hasField(fieldsToRetrieve, Field.RELATES_TO)) {
- // If RELATES_TO field has to be retrieved, add a filter for fetching
- // columns with RELATES_TO column prefix.
- infoFamilyColsFilter.addFilter(TimelineFilterUtils
- .createHBaseQualifierFilter(CompareOp.EQUAL,
- EntityColumnPrefix.RELATES_TO));
- } else if (relatesTo != null && !relatesTo.getFilterList().isEmpty()) {
- // Even if fields to retrieve does not contain RELATES_TO, we still
- // need to have a filter to fetch some of the column qualifiers if
- // relatesTo filters are specified. relatesTo filters will then be
- // matched after fetching rows from HBase.
- Set<String> relatesToCols =
- TimelineFilterUtils.fetchColumnsFromFilterList(relatesTo);
- infoFamilyColsFilter.addFilter(createFiltersFromColumnQualifiers(
- EntityColumnPrefix.RELATES_TO, relatesToCols));
- }
- TimelineFilterList isRelatedTo = getFilters().getIsRelatedTo();
- if (hasField(fieldsToRetrieve, Field.IS_RELATED_TO)) {
- // If IS_RELATED_TO field has to be retrieved, add a filter for fetching
- // columns with IS_RELATED_TO column prefix.
- infoFamilyColsFilter.addFilter(TimelineFilterUtils
- .createHBaseQualifierFilter(CompareOp.EQUAL,
- EntityColumnPrefix.IS_RELATED_TO));
- } else if (isRelatedTo != null && !isRelatedTo.getFilterList().isEmpty()) {
- // Even if fields to retrieve does not contain IS_RELATED_TO, we still
- // need to have a filter to fetch some of the column qualifiers if
- // isRelatedTo filters are specified. isRelatedTo filters will then be
- // matched after fetching rows from HBase.
- Set<String> isRelatedToCols =
- TimelineFilterUtils.fetchColumnsFromFilterList(isRelatedTo);
- infoFamilyColsFilter.addFilter(createFiltersFromColumnQualifiers(
- EntityColumnPrefix.IS_RELATED_TO, isRelatedToCols));
- }
- TimelineFilterList eventFilters = getFilters().getEventFilters();
- if (hasField(fieldsToRetrieve, Field.EVENTS)) {
- // If EVENTS field has to be retrieved, add a filter for fetching columns
- // with EVENT column prefix.
- infoFamilyColsFilter
- .addFilter(TimelineFilterUtils.createHBaseQualifierFilter(
- CompareOp.EQUAL, EntityColumnPrefix.EVENT));
- } else if (eventFilters != null &&
- !eventFilters.getFilterList().isEmpty()) {
- // Even if fields to retrieve does not contain EVENTS, we still need to
- // have a filter to fetch some of the column qualifiers on the basis of
- // event filters specified. Event filters will then be matched after
- // fetching rows from HBase.
- Set<String> eventCols =
- TimelineFilterUtils.fetchColumnsFromFilterList(eventFilters);
- infoFamilyColsFilter.addFilter(createFiltersFromColumnQualifiers(
- EntityColumnPrefix.EVENT, eventCols));
- }
- return infoFamilyColsFilter;
- }
-
- /**
- * Exclude column prefixes via filters which are not required(based on fields
- * to retrieve) from info column family. These filters are added to filter
- * list which contains a filter for getting info column family.
- *
- * @param infoColFamilyList filter list for info column family.
- */
- private void excludeFieldsFromInfoColFamily(FilterList infoColFamilyList) {
- EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
- // Events not required.
- if (!hasField(fieldsToRetrieve, Field.EVENTS)) {
- infoColFamilyList.addFilter(TimelineFilterUtils
- .createHBaseQualifierFilter(CompareOp.NOT_EQUAL,
- EntityColumnPrefix.EVENT));
- }
- // info not required.
- if (!hasField(fieldsToRetrieve, Field.INFO)) {
- infoColFamilyList.addFilter(TimelineFilterUtils
- .createHBaseQualifierFilter(CompareOp.NOT_EQUAL,
- EntityColumnPrefix.INFO));
- }
- // is related to not required.
- if (!hasField(fieldsToRetrieve, Field.IS_RELATED_TO)) {
- infoColFamilyList.addFilter(TimelineFilterUtils
- .createHBaseQualifierFilter(CompareOp.NOT_EQUAL,
- EntityColumnPrefix.IS_RELATED_TO));
- }
- // relates to not required.
- if (!hasField(fieldsToRetrieve, Field.RELATES_TO)) {
- infoColFamilyList.addFilter(TimelineFilterUtils
- .createHBaseQualifierFilter(CompareOp.NOT_EQUAL,
- EntityColumnPrefix.RELATES_TO));
- }
- }
-
- /**
- * Updates filter list based on fields for confs and metrics to retrieve.
- *
- * @param listBasedOnFields filter list based on fields.
- * @throws IOException if any problem occurs while updating filter list.
- */
- private void updateFilterForConfsAndMetricsToRetrieve(
- FilterList listBasedOnFields) throws IOException {
- TimelineDataToRetrieve dataToRetrieve = getDataToRetrieve();
- // Please note that if confsToRetrieve is specified, we would have added
- // CONFS to fields to retrieve in augmentParams() even if not specified.
- if (dataToRetrieve.getFieldsToRetrieve().contains(Field.CONFIGS)) {
- // Create a filter list for configs.
- listBasedOnFields.addFilter(TimelineFilterUtils
- .createFilterForConfsOrMetricsToRetrieve(
- dataToRetrieve.getConfsToRetrieve(), EntityColumnFamily.CONFIGS,
- EntityColumnPrefix.CONFIG));
- }
-
- // Please note that if metricsToRetrieve is specified, we would have added
- // METRICS to fields to retrieve in augmentParams() even if not specified.
- if (dataToRetrieve.getFieldsToRetrieve().contains(Field.METRICS)) {
- // Create a filter list for metrics.
- listBasedOnFields.addFilter(TimelineFilterUtils
- .createFilterForConfsOrMetricsToRetrieve(
- dataToRetrieve.getMetricsToRetrieve(),
- EntityColumnFamily.METRICS, EntityColumnPrefix.METRIC));
- }
- }
-
- @Override
- protected FilterList constructFilterListBasedOnFields() throws IOException {
- if (!needCreateFilterListBasedOnFields()) {
- // Fetch all the columns. No need of a filter.
- return null;
- }
- FilterList listBasedOnFields = new FilterList(Operator.MUST_PASS_ONE);
- FilterList infoColFamilyList = new FilterList();
- // By default fetch everything in INFO column family.
- FamilyFilter infoColumnFamily =
- new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(
- EntityColumnFamily.INFO.getBytes()));
- infoColFamilyList.addFilter(infoColumnFamily);
- if (!isSingleEntityRead() && fetchPartialColsFromInfoFamily()) {
- // We can fetch only some of the columns from info family.
- infoColFamilyList.addFilter(createFilterListForColsOfInfoFamily());
- } else {
- // Exclude column prefixes in info column family which are not required
- // based on fields to retrieve.
- excludeFieldsFromInfoColFamily(infoColFamilyList);
- }
- listBasedOnFields.addFilter(infoColFamilyList);
- updateFilterForConfsAndMetricsToRetrieve(listBasedOnFields);
- return listBasedOnFields;
- }
-
- @Override
- protected void validateParams() {
- Preconditions.checkNotNull(getContext(), "context shouldn't be null");
- Preconditions.checkNotNull(getDataToRetrieve(),
- "data to retrieve shouldn't be null");
- Preconditions.checkNotNull(getContext().getClusterId(),
- "clusterId shouldn't be null");
- Preconditions.checkNotNull(getContext().getAppId(),
- "appId shouldn't be null");
- Preconditions.checkNotNull(getContext().getEntityType(),
- "entityType shouldn't be null");
- if (isSingleEntityRead()) {
- Preconditions.checkNotNull(getContext().getEntityId(),
- "entityId shouldn't be null");
- }
- }
-
- @Override
- protected void augmentParams(Configuration hbaseConf, Connection conn)
- throws IOException {
- defaultAugmentParams(hbaseConf, conn);
- // Add configs/metrics to fields to retrieve if confsToRetrieve and/or
- // metricsToRetrieve are specified.
- getDataToRetrieve().addFieldsBasedOnConfsAndMetricsToRetrieve();
- if (!isSingleEntityRead()) {
- createFiltersIfNull();
- }
- }
-
- @Override
- protected Result getResult(Configuration hbaseConf, Connection conn,
- FilterList filterList) throws IOException {
- TimelineReaderContext context = getContext();
- Result result = null;
- if (context.getEntityIdPrefix() != null) {
- byte[] rowKey = new EntityRowKey(context.getClusterId(),
- context.getUserId(), context.getFlowName(), context.getFlowRunId(),
- context.getAppId(), context.getEntityType(),
- context.getEntityIdPrefix(), context.getEntityId()).getRowKey();
- Get get = new Get(rowKey);
- setMetricsTimeRange(get);
- get.setMaxVersions(getDataToRetrieve().getMetricsLimit());
- if (filterList != null && !filterList.getFilters().isEmpty()) {
- get.setFilter(filterList);
- }
- result = getTable().getResult(hbaseConf, conn, get);
-
- } else {
- // Prepare for range scan
- // create single SingleColumnValueFilter and add to existing filters.
- FilterList filter = new FilterList(Operator.MUST_PASS_ALL);
- if (filterList != null && !filterList.getFilters().isEmpty()) {
- filter.addFilter(filterList);
- }
- FilterList newFilter = new FilterList();
- newFilter.addFilter(TimelineFilterUtils.createHBaseSingleColValueFilter(
- EntityColumn.ID, context.getEntityId(), CompareOp.EQUAL));
- newFilter.addFilter(new PageFilter(1));
- filter.addFilter(newFilter);
-
- ResultScanner results = getResults(hbaseConf, conn, filter);
- try {
- Iterator<Result> iterator = results.iterator();
- if (iterator.hasNext()) {
- result = iterator.next();
- }
- } finally {
- results.close();
- }
- }
- return result;
- }
-
- private void setMetricsTimeRange(Query query) {
- // Set time range for metric values.
- HBaseTimelineStorageUtils.setMetricsTimeRange(
- query, EntityColumnFamily.METRICS.getBytes(),
- getDataToRetrieve().getMetricsTimeBegin(),
- getDataToRetrieve().getMetricsTimeEnd());
- }
-
- @Override
- protected ResultScanner getResults(Configuration hbaseConf, Connection conn,
- FilterList filterList) throws IOException {
- // Scan through part of the table to find the entities belong to one app
- // and one type
- Scan scan = new Scan();
- TimelineReaderContext context = getContext();
- RowKeyPrefix<EntityRowKey> entityRowKeyPrefix = null;
- // default mode, will always scans from beginning of entity type.
- if (getFilters() == null || getFilters().getFromId() == null) {
- entityRowKeyPrefix = new EntityRowKeyPrefix(context.getClusterId(),
- context.getUserId(), context.getFlowName(), context.getFlowRunId(),
- context.getAppId(), context.getEntityType(), null, null);
- scan.setRowPrefixFilter(entityRowKeyPrefix.getRowKeyPrefix());
- } else { // pagination mode, will scan from given entityIdPrefix!enitityId
-
- EntityRowKey entityRowKey = null;
- try {
- entityRowKey =
- EntityRowKey.parseRowKeyFromString(getFilters().getFromId());
- } catch (IllegalArgumentException e) {
- throw new BadRequestException("Invalid filter fromid is provided.");
- }
- if (!context.getClusterId().equals(entityRowKey.getClusterId())) {
- throw new BadRequestException(
- "fromid doesn't belong to clusterId=" + context.getClusterId());
- }
-
- // set start row
- scan.setStartRow(entityRowKey.getRowKey());
-
- // get the bytes for stop row
- entityRowKeyPrefix = new EntityRowKeyPrefix(context.getClusterId(),
- context.getUserId(), context.getFlowName(), context.getFlowRunId(),
- context.getAppId(), context.getEntityType(), null, null);
-
- // set stop row
- scan.setStopRow(
- HBaseTimelineStorageUtils.calculateTheClosestNextRowKeyForPrefix(
- entityRowKeyPrefix.getRowKeyPrefix()));
-
- // set page filter to limit. This filter has to set only in pagination
- // mode.
- filterList.addFilter(new PageFilter(getFilters().getLimit()));
- }
- setMetricsTimeRange(scan);
- scan.setMaxVersions(getDataToRetrieve().getMetricsLimit());
- if (filterList != null && !filterList.getFilters().isEmpty()) {
- scan.setFilter(filterList);
- }
- return getTable().getResultScanner(hbaseConf, conn, scan);
- }
-
- @Override
- protected TimelineEntity parseEntity(Result result) throws IOException {
- if (result == null || result.isEmpty()) {
- return null;
- }
- TimelineEntity entity = new TimelineEntity();
- EntityRowKey parseRowKey = EntityRowKey.parseRowKey(result.getRow());
- entity.setType(parseRowKey.getEntityType());
- entity.setId(parseRowKey.getEntityId());
- entity.setIdPrefix(parseRowKey.getEntityIdPrefix().longValue());
-
- TimelineEntityFilters filters = getFilters();
- // fetch created time
- Long createdTime = (Long) EntityColumn.CREATED_TIME.readResult(result);
- entity.setCreatedTime(createdTime);
-
- EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
- // fetch is related to entities and match isRelatedTo filter. If isRelatedTo
- // filters do not match, entity would be dropped. We have to match filters
- // locally as relevant HBase filters to filter out rows on the basis of
- // isRelatedTo are not set in HBase scan.
- boolean checkIsRelatedTo =
- !isSingleEntityRead() && filters.getIsRelatedTo() != null
- && filters.getIsRelatedTo().getFilterList().size() > 0;
- if (hasField(fieldsToRetrieve, Field.IS_RELATED_TO) || checkIsRelatedTo) {
- readRelationship(entity, result, EntityColumnPrefix.IS_RELATED_TO, true);
- if (checkIsRelatedTo
- && !TimelineStorageUtils.matchIsRelatedTo(entity,
- filters.getIsRelatedTo())) {
- return null;
- }
- if (!hasField(fieldsToRetrieve, Field.IS_RELATED_TO)) {
- entity.getIsRelatedToEntities().clear();
- }
- }
-
- // fetch relates to entities and match relatesTo filter. If relatesTo
- // filters do not match, entity would be dropped. We have to match filters
- // locally as relevant HBase filters to filter out rows on the basis of
- // relatesTo are not set in HBase scan.
- boolean checkRelatesTo =
- !isSingleEntityRead() && filters.getRelatesTo() != null
- && filters.getRelatesTo().getFilterList().size() > 0;
- if (hasField(fieldsToRetrieve, Field.RELATES_TO)
- || checkRelatesTo) {
- readRelationship(entity, result, EntityColumnPrefix.RELATES_TO, false);
- if (checkRelatesTo
- && !TimelineStorageUtils.matchRelatesTo(entity,
- filters.getRelatesTo())) {
- return null;
- }
- if (!hasField(fieldsToRetrieve, Field.RELATES_TO)) {
- entity.getRelatesToEntities().clear();
- }
- }
-
- // fetch info if fieldsToRetrieve contains INFO or ALL.
- if (hasField(fieldsToRetrieve, Field.INFO)) {
- readKeyValuePairs(entity, result, EntityColumnPrefix.INFO, false);
- }
-
- // fetch configs if fieldsToRetrieve contains CONFIGS or ALL.
- if (hasField(fieldsToRetrieve, Field.CONFIGS)) {
- readKeyValuePairs(entity, result, EntityColumnPrefix.CONFIG, true);
- }
-
- // fetch events and match event filters if they exist. If event filters do
- // not match, entity would be dropped. We have to match filters locally
- // as relevant HBase filters to filter out rows on the basis of events
- // are not set in HBase scan.
- boolean checkEvents =
- !isSingleEntityRead() && filters.getEventFilters() != null
- && filters.getEventFilters().getFilterList().size() > 0;
- if (hasField(fieldsToRetrieve, Field.EVENTS) || checkEvents) {
- readEvents(entity, result, EntityColumnPrefix.EVENT);
- if (checkEvents
- && !TimelineStorageUtils.matchEventFilters(entity,
- filters.getEventFilters())) {
- return null;
- }
- if (!hasField(fieldsToRetrieve, Field.EVENTS)) {
- entity.getEvents().clear();
- }
- }
-
- // fetch metrics if fieldsToRetrieve contains METRICS or ALL.
- if (hasField(fieldsToRetrieve, Field.METRICS)) {
- readMetrics(entity, result, EntityColumnPrefix.METRIC);
- }
-
- entity.getInfo().put(TimelineReaderUtils.FROMID_KEY,
- parseRowKey.getRowKeyAsString());
- return entity;
- }
-
- /**
- * Helper method for reading key-value pairs for either info or config.
- *
- * @param <T> Describes the type of column prefix.
- * @param entity entity to fill.
- * @param result result from HBase.
- * @param prefix column prefix.
- * @param isConfig if true, means we are reading configs, otherwise info.
- * @throws IOException if any problem is encountered while reading result.
- */
- protected <T> void readKeyValuePairs(TimelineEntity entity, Result result,
- ColumnPrefix<T> prefix, boolean isConfig) throws IOException {
- // info and configuration are of type Map<String, Object or String>
- Map<String, Object> columns =
- prefix.readResults(result, stringKeyConverter);
- if (isConfig) {
- for (Map.Entry<String, Object> column : columns.entrySet()) {
- entity.addConfig(column.getKey(), column.getValue().toString());
- }
- } else {
- entity.addInfo(columns);
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/18] hadoop git commit: YARN-7919. Refactor timelineservice-hbase
module into submodules. Contributed by Haibo Chen.
Posted by ro...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKey.java
new file mode 100644
index 0000000..fb1f774
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKey.java
@@ -0,0 +1,290 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication;
+
+import java.util.List;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverterToString;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+
+/**
+ * Represents a rowkey for the sub app table.
+ */
+public class SubApplicationRowKey {
+ private final String subAppUserId;
+ private final String clusterId;
+ private final String entityType;
+ private final Long entityIdPrefix;
+ private final String entityId;
+ private final String userId;
+ private final SubApplicationRowKeyConverter subAppRowKeyConverter =
+ new SubApplicationRowKeyConverter();
+
+ public SubApplicationRowKey(String subAppUserId, String clusterId,
+ String entityType, Long entityIdPrefix, String entityId, String userId) {
+ this.subAppUserId = subAppUserId;
+ this.clusterId = clusterId;
+ this.entityType = entityType;
+ this.entityIdPrefix = entityIdPrefix;
+ this.entityId = entityId;
+ this.userId = userId;
+ }
+
+ public String getClusterId() {
+ return clusterId;
+ }
+
+ public String getSubAppUserId() {
+ return subAppUserId;
+ }
+
+ public String getEntityType() {
+ return entityType;
+ }
+
+ public String getEntityId() {
+ return entityId;
+ }
+
+ public Long getEntityIdPrefix() {
+ return entityIdPrefix;
+ }
+
+ public String getUserId() {
+ return userId;
+ }
+
+ /**
+ * Constructs a row key for the sub app table as follows:
+ * {@code subAppUserId!clusterId!entityType
+ * !entityPrefix!entityId!userId}.
+ * Typically used while querying a specific sub app.
+ *
+ * subAppUserId is usually the doAsUser.
+ * userId is the yarn user that the AM runs as.
+ *
+ * @return byte array with the row key.
+ */
+ public byte[] getRowKey() {
+ return subAppRowKeyConverter.encode(this);
+ }
+
+ /**
+ * Given the raw row key as bytes, returns the row key as an object.
+ *
+ * @param rowKey byte representation of row key.
+ * @return An <cite>SubApplicationRowKey</cite> object.
+ */
+ public static SubApplicationRowKey parseRowKey(byte[] rowKey) {
+ return new SubApplicationRowKeyConverter().decode(rowKey);
+ }
+
+ /**
+ * Constructs a row key for the sub app table as follows:
+ * <p>
+ * {@code subAppUserId!clusterId!
+ * entityType!entityIdPrefix!entityId!userId}.
+ *
+ * subAppUserId is usually the doAsUser.
+ * userId is the yarn user that that the AM runs as.
+ *
+ * </p>
+ *
+ * @return String representation of row key.
+ */
+ public String getRowKeyAsString() {
+ return subAppRowKeyConverter.encodeAsString(this);
+ }
+
+ /**
+ * Given the encoded row key as string, returns the row key as an object.
+ *
+ * @param encodedRowKey String representation of row key.
+ * @return A <cite>SubApplicationRowKey</cite> object.
+ */
+ public static SubApplicationRowKey parseRowKeyFromString(
+ String encodedRowKey) {
+ return new SubApplicationRowKeyConverter().decodeFromString(encodedRowKey);
+ }
+
+ /**
+ * Encodes and decodes row key for sub app table.
+ * The row key is of the form :
+ * subAppUserId!clusterId!flowRunId!appId!entityType!entityId!userId
+ *
+ * subAppUserId is usually the doAsUser.
+ * userId is the yarn user that the AM runs as.
+ *
+ * <p>
+ */
+ final private static class SubApplicationRowKeyConverter
+ implements KeyConverter<SubApplicationRowKey>,
+ KeyConverterToString<SubApplicationRowKey> {
+
+ private SubApplicationRowKeyConverter() {
+ }
+
+ /**
+ * sub app row key is of the form
+ * subAppUserId!clusterId!entityType!entityPrefix!entityId!userId
+ * w. each segment separated by !.
+ *
+ * subAppUserId is usually the doAsUser.
+ * userId is the yarn user that the AM runs as.
+ *
+ * The sizes below indicate sizes of each one of these
+ * segments in sequence. clusterId, subAppUserId, entityType,
+ * entityId and userId are strings.
+ * entity prefix is a long hence 8 bytes in size. Strings are
+ * variable in size (i.e. end whenever separator is encountered).
+ * This is used while decoding and helps in determining where to split.
+ */
+ private static final int[] SEGMENT_SIZES = {Separator.VARIABLE_SIZE,
+ Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
+ Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE};
+
+ /*
+ * (non-Javadoc)
+ *
+ * Encodes SubApplicationRowKey object into a byte array with each
+ * component/field in SubApplicationRowKey separated by
+ * Separator#QUALIFIERS.
+ * This leads to an sub app table row key of the form
+ * subAppUserId!clusterId!entityType!entityPrefix!entityId!userId
+ *
+ * subAppUserId is usually the doAsUser.
+ * userId is the yarn user that the AM runs as.
+ *
+ * If entityType in passed SubApplicationRowKey object is null (and the
+ * fields preceding it are not null i.e. clusterId, subAppUserId), this
+ * returns a row key prefix of the form subAppUserId!clusterId!
+ * If entityId in SubApplicationRowKey is null
+ * (other components are not null), this returns a row key prefix
+ * of the form subAppUserId!clusterId!entityType!
+ *
+ * @see org.apache.hadoop.yarn.server.timelineservice.storage.common
+ * .KeyConverter#encode(java.lang.Object)
+ */
+ @Override
+ public byte[] encode(SubApplicationRowKey rowKey) {
+ byte[] subAppUser = Separator.encode(rowKey.getSubAppUserId(),
+ Separator.SPACE, Separator.TAB, Separator.QUALIFIERS);
+ byte[] cluster = Separator.encode(rowKey.getClusterId(), Separator.SPACE,
+ Separator.TAB, Separator.QUALIFIERS);
+ byte[] first = Separator.QUALIFIERS.join(subAppUser, cluster);
+ if (rowKey.getEntityType() == null) {
+ return first;
+ }
+ byte[] entityType = Separator.encode(rowKey.getEntityType(),
+ Separator.SPACE, Separator.TAB, Separator.QUALIFIERS);
+
+ if (rowKey.getEntityIdPrefix() == null) {
+ return Separator.QUALIFIERS.join(first, entityType,
+ Separator.EMPTY_BYTES);
+ }
+
+ byte[] entityIdPrefix = Bytes.toBytes(rowKey.getEntityIdPrefix());
+
+ if (rowKey.getEntityId() == null) {
+ return Separator.QUALIFIERS.join(first, entityType, entityIdPrefix,
+ Separator.EMPTY_BYTES);
+ }
+
+ byte[] entityId = Separator.encode(rowKey.getEntityId(), Separator.SPACE,
+ Separator.TAB, Separator.QUALIFIERS);
+
+ byte[] userId = Separator.encode(rowKey.getUserId(),
+ Separator.SPACE, Separator.TAB, Separator.QUALIFIERS);
+
+ byte[] second = Separator.QUALIFIERS.join(entityType, entityIdPrefix,
+ entityId, userId);
+
+ return Separator.QUALIFIERS.join(first, second);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * Decodes a sub application row key of the form
+ * subAppUserId!clusterId!entityType!entityPrefix!entityId!userId
+ *
+ * subAppUserId is usually the doAsUser.
+ * userId is the yarn user that the AM runs as.
+ *
+ * represented in byte format
+ * and converts it into an SubApplicationRowKey object.
+ *
+ * @see org.apache.hadoop.yarn.server.timelineservice.storage.common
+ * .KeyConverter#decode(byte[])
+ */
+ @Override
+ public SubApplicationRowKey decode(byte[] rowKey) {
+ byte[][] rowKeyComponents =
+ Separator.QUALIFIERS.split(rowKey, SEGMENT_SIZES);
+ if (rowKeyComponents.length != 6) {
+ throw new IllegalArgumentException(
+ "the row key is not valid for " + "a sub app");
+ }
+ String subAppUserId =
+ Separator.decode(Bytes.toString(rowKeyComponents[0]),
+ Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
+ String clusterId = Separator.decode(Bytes.toString(rowKeyComponents[1]),
+ Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
+ String entityType = Separator.decode(Bytes.toString(rowKeyComponents[2]),
+ Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
+
+ Long entityPrefixId = Bytes.toLong(rowKeyComponents[3]);
+
+ String entityId = Separator.decode(Bytes.toString(rowKeyComponents[4]),
+ Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
+ String userId =
+ Separator.decode(Bytes.toString(rowKeyComponents[5]),
+ Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
+
+ return new SubApplicationRowKey(subAppUserId, clusterId, entityType,
+ entityPrefixId, entityId, userId);
+ }
+
+ @Override
+ public String encodeAsString(SubApplicationRowKey key) {
+ if (key.subAppUserId == null || key.clusterId == null
+ || key.entityType == null || key.entityIdPrefix == null
+ || key.entityId == null || key.userId == null) {
+ throw new IllegalArgumentException();
+ }
+ return TimelineReaderUtils.joinAndEscapeStrings(
+ new String[] {key.subAppUserId, key.clusterId, key.entityType,
+ key.entityIdPrefix.toString(), key.entityId, key.userId});
+ }
+
+ @Override
+ public SubApplicationRowKey decodeFromString(String encodedRowKey) {
+ List<String> split = TimelineReaderUtils.split(encodedRowKey);
+ if (split == null || split.size() != 6) {
+ throw new IllegalArgumentException(
+ "Invalid row key for sub app table.");
+ }
+ Long entityIdPrefix = Long.valueOf(split.get(3));
+ return new SubApplicationRowKey(split.get(0), split.get(1),
+ split.get(2), entityIdPrefix, split.get(4), split.get(5));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKeyPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKeyPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKeyPrefix.java
new file mode 100644
index 0000000..0c04959
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationRowKeyPrefix.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication;
+
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.RowKeyPrefix;
+
+/**
+ * Represents a partial rowkey without the entityId or without entityType and
+ * entityId for the sub application table.
+ *
+ */
+public class SubApplicationRowKeyPrefix extends SubApplicationRowKey
+ implements RowKeyPrefix<SubApplicationRowKey> {
+
+ /**
+ * Creates a prefix which generates the following rowKeyPrefixes for the sub
+ * application table:
+ * {@code subAppUserId!clusterId!entityType!entityPrefix!userId}.
+ *
+ * @param subAppUserId
+ * identifying the subApp User
+ * @param clusterId
+ * identifying the cluster
+ * @param entityType
+ * which entity type
+ * @param entityIdPrefix
+ * for entityId
+ * @param entityId
+ * for an entity
+ * @param userId
+ * for the user who runs the AM
+ *
+ * subAppUserId is usually the doAsUser.
+ * userId is the yarn user that the AM runs as.
+ *
+ */
+ public SubApplicationRowKeyPrefix(String subAppUserId, String clusterId,
+ String entityType, Long entityIdPrefix, String entityId,
+ String userId) {
+ super(subAppUserId, clusterId, entityType, entityIdPrefix, entityId,
+ userId);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.
+ * RowKeyPrefix#getRowKeyPrefix()
+ */
+ public byte[] getRowKeyPrefix() {
+ return super.getRowKey();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationTable.java
new file mode 100644
index 0000000..de7dd4d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationTable.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication;
+
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+
+/**
+ * The sub application table has column families:
+ * info, config and metrics.
+ * Info stores information about a timeline entity object
+ * config stores configuration data of a timeline entity object
+ * metrics stores the metrics of a timeline entity object
+ *
+ * Example sub application table record:
+ *
+ * <pre>
+ * |-------------------------------------------------------------------------|
+ * | Row | Column Family | Column Family| Column Family|
+ * | key | info | metrics | config |
+ * |-------------------------------------------------------------------------|
+ * | subAppUserId! | id:entityId | metricId1: | configKey1: |
+ * | clusterId! | type:entityType | metricValue1 | configValue1 |
+ * | entityType! | | @timestamp1 | |
+ * | idPrefix!| | | | configKey2: |
+ * | entityId! | created_time: | metricId1: | configValue2 |
+ * | userId | 1392993084018 | metricValue2 | |
+ * | | | @timestamp2 | |
+ * | | i!infoKey: | | |
+ * | | infoValue | metricId1: | |
+ * | | | metricValue1 | |
+ * | | | @timestamp2 | |
+ * | | e!eventId=timestamp= | | |
+ * | | infoKey: | | |
+ * | | eventInfoValue | | |
+ * | | | | |
+ * | | r!relatesToKey: | | |
+ * | | id3=id4=id5 | | |
+ * | | | | |
+ * | | s!isRelatedToKey | | |
+ * | | id7=id9=id6 | | |
+ * | | | | |
+ * | | flowVersion: | | |
+ * | | versionValue | | |
+ * |-------------------------------------------------------------------------|
+ * </pre>
+ */
+public final class SubApplicationTable extends BaseTable<SubApplicationTable> {
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/package-info.java
new file mode 100644
index 0000000..52cc399
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication
+ * contains classes related to implementation for subapplication table.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestCustomApplicationIdConversion.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestCustomApplicationIdConversion.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestCustomApplicationIdConversion.java
new file mode 100644
index 0000000..0dc344f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestCustomApplicationIdConversion.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test for HBaseTimelineStorageUtils.convertApplicationIdToString(),
+ * a custom conversion from ApplicationId to String that avoids the
+ * incompatibility issue caused by mixing hadoop-common 2.5.1 and
+ * hadoop-yarn-api 3.0. See YARN-6905.
+ */
+public class TestCustomApplicationIdConversion {
+ @Test
+ public void testConvertAplicationIdToString() {
+ ApplicationId applicationId = ApplicationId.newInstance(0, 1);
+ String applicationIdStr =
+ HBaseTimelineSchemaUtils.convertApplicationIdToString(applicationId);
+ Assert.assertEquals(applicationId,
+ ApplicationId.fromString(applicationIdStr));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestKeyConverters.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestKeyConverters.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestKeyConverters.java
new file mode 100644
index 0000000..1bd363f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestKeyConverters.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.junit.Test;
+
+/**
+ * Unit tests for key converters for various tables' row keys.
+ *
+ */
+public class TestKeyConverters {
+
+ @Test
+ public void testAppIdKeyConverter() {
+ AppIdKeyConverter appIdKeyConverter = new AppIdKeyConverter();
+ long currentTs = System.currentTimeMillis();
+ ApplicationId appId1 = ApplicationId.newInstance(currentTs, 1);
+ ApplicationId appId2 = ApplicationId.newInstance(currentTs, 2);
+ ApplicationId appId3 = ApplicationId.newInstance(currentTs + 300, 1);
+ String appIdStr1 = appId1.toString();
+ String appIdStr2 = appId2.toString();
+ String appIdStr3 = appId3.toString();
+ byte[] appIdBytes1 = appIdKeyConverter.encode(appIdStr1);
+ byte[] appIdBytes2 = appIdKeyConverter.encode(appIdStr2);
+ byte[] appIdBytes3 = appIdKeyConverter.encode(appIdStr3);
+ // App ids' should be encoded in a manner wherein descending order
+ // is maintained.
+ assertTrue(
+ "Ordering of app ids' is incorrect",
+ Bytes.compareTo(appIdBytes1, appIdBytes2) > 0
+ && Bytes.compareTo(appIdBytes1, appIdBytes3) > 0
+ && Bytes.compareTo(appIdBytes2, appIdBytes3) > 0);
+ String decodedAppId1 = appIdKeyConverter.decode(appIdBytes1);
+ String decodedAppId2 = appIdKeyConverter.decode(appIdBytes2);
+ String decodedAppId3 = appIdKeyConverter.decode(appIdBytes3);
+ assertTrue("Decoded app id is not same as the app id encoded",
+ appIdStr1.equals(decodedAppId1));
+ assertTrue("Decoded app id is not same as the app id encoded",
+ appIdStr2.equals(decodedAppId2));
+ assertTrue("Decoded app id is not same as the app id encoded",
+ appIdStr3.equals(decodedAppId3));
+ }
+
+ @Test
+ public void testEventColumnNameConverter() {
+ String eventId = "=foo_=eve=nt=";
+ byte[] valSepBytes = Bytes.toBytes(Separator.VALUES.getValue());
+ byte[] maxByteArr =
+ Bytes.createMaxByteArray(Bytes.SIZEOF_LONG - valSepBytes.length);
+ byte[] ts = Bytes.add(valSepBytes, maxByteArr);
+ Long eventTs = Bytes.toLong(ts);
+ byte[] byteEventColName =
+ new EventColumnName(eventId, eventTs, null).getColumnQualifier();
+ KeyConverter<EventColumnName> eventColumnNameConverter =
+ new EventColumnNameConverter();
+ EventColumnName eventColName =
+ eventColumnNameConverter.decode(byteEventColName);
+ assertEquals(eventId, eventColName.getId());
+ assertEquals(eventTs, eventColName.getTimestamp());
+ assertNull(eventColName.getInfoKey());
+
+ String infoKey = "f=oo_event_in=fo=_key";
+ byteEventColName =
+ new EventColumnName(eventId, eventTs, infoKey).getColumnQualifier();
+ eventColName = eventColumnNameConverter.decode(byteEventColName);
+ assertEquals(eventId, eventColName.getId());
+ assertEquals(eventTs, eventColName.getTimestamp());
+ assertEquals(infoKey, eventColName.getInfoKey());
+ }
+
+ @Test
+ public void testLongKeyConverter() {
+ LongKeyConverter longKeyConverter = new LongKeyConverter();
+ confirmLongKeyConverter(longKeyConverter, Long.MIN_VALUE);
+ confirmLongKeyConverter(longKeyConverter, -1234567890L);
+ confirmLongKeyConverter(longKeyConverter, -128L);
+ confirmLongKeyConverter(longKeyConverter, -127L);
+ confirmLongKeyConverter(longKeyConverter, -1L);
+ confirmLongKeyConverter(longKeyConverter, 0L);
+ confirmLongKeyConverter(longKeyConverter, 1L);
+ confirmLongKeyConverter(longKeyConverter, 127L);
+ confirmLongKeyConverter(longKeyConverter, 128L);
+ confirmLongKeyConverter(longKeyConverter, 1234567890L);
+ confirmLongKeyConverter(longKeyConverter, Long.MAX_VALUE);
+ }
+
+ private void confirmLongKeyConverter(LongKeyConverter longKeyConverter,
+ Long testValue) {
+ Long decoded = longKeyConverter.decode(longKeyConverter.encode(testValue));
+ assertEquals(testValue, decoded);
+ }
+
+ @Test
+ public void testStringKeyConverter() {
+ StringKeyConverter stringKeyConverter = new StringKeyConverter();
+ String phrase = "QuackAttack now!";
+
+ for (int i = 0; i < phrase.length(); i++) {
+ String sub = phrase.substring(i, phrase.length());
+ confirmStrignKeyConverter(stringKeyConverter, sub);
+ confirmStrignKeyConverter(stringKeyConverter, sub + sub);
+ }
+ }
+
+ private void confirmStrignKeyConverter(StringKeyConverter stringKeyConverter,
+ String testValue) {
+ String decoded =
+ stringKeyConverter.decode(stringKeyConverter.encode(testValue));
+ assertEquals(testValue, decoded);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java
new file mode 100644
index 0000000..d05cbad
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java
@@ -0,0 +1,276 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKeyPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKeyPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKeyPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationRowKey;
+import org.junit.Test;
+
+
+/**
+ * Class to test the row key structures for various tables.
+ *
+ */
+public class TestRowKeys {
+
+ private final static String QUALIFIER_SEP = Separator.QUALIFIERS.getValue();
+ private final static byte[] QUALIFIER_SEP_BYTES = Bytes
+ .toBytes(QUALIFIER_SEP);
+ private final static String CLUSTER = "cl" + QUALIFIER_SEP + "uster";
+ private final static String USER = QUALIFIER_SEP + "user";
+ private final static String SUB_APP_USER = QUALIFIER_SEP + "subAppUser";
+ private final static String FLOW_NAME = "dummy_" + QUALIFIER_SEP + "flow"
+ + QUALIFIER_SEP;
+ private final static Long FLOW_RUN_ID;
+ private final static String APPLICATION_ID;
+ static {
+ long runid = Long.MAX_VALUE - 900L;
+ byte[] longMaxByteArr = Bytes.toBytes(Long.MAX_VALUE);
+ byte[] byteArr = Bytes.toBytes(runid);
+ int sepByteLen = QUALIFIER_SEP_BYTES.length;
+ if (sepByteLen <= byteArr.length) {
+ for (int i = 0; i < sepByteLen; i++) {
+ byteArr[i] = (byte) (longMaxByteArr[i] - QUALIFIER_SEP_BYTES[i]);
+ }
+ }
+ FLOW_RUN_ID = Bytes.toLong(byteArr);
+ long clusterTs = System.currentTimeMillis();
+ byteArr = Bytes.toBytes(clusterTs);
+ if (sepByteLen <= byteArr.length) {
+ for (int i = 0; i < sepByteLen; i++) {
+ byteArr[byteArr.length - sepByteLen + i] =
+ (byte) (longMaxByteArr[byteArr.length - sepByteLen + i] -
+ QUALIFIER_SEP_BYTES[i]);
+ }
+ }
+ clusterTs = Bytes.toLong(byteArr);
+ int seqId = 222;
+ APPLICATION_ID = ApplicationId.newInstance(clusterTs, seqId).toString();
+ }
+
+ private static void verifyRowPrefixBytes(byte[] byteRowKeyPrefix) {
+ int sepLen = QUALIFIER_SEP_BYTES.length;
+ for (int i = 0; i < sepLen; i++) {
+ assertTrue(
+ "Row key prefix not encoded properly.",
+ byteRowKeyPrefix[byteRowKeyPrefix.length - sepLen + i] ==
+ QUALIFIER_SEP_BYTES[i]);
+ }
+ }
+
+ @Test
+ public void testApplicationRowKey() {
+ byte[] byteRowKey =
+ new ApplicationRowKey(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID,
+ APPLICATION_ID).getRowKey();
+ ApplicationRowKey rowKey = ApplicationRowKey.parseRowKey(byteRowKey);
+ assertEquals(CLUSTER, rowKey.getClusterId());
+ assertEquals(USER, rowKey.getUserId());
+ assertEquals(FLOW_NAME, rowKey.getFlowName());
+ assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId());
+ assertEquals(APPLICATION_ID, rowKey.getAppId());
+
+ byte[] byteRowKeyPrefix =
+ new ApplicationRowKeyPrefix(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID)
+ .getRowKeyPrefix();
+ byte[][] splits =
+ Separator.QUALIFIERS.split(byteRowKeyPrefix,
+ new int[] {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
+ Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
+ Separator.VARIABLE_SIZE});
+ assertEquals(5, splits.length);
+ assertEquals(0, splits[4].length);
+ assertEquals(FLOW_NAME,
+ Separator.QUALIFIERS.decode(Bytes.toString(splits[2])));
+ assertEquals(FLOW_RUN_ID,
+ (Long) LongConverter.invertLong(Bytes.toLong(splits[3])));
+ verifyRowPrefixBytes(byteRowKeyPrefix);
+
+ byteRowKeyPrefix =
+ new ApplicationRowKeyPrefix(CLUSTER, USER, FLOW_NAME).getRowKeyPrefix();
+ splits =
+ Separator.QUALIFIERS.split(byteRowKeyPrefix, new int[] {
+ Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
+ Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE });
+ assertEquals(4, splits.length);
+ assertEquals(0, splits[3].length);
+ assertEquals(FLOW_NAME,
+ Separator.QUALIFIERS.decode(Bytes.toString(splits[2])));
+ verifyRowPrefixBytes(byteRowKeyPrefix);
+ }
+
+ /**
+ * Tests the converters indirectly through the public methods of the
+ * corresponding rowkey.
+ */
+ @Test
+ public void testAppToFlowRowKey() {
+ byte[] byteRowKey = new AppToFlowRowKey(APPLICATION_ID).getRowKey();
+ AppToFlowRowKey rowKey = AppToFlowRowKey.parseRowKey(byteRowKey);
+ assertEquals(APPLICATION_ID, rowKey.getAppId());
+ }
+
+ @Test
+ public void testEntityRowKey() {
+ TimelineEntity entity = new TimelineEntity();
+ entity.setId("!ent!ity!!id!");
+ entity.setType("entity!Type");
+ entity.setIdPrefix(54321);
+
+ byte[] byteRowKey =
+ new EntityRowKey(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID, APPLICATION_ID,
+ entity.getType(), entity.getIdPrefix(),
+ entity.getId()).getRowKey();
+ EntityRowKey rowKey = EntityRowKey.parseRowKey(byteRowKey);
+ assertEquals(CLUSTER, rowKey.getClusterId());
+ assertEquals(USER, rowKey.getUserId());
+ assertEquals(FLOW_NAME, rowKey.getFlowName());
+ assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId());
+ assertEquals(APPLICATION_ID, rowKey.getAppId());
+ assertEquals(entity.getType(), rowKey.getEntityType());
+ assertEquals(entity.getIdPrefix(), rowKey.getEntityIdPrefix().longValue());
+ assertEquals(entity.getId(), rowKey.getEntityId());
+
+ byte[] byteRowKeyPrefix =
+ new EntityRowKeyPrefix(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID,
+ APPLICATION_ID, entity.getType(), null, null)
+ .getRowKeyPrefix();
+ byte[][] splits =
+ Separator.QUALIFIERS.split(
+ byteRowKeyPrefix,
+ new int[] {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
+ Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
+ AppIdKeyConverter.getKeySize(), Separator.VARIABLE_SIZE,
+ Bytes.SIZEOF_LONG, Separator.VARIABLE_SIZE });
+ assertEquals(7, splits.length);
+ assertEquals(APPLICATION_ID, new AppIdKeyConverter().decode(splits[4]));
+ assertEquals(entity.getType(),
+ Separator.QUALIFIERS.decode(Bytes.toString(splits[5])));
+ verifyRowPrefixBytes(byteRowKeyPrefix);
+
+ byteRowKeyPrefix =
+ new EntityRowKeyPrefix(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID,
+ APPLICATION_ID).getRowKeyPrefix();
+ splits =
+ Separator.QUALIFIERS.split(
+ byteRowKeyPrefix,
+ new int[] {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
+ Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
+ AppIdKeyConverter.getKeySize(), Separator.VARIABLE_SIZE});
+ assertEquals(6, splits.length);
+ assertEquals(0, splits[5].length);
+ AppIdKeyConverter appIdKeyConverter = new AppIdKeyConverter();
+ assertEquals(APPLICATION_ID, appIdKeyConverter.decode(splits[4]));
+ verifyRowPrefixBytes(byteRowKeyPrefix);
+ }
+
+ @Test
+ public void testFlowActivityRowKey() {
+ Long ts = 1459900830000L;
+ Long dayTimestamp = HBaseTimelineSchemaUtils.getTopOfTheDayTimestamp(ts);
+ byte[] byteRowKey =
+ new FlowActivityRowKey(CLUSTER, ts, USER, FLOW_NAME).getRowKey();
+ FlowActivityRowKey rowKey = FlowActivityRowKey.parseRowKey(byteRowKey);
+ assertEquals(CLUSTER, rowKey.getClusterId());
+ assertEquals(dayTimestamp, rowKey.getDayTimestamp());
+ assertEquals(USER, rowKey.getUserId());
+ assertEquals(FLOW_NAME, rowKey.getFlowName());
+
+ byte[] byteRowKeyPrefix =
+ new FlowActivityRowKeyPrefix(CLUSTER).getRowKeyPrefix();
+ byte[][] splits =
+ Separator.QUALIFIERS.split(byteRowKeyPrefix, new int[] {
+ Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE });
+ assertEquals(2, splits.length);
+ assertEquals(0, splits[1].length);
+ assertEquals(CLUSTER,
+ Separator.QUALIFIERS.decode(Bytes.toString(splits[0])));
+ verifyRowPrefixBytes(byteRowKeyPrefix);
+
+ byteRowKeyPrefix =
+ new FlowActivityRowKeyPrefix(CLUSTER, ts).getRowKeyPrefix();
+ splits =
+ Separator.QUALIFIERS.split(byteRowKeyPrefix,
+ new int[] {Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
+ Separator.VARIABLE_SIZE});
+ assertEquals(3, splits.length);
+ assertEquals(0, splits[2].length);
+ assertEquals(CLUSTER,
+ Separator.QUALIFIERS.decode(Bytes.toString(splits[0])));
+ assertEquals(ts,
+ (Long) LongConverter.invertLong(Bytes.toLong(splits[1])));
+ verifyRowPrefixBytes(byteRowKeyPrefix);
+ }
+
+ @Test
+ public void testFlowRunRowKey() {
+ byte[] byteRowKey =
+ new FlowRunRowKey(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID).getRowKey();
+ FlowRunRowKey rowKey = FlowRunRowKey.parseRowKey(byteRowKey);
+ assertEquals(CLUSTER, rowKey.getClusterId());
+ assertEquals(USER, rowKey.getUserId());
+ assertEquals(FLOW_NAME, rowKey.getFlowName());
+ assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId());
+
+ byte[] byteRowKeyPrefix =
+ new FlowRunRowKey(CLUSTER, USER, FLOW_NAME, null).getRowKey();
+ byte[][] splits =
+ Separator.QUALIFIERS.split(byteRowKeyPrefix, new int[] {
+ Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
+ Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE });
+ assertEquals(4, splits.length);
+ assertEquals(0, splits[3].length);
+ assertEquals(FLOW_NAME,
+ Separator.QUALIFIERS.decode(Bytes.toString(splits[2])));
+ verifyRowPrefixBytes(byteRowKeyPrefix);
+ }
+
+ @Test
+ public void testSubAppRowKey() {
+ TimelineEntity entity = new TimelineEntity();
+ entity.setId("entity1");
+ entity.setType("DAG");
+ entity.setIdPrefix(54321);
+
+ byte[] byteRowKey =
+ new SubApplicationRowKey(SUB_APP_USER, CLUSTER,
+ entity.getType(), entity.getIdPrefix(),
+ entity.getId(), USER).getRowKey();
+ SubApplicationRowKey rowKey = SubApplicationRowKey.parseRowKey(byteRowKey);
+ assertEquals(CLUSTER, rowKey.getClusterId());
+ assertEquals(SUB_APP_USER, rowKey.getSubAppUserId());
+ assertEquals(entity.getType(), rowKey.getEntityType());
+ assertEquals(entity.getIdPrefix(), rowKey.getEntityIdPrefix().longValue());
+ assertEquals(entity.getId(), rowKey.getEntityId());
+ assertEquals(USER, rowKey.getUserId());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeysAsString.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeysAsString.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeysAsString.java
new file mode 100644
index 0000000..c4d07c7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeysAsString.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationRowKey;
+import org.junit.Test;
+
+/**
+ * Test for row key as string.
+ */
+public class TestRowKeysAsString {
+
+ private final static String CLUSTER =
+ "cl" + TimelineReaderUtils.DEFAULT_DELIMITER_CHAR + "uster"
+ + TimelineReaderUtils.DEFAULT_ESCAPE_CHAR;
+ private final static String USER =
+ TimelineReaderUtils.DEFAULT_ESCAPE_CHAR + "user";
+ private final static String SUB_APP_USER =
+ TimelineReaderUtils.DEFAULT_ESCAPE_CHAR + "subAppUser";
+
+ private final static String FLOW_NAME =
+ "dummy_" + TimelineReaderUtils.DEFAULT_DELIMITER_CHAR
+ + TimelineReaderUtils.DEFAULT_ESCAPE_CHAR + "flow"
+ + TimelineReaderUtils.DEFAULT_DELIMITER_CHAR;
+ private final static Long FLOW_RUN_ID = System.currentTimeMillis();
+ private final static String APPLICATION_ID =
+ ApplicationId.newInstance(System.currentTimeMillis(), 1).toString();
+
+ @Test(timeout = 10000)
+ public void testApplicationRow() {
+ String rowKeyAsString = new ApplicationRowKey(CLUSTER, USER, FLOW_NAME,
+ FLOW_RUN_ID, APPLICATION_ID).getRowKeyAsString();
+ ApplicationRowKey rowKey =
+ ApplicationRowKey.parseRowKeyFromString(rowKeyAsString);
+ assertEquals(CLUSTER, rowKey.getClusterId());
+ assertEquals(USER, rowKey.getUserId());
+ assertEquals(FLOW_NAME, rowKey.getFlowName());
+ assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId());
+ assertEquals(APPLICATION_ID, rowKey.getAppId());
+ }
+
+ @Test(timeout = 10000)
+ public void testEntityRowKey() {
+ char del = TimelineReaderUtils.DEFAULT_DELIMITER_CHAR;
+ char esc = TimelineReaderUtils.DEFAULT_ESCAPE_CHAR;
+ String id = del + esc + "ent" + esc + del + "ity" + esc + del + esc + "id"
+ + esc + del + esc;
+ String type = "entity" + esc + del + esc + "Type";
+ TimelineEntity entity = new TimelineEntity();
+ entity.setId(id);
+ entity.setType(type);
+ entity.setIdPrefix(54321);
+
+ String rowKeyAsString =
+ new EntityRowKey(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID, APPLICATION_ID,
+ entity.getType(), entity.getIdPrefix(), entity.getId())
+ .getRowKeyAsString();
+ EntityRowKey rowKey = EntityRowKey.parseRowKeyFromString(rowKeyAsString);
+ assertEquals(CLUSTER, rowKey.getClusterId());
+ assertEquals(USER, rowKey.getUserId());
+ assertEquals(FLOW_NAME, rowKey.getFlowName());
+ assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId());
+ assertEquals(APPLICATION_ID, rowKey.getAppId());
+ assertEquals(entity.getType(), rowKey.getEntityType());
+ assertEquals(entity.getIdPrefix(), rowKey.getEntityIdPrefix().longValue());
+ assertEquals(entity.getId(), rowKey.getEntityId());
+
+ }
+
+ @Test(timeout = 10000)
+ public void testFlowActivityRowKey() {
+ Long ts = 1459900830000L;
+ Long dayTimestamp = HBaseTimelineSchemaUtils.getTopOfTheDayTimestamp(ts);
+ String rowKeyAsString = new FlowActivityRowKey(CLUSTER, ts, USER, FLOW_NAME)
+ .getRowKeyAsString();
+ FlowActivityRowKey rowKey =
+ FlowActivityRowKey.parseRowKeyFromString(rowKeyAsString);
+ assertEquals(CLUSTER, rowKey.getClusterId());
+ assertEquals(dayTimestamp, rowKey.getDayTimestamp());
+ assertEquals(USER, rowKey.getUserId());
+ assertEquals(FLOW_NAME, rowKey.getFlowName());
+ }
+
+ @Test(timeout = 10000)
+ public void testFlowRunRowKey() {
+ String rowKeyAsString =
+ new FlowRunRowKey(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID)
+ .getRowKeyAsString();
+ FlowRunRowKey rowKey = FlowRunRowKey.parseRowKeyFromString(rowKeyAsString);
+ assertEquals(CLUSTER, rowKey.getClusterId());
+ assertEquals(USER, rowKey.getUserId());
+ assertEquals(FLOW_NAME, rowKey.getFlowName());
+ assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId());
+ }
+
+ @Test(timeout = 10000)
+ public void testSubApplicationRowKey() {
+ char del = TimelineReaderUtils.DEFAULT_DELIMITER_CHAR;
+ char esc = TimelineReaderUtils.DEFAULT_ESCAPE_CHAR;
+ String id = del + esc + "ent" + esc + del + "ity" + esc + del + esc + "id"
+ + esc + del + esc;
+ String type = "entity" + esc + del + esc + "Type";
+ TimelineEntity entity = new TimelineEntity();
+ entity.setId(id);
+ entity.setType(type);
+ entity.setIdPrefix(54321);
+
+ String rowKeyAsString = new SubApplicationRowKey(SUB_APP_USER, CLUSTER,
+ entity.getType(), entity.getIdPrefix(), entity.getId(), USER)
+ .getRowKeyAsString();
+ SubApplicationRowKey rowKey = SubApplicationRowKey
+ .parseRowKeyFromString(rowKeyAsString);
+ assertEquals(SUB_APP_USER, rowKey.getSubAppUserId());
+ assertEquals(CLUSTER, rowKey.getClusterId());
+ assertEquals(entity.getType(), rowKey.getEntityType());
+ assertEquals(entity.getIdPrefix(), rowKey.getEntityIdPrefix().longValue());
+ assertEquals(entity.getId(), rowKey.getEntityId());
+ assertEquals(USER, rowKey.getUserId());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestSeparator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestSeparator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestSeparator.java
new file mode 100644
index 0000000..7d37206
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestSeparator.java
@@ -0,0 +1,215 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Test;
+
+import com.google.common.collect.Iterables;
+
+public class TestSeparator {
+
+ private static String villain = "Dr. Heinz Doofenshmirtz";
+ private static String special =
+ ". * | ? + \t ( ) [ ] { } ^ $ \\ \" %";
+
+ /**
+ *
+ */
+ @Test
+ public void testEncodeDecodeString() {
+
+ for (Separator separator : Separator.values()) {
+ testEncodeDecode(separator, "");
+ testEncodeDecode(separator, " ");
+ testEncodeDecode(separator, "!");
+ testEncodeDecode(separator, "?");
+ testEncodeDecode(separator, "&");
+ testEncodeDecode(separator, "+");
+ testEncodeDecode(separator, "\t");
+ testEncodeDecode(separator, "Dr.");
+ testEncodeDecode(separator, "Heinz");
+ testEncodeDecode(separator, "Doofenshmirtz");
+ testEncodeDecode(separator, villain);
+ testEncodeDecode(separator, special);
+
+ assertNull(separator.encode(null));
+
+ }
+ }
+
+ private void testEncodeDecode(Separator separator, String token) {
+ String encoded = separator.encode(token);
+ String decoded = separator.decode(encoded);
+ String msg = "token:" + token + " separator:" + separator + ".";
+ assertEquals(msg, token, decoded);
+ }
+
+ @Test
+ public void testEncodeDecode() {
+ testEncodeDecode("Dr.", Separator.QUALIFIERS);
+ testEncodeDecode("Heinz", Separator.QUALIFIERS, Separator.QUALIFIERS);
+ testEncodeDecode("Doofenshmirtz", Separator.QUALIFIERS, null,
+ Separator.QUALIFIERS);
+ testEncodeDecode("&Perry", Separator.QUALIFIERS, Separator.VALUES, null);
+ testEncodeDecode("the ", Separator.QUALIFIERS, Separator.SPACE);
+ testEncodeDecode("Platypus...", (Separator) null);
+ testEncodeDecode("The what now ?!?", Separator.QUALIFIERS,
+ Separator.VALUES, Separator.SPACE);
+
+ }
+ @Test
+ public void testEncodedValues() {
+ testEncodeDecode("Double-escape %2$ and %9$ or %%2$ or %%3$, nor %%%2$" +
+ "= no problem!",
+ Separator.QUALIFIERS, Separator.VALUES, Separator.SPACE, Separator.TAB);
+ }
+
+ @Test
+ public void testSplits() {
+ byte[] maxLongBytes = Bytes.toBytes(Long.MAX_VALUE);
+ byte[] maxIntBytes = Bytes.toBytes(Integer.MAX_VALUE);
+ for (Separator separator : Separator.values()) {
+ String str1 = "cl" + separator.getValue() + "us";
+ String str2 = separator.getValue() + "rst";
+ byte[] sepByteArr = Bytes.toBytes(separator.getValue());
+ byte[] longVal1Arr = Bytes.add(sepByteArr, Bytes.copy(maxLongBytes,
+ sepByteArr.length, Bytes.SIZEOF_LONG - sepByteArr.length));
+ byte[] intVal1Arr = Bytes.add(sepByteArr, Bytes.copy(maxIntBytes,
+ sepByteArr.length, Bytes.SIZEOF_INT - sepByteArr.length));
+ byte[] arr = separator.join(
+ Bytes.toBytes(separator.encode(str1)), longVal1Arr,
+ Bytes.toBytes(separator.encode(str2)), intVal1Arr);
+ int[] sizes = {Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
+ Separator.VARIABLE_SIZE, Bytes.SIZEOF_INT};
+ byte[][] splits = separator.split(arr, sizes);
+ assertEquals(4, splits.length);
+ assertEquals(str1, separator.decode(Bytes.toString(splits[0])));
+ assertEquals(Bytes.toLong(longVal1Arr), Bytes.toLong(splits[1]));
+ assertEquals(str2, separator.decode(Bytes.toString(splits[2])));
+ assertEquals(Bytes.toInt(intVal1Arr), Bytes.toInt(splits[3]));
+
+ longVal1Arr = Bytes.add(Bytes.copy(maxLongBytes, 0, Bytes.SIZEOF_LONG -
+ sepByteArr.length), sepByteArr);
+ intVal1Arr = Bytes.add(Bytes.copy(maxIntBytes, 0, Bytes.SIZEOF_INT -
+ sepByteArr.length), sepByteArr);
+ arr = separator.join(Bytes.toBytes(separator.encode(str1)), longVal1Arr,
+ Bytes.toBytes(separator.encode(str2)), intVal1Arr);
+ splits = separator.split(arr, sizes);
+ assertEquals(4, splits.length);
+ assertEquals(str1, separator.decode(Bytes.toString(splits[0])));
+ assertEquals(Bytes.toLong(longVal1Arr), Bytes.toLong(splits[1]));
+ assertEquals(str2, separator.decode(Bytes.toString(splits[2])));
+ assertEquals(Bytes.toInt(intVal1Arr), Bytes.toInt(splits[3]));
+
+ longVal1Arr = Bytes.add(sepByteArr, Bytes.copy(maxLongBytes,
+ sepByteArr.length, 4 - sepByteArr.length), sepByteArr);
+ longVal1Arr = Bytes.add(longVal1Arr, Bytes.copy(maxLongBytes, 4, 3 -
+ sepByteArr.length), sepByteArr);
+ arr = separator.join(Bytes.toBytes(separator.encode(str1)), longVal1Arr,
+ Bytes.toBytes(separator.encode(str2)), intVal1Arr);
+ splits = separator.split(arr, sizes);
+ assertEquals(4, splits.length);
+ assertEquals(str1, separator.decode(Bytes.toString(splits[0])));
+ assertEquals(Bytes.toLong(longVal1Arr), Bytes.toLong(splits[1]));
+ assertEquals(str2, separator.decode(Bytes.toString(splits[2])));
+ assertEquals(Bytes.toInt(intVal1Arr), Bytes.toInt(splits[3]));
+
+ arr = separator.join(Bytes.toBytes(separator.encode(str1)),
+ Bytes.toBytes(separator.encode(str2)), intVal1Arr, longVal1Arr);
+ int[] sizes1 = {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
+ Bytes.SIZEOF_INT, Bytes.SIZEOF_LONG};
+ splits = separator.split(arr, sizes1);
+ assertEquals(4, splits.length);
+ assertEquals(str1, separator.decode(Bytes.toString(splits[0])));
+ assertEquals(str2, separator.decode(Bytes.toString(splits[1])));
+ assertEquals(Bytes.toInt(intVal1Arr), Bytes.toInt(splits[2]));
+ assertEquals(Bytes.toLong(longVal1Arr), Bytes.toLong(splits[3]));
+
+ try {
+ int[] sizes2 = {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
+ Bytes.SIZEOF_INT, 7};
+ splits = separator.split(arr, sizes2);
+ fail("Exception should have been thrown.");
+ } catch (IllegalArgumentException e) {}
+
+ try {
+ int[] sizes2 = {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE, 2,
+ Bytes.SIZEOF_LONG};
+ splits = separator.split(arr, sizes2);
+ fail("Exception should have been thrown.");
+ } catch (IllegalArgumentException e) {}
+ }
+ }
+
+ /**
+ * Simple test to encode and decode using the same separators and confirm that
+ * we end up with the same as what we started with.
+ *
+ * @param token
+ * @param separators
+ */
+ private static void testEncodeDecode(String token, Separator... separators) {
+ byte[] encoded = Separator.encode(token, separators);
+ String decoded = Separator.decode(encoded, separators);
+ assertEquals(token, decoded);
+ }
+
+ @Test
+ public void testJoinStripped() {
+ List<String> stringList = new ArrayList<String>(0);
+ stringList.add("nothing");
+
+ String joined = Separator.VALUES.joinEncoded(stringList);
+ Iterable<String> split = Separator.VALUES.splitEncoded(joined);
+ assertTrue(Iterables.elementsEqual(stringList, split));
+
+ stringList = new ArrayList<String>(3);
+ stringList.add("a");
+ stringList.add("b?");
+ stringList.add("c");
+
+ joined = Separator.VALUES.joinEncoded(stringList);
+ split = Separator.VALUES.splitEncoded(joined);
+ assertTrue(Iterables.elementsEqual(stringList, split));
+
+ String[] stringArray1 = {"else"};
+ joined = Separator.VALUES.joinEncoded(stringArray1);
+ split = Separator.VALUES.splitEncoded(joined);
+ assertTrue(Iterables.elementsEqual(Arrays.asList(stringArray1), split));
+
+ String[] stringArray2 = {"d", "e?", "f"};
+ joined = Separator.VALUES.joinEncoded(stringArray2);
+ split = Separator.VALUES.splitEncoded(joined);
+ assertTrue(Iterables.elementsEqual(Arrays.asList(stringArray2), split));
+
+ List<String> empty = new ArrayList<String>(0);
+ split = Separator.VALUES.splitEncoded(null);
+ assertTrue(Iterables.elementsEqual(empty, split));
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/pom.xml
new file mode 100644
index 0000000..d06907d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/pom.xml
@@ -0,0 +1,161 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+ http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>hadoop-yarn-server-timelineservice-hbase</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <version>3.2.0-SNAPSHOT</version>
+ </parent>
+
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hadoop-yarn-server-timelineservice-hbase-server</artifactId>
+ <name>Apache Hadoop YARN TimelineService HBase Server</name>
+ <version>3.2.0-SNAPSHOT</version>
+
+ <properties>
+ <!-- Needed for generating FindBugs warnings using parent pom -->
+ <yarn.basedir>${project.parent.parent.parent.basedir}</yarn.basedir>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-timelineservice-hbase-common</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-sslengine</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptor>src/assembly/coprocessor.xml</descriptor>
+ <attach>true</attach>
+ </configuration>
+ <executions>
+ <execution>
+ <id>create-coprocessor-jar</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/assembly/coprocessor.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/assembly/coprocessor.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/assembly/coprocessor.xml
new file mode 100644
index 0000000..01ff0dd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/assembly/coprocessor.xml
@@ -0,0 +1,37 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.01
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
+ <id>coprocessor</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>/</outputDirectory>
+ <useProjectArtifact>true</useProjectArtifact>
+ <unpack>true</unpack>
+ <scope>runtime</scope>
+ <includes>
+ <include>org.apache.hadoop:hadoop-yarn-server-timelineservice-hbase-common</include>
+ <include>org.apache.hadoop:hadoop-yarn-server-timelineservice-hbase-server</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+</assembly>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineServerUtils.java
new file mode 100644
index 0000000..5c07670
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineServerUtils.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.Tag;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationCompactionDimension;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationOperation;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A utility class used by hbase-server module.
+ */
+public final class HBaseTimelineServerUtils {
+ private HBaseTimelineServerUtils() {
+ }
+
+ /**
+ * Creates a {@link Tag} from the input attribute.
+ *
+ * @param attribute Attribute from which tag has to be fetched.
+ * @return a HBase Tag.
+ */
+ public static Tag getTagFromAttribute(Map.Entry<String, byte[]> attribute) {
+ // attribute could be either an Aggregation Operation or
+ // an Aggregation Dimension
+ // Get the Tag type from either
+ AggregationOperation aggOp = AggregationOperation
+ .getAggregationOperation(attribute.getKey());
+ if (aggOp != null) {
+ Tag t = new Tag(aggOp.getTagType(), attribute.getValue());
+ return t;
+ }
+
+ AggregationCompactionDimension aggCompactDim =
+ AggregationCompactionDimension.getAggregationCompactionDimension(
+ attribute.getKey());
+ if (aggCompactDim != null) {
+ Tag t = new Tag(aggCompactDim.getTagType(), attribute.getValue());
+ return t;
+ }
+ return null;
+ }
+
+ /**
+ * creates a new cell based on the input cell but with the new value.
+ *
+ * @param origCell Original cell
+ * @param newValue new cell value
+ * @return cell
+ * @throws IOException while creating new cell.
+ */
+ public static Cell createNewCell(Cell origCell, byte[] newValue)
+ throws IOException {
+ return CellUtil.createCell(CellUtil.cloneRow(origCell),
+ CellUtil.cloneFamily(origCell), CellUtil.cloneQualifier(origCell),
+ origCell.getTimestamp(), KeyValue.Type.Put.getCode(), newValue);
+ }
+
+ /**
+ * creates a cell with the given inputs.
+ *
+ * @param row row of the cell to be created
+ * @param family column family name of the new cell
+ * @param qualifier qualifier for the new cell
+ * @param ts timestamp of the new cell
+ * @param newValue value of the new cell
+ * @param tags tags in the new cell
+ * @return cell
+ * @throws IOException while creating the cell.
+ */
+ public static Cell createNewCell(byte[] row, byte[] family, byte[] qualifier,
+ long ts, byte[] newValue, byte[] tags) throws IOException {
+ return CellUtil.createCell(row, family, qualifier, ts, KeyValue.Type.Put,
+ newValue, tags);
+ }
+
+ /**
+ * returns app id from the list of tags.
+ *
+ * @param tags cell tags to be looked into
+ * @return App Id as the AggregationCompactionDimension
+ */
+ public static String getAggregationCompactionDimension(List<Tag> tags) {
+ String appId = null;
+ for (Tag t : tags) {
+ if (AggregationCompactionDimension.APPLICATION_ID.getTagType() == t
+ .getType()) {
+ appId = Bytes.toString(t.getValue());
+ return appId;
+ }
+ }
+ return appId;
+ }
+
+ /**
+ * Returns the first seen aggregation operation as seen in the list of input
+ * tags or null otherwise.
+ *
+ * @param tags list of HBase tags.
+ * @return AggregationOperation
+ */
+ public static AggregationOperation getAggregationOperationFromTagsList(
+ List<Tag> tags) {
+ for (AggregationOperation aggOp : AggregationOperation.values()) {
+ for (Tag tag : tags) {
+ if (tag.getType() == aggOp.getTagType()) {
+ return aggOp;
+ }
+ }
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java
new file mode 100644
index 0000000..0df5b8a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.storage.common contains
+ * a set of utility classes used across backend storage reader and writer.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/18] hadoop git commit: YARN-7919. Refactor timelineservice-hbase
module into submodules. Contributed by Haibo Chen.
Posted by ro...@apache.org.
YARN-7919. Refactor timelineservice-hbase module into submodules. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9af30d46
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9af30d46
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9af30d46
Branch: refs/heads/trunk
Commit: 9af30d46c6e82332a8eda20cb3eb5f987e25e7a2
Parents: a1e56a6
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Sat Feb 17 20:30:28 2018 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Sat Feb 17 20:30:28 2018 +0530
----------------------------------------------------------------------
.../resources/assemblies/hadoop-yarn-dist.xml | 22 +-
hadoop-project/pom.xml | 14 +-
.../pom.xml | 26 +-
...stTimelineReaderWebServicesHBaseStorage.java | 8 +-
.../storage/TestHBaseTimelineStorageApps.java | 50 +-
.../TestHBaseTimelineStorageEntities.java | 89 +--
.../storage/TestHBaseTimelineStorageSchema.java | 61 +-
.../flow/TestHBaseStorageFlowActivity.java | 24 +-
.../storage/flow/TestHBaseStorageFlowRun.java | 42 +-
.../flow/TestHBaseStorageFlowRunCompaction.java | 58 +-
.../pom.xml | 219 ++++++
.../reader/filter/TimelineFilterUtils.java | 313 ++++++++
.../reader/filter/package-info.java | 28 +
.../storage/HBaseTimelineReaderImpl.java | 96 +++
.../storage/HBaseTimelineWriterImpl.java | 611 ++++++++++++++++
.../storage/TimelineSchemaCreator.java | 368 ++++++++++
.../storage/application/ApplicationTableRW.java | 137 ++++
.../storage/application/package-info.java | 28 +
.../storage/apptoflow/AppToFlowTableRW.java | 92 +++
.../storage/apptoflow/package-info.java | 28 +
.../storage/common/BaseTableRW.java | 167 +++++
.../storage/common/ColumnRWHelper.java | 487 +++++++++++++
.../common/HBaseTimelineStorageUtils.java | 121 +++
.../common/TimelineHBaseSchemaConstants.java | 71 ++
.../storage/common/TypedBufferedMutator.java | 73 ++
.../storage/common/package-info.java | 28 +
.../storage/entity/EntityTableRW.java | 136 ++++
.../storage/entity/package-info.java | 28 +
.../storage/flow/FlowActivityTableRW.java | 91 +++
.../storage/flow/FlowRunTableRW.java | 102 +++
.../storage/flow/package-info.java | 29 +
.../timelineservice/storage/package-info.java | 28 +
.../reader/AbstractTimelineStorageReader.java | 159 ++++
.../storage/reader/ApplicationEntityReader.java | 523 +++++++++++++
.../storage/reader/EntityTypeReader.java | 175 +++++
.../reader/FlowActivityEntityReader.java | 186 +++++
.../storage/reader/FlowRunEntityReader.java | 298 ++++++++
.../storage/reader/GenericEntityReader.java | 655 +++++++++++++++++
.../reader/SubApplicationEntityReader.java | 489 +++++++++++++
.../storage/reader/TimelineEntityReader.java | 464 ++++++++++++
.../reader/TimelineEntityReaderFactory.java | 105 +++
.../storage/reader/package-info.java | 28 +
.../subapplication/SubApplicationTableRW.java | 137 ++++
.../storage/subapplication/package-info.java | 28 +
.../common/TestHBaseTimelineStorageUtils.java | 33 +
.../pom.xml | 132 ++++
.../storage/application/ApplicationColumn.java | 101 +++
.../application/ApplicationColumnFamily.java | 65 ++
.../application/ApplicationColumnPrefix.java | 150 ++++
.../storage/application/ApplicationRowKey.java | 251 +++++++
.../application/ApplicationRowKeyPrefix.java | 69 ++
.../storage/application/ApplicationTable.java | 60 ++
.../storage/application/package-info.java | 28 +
.../storage/apptoflow/AppToFlowColumn.java | 95 +++
.../apptoflow/AppToFlowColumnFamily.java | 51 ++
.../apptoflow/AppToFlowColumnPrefix.java | 105 +++
.../storage/apptoflow/AppToFlowRowKey.java | 58 ++
.../storage/apptoflow/AppToFlowTable.java | 60 ++
.../storage/apptoflow/package-info.java | 28 +
.../storage/common/AppIdKeyConverter.java | 97 +++
.../storage/common/BaseTable.java | 27 +
.../timelineservice/storage/common/Column.java | 56 ++
.../storage/common/ColumnFamily.java | 34 +
.../storage/common/ColumnHelper.java | 101 +++
.../storage/common/ColumnPrefix.java | 71 ++
.../storage/common/EventColumnName.java | 63 ++
.../common/EventColumnNameConverter.java | 99 +++
.../storage/common/GenericConverter.java | 48 ++
.../common/HBaseTimelineSchemaUtils.java | 156 ++++
.../storage/common/KeyConverter.java | 41 ++
.../storage/common/KeyConverterToString.java | 38 +
.../storage/common/LongConverter.java | 94 +++
.../storage/common/LongKeyConverter.java | 68 ++
.../storage/common/NumericValueConverter.java | 39 +
.../timelineservice/storage/common/Range.java | 62 ++
.../storage/common/RowKeyPrefix.java | 42 ++
.../storage/common/Separator.java | 575 +++++++++++++++
.../storage/common/StringKeyConverter.java | 54 ++
.../storage/common/TimestampGenerator.java | 116 +++
.../storage/common/ValueConverter.java | 47 ++
.../storage/common/package-info.java | 28 +
.../storage/entity/EntityColumn.java | 105 +++
.../storage/entity/EntityColumnFamily.java | 65 ++
.../storage/entity/EntityColumnPrefix.java | 162 +++++
.../storage/entity/EntityRowKey.java | 299 ++++++++
.../storage/entity/EntityRowKeyPrefix.java | 77 ++
.../storage/entity/EntityTable.java | 61 ++
.../storage/entity/package-info.java | 28 +
.../flow/AggregationCompactionDimension.java | 63 ++
.../storage/flow/AggregationOperation.java | 94 +++
.../timelineservice/storage/flow/Attribute.java | 39 +
.../storage/flow/FlowActivityColumnFamily.java | 55 ++
.../storage/flow/FlowActivityColumnPrefix.java | 133 ++++
.../storage/flow/FlowActivityRowKey.java | 247 +++++++
.../storage/flow/FlowActivityRowKeyPrefix.java | 60 ++
.../storage/flow/FlowActivityTable.java | 45 ++
.../storage/flow/FlowRunColumn.java | 112 +++
.../storage/flow/FlowRunColumnFamily.java | 54 ++
.../storage/flow/FlowRunColumnPrefix.java | 129 ++++
.../storage/flow/FlowRunRowKey.java | 233 ++++++
.../storage/flow/FlowRunRowKeyPrefix.java | 54 ++
.../storage/flow/FlowRunTable.java | 77 ++
.../storage/flow/package-info.java | 29 +
.../timelineservice/storage/package-info.java | 28 +
.../subapplication/SubApplicationColumn.java | 99 +++
.../SubApplicationColumnFamily.java | 68 ++
.../SubApplicationColumnPrefix.java | 163 +++++
.../subapplication/SubApplicationRowKey.java | 290 ++++++++
.../SubApplicationRowKeyPrefix.java | 69 ++
.../subapplication/SubApplicationTable.java | 64 ++
.../storage/subapplication/package-info.java | 28 +
.../TestCustomApplicationIdConversion.java | 39 +
.../storage/common/TestKeyConverters.java | 134 ++++
.../storage/common/TestRowKeys.java | 276 +++++++
.../storage/common/TestRowKeysAsString.java | 144 ++++
.../storage/common/TestSeparator.java | 215 ++++++
.../pom.xml | 161 ++++
.../src/assembly/coprocessor.xml | 37 +
.../common/HBaseTimelineServerUtils.java | 135 ++++
.../storage/common/package-info.java | 28 +
.../storage/flow/FlowRunCoprocessor.java | 277 +++++++
.../storage/flow/FlowScanner.java | 723 ++++++++++++++++++
.../storage/flow/FlowScannerOperation.java | 46 ++
.../storage/flow/package-info.java | 29 +
.../timelineservice/storage/package-info.java | 28 +
.../pom.xml | 193 +----
.../reader/filter/TimelineFilterUtils.java | 308 --------
.../reader/filter/package-info.java | 28 -
.../storage/HBaseTimelineReaderImpl.java | 96 ---
.../storage/HBaseTimelineWriterImpl.java | 593 ---------------
.../storage/TimelineSchemaCreator.java | 367 ----------
.../storage/application/ApplicationColumn.java | 108 ---
.../application/ApplicationColumnFamily.java | 65 --
.../application/ApplicationColumnPrefix.java | 236 ------
.../storage/application/ApplicationRowKey.java | 251 -------
.../application/ApplicationRowKeyPrefix.java | 69 --
.../storage/application/ApplicationTable.java | 170 -----
.../storage/application/package-info.java | 28 -
.../storage/apptoflow/AppToFlowColumn.java | 101 ---
.../apptoflow/AppToFlowColumnFamily.java | 51 --
.../apptoflow/AppToFlowColumnPrefix.java | 206 ------
.../storage/apptoflow/AppToFlowRowKey.java | 58 --
.../storage/apptoflow/AppToFlowTable.java | 125 ----
.../storage/apptoflow/package-info.java | 28 -
.../storage/common/AppIdKeyConverter.java | 97 ---
.../storage/common/BaseTable.java | 167 -----
.../common/BufferedMutatorDelegator.java | 73 --
.../timelineservice/storage/common/Column.java | 80 --
.../storage/common/ColumnFamily.java | 34 -
.../storage/common/ColumnHelper.java | 414 -----------
.../storage/common/ColumnPrefix.java | 145 ----
.../storage/common/EventColumnName.java | 63 --
.../common/EventColumnNameConverter.java | 99 ---
.../storage/common/GenericConverter.java | 48 --
.../common/HBaseTimelineStorageUtils.java | 354 ---------
.../storage/common/KeyConverter.java | 41 --
.../storage/common/KeyConverterToString.java | 38 -
.../storage/common/LongConverter.java | 94 ---
.../storage/common/LongKeyConverter.java | 68 --
.../storage/common/NumericValueConverter.java | 39 -
.../timelineservice/storage/common/Range.java | 62 --
.../storage/common/RowKeyPrefix.java | 42 --
.../storage/common/Separator.java | 575 ---------------
.../storage/common/StringKeyConverter.java | 54 --
.../common/TimelineHBaseSchemaConstants.java | 71 --
.../storage/common/TimestampGenerator.java | 116 ---
.../storage/common/TypedBufferedMutator.java | 28 -
.../storage/common/ValueConverter.java | 47 --
.../storage/common/package-info.java | 28 -
.../storage/entity/EntityColumn.java | 112 ---
.../storage/entity/EntityColumnFamily.java | 65 --
.../storage/entity/EntityColumnPrefix.java | 249 -------
.../storage/entity/EntityRowKey.java | 299 --------
.../storage/entity/EntityRowKeyPrefix.java | 77 --
.../storage/entity/EntityTable.java | 170 -----
.../storage/entity/package-info.java | 28 -
.../flow/AggregationCompactionDimension.java | 63 --
.../storage/flow/AggregationOperation.java | 94 ---
.../timelineservice/storage/flow/Attribute.java | 39 -
.../storage/flow/FlowActivityColumnFamily.java | 55 --
.../storage/flow/FlowActivityColumnPrefix.java | 221 ------
.../storage/flow/FlowActivityRowKey.java | 247 -------
.../storage/flow/FlowActivityRowKeyPrefix.java | 60 --
.../storage/flow/FlowActivityTable.java | 109 ---
.../storage/flow/FlowRunColumn.java | 131 ----
.../storage/flow/FlowRunColumnFamily.java | 54 --
.../storage/flow/FlowRunColumnPrefix.java | 217 ------
.../storage/flow/FlowRunCoprocessor.java | 277 -------
.../storage/flow/FlowRunRowKey.java | 233 ------
.../storage/flow/FlowRunRowKeyPrefix.java | 54 --
.../storage/flow/FlowRunTable.java | 151 ----
.../storage/flow/FlowScanner.java | 729 -------------------
.../storage/flow/FlowScannerOperation.java | 46 --
.../storage/flow/package-info.java | 29 -
.../timelineservice/storage/package-info.java | 28 -
.../reader/AbstractTimelineStorageReader.java | 158 ----
.../storage/reader/ApplicationEntityReader.java | 520 -------------
.../storage/reader/EntityTypeReader.java | 175 -----
.../reader/FlowActivityEntityReader.java | 185 -----
.../storage/reader/FlowRunEntityReader.java | 294 --------
.../storage/reader/GenericEntityReader.java | 651 -----------------
.../reader/SubApplicationEntityReader.java | 488 -------------
.../storage/reader/TimelineEntityReader.java | 459 ------------
.../reader/TimelineEntityReaderFactory.java | 105 ---
.../storage/reader/package-info.java | 28 -
.../subapplication/SubApplicationColumn.java | 108 ---
.../SubApplicationColumnFamily.java | 68 --
.../SubApplicationColumnPrefix.java | 250 -------
.../subapplication/SubApplicationRowKey.java | 290 --------
.../SubApplicationRowKeyPrefix.java | 69 --
.../subapplication/SubApplicationTable.java | 174 -----
.../storage/subapplication/package-info.java | 28 -
.../TestCustomApplicationIdConversion.java | 39 -
.../common/TestHBaseTimelineStorageUtils.java | 33 -
.../storage/common/TestKeyConverters.java | 134 ----
.../storage/common/TestRowKeys.java | 276 -------
.../storage/common/TestRowKeysAsString.java | 144 ----
.../storage/common/TestSeparator.java | 215 ------
218 files changed, 15320 insertions(+), 14832 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
index a77dd20..2c266b6 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -213,7 +213,11 @@
</fileSet>
<!-- Copy dependecies from hadoop-yarn-server-timelineservice as well -->
<fileSet>
- <directory>hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/target/lib</directory>
+ <directory>hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/target/lib</directory>
+ <outputDirectory>share/hadoop/${hadoop.component}/timelineservice/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/target/lib</directory>
<outputDirectory>share/hadoop/${hadoop.component}/timelineservice/lib</outputDirectory>
</fileSet>
</fileSets>
@@ -221,12 +225,26 @@
<moduleSet>
<includes>
<include>org.apache.hadoop:hadoop-yarn-server-timelineservice</include>
- <include>org.apache.hadoop:hadoop-yarn-server-timelineservice-hbase</include>
+ <include>org.apache.hadoop:hadoop-yarn-server-timelineservice-hbase-client</include>
+ <include>org.apache.hadoop:hadoop-yarn-server-timelineservice-hbase-common</include>
+ </includes>
+ <binaries>
+ <outputDirectory>share/hadoop/${hadoop.component}/timelineservice</outputDirectory>
+ <includeDependencies>false</includeDependencies>
+ <unpack>false</unpack>
+ </binaries>
+ </moduleSet>
+ <moduleSet>
+ <includes>
+ <include>org.apache.hadoop:hadoop-yarn-server-timelineservice-hbase-server</include>
</includes>
<binaries>
<outputDirectory>share/hadoop/${hadoop.component}/timelineservice</outputDirectory>
<includeDependencies>false</includeDependencies>
+ <!-- This is the id of the timelineservice-hbase-coprocessor assembly descriptor -->
+ <attachmentClassifier>coprocessor</attachmentClassifier>
<unpack>false</unpack>
+ <outputFileNameMapping>hadoop-yarn-server-timelineservice-hbase-coprocessor-${module.version}.${module.extension}</outputFileNameMapping>
</binaries>
</moduleSet>
<moduleSet>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index c27596c..ce51c99 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -397,7 +397,19 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-timelineservice-hbase</artifactId>
+ <artifactId>hadoop-yarn-server-timelineservice-hbase-client</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-timelineservice-hbase-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-timelineservice-hbase-server</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
index f36897b..d9f992d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
@@ -60,7 +60,31 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-timelineservice-hbase</artifactId>
+ <artifactId>hadoop-yarn-server-timelineservice-hbase-client</artifactId>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-timelineservice-hbase-common</artifactId>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-timelineservice-hbase-server</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
index b2029ca..33d8dcd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
@@ -50,7 +50,7 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineSchemaUtils;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -70,7 +70,7 @@ public class TestTimelineReaderWebServicesHBaseStorage
extends AbstractTimelineReaderHBaseTestBase {
private static long ts = System.currentTimeMillis();
private static long dayTs =
- HBaseTimelineStorageUtils.getTopOfTheDayTimestamp(ts);
+ HBaseTimelineSchemaUtils.getTopOfTheDayTimestamp(ts);
private static String doAsUser = "remoteuser";
@BeforeClass
@@ -371,7 +371,7 @@ public class TestTimelineReaderWebServicesHBaseStorage
BuilderUtils.newApplicationId(timestamp, count++);
ApplicationEntity appEntity = new ApplicationEntity();
appEntity.setId(
- HBaseTimelineStorageUtils.convertApplicationIdToString(appId));
+ HBaseTimelineSchemaUtils.convertApplicationIdToString(appId));
appEntity.setCreatedTime(timestamp);
TimelineEvent created = new TimelineEvent();
@@ -929,7 +929,7 @@ public class TestTimelineReaderWebServicesHBaseStorage
new String[] {"flow1"});
long firstFlowActivity =
- HBaseTimelineStorageUtils.getTopOfTheDayTimestamp(1425016501000L);
+ HBaseTimelineSchemaUtils.getTopOfTheDayTimestamp(1425016501000L);
DateFormat fmt = TimelineReaderWebServices.DATE_FORMAT.get();
uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
index 111008a..bc33427 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
@@ -68,10 +68,11 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Fiel
import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn;
import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTableRW;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnRWHelper;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.EventColumnName;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.EventColumnNameConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineSchemaUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.StringKeyConverter;
@@ -173,7 +174,7 @@ public class TestHBaseTimelineStorageApps {
scan.setStartRow(Bytes.toBytes(cluster));
scan.setStopRow(Bytes.toBytes(cluster + "1"));
Connection conn = ConnectionFactory.createConnection(c1);
- ResultScanner resultScanner = new ApplicationTable()
+ ResultScanner resultScanner = new ApplicationTableRW()
.getResultScanner(c1, conn, scan);
assertTrue(resultScanner != null);
@@ -308,7 +309,7 @@ public class TestHBaseTimelineStorageApps {
Get get = new Get(rowKey);
get.setMaxVersions(Integer.MAX_VALUE);
Connection conn = ConnectionFactory.createConnection(c1);
- Result result = new ApplicationTable().getResult(c1, conn, get);
+ Result result = new ApplicationTableRW().getResult(c1, conn, get);
assertTrue(result != null);
assertEquals(17, result.size());
@@ -319,24 +320,24 @@ public class TestHBaseTimelineStorageApps {
appId));
// check info column family
- String id1 = ApplicationColumn.ID.readResult(result).toString();
+ String id1 =
+ ColumnRWHelper.readResult(result, ApplicationColumn.ID).toString();
assertEquals(appId, id1);
- Long cTime1 =
- (Long) ApplicationColumn.CREATED_TIME.readResult(result);
+ Long cTime1 = (Long)
+ ColumnRWHelper.readResult(result, ApplicationColumn.CREATED_TIME);
assertEquals(cTime, cTime1);
- Map<String, Object> infoColumns =
- ApplicationColumnPrefix.INFO.readResults(result,
- new StringKeyConverter());
+ Map<String, Object> infoColumns = ColumnRWHelper.readResults(
+ result, ApplicationColumnPrefix.INFO, new StringKeyConverter());
assertEquals(infoMap, infoColumns);
// Remember isRelatedTo is of type Map<String, Set<String>>
for (Map.Entry<String, Set<String>> isRelatedToEntry : isRelatedTo
.entrySet()) {
- Object isRelatedToValue =
- ApplicationColumnPrefix.IS_RELATED_TO.readResult(result,
- isRelatedToEntry.getKey());
+ Object isRelatedToValue = ColumnRWHelper.readResult(
+ result, ApplicationColumnPrefix.IS_RELATED_TO,
+ isRelatedToEntry.getKey());
String compoundValue = isRelatedToValue.toString();
// id7?id9?id6
Set<String> isRelatedToValues =
@@ -351,9 +352,9 @@ public class TestHBaseTimelineStorageApps {
// RelatesTo
for (Map.Entry<String, Set<String>> relatesToEntry : relatesTo
.entrySet()) {
- String compoundValue =
- ApplicationColumnPrefix.RELATES_TO.readResult(result,
- relatesToEntry.getKey()).toString();
+ String compoundValue = ColumnRWHelper.readResult(result,
+ ApplicationColumnPrefix.RELATES_TO, relatesToEntry.getKey())
+ .toString();
// id3?id4?id5
Set<String> relatesToValues =
new HashSet<String>(Separator.VALUES.splitEncoded(compoundValue));
@@ -366,14 +367,13 @@ public class TestHBaseTimelineStorageApps {
KeyConverter<String> stringKeyConverter = new StringKeyConverter();
// Configuration
- Map<String, Object> configColumns =
- ApplicationColumnPrefix.CONFIG
- .readResults(result, stringKeyConverter);
+ Map<String, Object> configColumns = ColumnRWHelper.readResults(
+ result, ApplicationColumnPrefix.CONFIG, stringKeyConverter);
assertEquals(conf, configColumns);
NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
- ApplicationColumnPrefix.METRIC.readResultsWithTimestamps(result,
- stringKeyConverter);
+ ColumnRWHelper.readResultsWithTimestamps(
+ result, ApplicationColumnPrefix.METRIC, stringKeyConverter);
NavigableMap<Long, Number> metricMap = metricsResult.get(m1.getId());
matchMetrics(metricValues, metricMap);
@@ -500,7 +500,7 @@ public class TestHBaseTimelineStorageApps {
event.addInfo(expKey, expVal);
final TimelineEntity entity = new ApplicationEntity();
- entity.setId(HBaseTimelineStorageUtils.convertApplicationIdToString(
+ entity.setId(HBaseTimelineSchemaUtils.convertApplicationIdToString(
ApplicationId.newInstance(0, 1)));
entity.addEvent(event);
@@ -531,7 +531,7 @@ public class TestHBaseTimelineStorageApps {
Get get = new Get(rowKey);
get.setMaxVersions(Integer.MAX_VALUE);
Connection conn = ConnectionFactory.createConnection(c1);
- Result result = new ApplicationTable().getResult(c1, conn, get);
+ Result result = new ApplicationTableRW().getResult(c1, conn, get);
assertTrue(result != null);
@@ -541,8 +541,8 @@ public class TestHBaseTimelineStorageApps {
appName));
Map<EventColumnName, Object> eventsResult =
- ApplicationColumnPrefix.EVENT.readResults(result,
- new EventColumnNameConverter());
+ ColumnRWHelper.readResults(result,
+ ApplicationColumnPrefix.EVENT, new EventColumnNameConverter());
// there should be only one event
assertEquals(1, eventsResult.size());
for (Map.Entry<EventColumnName, Object> e : eventsResult.entrySet()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java
index 5e08999..90a6959 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java
@@ -62,9 +62,10 @@ import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyVa
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValuesFilter;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelinePrefixFilter;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnRWHelper;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.EventColumnName;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.EventColumnNameConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineSchemaUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.StringKeyConverter;
@@ -73,12 +74,12 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKeyPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTableRW;
import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationColumn;
import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationColumnPrefix;
import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationRowKey;
import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationRowKeyPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationTableRW;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -208,7 +209,7 @@ public class TestHBaseTimelineStorageEntities {
String flow = "some_flow_name";
String flowVersion = "AB7822C10F1111";
long runid = 1002345678919L;
- String appName = HBaseTimelineStorageUtils.convertApplicationIdToString(
+ String appName = HBaseTimelineSchemaUtils.convertApplicationIdToString(
ApplicationId.newInstance(System.currentTimeMillis() + 9000000L, 1)
);
hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion,
@@ -224,7 +225,7 @@ public class TestHBaseTimelineStorageEntities {
s.setStartRow(startRow);
s.setMaxVersions(Integer.MAX_VALUE);
Connection conn = ConnectionFactory.createConnection(c1);
- ResultScanner scanner = new EntityTable().getResultScanner(c1, conn, s);
+ ResultScanner scanner = new EntityTableRW().getResultScanner(c1, conn, s);
int rowCount = 0;
int colCount = 0;
@@ -238,26 +239,27 @@ public class TestHBaseTimelineStorageEntities {
entity));
// check info column family
- String id1 = EntityColumn.ID.readResult(result).toString();
+ String id1 =
+ ColumnRWHelper.readResult(result, EntityColumn.ID).toString();
assertEquals(id, id1);
- String type1 = EntityColumn.TYPE.readResult(result).toString();
+ String type1 =
+ ColumnRWHelper.readResult(result, EntityColumn.TYPE).toString();
assertEquals(type, type1);
- Long cTime1 = (Long) EntityColumn.CREATED_TIME.readResult(result);
+ Long cTime1 = (Long)
+ ColumnRWHelper.readResult(result, EntityColumn.CREATED_TIME);
assertEquals(cTime1, cTime);
- Map<String, Object> infoColumns =
- EntityColumnPrefix.INFO.readResults(result,
- new StringKeyConverter());
+ Map<String, Object> infoColumns = ColumnRWHelper.readResults(
+ result, EntityColumnPrefix.INFO, new StringKeyConverter());
assertEquals(infoMap, infoColumns);
// Remember isRelatedTo is of type Map<String, Set<String>>
for (Map.Entry<String, Set<String>> isRelatedToEntry : isRelatedTo
.entrySet()) {
- Object isRelatedToValue =
- EntityColumnPrefix.IS_RELATED_TO.readResult(result,
- isRelatedToEntry.getKey());
+ Object isRelatedToValue = ColumnRWHelper.readResult(result,
+ EntityColumnPrefix.IS_RELATED_TO, isRelatedToEntry.getKey());
String compoundValue = isRelatedToValue.toString();
// id7?id9?id6
Set<String> isRelatedToValues =
@@ -273,8 +275,9 @@ public class TestHBaseTimelineStorageEntities {
// RelatesTo
for (Map.Entry<String, Set<String>> relatesToEntry : relatesTo
.entrySet()) {
- String compoundValue = EntityColumnPrefix.RELATES_TO
- .readResult(result, relatesToEntry.getKey()).toString();
+ String compoundValue = ColumnRWHelper.readResult(result,
+ EntityColumnPrefix.RELATES_TO, relatesToEntry.getKey())
+ .toString();
// id3?id4?id5
Set<String> relatesToValues =
new HashSet<String>(
@@ -287,13 +290,13 @@ public class TestHBaseTimelineStorageEntities {
}
// Configuration
- Map<String, Object> configColumns =
- EntityColumnPrefix.CONFIG.readResults(result, stringKeyConverter);
+ Map<String, Object> configColumns = ColumnRWHelper.readResults(
+ result, EntityColumnPrefix.CONFIG, stringKeyConverter);
assertEquals(conf, configColumns);
NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
- EntityColumnPrefix.METRIC.readResultsWithTimestamps(result,
- stringKeyConverter);
+ ColumnRWHelper.readResultsWithTimestamps(
+ result, EntityColumnPrefix.METRIC, stringKeyConverter);
NavigableMap<Long, Number> metricMap = metricsResult.get(m1.getId());
matchMetrics(metricValues, metricMap);
@@ -386,14 +389,14 @@ public class TestHBaseTimelineStorageEntities {
Set<TimelineMetric> metrics, Long cTime, TimelineMetric m1)
throws IOException {
Scan s = new Scan();
- // read from SubApplicationTable
+ // read from SubApplicationTableRW
byte[] startRow = new SubApplicationRowKeyPrefix(cluster, subAppUser, null,
null, null, null).getRowKeyPrefix();
s.setStartRow(startRow);
s.setMaxVersions(Integer.MAX_VALUE);
Connection conn = ConnectionFactory.createConnection(c1);
ResultScanner scanner =
- new SubApplicationTable().getResultScanner(c1, conn, s);
+ new SubApplicationTableRW().getResultScanner(c1, conn, s);
int rowCount = 0;
int colCount = 0;
@@ -407,25 +410,28 @@ public class TestHBaseTimelineStorageEntities {
user, entity));
// check info column family
- String id1 = SubApplicationColumn.ID.readResult(result).toString();
+ String id1 = ColumnRWHelper.readResult(result, SubApplicationColumn.ID)
+ .toString();
assertEquals(id, id1);
- String type1 = SubApplicationColumn.TYPE.readResult(result).toString();
+ String type1 = ColumnRWHelper.readResult(result,
+ SubApplicationColumn.TYPE).toString();
assertEquals(type, type1);
- Long cTime1 =
- (Long) SubApplicationColumn.CREATED_TIME.readResult(result);
+ Long cTime1 = (Long) ColumnRWHelper.readResult(result,
+ SubApplicationColumn.CREATED_TIME);
assertEquals(cTime1, cTime);
- Map<String, Object> infoColumns = SubApplicationColumnPrefix.INFO
- .readResults(result, new StringKeyConverter());
+ Map<String, Object> infoColumns = ColumnRWHelper.readResults(
+ result, SubApplicationColumnPrefix.INFO, new StringKeyConverter());
assertEquals(infoMap, infoColumns);
// Remember isRelatedTo is of type Map<String, Set<String>>
for (Map.Entry<String, Set<String>> isRelatedToEntry : isRelatedTo
.entrySet()) {
- Object isRelatedToValue = SubApplicationColumnPrefix.IS_RELATED_TO
- .readResult(result, isRelatedToEntry.getKey());
+ Object isRelatedToValue = ColumnRWHelper.readResult(
+ result, SubApplicationColumnPrefix.IS_RELATED_TO,
+ isRelatedToEntry.getKey());
String compoundValue = isRelatedToValue.toString();
// id7?id9?id6
Set<String> isRelatedToValues =
@@ -440,8 +446,9 @@ public class TestHBaseTimelineStorageEntities {
// RelatesTo
for (Map.Entry<String, Set<String>> relatesToEntry : relatesTo
.entrySet()) {
- String compoundValue = SubApplicationColumnPrefix.RELATES_TO
- .readResult(result, relatesToEntry.getKey()).toString();
+ String compoundValue = ColumnRWHelper.readResult(result,
+ SubApplicationColumnPrefix.RELATES_TO, relatesToEntry.getKey())
+ .toString();
// id3?id4?id5
Set<String> relatesToValues =
new HashSet<String>(Separator.VALUES.splitEncoded(compoundValue));
@@ -453,13 +460,13 @@ public class TestHBaseTimelineStorageEntities {
}
// Configuration
- Map<String, Object> configColumns = SubApplicationColumnPrefix.CONFIG
- .readResults(result, stringKeyConverter);
+ Map<String, Object> configColumns = ColumnRWHelper.readResults(
+ result, SubApplicationColumnPrefix.CONFIG, stringKeyConverter);
assertEquals(conf, configColumns);
NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
- SubApplicationColumnPrefix.METRIC.readResultsWithTimestamps(result,
- stringKeyConverter);
+ ColumnRWHelper.readResultsWithTimestamps(
+ result, SubApplicationColumnPrefix.METRIC, stringKeyConverter);
NavigableMap<Long, Number> metricMap = metricsResult.get(m1.getId());
matchMetrics(metricValues, metricMap);
@@ -511,7 +518,7 @@ public class TestHBaseTimelineStorageEntities {
String flow = "other_flow_name";
String flowVersion = "1111F01C2287BA";
long runid = 1009876543218L;
- String appName = HBaseTimelineStorageUtils.convertApplicationIdToString(
+ String appName = HBaseTimelineSchemaUtils.convertApplicationIdToString(
ApplicationId.newInstance(System.currentTimeMillis() + 9000000L, 1));
byte[] startRow =
new EntityRowKeyPrefix(cluster, user, flow, runid, appName)
@@ -525,7 +532,7 @@ public class TestHBaseTimelineStorageEntities {
s.setStartRow(startRow);
s.addFamily(EntityColumnFamily.INFO.getBytes());
Connection conn = ConnectionFactory.createConnection(c1);
- ResultScanner scanner = new EntityTable().getResultScanner(c1, conn, s);
+ ResultScanner scanner = new EntityTableRW().getResultScanner(c1, conn, s);
int rowCount = 0;
for (Result result : scanner) {
@@ -538,8 +545,8 @@ public class TestHBaseTimelineStorageEntities {
entity));
Map<EventColumnName, Object> eventsResult =
- EntityColumnPrefix.EVENT.readResults(result,
- new EventColumnNameConverter());
+ ColumnRWHelper.readResults(result,
+ EntityColumnPrefix.EVENT, new EventColumnNameConverter());
// there should be only one event
assertEquals(1, eventsResult.size());
for (Map.Entry<EventColumnName, Object> e : eventsResult.entrySet()) {
@@ -604,7 +611,7 @@ public class TestHBaseTimelineStorageEntities {
final TimelineEntity entity = new ApplicationEntity();
entity.setId(
- HBaseTimelineStorageUtils.convertApplicationIdToString(
+ HBaseTimelineSchemaUtils.convertApplicationIdToString(
ApplicationId.newInstance(0, 1)));
entity.addEvent(event);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageSchema.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageSchema.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageSchema.java
index 0dcd171..f838178 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageSchema.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageSchema.java
@@ -21,6 +21,9 @@ package org.apache.hadoop.yarn.server.timelineservice.storage;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTableRW;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTableRW;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTableRW;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -35,10 +38,6 @@ import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTable;
-
/**
* Unit tests for checking different schema prefixes.
*/
@@ -61,22 +60,24 @@ public class TestHBaseTimelineStorageSchema {
conn = ConnectionFactory.createConnection(hbaseConf);
Admin admin = conn.getAdmin();
- TableName entityTableName = BaseTable.getTableName(hbaseConf,
- EntityTable.TABLE_NAME_CONF_NAME, EntityTable.DEFAULT_TABLE_NAME);
+ TableName entityTableName = BaseTableRW.getTableName(hbaseConf,
+ EntityTableRW.TABLE_NAME_CONF_NAME, EntityTableRW.DEFAULT_TABLE_NAME);
assertTrue(admin.tableExists(entityTableName));
assertTrue(entityTableName.getNameAsString().startsWith(
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX));
- Table entityTable = conn.getTable(BaseTable.getTableName(hbaseConf,
- EntityTable.TABLE_NAME_CONF_NAME, EntityTable.DEFAULT_TABLE_NAME));
+ Table entityTable = conn.getTable(BaseTableRW.getTableName(hbaseConf,
+ EntityTableRW.TABLE_NAME_CONF_NAME, EntityTableRW.DEFAULT_TABLE_NAME));
assertNotNull(entityTable);
- TableName flowRunTableName = BaseTable.getTableName(hbaseConf,
- FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME);
+ TableName flowRunTableName = BaseTableRW.getTableName(hbaseConf,
+ FlowRunTableRW.TABLE_NAME_CONF_NAME, FlowRunTableRW.DEFAULT_TABLE_NAME);
assertTrue(admin.tableExists(flowRunTableName));
assertTrue(flowRunTableName.getNameAsString().startsWith(
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX));
- Table flowRunTable = conn.getTable(BaseTable.getTableName(hbaseConf,
- FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME));
+ Table flowRunTable = conn.getTable(
+ BaseTableRW.getTableName(hbaseConf,
+ FlowRunTableRW.TABLE_NAME_CONF_NAME,
+ FlowRunTableRW.DEFAULT_TABLE_NAME));
assertNotNull(flowRunTable);
}
@@ -91,20 +92,22 @@ public class TestHBaseTimelineStorageSchema {
conn = ConnectionFactory.createConnection(hbaseConf);
Admin admin = conn.getAdmin();
- TableName entityTableName = BaseTable.getTableName(hbaseConf,
- EntityTable.TABLE_NAME_CONF_NAME, EntityTable.DEFAULT_TABLE_NAME);
+ TableName entityTableName = BaseTableRW.getTableName(hbaseConf,
+ EntityTableRW.TABLE_NAME_CONF_NAME, EntityTableRW.DEFAULT_TABLE_NAME);
assertTrue(admin.tableExists(entityTableName));
assertTrue(entityTableName.getNameAsString().startsWith(prefix));
- Table entityTable = conn.getTable(BaseTable.getTableName(hbaseConf,
- EntityTable.TABLE_NAME_CONF_NAME, EntityTable.DEFAULT_TABLE_NAME));
+ Table entityTable = conn.getTable(BaseTableRW.getTableName(hbaseConf,
+ EntityTableRW.TABLE_NAME_CONF_NAME, EntityTableRW.DEFAULT_TABLE_NAME));
assertNotNull(entityTable);
- TableName flowRunTableName = BaseTable.getTableName(hbaseConf,
- FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME);
+ TableName flowRunTableName = BaseTableRW.getTableName(hbaseConf,
+ FlowRunTableRW.TABLE_NAME_CONF_NAME, FlowRunTableRW.DEFAULT_TABLE_NAME);
assertTrue(admin.tableExists(flowRunTableName));
assertTrue(flowRunTableName.getNameAsString().startsWith(prefix));
- Table flowRunTable = conn.getTable(BaseTable.getTableName(hbaseConf,
- FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME));
+ Table flowRunTable = conn.getTable(
+ BaseTableRW.getTableName(hbaseConf,
+ FlowRunTableRW.TABLE_NAME_CONF_NAME,
+ FlowRunTableRW.DEFAULT_TABLE_NAME));
assertNotNull(flowRunTable);
// create another set with a diff prefix
@@ -114,20 +117,22 @@ public class TestHBaseTimelineStorageSchema {
hbaseConf.set(YarnConfiguration.TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX_NAME,
prefix);
DataGeneratorForTest.createSchema(hbaseConf);
- entityTableName = BaseTable.getTableName(hbaseConf,
- EntityTable.TABLE_NAME_CONF_NAME, EntityTable.DEFAULT_TABLE_NAME);
+ entityTableName = BaseTableRW.getTableName(hbaseConf,
+ EntityTableRW.TABLE_NAME_CONF_NAME, EntityTableRW.DEFAULT_TABLE_NAME);
assertTrue(admin.tableExists(entityTableName));
assertTrue(entityTableName.getNameAsString().startsWith(prefix));
- entityTable = conn.getTable(BaseTable.getTableName(hbaseConf,
- EntityTable.TABLE_NAME_CONF_NAME, EntityTable.DEFAULT_TABLE_NAME));
+ entityTable = conn.getTable(BaseTableRW.getTableName(hbaseConf,
+ EntityTableRW.TABLE_NAME_CONF_NAME, EntityTableRW.DEFAULT_TABLE_NAME));
assertNotNull(entityTable);
- flowRunTableName = BaseTable.getTableName(hbaseConf,
- FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME);
+ flowRunTableName = BaseTableRW.getTableName(hbaseConf,
+ FlowRunTableRW.TABLE_NAME_CONF_NAME, FlowRunTableRW.DEFAULT_TABLE_NAME);
assertTrue(admin.tableExists(flowRunTableName));
assertTrue(flowRunTableName.getNameAsString().startsWith(prefix));
- flowRunTable = conn.getTable(BaseTable.getTableName(hbaseConf,
- FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME));
+ flowRunTable = conn.getTable(
+ BaseTableRW.getTableName(hbaseConf,
+ FlowRunTableRW.TABLE_NAME_CONF_NAME,
+ FlowRunTableRW.DEFAULT_TABLE_NAME));
assertNotNull(flowRunTable);
hbaseConf
.unset(YarnConfiguration.TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX_NAME);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
index 4bf221e..645b7d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
@@ -52,9 +52,9 @@ import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContex
import org.apache.hadoop.yarn.server.timelineservice.storage.DataGeneratorForTest;
import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl;
import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTableRW;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineSchemaUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -161,8 +161,8 @@ public class TestHBaseStorageFlowActivity {
Connection conn = ConnectionFactory.createConnection(c1);
// check in flow activity table
Table table1 = conn.getTable(
- BaseTable.getTableName(c1, FlowActivityTable.TABLE_NAME_CONF_NAME,
- FlowActivityTable.DEFAULT_TABLE_NAME));
+ BaseTableRW.getTableName(c1, FlowActivityTableRW.TABLE_NAME_CONF_NAME,
+ FlowActivityTableRW.DEFAULT_TABLE_NAME));
byte[] startRow =
new FlowActivityRowKey(cluster, minStartTs, user, flow).getRowKey();
Get g = new Get(startRow);
@@ -178,7 +178,7 @@ public class TestHBaseStorageFlowActivity {
assertEquals(cluster, flowActivityRowKey.getClusterId());
assertEquals(user, flowActivityRowKey.getUserId());
assertEquals(flow, flowActivityRowKey.getFlowName());
- Long dayTs = HBaseTimelineStorageUtils.getTopOfTheDayTimestamp(minStartTs);
+ Long dayTs = HBaseTimelineSchemaUtils.getTopOfTheDayTimestamp(minStartTs);
assertEquals(dayTs, flowActivityRowKey.getDayTimestamp());
assertEquals(1, values.size());
checkFlowActivityRunId(runid, flowVersion, values);
@@ -292,8 +292,8 @@ public class TestHBaseStorageFlowActivity {
s.setStopRow(stopRow);
Connection conn = ConnectionFactory.createConnection(c1);
Table table1 = conn.getTable(
- BaseTable.getTableName(c1, FlowActivityTable.TABLE_NAME_CONF_NAME,
- FlowActivityTable.DEFAULT_TABLE_NAME));
+ BaseTableRW.getTableName(c1, FlowActivityTableRW.TABLE_NAME_CONF_NAME,
+ FlowActivityTableRW.DEFAULT_TABLE_NAME));
ResultScanner scanner = table1.getScanner(s);
int rowCount = 0;
for (Result result : scanner) {
@@ -309,7 +309,7 @@ public class TestHBaseStorageFlowActivity {
assertEquals(cluster, flowActivityRowKey.getClusterId());
assertEquals(user, flowActivityRowKey.getUserId());
assertEquals(flow, flowActivityRowKey.getFlowName());
- Long dayTs = HBaseTimelineStorageUtils.getTopOfTheDayTimestamp(
+ Long dayTs = HBaseTimelineSchemaUtils.getTopOfTheDayTimestamp(
appCreatedTime);
assertEquals(dayTs, flowActivityRowKey.getDayTimestamp());
assertEquals(1, values.size());
@@ -401,7 +401,7 @@ public class TestHBaseStorageFlowActivity {
assertEquals(user, flowActivity.getUser());
assertEquals(flow, flowActivity.getFlowName());
long dayTs =
- HBaseTimelineStorageUtils.getTopOfTheDayTimestamp(appCreatedTime);
+ HBaseTimelineSchemaUtils.getTopOfTheDayTimestamp(appCreatedTime);
assertEquals(dayTs, flowActivity.getDate().getTime());
Set<FlowRunEntity> flowRuns = flowActivity.getFlowRuns();
assertEquals(3, flowRuns.size());
@@ -442,8 +442,8 @@ public class TestHBaseStorageFlowActivity {
s.setStopRow(stopRow);
Connection conn = ConnectionFactory.createConnection(c1);
Table table1 = conn.getTable(
- BaseTable.getTableName(c1, FlowActivityTable.TABLE_NAME_CONF_NAME,
- FlowActivityTable.DEFAULT_TABLE_NAME));
+ BaseTableRW.getTableName(c1, FlowActivityTableRW.TABLE_NAME_CONF_NAME,
+ FlowActivityTableRW.DEFAULT_TABLE_NAME));
ResultScanner scanner = table1.getScanner(s);
int rowCount = 0;
for (Result result : scanner) {
@@ -456,7 +456,7 @@ public class TestHBaseStorageFlowActivity {
assertEquals(cluster, flowActivityRowKey.getClusterId());
assertEquals(user, flowActivityRowKey.getUserId());
assertEquals(flow, flowActivityRowKey.getFlowName());
- Long dayTs = HBaseTimelineStorageUtils.getTopOfTheDayTimestamp(
+ Long dayTs = HBaseTimelineSchemaUtils.getTopOfTheDayTimestamp(
appCreatedTime);
assertEquals(dayTs, flowActivityRowKey.getDayTimestamp());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
index 1ad02e1..622b0eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
@@ -62,9 +62,9 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.DataGeneratorForTes
import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl;
import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTableRW;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTableRW;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -91,8 +91,8 @@ public class TestHBaseStorageFlowRun {
@Test
public void checkCoProcessorOff() throws IOException, InterruptedException {
Configuration hbaseConf = util.getConfiguration();
- TableName table = BaseTable.getTableName(hbaseConf,
- FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME);
+ TableName table = BaseTableRW.getTableName(hbaseConf,
+ FlowRunTableRW.TABLE_NAME_CONF_NAME, FlowRunTableRW.DEFAULT_TABLE_NAME);
Connection conn = null;
conn = ConnectionFactory.createConnection(hbaseConf);
Admin admin = conn.getAdmin();
@@ -106,9 +106,9 @@ public class TestHBaseStorageFlowRun {
checkCoprocessorExists(table, true);
}
- table = BaseTable.getTableName(hbaseConf,
- FlowActivityTable.TABLE_NAME_CONF_NAME,
- FlowActivityTable.DEFAULT_TABLE_NAME);
+ table = BaseTableRW.getTableName(hbaseConf,
+ FlowActivityTableRW.TABLE_NAME_CONF_NAME,
+ FlowActivityTableRW.DEFAULT_TABLE_NAME);
if (admin.tableExists(table)) {
// check the regions.
// check in flow activity table
@@ -116,8 +116,8 @@ public class TestHBaseStorageFlowRun {
checkCoprocessorExists(table, false);
}
- table = BaseTable.getTableName(hbaseConf, EntityTable.TABLE_NAME_CONF_NAME,
- EntityTable.DEFAULT_TABLE_NAME);
+ table = BaseTableRW.getTableName(hbaseConf,
+ EntityTableRW.TABLE_NAME_CONF_NAME, EntityTableRW.DEFAULT_TABLE_NAME);
if (admin.tableExists(table)) {
// check the regions.
// check in entity run table
@@ -224,8 +224,10 @@ public class TestHBaseStorageFlowRun {
Connection conn = ConnectionFactory.createConnection(c1);
// check in flow run table
- Table table1 = conn.getTable(BaseTable.getTableName(c1,
- FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME));
+ Table table1 = conn.getTable(
+ BaseTableRW.getTableName(c1,
+ FlowRunTableRW.TABLE_NAME_CONF_NAME,
+ FlowRunTableRW.DEFAULT_TABLE_NAME));
// scan the table and see that we get back the right min and max
// timestamps
byte[] startRow = new FlowRunRowKey(cluster, user, flow, runid).getRowKey();
@@ -380,8 +382,10 @@ public class TestHBaseStorageFlowRun {
.getRowKey();
s.setStopRow(stopRow);
Connection conn = ConnectionFactory.createConnection(c1);
- Table table1 = conn.getTable(BaseTable.getTableName(c1,
- FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME));
+ Table table1 = conn.getTable(
+ BaseTableRW.getTableName(c1,
+ FlowRunTableRW.TABLE_NAME_CONF_NAME,
+ FlowRunTableRW.DEFAULT_TABLE_NAME));
ResultScanner scanner = table1.getScanner(s);
int loopCount = 0;
@@ -525,8 +529,10 @@ public class TestHBaseStorageFlowRun {
new FlowRunRowKey(clusterStop, user, flow, runid).getRowKey();
s.setStopRow(stopRow);
Connection conn = ConnectionFactory.createConnection(c1);
- Table table1 = conn.getTable(BaseTable.getTableName(c1,
- FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME));
+ Table table1 = conn.getTable(
+ BaseTableRW.getTableName(c1,
+ FlowRunTableRW.TABLE_NAME_CONF_NAME,
+ FlowRunTableRW.DEFAULT_TABLE_NAME));
ResultScanner scanner = table1.getScanner(s);
int rowCount = 0;
@@ -810,8 +816,10 @@ public class TestHBaseStorageFlowRun {
boolean checkMax) throws IOException {
Connection conn = ConnectionFactory.createConnection(c1);
// check in flow run table
- Table table1 = conn.getTable(BaseTable.getTableName(c1,
- FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME));
+ Table table1 = conn.getTable(
+ BaseTableRW.getTableName(c1,
+ FlowRunTableRW.TABLE_NAME_CONF_NAME,
+ FlowRunTableRW.DEFAULT_TABLE_NAME));
// scan the table and see that we get back the right min and max
// timestamps
byte[] startRow = new FlowRunRowKey(cluster, user, flow, runid).getRowKey();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
index 0ef8260..31be285 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
@@ -54,9 +54,9 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
import org.apache.hadoop.yarn.server.timelineservice.storage.DataGeneratorForTest;
import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTableRW;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineServerUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimestampGenerator;
import org.junit.AfterClass;
@@ -107,8 +107,10 @@ public class TestHBaseStorageFlowRunCompaction {
Configuration hbaseConf = util.getConfiguration();
Connection conn = null;
conn = ConnectionFactory.createConnection(hbaseConf);
- Table flowRunTable = conn.getTable(BaseTable.getTableName(hbaseConf,
- FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME));
+ Table flowRunTable = conn.getTable(
+ BaseTableRW.getTableName(hbaseConf,
+ FlowRunTableRW.TABLE_NAME_CONF_NAME,
+ FlowRunTableRW.DEFAULT_TABLE_NAME));
flowRunTable.put(p);
Get g = new Get(rowKeyBytes);
@@ -156,8 +158,10 @@ public class TestHBaseStorageFlowRunCompaction {
Configuration hbaseConf = util.getConfiguration();
Connection conn = null;
conn = ConnectionFactory.createConnection(hbaseConf);
- Table flowRunTable = conn.getTable(BaseTable.getTableName(hbaseConf,
- FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME));
+ Table flowRunTable = conn.getTable(
+ BaseTableRW.getTableName(hbaseConf,
+ FlowRunTableRW.TABLE_NAME_CONF_NAME,
+ FlowRunTableRW.DEFAULT_TABLE_NAME));
flowRunTable.put(p);
String rowKey2 = "nonNumericRowKey2";
@@ -324,10 +328,12 @@ public class TestHBaseStorageFlowRunCompaction {
// check in flow run table
HRegionServer server = util.getRSForFirstRegionInTable(
- BaseTable.getTableName(c1, FlowRunTable.TABLE_NAME_CONF_NAME,
- FlowRunTable.DEFAULT_TABLE_NAME));
- List<Region> regions = server.getOnlineRegions(BaseTable.getTableName(c1,
- FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME));
+ BaseTableRW.getTableName(c1, FlowRunTableRW.TABLE_NAME_CONF_NAME,
+ FlowRunTableRW.DEFAULT_TABLE_NAME));
+ List<Region> regions = server.getOnlineRegions(
+ BaseTableRW.getTableName(c1,
+ FlowRunTableRW.TABLE_NAME_CONF_NAME,
+ FlowRunTableRW.DEFAULT_TABLE_NAME));
assertTrue("Didn't find any regions for primary table!",
regions.size() > 0);
// flush and compact all the regions of the primary table
@@ -352,8 +358,10 @@ public class TestHBaseStorageFlowRunCompaction {
new FlowRunRowKey(clusterStop, user, flow, runid).getRowKey();
s.setStopRow(stopRow);
Connection conn = ConnectionFactory.createConnection(c1);
- Table table1 = conn.getTable(BaseTable.getTableName(c1,
- FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME));
+ Table table1 = conn.getTable(
+ BaseTableRW.getTableName(c1,
+ FlowRunTableRW.TABLE_NAME_CONF_NAME,
+ FlowRunTableRW.DEFAULT_TABLE_NAME));
ResultScanner scanner = table1.getScanner(s);
int rowCount = 0;
@@ -420,7 +428,7 @@ public class TestHBaseStorageFlowRunCompaction {
tags.add(t);
byte[] tagByteArray = Tag.fromList(tags);
// create a cell with a VERY old timestamp and attribute SUM_FINAL
- Cell c1 = HBaseTimelineStorageUtils.createNewCell(aRowKey, aFamily,
+ Cell c1 = HBaseTimelineServerUtils.createNewCell(aRowKey, aFamily,
aQualifier, cell1Ts, Bytes.toBytes(cellValue1), tagByteArray);
currentColumnCells.add(c1);
@@ -430,7 +438,7 @@ public class TestHBaseStorageFlowRunCompaction {
tags.add(t);
tagByteArray = Tag.fromList(tags);
// create a cell with a recent timestamp and attribute SUM_FINAL
- Cell c2 = HBaseTimelineStorageUtils.createNewCell(aRowKey, aFamily,
+ Cell c2 = HBaseTimelineServerUtils.createNewCell(aRowKey, aFamily,
aQualifier, cell2Ts, Bytes.toBytes(cellValue2), tagByteArray);
currentColumnCells.add(c2);
@@ -440,7 +448,7 @@ public class TestHBaseStorageFlowRunCompaction {
tags.add(t);
tagByteArray = Tag.fromList(tags);
// create a cell with a VERY old timestamp but has attribute SUM
- Cell c3 = HBaseTimelineStorageUtils.createNewCell(aRowKey, aFamily,
+ Cell c3 = HBaseTimelineServerUtils.createNewCell(aRowKey, aFamily,
aQualifier, cell3Ts, Bytes.toBytes(cellValue3), tagByteArray);
currentColumnCells.add(c3);
@@ -450,7 +458,7 @@ public class TestHBaseStorageFlowRunCompaction {
tags.add(t);
tagByteArray = Tag.fromList(tags);
// create a cell with a VERY old timestamp but has attribute SUM
- Cell c4 = HBaseTimelineStorageUtils.createNewCell(aRowKey, aFamily,
+ Cell c4 = HBaseTimelineServerUtils.createNewCell(aRowKey, aFamily,
aQualifier, cell4Ts, Bytes.toBytes(cellValue4), tagByteArray);
currentColumnCells.add(c4);
@@ -520,7 +528,7 @@ public class TestHBaseStorageFlowRunCompaction {
tags.add(t);
byte[] tagByteArray = Tag.fromList(tags);
// create a cell with a VERY old timestamp and attribute SUM_FINAL
- c1 = HBaseTimelineStorageUtils.createNewCell(aRowKey, aFamily, aQualifier,
+ c1 = HBaseTimelineServerUtils.createNewCell(aRowKey, aFamily, aQualifier,
cellTsFinal, Bytes.toBytes(cellValueFinal), tagByteArray);
currentColumnCells.add(c1);
cellTsFinal++;
@@ -534,7 +542,7 @@ public class TestHBaseStorageFlowRunCompaction {
tags.add(t);
byte[] tagByteArray = Tag.fromList(tags);
// create a cell with attribute SUM
- c1 = HBaseTimelineStorageUtils.createNewCell(aRowKey, aFamily, aQualifier,
+ c1 = HBaseTimelineServerUtils.createNewCell(aRowKey, aFamily, aQualifier,
cellTsNotFinal, Bytes.toBytes(cellValueNotFinal), tagByteArray);
currentColumnCells.add(c1);
cellTsNotFinal++;
@@ -611,7 +619,7 @@ public class TestHBaseStorageFlowRunCompaction {
tags.add(t);
byte[] tagByteArray = Tag.fromList(tags);
// create a cell with a VERY old timestamp and attribute SUM_FINAL
- c1 = HBaseTimelineStorageUtils.createNewCell(aRowKey, aFamily, aQualifier,
+ c1 = HBaseTimelineServerUtils.createNewCell(aRowKey, aFamily, aQualifier,
cellTsFinal, Bytes.toBytes(cellValueFinal), tagByteArray);
currentColumnCells.add(c1);
cellTsFinal++;
@@ -625,7 +633,7 @@ public class TestHBaseStorageFlowRunCompaction {
tags.add(t);
byte[] tagByteArray = Tag.fromList(tags);
// create a cell with a VERY old timestamp and attribute SUM_FINAL
- c1 = HBaseTimelineStorageUtils.createNewCell(aRowKey, aFamily, aQualifier,
+ c1 = HBaseTimelineServerUtils.createNewCell(aRowKey, aFamily, aQualifier,
cellTsFinalNotExpire, Bytes.toBytes(cellValueFinal), tagByteArray);
currentColumnCells.add(c1);
cellTsFinalNotExpire++;
@@ -639,7 +647,7 @@ public class TestHBaseStorageFlowRunCompaction {
tags.add(t);
byte[] tagByteArray = Tag.fromList(tags);
// create a cell with attribute SUM
- c1 = HBaseTimelineStorageUtils.createNewCell(aRowKey, aFamily, aQualifier,
+ c1 = HBaseTimelineServerUtils.createNewCell(aRowKey, aFamily, aQualifier,
cellTsNotFinal, Bytes.toBytes(cellValueNotFinal), tagByteArray);
currentColumnCells.add(c1);
cellTsNotFinal++;
@@ -696,7 +704,7 @@ public class TestHBaseStorageFlowRunCompaction {
SortedSet<Cell> currentColumnCells = new TreeSet<Cell>(KeyValue.COMPARATOR);
// create a cell with a VERY old timestamp and attribute SUM_FINAL
- Cell c1 = HBaseTimelineStorageUtils.createNewCell(aRowKey, aFamily,
+ Cell c1 = HBaseTimelineServerUtils.createNewCell(aRowKey, aFamily,
aQualifier, 120L, Bytes.toBytes(cellValue1), tagByteArray);
currentColumnCells.add(c1);
@@ -707,7 +715,7 @@ public class TestHBaseStorageFlowRunCompaction {
tagByteArray = Tag.fromList(tags);
// create a cell with a VERY old timestamp but has attribute SUM
- Cell c2 = HBaseTimelineStorageUtils.createNewCell(aRowKey, aFamily,
+ Cell c2 = HBaseTimelineServerUtils.createNewCell(aRowKey, aFamily,
aQualifier, 130L, Bytes.toBytes(cellValue2), tagByteArray);
currentColumnCells.add(c2);
List<Cell> cells = fs.processSummationMajorCompaction(currentColumnCells,
@@ -754,7 +762,7 @@ public class TestHBaseStorageFlowRunCompaction {
SortedSet<Cell> currentColumnCells = new TreeSet<Cell>(KeyValue.COMPARATOR);
// create a cell with a VERY old timestamp
- Cell c1 = HBaseTimelineStorageUtils.createNewCell(aRowKey, aFamily,
+ Cell c1 = HBaseTimelineServerUtils.createNewCell(aRowKey, aFamily,
aQualifier, 120L, Bytes.toBytes(1110L), tagByteArray);
currentColumnCells.add(c1);
@@ -792,7 +800,7 @@ public class TestHBaseStorageFlowRunCompaction {
SortedSet<Cell> currentColumnCells = new TreeSet<Cell>(KeyValue.COMPARATOR);
- Cell c1 = HBaseTimelineStorageUtils.createNewCell(aRowKey, aFamily,
+ Cell c1 = HBaseTimelineServerUtils.createNewCell(aRowKey, aFamily,
aQualifier, currentTimestamp, Bytes.toBytes(1110L), tagByteArray);
currentColumnCells.add(c1);
List<Cell> cells = fs.processSummationMajorCompaction(currentColumnCells,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml
new file mode 100644
index 0000000..a1db497
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml
@@ -0,0 +1,219 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+ http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>hadoop-yarn-server-timelineservice-hbase</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <version>3.2.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hadoop-yarn-server-timelineservice-hbase-client</artifactId>
+ <name>Apache Hadoop YARN TimelineService HBase Client</name>
+
+ <properties>
+ <!-- Needed for generating FindBugs warnings using parent pom -->
+ <yarn.basedir>${project.parent.parent.parent.basedir}</yarn.basedir>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-timelineservice-hbase-common</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-applicationhistoryservice</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-timelineservice</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-sslengine</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ <phase>test-compile</phase>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <configuration>
+ <additionalDependencies>
+ <additionnalDependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.11</version>
+ </additionnalDependency>
+ </additionalDependencies>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <includeScope>runtime</includeScope>
+ <excludeGroupIds>org.slf4j,org.apache.hadoop,com.github.stephenc.findbugs</excludeGroupIds>
+ <outputDirectory>${project.build.directory}/lib</outputDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/18] hadoop git commit: YARN-7919. Refactor timelineservice-hbase
module into submodules. Contributed by Haibo Chen.
Posted by ro...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnFamily.java
deleted file mode 100644
index f3f045e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnFamily.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow;
-
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-
-/**
- * Represents the app_flow table column families.
- */
-public enum AppToFlowColumnFamily implements ColumnFamily<AppToFlowTable> {
- /**
- * Mapping column family houses known columns such as flowName and flowRunId.
- */
- MAPPING("m");
-
- /**
- * Byte representation of this column family.
- */
- private final byte[] bytes;
-
- /**
- * @param value create a column family with this name. Must be lower case and
- * without spaces.
- */
- AppToFlowColumnFamily(String value) {
- // column families should be lower case and not contain any spaces.
- this.bytes = Bytes.toBytes(Separator.SPACE.encode(value));
- }
-
- public byte[] getBytes() {
- return Bytes.copy(bytes);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnPrefix.java
deleted file mode 100644
index 752a380..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnPrefix.java
+++ /dev/null
@@ -1,206 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.NavigableMap;
-
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
-
-/**
- * Identifies partially qualified columns for the app-to-flow table.
- */
-public enum AppToFlowColumnPrefix implements ColumnPrefix<AppToFlowTable> {
-
- /**
- * The flow name.
- */
- FLOW_NAME(AppToFlowColumnFamily.MAPPING, "flow_name"),
-
- /**
- * The flow run ID.
- */
- FLOW_RUN_ID(AppToFlowColumnFamily.MAPPING, "flow_run_id"),
-
- /**
- * The user.
- */
- USER_ID(AppToFlowColumnFamily.MAPPING, "user_id");
-
- private final ColumnHelper<AppToFlowTable> column;
- private final ColumnFamily<AppToFlowTable> columnFamily;
- private final String columnPrefix;
- private final byte[] columnPrefixBytes;
-
- AppToFlowColumnPrefix(ColumnFamily<AppToFlowTable> columnFamily,
- String columnPrefix) {
- this.columnFamily = columnFamily;
- this.columnPrefix = columnPrefix;
- if (columnPrefix == null) {
- this.columnPrefixBytes = null;
- } else {
- // Future-proof by ensuring the right column prefix hygiene.
- this.columnPrefixBytes =
- Bytes.toBytes(Separator.SPACE.encode(columnPrefix));
- }
- this.column = new ColumnHelper<AppToFlowTable>(columnFamily);
- }
-
- @Override
- public byte[] getColumnPrefixBytes(String qualifierPrefix) {
- return ColumnHelper.getColumnQualifier(
- columnPrefixBytes, qualifierPrefix);
- }
-
- @Override
- public byte[] getColumnPrefixBytes(byte[] qualifierPrefix) {
- return ColumnHelper.getColumnQualifier(
- columnPrefixBytes, qualifierPrefix);
- }
-
- @Override
- public byte[] getColumnFamilyBytes() {
- return columnFamily.getBytes();
- }
-
- @Override
- public void store(byte[] rowKey,
- TypedBufferedMutator<AppToFlowTable> tableMutator, byte[] qualifier,
- Long timestamp, Object inputValue, Attribute... attributes)
- throws IOException {
-
- // Null check
- if (qualifier == null) {
- throw new IOException("Cannot store column with null qualifier in "
- + tableMutator.getName().getNameAsString());
- }
-
- byte[] columnQualifier = getColumnPrefixBytes(qualifier);
-
- column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue,
- attributes);
- }
-
- @Override
- public void store(byte[] rowKey,
- TypedBufferedMutator<AppToFlowTable> tableMutator, String qualifier,
- Long timestamp, Object inputValue, Attribute... attributes)
- throws IOException {
-
- // Null check
- if (qualifier == null) {
- throw new IOException("Cannot store column with null qualifier in "
- + tableMutator.getName().getNameAsString());
- }
-
- byte[] columnQualifier = getColumnPrefixBytes(qualifier);
-
- column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue,
- attributes);
- }
-
- @Override
- public ValueConverter getValueConverter() {
- return column.getValueConverter();
- }
-
- @Override
- public Object readResult(Result result, String qualifier) throws IOException {
- byte[] columnQualifier =
- ColumnHelper.getColumnQualifier(columnPrefixBytes, qualifier);
- return column.readResult(result, columnQualifier);
- }
-
- @Override
- public <K> Map<K, Object> readResults(Result result,
- KeyConverter<K> keyConverter)
- throws IOException {
- return column.readResults(result, columnPrefixBytes, keyConverter);
- }
-
- @Override
- public <K, V> NavigableMap<K, NavigableMap<Long, V>>
- readResultsWithTimestamps(Result result,
- KeyConverter<K> keyConverter) throws IOException {
- return column.readResultsWithTimestamps(result, columnPrefixBytes,
- keyConverter);
- }
-
- /**
- * Retrieve an {@link AppToFlowColumnPrefix} given a name, or null if there
- * is no match. The following holds true: {@code columnFor(x) == columnFor(y)}
- * if and only if {@code x.equals(y)} or {@code (x == y == null)}
- *
- * @param columnPrefix Name of the column to retrieve
- * @return the corresponding {@link AppToFlowColumnPrefix} or null
- */
- public static final AppToFlowColumnPrefix columnFor(String columnPrefix) {
-
- // Match column based on value, assume column family matches.
- for (AppToFlowColumnPrefix afcp : AppToFlowColumnPrefix.values()) {
- // Find a match based only on name.
- if (afcp.columnPrefix.equals(columnPrefix)) {
- return afcp;
- }
- }
-
- // Default to null
- return null;
- }
-
- /**
- * Retrieve an {@link AppToFlowColumnPrefix} given a name, or null if there
- * is no match. The following holds true:
- * {@code columnFor(a,x) == columnFor(b,y)} if and only if
- * {@code (x == y == null)} or {@code a.equals(b) & x.equals(y)}
- *
- * @param columnFamily The columnFamily for which to retrieve the column.
- * @param columnPrefix Name of the column to retrieve
- * @return the corresponding {@link AppToFlowColumnPrefix} or null if both
- * arguments don't match.
- */
- public static final AppToFlowColumnPrefix columnFor(
- AppToFlowColumnFamily columnFamily, String columnPrefix) {
-
- // TODO: needs unit test to confirm and need to update javadoc to explain
- // null prefix case.
-
- for (AppToFlowColumnPrefix afcp : AppToFlowColumnPrefix.values()) {
- // Find a match based column family and on name.
- if (afcp.columnFamily.equals(columnFamily)
- && (((columnPrefix == null) && (afcp.columnPrefix == null)) ||
- (afcp.columnPrefix.equals(columnPrefix)))) {
- return afcp;
- }
- }
-
- // Default to null
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java
deleted file mode 100644
index 146c475..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow;
-
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.AppIdKeyConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
-
-/**
- * Represents a row key for the app_flow table, which is the app id.
- */
-public class AppToFlowRowKey {
- private final String appId;
- private final KeyConverter<String> appIdKeyConverter =
- new AppIdKeyConverter();
-
- public AppToFlowRowKey(String appId) {
- this.appId = appId;
- }
-
- public String getAppId() {
- return appId;
- }
-
- /**
- * Constructs a row key prefix for the app_flow table.
- *
- * @return byte array with the row key
- */
- public byte[] getRowKey() {
- return appIdKeyConverter.encode(appId);
- }
-
- /**
- * Given the raw row key as bytes, returns the row key as an object.
- *
- * @param rowKey a rowkey represented as a byte array.
- * @return an <cite>AppToFlowRowKey</cite> object.
- */
- public static AppToFlowRowKey parseRowKey(byte[] rowKey) {
- String appId = new AppIdKeyConverter().decode(rowKey);
- return new AppToFlowRowKey(appId);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java
deleted file mode 100644
index 04da5c7..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow;
-
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.regionserver.BloomType;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBaseSchemaConstants;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-/**
- * The app_flow table as column families mapping. Mapping stores
- * appId to flowName and flowRunId mapping information
- *
- * Example app_flow table record:
- *
- * <pre>
- * |--------------------------------------|
- * | Row | Column Family |
- * | key | mapping |
- * |--------------------------------------|
- * | appId | flow_name!cluster1: |
- * | | foo@daily_hive_report |
- * | | |
- * | | flow_run_id!cluster1: |
- * | | 1452828720457 |
- * | | |
- * | | user_id!cluster1: |
- * | | admin |
- * | | |
- * | | flow_name!cluster2: |
- * | | bar@ad_hoc_query |
- * | | |
- * | | flow_run_id!cluster2: |
- * | | 1452828498752 |
- * | | |
- * | | user_id!cluster2: |
- * | | joe |
- * | | |
- * |--------------------------------------|
- * </pre>
- *
- * It is possible (although unlikely) in a multi-cluster environment that there
- * may be more than one applications for a given app id. Different clusters are
- * recorded as different sets of columns.
- */
-public class AppToFlowTable extends BaseTable<AppToFlowTable> {
- /** app_flow prefix. */
- private static final String PREFIX =
- YarnConfiguration.TIMELINE_SERVICE_PREFIX + "app-flow";
-
- /** config param name that specifies the app_flow table name. */
- public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
-
- /** default value for app_flow table name. */
- private static final String DEFAULT_TABLE_NAME = "timelineservice.app_flow";
-
- private static final Logger LOG =
- LoggerFactory.getLogger(AppToFlowTable.class);
-
- public AppToFlowTable() {
- super(TABLE_NAME_CONF_NAME, DEFAULT_TABLE_NAME);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.BaseTable#createTable
- * (org.apache.hadoop.hbase.client.Admin,
- * org.apache.hadoop.conf.Configuration)
- */
- public void createTable(Admin admin, Configuration hbaseConf)
- throws IOException {
-
- TableName table = getTableName(hbaseConf);
- if (admin.tableExists(table)) {
- // do not disable / delete existing table
- // similar to the approach taken by map-reduce jobs when
- // output directory exists
- throw new IOException("Table " + table.getNameAsString()
- + " already exists.");
- }
-
- HTableDescriptor appToFlowTableDescp = new HTableDescriptor(table);
- HColumnDescriptor mappCF =
- new HColumnDescriptor(AppToFlowColumnFamily.MAPPING.getBytes());
- mappCF.setBloomFilterType(BloomType.ROWCOL);
- appToFlowTableDescp.addFamily(mappCF);
-
- appToFlowTableDescp
- .setRegionSplitPolicyClassName(
- "org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy");
- appToFlowTableDescp.setValue("KeyPrefixRegionSplitPolicy.prefix_length",
- TimelineHBaseSchemaConstants.USERNAME_SPLIT_KEY_PREFIX_LENGTH);
- admin.createTable(appToFlowTableDescp,
- TimelineHBaseSchemaConstants.getUsernameSplits());
- LOG.info("Status of table creation for " + table.getNameAsString() + "="
- + admin.tableExists(table));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/package-info.java
deleted file mode 100644
index f01d982..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/package-info.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow
- * contains classes related to implementation for app to flow table.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/AppIdKeyConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/AppIdKeyConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/AppIdKeyConverter.java
deleted file mode 100644
index 51604f0..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/AppIdKeyConverter.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-
-/**
- * Encodes and decodes {@link ApplicationId} for row keys.
- * App ID is stored in row key as 12 bytes, cluster timestamp section of app id
- * (long - 8 bytes) followed by sequence id section of app id (int - 4 bytes).
- */
-public final class AppIdKeyConverter implements KeyConverter<String> {
-
- public AppIdKeyConverter() {
- }
-
- /*
- * (non-Javadoc)
- *
- * Converts/encodes a string app Id into a byte representation for (row) keys.
- * For conversion, we extract cluster timestamp and sequence id from the
- * string app id (calls ConverterUtils#toApplicationId(String) for
- * conversion) and then store it in a byte array of length 12 (8 bytes (long)
- * for cluster timestamp followed 4 bytes(int) for sequence id). Both cluster
- * timestamp and sequence id are inverted so that the most recent cluster
- * timestamp and highest sequence id appears first in the table (i.e.
- * application id appears in a descending order).
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter
- * #encode(java.lang.Object)
- */
- @Override
- public byte[] encode(String appIdStr) {
- ApplicationId appId = ApplicationId.fromString(appIdStr);
- byte[] appIdBytes = new byte[getKeySize()];
- byte[] clusterTs = Bytes.toBytes(
- LongConverter.invertLong(appId.getClusterTimestamp()));
- System.arraycopy(clusterTs, 0, appIdBytes, 0, Bytes.SIZEOF_LONG);
- byte[] seqId = Bytes.toBytes(
- HBaseTimelineStorageUtils.invertInt(appId.getId()));
- System.arraycopy(seqId, 0, appIdBytes, Bytes.SIZEOF_LONG, Bytes.SIZEOF_INT);
- return appIdBytes;
- }
-
- /*
- * (non-Javadoc)
- *
- * Converts/decodes a 12 byte representation of app id for (row) keys to an
- * app id in string format which can be returned back to client.
- * For decoding, 12 bytes are interpreted as 8 bytes of inverted cluster
- * timestamp(long) followed by 4 bytes of inverted sequence id(int). Calls
- * ApplicationId#toString to generate string representation of app id.
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter
- * #decode(byte[])
- */
- @Override
- public String decode(byte[] appIdBytes) {
- if (appIdBytes.length != getKeySize()) {
- throw new IllegalArgumentException("Invalid app id in byte format");
- }
- long clusterTs = LongConverter.invertLong(
- Bytes.toLong(appIdBytes, 0, Bytes.SIZEOF_LONG));
- int seqId = HBaseTimelineStorageUtils.invertInt(
- Bytes.toInt(appIdBytes, Bytes.SIZEOF_LONG, Bytes.SIZEOF_INT));
- return HBaseTimelineStorageUtils.convertApplicationIdToString(
- ApplicationId.newInstance(clusterTs, seqId));
- }
-
- /**
- * Returns the size of app id after encoding.
- *
- * @return size of app id after encoding.
- */
- public static int getKeySize() {
- return Bytes.SIZEOF_LONG + Bytes.SIZEOF_INT;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
deleted file mode 100644
index 93d809c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.BufferedMutator;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-
-/**
- * Implements behavior common to tables used in the timeline service storage. It
- * is thread-safe, and can be used by multiple threads concurrently.
- *
- * @param <T> reference to the table instance class itself for type safety.
- */
-public abstract class BaseTable<T> {
-
- /**
- * Name of config variable that is used to point to this table.
- */
- private final String tableNameConfName;
-
- /**
- * Unless the configuration overrides, this will be the default name for the
- * table when it is created.
- */
- private final String defaultTableName;
-
- /**
- * @param tableNameConfName name of config variable that is used to point to
- * this table.
- * @param defaultTableName Default table name if table from config is not
- * found.
- */
- protected BaseTable(String tableNameConfName, String defaultTableName) {
- this.tableNameConfName = tableNameConfName;
- this.defaultTableName = defaultTableName;
- }
-
- /**
- * Used to create a type-safe mutator for this table.
- *
- * @param hbaseConf used to read table name.
- * @param conn used to create a table from.
- * @return a type safe {@link BufferedMutator} for the entity table.
- * @throws IOException if any exception occurs while creating mutator for the
- * table.
- */
- public TypedBufferedMutator<T> getTableMutator(Configuration hbaseConf,
- Connection conn) throws IOException {
-
- TableName tableName = this.getTableName(hbaseConf);
-
- // Plain buffered mutator
- BufferedMutator bufferedMutator = conn.getBufferedMutator(tableName);
-
- // Now make this thing type safe.
- // This is how service initialization should hang on to this variable, with
- // the proper type
- TypedBufferedMutator<T> table =
- new BufferedMutatorDelegator<T>(bufferedMutator);
-
- return table;
- }
-
- /**
- * @param hbaseConf used to read settings that override defaults
- * @param conn used to create table from
- * @param scan that specifies what you want to read from this table.
- * @return scanner for the table.
- * @throws IOException if any exception occurs while getting the scanner.
- */
- public ResultScanner getResultScanner(Configuration hbaseConf,
- Connection conn, Scan scan) throws IOException {
- Table table = conn.getTable(getTableName(hbaseConf));
- return table.getScanner(scan);
- }
-
- /**
- *
- * @param hbaseConf used to read settings that override defaults
- * @param conn used to create table from
- * @param get that specifies what single row you want to get from this table
- * @return result of get operation
- * @throws IOException if any exception occurs while getting the result.
- */
- public Result getResult(Configuration hbaseConf, Connection conn, Get get)
- throws IOException {
- Table table = conn.getTable(getTableName(hbaseConf));
- return table.get(get);
- }
-
- /**
- * Get the table name for the input table.
- *
- * @param conf HBase configuration from which table name will be fetched.
- * @param tableName name of the table to be fetched
- * @return A {@link TableName} object.
- */
- public static TableName getTableName(Configuration conf, String tableName) {
- String tableSchemaPrefix = conf.get(
- YarnConfiguration.TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX_NAME,
- YarnConfiguration.DEFAULT_TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX);
- return TableName.valueOf(tableSchemaPrefix + tableName);
- }
-
- /**
- * Get the table name for this table.
- *
- * @param conf HBase configuration from which table name will be fetched.
- * @return A {@link TableName} object.
- */
- public TableName getTableName(Configuration conf) {
- String tableName = conf.get(tableNameConfName, defaultTableName);
- return getTableName(conf, tableName);
- }
-
- /**
- * Get the table name based on the input config parameters.
- *
- * @param conf HBase configuration from which table name will be fetched.
- * @param tableNameInConf the table name parameter in conf.
- * @param defaultTableName the default table name.
- * @return A {@link TableName} object.
- */
- public static TableName getTableName(Configuration conf,
- String tableNameInConf, String defaultTableName) {
- String tableName = conf.get(tableNameInConf, defaultTableName);
- return getTableName(conf, tableName);
- }
-
- /**
- * Used to create the table in HBase. Should be called only once (per HBase
- * instance).
- *
- * @param admin Used for doing HBase table operations.
- * @param hbaseConf Hbase configuration.
- * @throws IOException if any exception occurs while creating the table.
- */
- public abstract void createTable(Admin admin, Configuration hbaseConf)
- throws IOException;
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BufferedMutatorDelegator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BufferedMutatorDelegator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BufferedMutatorDelegator.java
deleted file mode 100644
index cf469a5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BufferedMutatorDelegator.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.BufferedMutator;
-import org.apache.hadoop.hbase.client.Mutation;
-
-/**
- * To be used to wrap an actual {@link BufferedMutator} in a type safe manner.
- *
- * @param <T> The class referring to the table to be written to.
- */
-class BufferedMutatorDelegator<T> implements TypedBufferedMutator<T> {
-
- private final BufferedMutator bufferedMutator;
-
- /**
- * @param bufferedMutator the mutator to be wrapped for delegation. Shall not
- * be null.
- */
- public BufferedMutatorDelegator(BufferedMutator bufferedMutator) {
- this.bufferedMutator = bufferedMutator;
- }
-
- public TableName getName() {
- return bufferedMutator.getName();
- }
-
- public Configuration getConfiguration() {
- return bufferedMutator.getConfiguration();
- }
-
- public void mutate(Mutation mutation) throws IOException {
- bufferedMutator.mutate(mutation);
- }
-
- public void mutate(List<? extends Mutation> mutations) throws IOException {
- bufferedMutator.mutate(mutations);
- }
-
- public void close() throws IOException {
- bufferedMutator.close();
- }
-
- public void flush() throws IOException {
- bufferedMutator.flush();
- }
-
- public long getWriteBufferSize() {
- return bufferedMutator.getWriteBufferSize();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java
deleted file mode 100644
index 90f2de4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
-
-/**
- * A Column represents the way to store a fully qualified column in a specific
- * table.
- */
-public interface Column<T> {
-
- /**
- * Sends a Mutation to the table. The mutations will be buffered and sent over
- * the wire as part of a batch.
- *
- * @param rowKey identifying the row to write. Nothing gets written when null.
- * @param tableMutator used to modify the underlying HBase table. Caller is
- * responsible to pass a mutator for the table that actually has this
- * column.
- * @param timestamp version timestamp. When null the server timestamp will be
- * used.
- * @param attributes Map of attributes for this mutation. used in the
- * coprocessor to set/read the cell tags. Can be null.
- * @param inputValue the value to write to the rowKey and column qualifier.
- * Nothing gets written when null.
- * @throws IOException if there is any exception encountered during store.
- */
- void store(byte[] rowKey, TypedBufferedMutator<T> tableMutator,
- Long timestamp, Object inputValue, Attribute... attributes)
- throws IOException;
-
- /**
- * Get the latest version of this specified column. Note: this call clones the
- * value content of the hosting {@link org.apache.hadoop.hbase.Cell Cell}.
- *
- * @param result Cannot be null
- * @return result object (can be cast to whatever object was written to), or
- * null when result doesn't contain this column.
- * @throws IOException if there is any exception encountered while reading
- * result.
- */
- Object readResult(Result result) throws IOException;
-
- /**
- * Returns column family name(as bytes) associated with this column.
- * @return a byte array encoding column family for this column qualifier.
- */
- byte[] getColumnFamilyBytes();
-
- /**
- * Get byte representation for this column qualifier.
- * @return a byte array representing column qualifier.
- */
- byte[] getColumnQualifierBytes();
-
- /**
- * Returns value converter implementation associated with this column.
- * @return a {@link ValueConverter} implementation.
- */
- ValueConverter getValueConverter();
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnFamily.java
deleted file mode 100644
index 452adcd..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnFamily.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-/**
- * Type safe column family.
- *
- * @param <T> refers to the table for which this column family is used for.
- */
-public interface ColumnFamily<T> {
-
- /**
- * Keep a local copy if you need to avoid overhead of repeated cloning.
- *
- * @return a clone of the byte representation of the column family.
- */
- byte[] getBytes();
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
deleted file mode 100644
index 9f95d44..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
+++ /dev/null
@@ -1,414 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.NavigableMap;
-import java.util.TreeMap;
-
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationCompactionDimension;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class is meant to be used only by explicit Columns, and not directly to
- * write by clients.
- *
- * @param <T> refers to the table.
- */
-public class ColumnHelper<T> {
- private static final Logger LOG =
- LoggerFactory.getLogger(ColumnHelper.class);
-
- private final ColumnFamily<T> columnFamily;
-
- /**
- * Local copy of bytes representation of columnFamily so that we can avoid
- * cloning a new copy over and over.
- */
- private final byte[] columnFamilyBytes;
-
- private final ValueConverter converter;
-
- private final boolean supplementTs;
-
- public ColumnHelper(ColumnFamily<T> columnFamily) {
- this(columnFamily, GenericConverter.getInstance());
- }
-
- public ColumnHelper(ColumnFamily<T> columnFamily, ValueConverter converter) {
- this(columnFamily, converter, false);
- }
-
- /**
- * @param columnFamily column family implementation.
- * @param converter converter use to encode/decode values stored in the column
- * or column prefix.
- * @param needSupplementTs flag to indicate if cell timestamp needs to be
- * modified for this column by calling
- * {@link TimestampGenerator#getSupplementedTimestamp(long, String)}. This
- * would be required for columns(such as metrics in flow run table) where
- * potential collisions can occur due to same timestamp.
- */
- public ColumnHelper(ColumnFamily<T> columnFamily, ValueConverter converter,
- boolean needSupplementTs) {
- this.columnFamily = columnFamily;
- columnFamilyBytes = columnFamily.getBytes();
- if (converter == null) {
- this.converter = GenericConverter.getInstance();
- } else {
- this.converter = converter;
- }
- this.supplementTs = needSupplementTs;
- }
-
- /**
- * Sends a Mutation to the table. The mutations will be buffered and sent over
- * the wire as part of a batch.
- *
- * @param rowKey
- * identifying the row to write. Nothing gets written when null.
- * @param tableMutator
- * used to modify the underlying HBase table
- * @param columnQualifier
- * column qualifier. Nothing gets written when null.
- * @param timestamp
- * version timestamp. When null the current timestamp multiplied with
- * TimestampGenerator.TS_MULTIPLIER and added with last 3 digits of
- * app id will be used
- * @param inputValue
- * the value to write to the rowKey and column qualifier. Nothing
- * gets written when null.
- * @param attributes Attributes to be set for HBase Put.
- * @throws IOException if any problem occurs during store operation(sending
- * mutation to table).
- */
- public void store(byte[] rowKey, TypedBufferedMutator<?> tableMutator,
- byte[] columnQualifier, Long timestamp, Object inputValue,
- Attribute... attributes) throws IOException {
- if ((rowKey == null) || (columnQualifier == null) || (inputValue == null)) {
- return;
- }
- Put p = new Put(rowKey);
- timestamp = getPutTimestamp(timestamp, attributes);
- p.addColumn(columnFamilyBytes, columnQualifier, timestamp,
- converter.encodeValue(inputValue));
- if ((attributes != null) && (attributes.length > 0)) {
- for (Attribute attribute : attributes) {
- p.setAttribute(attribute.getName(), attribute.getValue());
- }
- }
- tableMutator.mutate(p);
- }
-
- /*
- * Figures out the cell timestamp used in the Put For storing.
- * Will supplement the timestamp if required. Typically done for flow run
- * table.If we supplement the timestamp, we left shift the timestamp and
- * supplement it with the AppId id so that there are no collisions in the flow
- * run table's cells.
- */
- private long getPutTimestamp(Long timestamp, Attribute[] attributes) {
- if (timestamp == null) {
- timestamp = System.currentTimeMillis();
- }
- if (!this.supplementTs) {
- return timestamp;
- } else {
- String appId = getAppIdFromAttributes(attributes);
- long supplementedTS = TimestampGenerator.getSupplementedTimestamp(
- timestamp, appId);
- return supplementedTS;
- }
- }
-
- private String getAppIdFromAttributes(Attribute[] attributes) {
- if (attributes == null) {
- return null;
- }
- String appId = null;
- for (Attribute attribute : attributes) {
- if (AggregationCompactionDimension.APPLICATION_ID.toString().equals(
- attribute.getName())) {
- appId = Bytes.toString(attribute.getValue());
- }
- }
- return appId;
- }
-
- /**
- * @return the column family for this column implementation.
- */
- public ColumnFamily<T> getColumnFamily() {
- return columnFamily;
- }
-
- /**
- * Get the latest version of this specified column. Note: this call clones the
- * value content of the hosting {@link org.apache.hadoop.hbase.Cell Cell}.
- *
- * @param result from which to read the value. Cannot be null
- * @param columnQualifierBytes referring to the column to be read.
- * @return latest version of the specified column of whichever object was
- * written.
- * @throws IOException if any problem occurs while reading result.
- */
- public Object readResult(Result result, byte[] columnQualifierBytes)
- throws IOException {
- if (result == null || columnQualifierBytes == null) {
- return null;
- }
-
- // Would have preferred to be able to use getValueAsByteBuffer and get a
- // ByteBuffer to avoid copy, but GenericObjectMapper doesn't seem to like
- // that.
- byte[] value = result.getValue(columnFamilyBytes, columnQualifierBytes);
- return converter.decodeValue(value);
- }
-
- /**
- * @param result from which to reads data with timestamps
- * @param columnPrefixBytes optional prefix to limit columns. If null all
- * columns are returned.
- * @param <K> identifies the type of column name(indicated by type of key
- * converter).
- * @param <V> the type of the values. The values will be cast into that type.
- * @param keyConverter used to convert column bytes to the appropriate key
- * type.
- * @return the cell values at each respective time in for form
- * {@literal {idA={timestamp1->value1}, idA={timestamp2->value2},
- * idB={timestamp3->value3}, idC={timestamp1->value4}}}
- * @throws IOException if any problem occurs while reading results.
- */
- @SuppressWarnings("unchecked")
- public <K, V> NavigableMap<K, NavigableMap<Long, V>>
- readResultsWithTimestamps(Result result, byte[] columnPrefixBytes,
- KeyConverter<K> keyConverter) throws IOException {
-
- NavigableMap<K, NavigableMap<Long, V>> results = new TreeMap<>();
-
- if (result != null) {
- NavigableMap<
- byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> resultMap =
- result.getMap();
-
- NavigableMap<byte[], NavigableMap<Long, byte[]>> columnCellMap =
- resultMap.get(columnFamilyBytes);
- // could be that there is no such column family.
- if (columnCellMap != null) {
- for (Entry<byte[], NavigableMap<Long, byte[]>> entry : columnCellMap
- .entrySet()) {
- K converterColumnKey = null;
- if (columnPrefixBytes == null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("null prefix was specified; returning all columns");
- }
- try {
- converterColumnKey = keyConverter.decode(entry.getKey());
- } catch (IllegalArgumentException iae) {
- LOG.error("Illegal column found, skipping this column.", iae);
- continue;
- }
- } else {
- // A non-null prefix means columns are actually of the form
- // prefix!columnNameRemainder
- byte[][] columnNameParts =
- Separator.QUALIFIERS.split(entry.getKey(), 2);
- byte[] actualColumnPrefixBytes = columnNameParts[0];
- if (Bytes.equals(columnPrefixBytes, actualColumnPrefixBytes)
- && columnNameParts.length == 2) {
- try {
- // This is the prefix that we want
- converterColumnKey = keyConverter.decode(columnNameParts[1]);
- } catch (IllegalArgumentException iae) {
- LOG.error("Illegal column found, skipping this column.", iae);
- continue;
- }
- }
- }
-
- // If this column has the prefix we want
- if (converterColumnKey != null) {
- NavigableMap<Long, V> cellResults =
- new TreeMap<Long, V>();
- NavigableMap<Long, byte[]> cells = entry.getValue();
- if (cells != null) {
- for (Entry<Long, byte[]> cell : cells.entrySet()) {
- V value =
- (V) converter.decodeValue(cell.getValue());
- Long ts = supplementTs ? TimestampGenerator.
- getTruncatedTimestamp(cell.getKey()) : cell.getKey();
- cellResults.put(ts, value);
- }
- }
- results.put(converterColumnKey, cellResults);
- }
- } // for entry : columnCellMap
- } // if columnCellMap != null
- } // if result != null
- return results;
- }
-
- /**
- * @param <K> identifies the type of column name(indicated by type of key
- * converter).
- * @param result from which to read columns
- * @param columnPrefixBytes optional prefix to limit columns. If null all
- * columns are returned.
- * @param keyConverter used to convert column bytes to the appropriate key
- * type.
- * @return the latest values of columns in the column family. If the column
- * prefix is null, the column qualifier is returned as Strings. For a
- * non-null column prefix bytes, the column qualifier is returned as
- * a list of parts, each part a byte[]. This is to facilitate
- * returning byte arrays of values that were not Strings.
- * @throws IOException if any problem occurs while reading results.
- */
- public <K> Map<K, Object> readResults(Result result,
- byte[] columnPrefixBytes, KeyConverter<K> keyConverter)
- throws IOException {
- Map<K, Object> results = new HashMap<K, Object>();
-
- if (result != null) {
- Map<byte[], byte[]> columns = result.getFamilyMap(columnFamilyBytes);
- for (Entry<byte[], byte[]> entry : columns.entrySet()) {
- byte[] columnKey = entry.getKey();
- if (columnKey != null && columnKey.length > 0) {
-
- K converterColumnKey = null;
- if (columnPrefixBytes == null) {
- try {
- converterColumnKey = keyConverter.decode(columnKey);
- } catch (IllegalArgumentException iae) {
- LOG.error("Illegal column found, skipping this column.", iae);
- continue;
- }
- } else {
- // A non-null prefix means columns are actually of the form
- // prefix!columnNameRemainder
- byte[][] columnNameParts = Separator.QUALIFIERS.split(columnKey, 2);
- if (columnNameParts.length > 0) {
- byte[] actualColumnPrefixBytes = columnNameParts[0];
- // If this is the prefix that we want
- if (Bytes.equals(columnPrefixBytes, actualColumnPrefixBytes)
- && columnNameParts.length == 2) {
- try {
- converterColumnKey = keyConverter.decode(columnNameParts[1]);
- } catch (IllegalArgumentException iae) {
- LOG.error("Illegal column found, skipping this column.", iae);
- continue;
- }
- }
- }
- } // if-else
-
- // If the columnPrefix is null (we want all columns), or the actual
- // prefix matches the given prefix we want this column
- if (converterColumnKey != null) {
- Object value = converter.decodeValue(entry.getValue());
- // we return the columnQualifier in parts since we don't know
- // which part is of which data type.
- results.put(converterColumnKey, value);
- }
- }
- } // for entry
- }
- return results;
- }
-
- /**
- * @param columnPrefixBytes The byte representation for the column prefix.
- * Should not contain {@link Separator#QUALIFIERS}.
- * @param qualifier for the remainder of the column.
- * {@link Separator#QUALIFIERS} is permissible in the qualifier
- * as it is joined only with the column prefix bytes.
- * @return fully sanitized column qualifier that is a combination of prefix
- * and qualifier. If prefix is null, the result is simply the encoded
- * qualifier without any separator.
- */
- public static byte[] getColumnQualifier(byte[] columnPrefixBytes,
- String qualifier) {
-
- // We don't want column names to have spaces / tabs.
- byte[] encodedQualifier =
- Separator.encode(qualifier, Separator.SPACE, Separator.TAB);
- if (columnPrefixBytes == null) {
- return encodedQualifier;
- }
-
- // Convert qualifier to lower case, strip of separators and tag on column
- // prefix.
- byte[] columnQualifier =
- Separator.QUALIFIERS.join(columnPrefixBytes, encodedQualifier);
- return columnQualifier;
- }
-
- /**
- * @param columnPrefixBytes The byte representation for the column prefix.
- * Should not contain {@link Separator#QUALIFIERS}.
- * @param qualifier for the remainder of the column.
- * @return fully sanitized column qualifier that is a combination of prefix
- * and qualifier. If prefix is null, the result is simply the encoded
- * qualifier without any separator.
- */
- public static byte[] getColumnQualifier(byte[] columnPrefixBytes,
- long qualifier) {
-
- if (columnPrefixBytes == null) {
- return Bytes.toBytes(qualifier);
- }
-
- // Convert qualifier to lower case, strip of separators and tag on column
- // prefix.
- byte[] columnQualifier =
- Separator.QUALIFIERS.join(columnPrefixBytes, Bytes.toBytes(qualifier));
- return columnQualifier;
- }
-
- public ValueConverter getValueConverter() {
- return converter;
- }
-
- /**
- * @param columnPrefixBytes The byte representation for the column prefix.
- * Should not contain {@link Separator#QUALIFIERS}.
- * @param qualifier the byte representation for the remainder of the column.
- * @return fully sanitized column qualifier that is a combination of prefix
- * and qualifier. If prefix is null, the result is simply the encoded
- * qualifier without any separator.
- */
- public static byte[] getColumnQualifier(byte[] columnPrefixBytes,
- byte[] qualifier) {
-
- if (columnPrefixBytes == null) {
- return qualifier;
- }
-
- byte[] columnQualifier =
- Separator.QUALIFIERS.join(columnPrefixBytes, qualifier);
- return columnQualifier;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
deleted file mode 100644
index 89aa013..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.NavigableMap;
-
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
-
-/**
- * Used to represent a partially qualified column, where the actual column name
- * will be composed of a prefix and the remainder of the column qualifier. The
- * prefix can be null, in which case the column qualifier will be completely
- * determined when the values are stored.
- */
-public interface ColumnPrefix<T> {
-
- /**
- * Sends a Mutation to the table. The mutations will be buffered and sent over
- * the wire as part of a batch.
- *
- * @param rowKey identifying the row to write. Nothing gets written when null.
- * @param tableMutator used to modify the underlying HBase table. Caller is
- * responsible to pass a mutator for the table that actually has this
- * column.
- * @param qualifier column qualifier. Nothing gets written when null.
- * @param timestamp version timestamp. When null the server timestamp will be
- * used.
- * @param attributes attributes for the mutation that are used by the
- * coprocessor to set/read the cell tags.
- * @param inputValue the value to write to the rowKey and column qualifier.
- * Nothing gets written when null.
- * @throws IOException if there is any exception encountered while doing
- * store operation(sending mutation to the table).
- */
- void store(byte[] rowKey, TypedBufferedMutator<T> tableMutator,
- byte[] qualifier, Long timestamp, Object inputValue,
- Attribute... attributes) throws IOException;
-
- /**
- * Sends a Mutation to the table. The mutations will be buffered and sent over
- * the wire as part of a batch.
- *
- * @param rowKey identifying the row to write. Nothing gets written when null.
- * @param tableMutator used to modify the underlying HBase table. Caller is
- * responsible to pass a mutator for the table that actually has this
- * column.
- * @param qualifier column qualifier. Nothing gets written when null.
- * @param timestamp version timestamp. When null the server timestamp will be
- * used.
- * @param attributes attributes for the mutation that are used by the
- * coprocessor to set/read the cell tags.
- * @param inputValue the value to write to the rowKey and column qualifier.
- * Nothing gets written when null.
- * @throws IOException if there is any exception encountered while doing
- * store operation(sending mutation to the table).
- */
- void store(byte[] rowKey, TypedBufferedMutator<T> tableMutator,
- String qualifier, Long timestamp, Object inputValue,
- Attribute... attributes) throws IOException;
-
- /**
- * Get the latest version of this specified column. Note: this call clones the
- * value content of the hosting {@link org.apache.hadoop.hbase.Cell Cell}.
- *
- * @param result Cannot be null
- * @param qualifier column qualifier. Nothing gets read when null.
- * @return result object (can be cast to whatever object was written to) or
- * null when specified column qualifier for this prefix doesn't exist
- * in the result.
- * @throws IOException if there is any exception encountered while reading
- * result.
- */
- Object readResult(Result result, String qualifier) throws IOException;
-
- /**
- *
- * @param <K> identifies the type of key converter.
- * @param result from which to read columns.
- * @param keyConverter used to convert column bytes to the appropriate key
- * type
- * @return the latest values of columns in the column family with this prefix
- * (or all of them if the prefix value is null).
- * @throws IOException if there is any exception encountered while reading
- * results.
- */
- <K> Map<K, Object> readResults(Result result, KeyConverter<K> keyConverter)
- throws IOException;
-
- /**
- * @param result from which to reads data with timestamps.
- * @param <K> identifies the type of key converter.
- * @param <V> the type of the values. The values will be cast into that type.
- * @param keyConverter used to convert column bytes to the appropriate key
- * type.
- * @return the cell values at each respective time in for form
- * {@literal {idA={timestamp1->value1}, idA={timestamp2->value2},
- * idB={timestamp3->value3}, idC={timestamp1->value4}}}
- * @throws IOException if there is any exception encountered while reading
- * result.
- */
- <K, V> NavigableMap<K, NavigableMap<Long, V>> readResultsWithTimestamps(
- Result result, KeyConverter<K> keyConverter) throws IOException;
-
- /**
- * @param qualifierPrefix Column qualifier or prefix of qualifier.
- * @return a byte array encoding column prefix and qualifier/prefix passed.
- */
- byte[] getColumnPrefixBytes(String qualifierPrefix);
-
- /**
- * @param qualifierPrefix Column qualifier or prefix of qualifier.
- * @return a byte array encoding column prefix and qualifier/prefix passed.
- */
- byte[] getColumnPrefixBytes(byte[] qualifierPrefix);
-
- /**
- * Returns column family name(as bytes) associated with this column prefix.
- * @return a byte array encoding column family for this prefix.
- */
- byte[] getColumnFamilyBytes();
-
- /**
- * Returns value converter implementation associated with this column prefix.
- * @return a {@link ValueConverter} implementation.
- */
- ValueConverter getValueConverter();
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/EventColumnName.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/EventColumnName.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/EventColumnName.java
deleted file mode 100644
index 8445575..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/EventColumnName.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-/**
- * Encapsulates information about Event column names for application and entity
- * tables. Used while encoding/decoding event column names.
- */
-public class EventColumnName {
-
- private final String id;
- private final Long timestamp;
- private final String infoKey;
- private final KeyConverter<EventColumnName> eventColumnNameConverter =
- new EventColumnNameConverter();
-
- public EventColumnName(String id, Long timestamp, String infoKey) {
- this.id = id;
- this.timestamp = timestamp;
- this.infoKey = infoKey;
- }
-
- public String getId() {
- return id;
- }
-
- public Long getTimestamp() {
- return timestamp;
- }
-
- public String getInfoKey() {
- return infoKey;
- }
-
- /**
- * @return a byte array with each components/fields separated by
- * Separator#VALUES. This leads to an event column name of the form
- * eventId=timestamp=infokey. If both timestamp and infokey are null,
- * then a qualifier of the form eventId=timestamp= is returned. If
- * only infokey is null, then a qualifier of the form eventId= is
- * returned. These prefix forms are useful for queries that intend to
- * retrieve more than one specific column name.
- */
- public byte[] getColumnQualifier() {
- return eventColumnNameConverter.encode(this);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/EventColumnNameConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/EventColumnNameConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/EventColumnNameConverter.java
deleted file mode 100644
index d3ef897..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/EventColumnNameConverter.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import org.apache.hadoop.hbase.util.Bytes;
-
-/**
- * Encodes and decodes event column names for application and entity tables.
- * The event column name is of the form : eventId=timestamp=infokey.
- * If info is not associated with the event, event column name is of the form :
- * eventId=timestamp=
- * Event timestamp is long and rest are strings.
- * Column prefixes are not part of the eventcolumn name passed for encoding. It
- * is added later, if required in the associated ColumnPrefix implementations.
- */
-public final class EventColumnNameConverter
- implements KeyConverter<EventColumnName> {
-
- public EventColumnNameConverter() {
- }
-
- // eventId=timestamp=infokey are of types String, Long String
- // Strings are variable in size (i.e. end whenever separator is encountered).
- // This is used while decoding and helps in determining where to split.
- private static final int[] SEGMENT_SIZES = {
- Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG, Separator.VARIABLE_SIZE };
-
- /*
- * (non-Javadoc)
- *
- * Encodes EventColumnName into a byte array with each component/field in
- * EventColumnName separated by Separator#VALUES. This leads to an event
- * column name of the form eventId=timestamp=infokey.
- * If timestamp in passed EventColumnName object is null (eventId is not null)
- * this returns a column prefix of the form eventId= and if infokey in
- * EventColumnName is null (other 2 components are not null), this returns a
- * column name of the form eventId=timestamp=
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter
- * #encode(java.lang.Object)
- */
- @Override
- public byte[] encode(EventColumnName key) {
- byte[] first = Separator.encode(key.getId(), Separator.SPACE, Separator.TAB,
- Separator.VALUES);
- if (key.getTimestamp() == null) {
- return Separator.VALUES.join(first, Separator.EMPTY_BYTES);
- }
- byte[] second = Bytes.toBytes(
- LongConverter.invertLong(key.getTimestamp()));
- if (key.getInfoKey() == null) {
- return Separator.VALUES.join(first, second, Separator.EMPTY_BYTES);
- }
- return Separator.VALUES.join(first, second, Separator.encode(
- key.getInfoKey(), Separator.SPACE, Separator.TAB, Separator.VALUES));
- }
-
- /*
- * (non-Javadoc)
- *
- * Decodes an event column name of the form eventId=timestamp= or
- * eventId=timestamp=infoKey represented in byte format and converts it into
- * an EventColumnName object.
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter
- * #decode(byte[])
- */
- @Override
- public EventColumnName decode(byte[] bytes) {
- byte[][] components = Separator.VALUES.split(bytes, SEGMENT_SIZES);
- if (components.length != 3) {
- throw new IllegalArgumentException("the column name is not valid");
- }
- String id = Separator.decode(Bytes.toString(components[0]),
- Separator.VALUES, Separator.TAB, Separator.SPACE);
- Long ts = LongConverter.invertLong(Bytes.toLong(components[1]));
- String infoKey = components[2].length == 0 ? null :
- Separator.decode(Bytes.toString(components[2]),
- Separator.VALUES, Separator.TAB, Separator.SPACE);
- return new EventColumnName(id, ts, infoKey);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/GenericConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/GenericConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/GenericConverter.java
deleted file mode 100644
index c34bfcb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/GenericConverter.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import java.io.IOException;
-
-import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
-
-/**
- * Uses GenericObjectMapper to encode objects as bytes and decode bytes as
- * objects.
- */
-public final class GenericConverter implements ValueConverter {
- private static final GenericConverter INSTANCE = new GenericConverter();
-
- private GenericConverter() {
- }
-
- public static GenericConverter getInstance() {
- return INSTANCE;
- }
-
- @Override
- public byte[] encodeValue(Object value) throws IOException {
- return GenericObjectMapper.write(value);
- }
-
- @Override
- public Object decodeValue(byte[] bytes) throws IOException {
- return GenericObjectMapper.read(bytes);
- }
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/18] hadoop git commit: YARN-7919. Refactor timelineservice-hbase
module into submodules. Contributed by Haibo Chen.
Posted by ro...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterUtils.java
new file mode 100644
index 0000000..2b98eec
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterUtils.java
@@ -0,0 +1,313 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader.filter;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.hadoop.hbase.filter.BinaryComparator;
+import org.apache.hadoop.hbase.filter.BinaryPrefixComparator;
+import org.apache.hadoop.hbase.filter.FamilyFilter;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.FilterList.Operator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Column;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
+import org.apache.hadoop.hbase.filter.QualifierFilter;
+import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Set of utility methods used by timeline filter classes.
+ */
+public final class TimelineFilterUtils {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TimelineFilterUtils.class);
+
+ private TimelineFilterUtils() {
+ }
+
+ /**
+ * Returns the equivalent HBase filter list's {@link Operator}.
+ *
+ * @param op timeline filter list operator.
+ * @return HBase filter list's Operator.
+ */
+ private static Operator getHBaseOperator(TimelineFilterList.Operator op) {
+ switch (op) {
+ case AND:
+ return Operator.MUST_PASS_ALL;
+ case OR:
+ return Operator.MUST_PASS_ONE;
+ default:
+ throw new IllegalArgumentException("Invalid operator");
+ }
+ }
+
+ /**
+ * Returns the equivalent HBase compare filter's {@link CompareOp}.
+ *
+ * @param op timeline compare op.
+ * @return HBase compare filter's CompareOp.
+ */
+ private static CompareOp getHBaseCompareOp(
+ TimelineCompareOp op) {
+ switch (op) {
+ case LESS_THAN:
+ return CompareOp.LESS;
+ case LESS_OR_EQUAL:
+ return CompareOp.LESS_OR_EQUAL;
+ case EQUAL:
+ return CompareOp.EQUAL;
+ case NOT_EQUAL:
+ return CompareOp.NOT_EQUAL;
+ case GREATER_OR_EQUAL:
+ return CompareOp.GREATER_OR_EQUAL;
+ case GREATER_THAN:
+ return CompareOp.GREATER;
+ default:
+ throw new IllegalArgumentException("Invalid compare operator");
+ }
+ }
+
+ /**
+ * Converts a {@link TimelinePrefixFilter} to an equivalent HBase
+ * {@link QualifierFilter}.
+ * @param colPrefix
+ * @param filter
+ * @return a {@link QualifierFilter} object
+ */
+ private static <T extends BaseTable<T>> Filter createHBaseColQualPrefixFilter(
+ ColumnPrefix<T> colPrefix, TimelinePrefixFilter filter) {
+ return new QualifierFilter(getHBaseCompareOp(filter.getCompareOp()),
+ new BinaryPrefixComparator(
+ colPrefix.getColumnPrefixBytes(filter.getPrefix())));
+ }
+
+ /**
+ * Create a HBase {@link QualifierFilter} for the passed column prefix and
+ * compare op.
+ *
+ * @param <T> Describes the type of column prefix.
+ * @param compareOp compare op.
+ * @param columnPrefix column prefix.
+ * @return a column qualifier filter.
+ */
+ public static <T extends BaseTable<T>> Filter createHBaseQualifierFilter(
+ CompareOp compareOp, ColumnPrefix<T> columnPrefix) {
+ return new QualifierFilter(compareOp,
+ new BinaryPrefixComparator(
+ columnPrefix.getColumnPrefixBytes("")));
+ }
+
+ /**
+ * Create filters for confs or metrics to retrieve. This list includes a
+ * configs/metrics family filter and relevant filters for confs/metrics to
+ * retrieve, if present.
+ *
+ * @param <T> Describes the type of column prefix.
+ * @param confsOrMetricToRetrieve configs/metrics to retrieve.
+ * @param columnFamily config or metric column family.
+ * @param columnPrefix config or metric column prefix.
+ * @return a filter list.
+ * @throws IOException if any problem occurs while creating the filters.
+ */
+ public static <T extends BaseTable<T>> Filter
+ createFilterForConfsOrMetricsToRetrieve(
+ TimelineFilterList confsOrMetricToRetrieve, ColumnFamily<T> columnFamily,
+ ColumnPrefix<T> columnPrefix) throws IOException {
+ Filter familyFilter = new FamilyFilter(CompareOp.EQUAL,
+ new BinaryComparator(columnFamily.getBytes()));
+ if (confsOrMetricToRetrieve != null &&
+ !confsOrMetricToRetrieve.getFilterList().isEmpty()) {
+ // If confsOrMetricsToRetrive are specified, create a filter list based
+ // on it and family filter.
+ FilterList filter = new FilterList(familyFilter);
+ filter.addFilter(
+ createHBaseFilterList(columnPrefix, confsOrMetricToRetrieve));
+ return filter;
+ } else {
+ // Only the family filter needs to be added.
+ return familyFilter;
+ }
+ }
+
+ /**
+ * Create 2 HBase {@link SingleColumnValueFilter} filters for the specified
+ * value range represented by start and end value and wraps them inside a
+ * filter list. Start and end value should not be null.
+ *
+ * @param <T> Describes the type of column prefix.
+ * @param column Column for which single column value filter is to be created.
+ * @param startValue Start value.
+ * @param endValue End value.
+ * @return 2 single column value filters wrapped in a filter list.
+ * @throws IOException if any problem is encountered while encoding value.
+ */
+ public static <T extends BaseTable<T>> FilterList
+ createSingleColValueFiltersByRange(Column<T> column,
+ Object startValue, Object endValue) throws IOException {
+ FilterList list = new FilterList();
+ Filter singleColValFilterStart = createHBaseSingleColValueFilter(
+ column.getColumnFamilyBytes(), column.getColumnQualifierBytes(),
+ column.getValueConverter().encodeValue(startValue),
+ CompareOp.GREATER_OR_EQUAL, true);
+ list.addFilter(singleColValFilterStart);
+
+ Filter singleColValFilterEnd = createHBaseSingleColValueFilter(
+ column.getColumnFamilyBytes(), column.getColumnQualifierBytes(),
+ column.getValueConverter().encodeValue(endValue),
+ CompareOp.LESS_OR_EQUAL, true);
+ list.addFilter(singleColValFilterEnd);
+ return list;
+ }
+
+ /**
+ * Creates a HBase {@link SingleColumnValueFilter} with specified column.
+ * @param <T> Describes the type of column prefix.
+ * @param column Column which value to be filtered.
+ * @param value Value to be filtered.
+ * @param op Compare operator
+ * @return a SingleColumnValue Filter
+ * @throws IOException if any exception.
+ */
+ public static <T extends BaseTable<T>> Filter
+ createHBaseSingleColValueFilter(Column<T> column,
+ Object value, CompareOp op) throws IOException {
+ Filter singleColValFilter = createHBaseSingleColValueFilter(
+ column.getColumnFamilyBytes(), column.getColumnQualifierBytes(),
+ column.getValueConverter().encodeValue(value), op, true);
+ return singleColValFilter;
+ }
+
+ /**
+ * Creates a HBase {@link SingleColumnValueFilter}.
+ *
+ * @param columnFamily Column Family represented as bytes.
+ * @param columnQualifier Column Qualifier represented as bytes.
+ * @param value Value.
+ * @param compareOp Compare operator.
+ * @param filterIfMissing This flag decides if we should filter the row if the
+ * specified column is missing. This is based on the filter's keyMustExist
+ * field.
+ * @return a {@link SingleColumnValueFilter} object
+ * @throws IOException
+ */
+ private static SingleColumnValueFilter createHBaseSingleColValueFilter(
+ byte[] columnFamily, byte[] columnQualifier, byte[] value,
+ CompareOp compareOp, boolean filterIfMissing) throws IOException {
+ SingleColumnValueFilter singleColValFilter =
+ new SingleColumnValueFilter(columnFamily, columnQualifier, compareOp,
+ new BinaryComparator(value));
+ singleColValFilter.setLatestVersionOnly(true);
+ singleColValFilter.setFilterIfMissing(filterIfMissing);
+ return singleColValFilter;
+ }
+
+ /**
+ * Fetch columns from filter list containing exists and multivalue equality
+ * filters. This is done to fetch only required columns from back-end and
+ * then match event filters or relationships in reader.
+ *
+ * @param filterList filter list.
+ * @return set of columns.
+ */
+ public static Set<String> fetchColumnsFromFilterList(
+ TimelineFilterList filterList) {
+ Set<String> strSet = new HashSet<String>();
+ for (TimelineFilter filter : filterList.getFilterList()) {
+ switch(filter.getFilterType()) {
+ case LIST:
+ strSet.addAll(fetchColumnsFromFilterList((TimelineFilterList)filter));
+ break;
+ case KEY_VALUES:
+ strSet.add(((TimelineKeyValuesFilter)filter).getKey());
+ break;
+ case EXISTS:
+ strSet.add(((TimelineExistsFilter)filter).getValue());
+ break;
+ default:
+ LOG.info("Unexpected filter type " + filter.getFilterType());
+ break;
+ }
+ }
+ return strSet;
+ }
+
+ /**
+ * Creates equivalent HBase {@link FilterList} from {@link TimelineFilterList}
+ * while converting different timeline filters(of type {@link TimelineFilter})
+ * into their equivalent HBase filters.
+ *
+ * @param <T> Describes the type of column prefix.
+ * @param colPrefix column prefix which will be used for conversion.
+ * @param filterList timeline filter list which has to be converted.
+ * @return A {@link FilterList} object.
+ * @throws IOException if any problem occurs while creating the filter list.
+ */
+ public static <T extends BaseTable<T>> FilterList createHBaseFilterList(
+ ColumnPrefix<T> colPrefix,
+ TimelineFilterList filterList) throws IOException {
+ FilterList list =
+ new FilterList(getHBaseOperator(filterList.getOperator()));
+ for (TimelineFilter filter : filterList.getFilterList()) {
+ switch(filter.getFilterType()) {
+ case LIST:
+ list.addFilter(createHBaseFilterList(colPrefix,
+ (TimelineFilterList)filter));
+ break;
+ case PREFIX:
+ list.addFilter(createHBaseColQualPrefixFilter(colPrefix,
+ (TimelinePrefixFilter)filter));
+ break;
+ case COMPARE:
+ TimelineCompareFilter compareFilter = (TimelineCompareFilter)filter;
+ list.addFilter(
+ createHBaseSingleColValueFilter(
+ colPrefix.getColumnFamilyBytes(),
+ colPrefix.getColumnPrefixBytes(compareFilter.getKey()),
+ colPrefix.getValueConverter().
+ encodeValue(compareFilter.getValue()),
+ getHBaseCompareOp(compareFilter.getCompareOp()),
+ compareFilter.getKeyMustExist()));
+ break;
+ case KEY_VALUE:
+ TimelineKeyValueFilter kvFilter = (TimelineKeyValueFilter)filter;
+ list.addFilter(
+ createHBaseSingleColValueFilter(
+ colPrefix.getColumnFamilyBytes(),
+ colPrefix.getColumnPrefixBytes(kvFilter.getKey()),
+ colPrefix.getValueConverter().encodeValue(kvFilter.getValue()),
+ getHBaseCompareOp(kvFilter.getCompareOp()),
+ kvFilter.getKeyMustExist()));
+ break;
+ default:
+ LOG.info("Unexpected filter type " + filter.getFilterType());
+ break;
+ }
+ }
+ return list;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/package-info.java
new file mode 100644
index 0000000..f7c0705
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.server.timelineservice.reader.filter stores
+ * timeline filter implementations.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.reader.filter;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
new file mode 100644
index 0000000..1ebfab2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.reader.EntityTypeReader;
+import org.apache.hadoop.yarn.server.timelineservice.storage.reader.TimelineEntityReader;
+import org.apache.hadoop.yarn.server.timelineservice.storage.reader.TimelineEntityReaderFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * HBase based implementation for {@link TimelineReader}.
+ */
+public class HBaseTimelineReaderImpl
+ extends AbstractService implements TimelineReader {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(HBaseTimelineReaderImpl.class);
+
+ private Configuration hbaseConf = null;
+ private Connection conn;
+
+ public HBaseTimelineReaderImpl() {
+ super(HBaseTimelineReaderImpl.class.getName());
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ super.serviceInit(conf);
+ hbaseConf = HBaseTimelineStorageUtils.getTimelineServiceHBaseConf(conf);
+ conn = ConnectionFactory.createConnection(hbaseConf);
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (conn != null) {
+ LOG.info("closing the hbase Connection");
+ conn.close();
+ }
+ super.serviceStop();
+ }
+
+ @Override
+ public TimelineEntity getEntity(TimelineReaderContext context,
+ TimelineDataToRetrieve dataToRetrieve) throws IOException {
+ TimelineEntityReader reader =
+ TimelineEntityReaderFactory.createSingleEntityReader(context,
+ dataToRetrieve);
+ return reader.readEntity(hbaseConf, conn);
+ }
+
+ @Override
+ public Set<TimelineEntity> getEntities(TimelineReaderContext context,
+ TimelineEntityFilters filters, TimelineDataToRetrieve dataToRetrieve)
+ throws IOException {
+ TimelineEntityReader reader =
+ TimelineEntityReaderFactory.createMultipleEntitiesReader(context,
+ filters, dataToRetrieve);
+ return reader.readEntities(hbaseConf, conn);
+ }
+
+ @Override
+ public Set<String> getEntityTypes(TimelineReaderContext context)
+ throws IOException {
+ EntityTypeReader reader = new EntityTypeReader(context);
+ return reader.readEntityTypes(hbaseConf, conn);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
new file mode 100644
index 0000000..027505b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -0,0 +1,611 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTableRW;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTableRW;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnRWHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.EventColumnName;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongKeyConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.StringKeyConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTableRW;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationCompactionDimension;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationOperation;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityTableRW;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTableRW;
+import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationTableRW;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This implements a hbase based backend for storing the timeline entity
+ * information.
+ * It writes to multiple tables at the backend
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class HBaseTimelineWriterImpl extends AbstractService implements
+ TimelineWriter {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(HBaseTimelineWriterImpl.class);
+
+ private Connection conn;
+ private TypedBufferedMutator<EntityTable> entityTable;
+ private TypedBufferedMutator<AppToFlowTable> appToFlowTable;
+ private TypedBufferedMutator<ApplicationTable> applicationTable;
+ private TypedBufferedMutator<FlowActivityTable> flowActivityTable;
+ private TypedBufferedMutator<FlowRunTable> flowRunTable;
+ private TypedBufferedMutator<SubApplicationTable> subApplicationTable;
+
+ /**
+ * Used to convert strings key components to and from storage format.
+ */
+ private final KeyConverter<String> stringKeyConverter =
+ new StringKeyConverter();
+
+ /**
+ * Used to convert Long key components to and from storage format.
+ */
+ private final KeyConverter<Long> longKeyConverter = new LongKeyConverter();
+
+ private enum Tables {
+ APPLICATION_TABLE, ENTITY_TABLE, SUBAPPLICATION_TABLE
+ };
+
+ public HBaseTimelineWriterImpl() {
+ super(HBaseTimelineWriterImpl.class.getName());
+ }
+
+ /**
+ * initializes the hbase connection to write to the entity table.
+ */
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ super.serviceInit(conf);
+ Configuration hbaseConf =
+ HBaseTimelineStorageUtils.getTimelineServiceHBaseConf(conf);
+ conn = ConnectionFactory.createConnection(hbaseConf);
+ entityTable = new EntityTableRW().getTableMutator(hbaseConf, conn);
+ appToFlowTable = new AppToFlowTableRW().getTableMutator(hbaseConf, conn);
+ applicationTable =
+ new ApplicationTableRW().getTableMutator(hbaseConf, conn);
+ flowRunTable = new FlowRunTableRW().getTableMutator(hbaseConf, conn);
+ flowActivityTable =
+ new FlowActivityTableRW().getTableMutator(hbaseConf, conn);
+ subApplicationTable =
+ new SubApplicationTableRW().getTableMutator(hbaseConf, conn);
+
+ UserGroupInformation ugi = UserGroupInformation.isSecurityEnabled() ?
+ UserGroupInformation.getLoginUser() :
+ UserGroupInformation.getCurrentUser();
+ LOG.info("Initialized HBaseTimelineWriterImpl UGI to " + ugi);
+ }
+
+ /**
+ * Stores the entire information in TimelineEntities to the timeline store.
+ */
+ @Override
+ public TimelineWriteResponse write(TimelineCollectorContext context,
+ TimelineEntities data, UserGroupInformation callerUgi)
+ throws IOException {
+
+ TimelineWriteResponse putStatus = new TimelineWriteResponse();
+
+ String clusterId = context.getClusterId();
+ String userId = context.getUserId();
+ String flowName = context.getFlowName();
+ String flowVersion = context.getFlowVersion();
+ long flowRunId = context.getFlowRunId();
+ String appId = context.getAppId();
+ String subApplicationUser = callerUgi.getShortUserName();
+
+ // defensive coding to avoid NPE during row key construction
+ if ((flowName == null) || (appId == null) || (clusterId == null)
+ || (userId == null)) {
+ LOG.warn("Found null for one of: flowName=" + flowName + " appId=" + appId
+ + " userId=" + userId + " clusterId=" + clusterId
+ + " . Not proceeding with writing to hbase");
+ return putStatus;
+ }
+
+ for (TimelineEntity te : data.getEntities()) {
+
+ // a set can have at most 1 null
+ if (te == null) {
+ continue;
+ }
+
+ // if the entity is the application, the destination is the application
+ // table
+ boolean isApplication = ApplicationEntity.isApplicationEntity(te);
+ byte[] rowKey;
+ if (isApplication) {
+ ApplicationRowKey applicationRowKey =
+ new ApplicationRowKey(clusterId, userId, flowName, flowRunId,
+ appId);
+ rowKey = applicationRowKey.getRowKey();
+ store(rowKey, te, flowVersion, Tables.APPLICATION_TABLE);
+ } else {
+ EntityRowKey entityRowKey =
+ new EntityRowKey(clusterId, userId, flowName, flowRunId, appId,
+ te.getType(), te.getIdPrefix(), te.getId());
+ rowKey = entityRowKey.getRowKey();
+ store(rowKey, te, flowVersion, Tables.ENTITY_TABLE);
+ }
+
+ if (!isApplication && !userId.equals(subApplicationUser)) {
+ SubApplicationRowKey subApplicationRowKey =
+ new SubApplicationRowKey(subApplicationUser, clusterId,
+ te.getType(), te.getIdPrefix(), te.getId(), userId);
+ rowKey = subApplicationRowKey.getRowKey();
+ store(rowKey, te, flowVersion, Tables.SUBAPPLICATION_TABLE);
+ }
+
+ if (isApplication) {
+ TimelineEvent event =
+ ApplicationEntity.getApplicationEvent(te,
+ ApplicationMetricsConstants.CREATED_EVENT_TYPE);
+ FlowRunRowKey flowRunRowKey =
+ new FlowRunRowKey(clusterId, userId, flowName, flowRunId);
+ if (event != null) {
+ onApplicationCreated(flowRunRowKey, clusterId, appId, userId,
+ flowVersion, te, event.getTimestamp());
+ }
+ // if it's an application entity, store metrics
+ storeFlowMetricsAppRunning(flowRunRowKey, appId, te);
+ // if application has finished, store it's finish time and write final
+ // values of all metrics
+ event = ApplicationEntity.getApplicationEvent(te,
+ ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
+ if (event != null) {
+ onApplicationFinished(flowRunRowKey, flowVersion, appId, te,
+ event.getTimestamp());
+ }
+ }
+ }
+ return putStatus;
+ }
+
+ private void onApplicationCreated(FlowRunRowKey flowRunRowKey,
+ String clusterId, String appId, String userId, String flowVersion,
+ TimelineEntity te, long appCreatedTimeStamp)
+ throws IOException {
+
+ String flowName = flowRunRowKey.getFlowName();
+ Long flowRunId = flowRunRowKey.getFlowRunId();
+
+ // store in App to flow table
+ AppToFlowRowKey appToFlowRowKey = new AppToFlowRowKey(appId);
+ byte[] rowKey = appToFlowRowKey.getRowKey();
+ ColumnRWHelper.store(rowKey, appToFlowTable,
+ AppToFlowColumnPrefix.FLOW_NAME, clusterId, null, flowName);
+ ColumnRWHelper.store(rowKey, appToFlowTable,
+ AppToFlowColumnPrefix.FLOW_RUN_ID, clusterId, null, flowRunId);
+ ColumnRWHelper.store(rowKey, appToFlowTable, AppToFlowColumnPrefix.USER_ID,
+ clusterId, null, userId);
+
+ // store in flow run table
+ storeAppCreatedInFlowRunTable(flowRunRowKey, appId, te);
+
+ // store in flow activity table
+ byte[] flowActivityRowKeyBytes =
+ new FlowActivityRowKey(flowRunRowKey.getClusterId(),
+ appCreatedTimeStamp, flowRunRowKey.getUserId(), flowName)
+ .getRowKey();
+ byte[] qualifier = longKeyConverter.encode(flowRunRowKey.getFlowRunId());
+ ColumnRWHelper.store(flowActivityRowKeyBytes, flowActivityTable,
+ FlowActivityColumnPrefix.RUN_ID, qualifier, null, flowVersion,
+ AggregationCompactionDimension.APPLICATION_ID.getAttribute(appId));
+ }
+
+ /*
+ * updates the {@link FlowRunTable} with Application Created information
+ */
+ private void storeAppCreatedInFlowRunTable(FlowRunRowKey flowRunRowKey,
+ String appId, TimelineEntity te) throws IOException {
+ byte[] rowKey = flowRunRowKey.getRowKey();
+ ColumnRWHelper.store(rowKey, flowRunTable, FlowRunColumn.MIN_START_TIME,
+ null, te.getCreatedTime(),
+ AggregationCompactionDimension.APPLICATION_ID.getAttribute(appId));
+ }
+
+
+ /*
+ * updates the {@link FlowRunTable} and {@link FlowActivityTable} when an
+ * application has finished
+ */
+ private void onApplicationFinished(FlowRunRowKey flowRunRowKey,
+ String flowVersion, String appId, TimelineEntity te,
+ long appFinishedTimeStamp) throws IOException {
+ // store in flow run table
+ storeAppFinishedInFlowRunTable(flowRunRowKey, appId, te,
+ appFinishedTimeStamp);
+
+ // indicate in the flow activity table that the app has finished
+ byte[] rowKey =
+ new FlowActivityRowKey(flowRunRowKey.getClusterId(),
+ appFinishedTimeStamp, flowRunRowKey.getUserId(),
+ flowRunRowKey.getFlowName()).getRowKey();
+ byte[] qualifier = longKeyConverter.encode(flowRunRowKey.getFlowRunId());
+ ColumnRWHelper.store(rowKey, flowActivityTable,
+ FlowActivityColumnPrefix.RUN_ID, qualifier, null, flowVersion,
+ AggregationCompactionDimension.APPLICATION_ID.getAttribute(appId));
+ }
+
+ /*
+ * Update the {@link FlowRunTable} with Application Finished information
+ */
+ private void storeAppFinishedInFlowRunTable(FlowRunRowKey flowRunRowKey,
+ String appId, TimelineEntity te, long appFinishedTimeStamp)
+ throws IOException {
+ byte[] rowKey = flowRunRowKey.getRowKey();
+ Attribute attributeAppId =
+ AggregationCompactionDimension.APPLICATION_ID.getAttribute(appId);
+ ColumnRWHelper.store(rowKey, flowRunTable, FlowRunColumn.MAX_END_TIME,
+ null, appFinishedTimeStamp, attributeAppId);
+
+ // store the final value of metrics since application has finished
+ Set<TimelineMetric> metrics = te.getMetrics();
+ if (metrics != null) {
+ storeFlowMetrics(rowKey, metrics, attributeAppId,
+ AggregationOperation.SUM_FINAL.getAttribute());
+ }
+ }
+
+ /*
+ * Updates the {@link FlowRunTable} with Application Metrics
+ */
+ private void storeFlowMetricsAppRunning(FlowRunRowKey flowRunRowKey,
+ String appId, TimelineEntity te) throws IOException {
+ Set<TimelineMetric> metrics = te.getMetrics();
+ if (metrics != null) {
+ byte[] rowKey = flowRunRowKey.getRowKey();
+ storeFlowMetrics(rowKey, metrics,
+ AggregationCompactionDimension.APPLICATION_ID.getAttribute(appId),
+ AggregationOperation.SUM.getAttribute());
+ }
+ }
+
+ private void storeFlowMetrics(byte[] rowKey, Set<TimelineMetric> metrics,
+ Attribute... attributes) throws IOException {
+ for (TimelineMetric metric : metrics) {
+ byte[] metricColumnQualifier = stringKeyConverter.encode(metric.getId());
+ Map<Long, Number> timeseries = metric.getValues();
+ for (Map.Entry<Long, Number> timeseriesEntry : timeseries.entrySet()) {
+ Long timestamp = timeseriesEntry.getKey();
+ ColumnRWHelper.store(rowKey, flowRunTable, FlowRunColumnPrefix.METRIC,
+ metricColumnQualifier, timestamp, timeseriesEntry.getValue(),
+ attributes);
+ }
+ }
+ }
+
+ /**
+ * Stores the Relations from the {@linkplain TimelineEntity} object.
+ */
+ private <T extends BaseTable<T>> void storeRelations(byte[] rowKey,
+ Map<String, Set<String>> connectedEntities, ColumnPrefix<T> columnPrefix,
+ TypedBufferedMutator<T> table) throws IOException {
+ if (connectedEntities != null) {
+ for (Map.Entry<String, Set<String>> connectedEntity : connectedEntities
+ .entrySet()) {
+ // id3?id4?id5
+ String compoundValue =
+ Separator.VALUES.joinEncoded(connectedEntity.getValue());
+ ColumnRWHelper.store(rowKey, table, columnPrefix,
+ stringKeyConverter.encode(connectedEntity.getKey()),
+ null, compoundValue);
+ }
+ }
+ }
+
+ /**
+ * Stores information from the {@linkplain TimelineEntity} object.
+ */
+ private void store(byte[] rowKey, TimelineEntity te,
+ String flowVersion,
+ Tables table) throws IOException {
+ switch (table) {
+ case APPLICATION_TABLE:
+ ColumnRWHelper.store(rowKey, applicationTable,
+ ApplicationColumn.ID, null, te.getId());
+ ColumnRWHelper.store(rowKey, applicationTable,
+ ApplicationColumn.CREATED_TIME, null, te.getCreatedTime());
+ ColumnRWHelper.store(rowKey, applicationTable,
+ ApplicationColumn.FLOW_VERSION, null, flowVersion);
+ storeInfo(rowKey, te.getInfo(), flowVersion, ApplicationColumnPrefix.INFO,
+ applicationTable);
+ storeMetrics(rowKey, te.getMetrics(), ApplicationColumnPrefix.METRIC,
+ applicationTable);
+ storeEvents(rowKey, te.getEvents(), ApplicationColumnPrefix.EVENT,
+ applicationTable);
+ storeConfig(rowKey, te.getConfigs(), ApplicationColumnPrefix.CONFIG,
+ applicationTable);
+ storeRelations(rowKey, te.getIsRelatedToEntities(),
+ ApplicationColumnPrefix.IS_RELATED_TO, applicationTable);
+ storeRelations(rowKey, te.getRelatesToEntities(),
+ ApplicationColumnPrefix.RELATES_TO, applicationTable);
+ break;
+ case ENTITY_TABLE:
+ ColumnRWHelper.store(rowKey, entityTable,
+ EntityColumn.ID, null, te.getId());
+ ColumnRWHelper.store(rowKey, entityTable,
+ EntityColumn.TYPE, null, te.getType());
+ ColumnRWHelper.store(rowKey, entityTable,
+ EntityColumn.CREATED_TIME, null, te.getCreatedTime());
+ ColumnRWHelper.store(rowKey, entityTable,
+ EntityColumn.FLOW_VERSION, null, flowVersion);
+ storeInfo(rowKey, te.getInfo(), flowVersion, EntityColumnPrefix.INFO,
+ entityTable);
+ storeMetrics(rowKey, te.getMetrics(), EntityColumnPrefix.METRIC,
+ entityTable);
+ storeEvents(rowKey, te.getEvents(), EntityColumnPrefix.EVENT,
+ entityTable);
+ storeConfig(rowKey, te.getConfigs(), EntityColumnPrefix.CONFIG,
+ entityTable);
+ storeRelations(rowKey, te.getIsRelatedToEntities(),
+ EntityColumnPrefix.IS_RELATED_TO, entityTable);
+ storeRelations(rowKey, te.getRelatesToEntities(),
+ EntityColumnPrefix.RELATES_TO, entityTable);
+ break;
+ case SUBAPPLICATION_TABLE:
+ ColumnRWHelper.store(rowKey, subApplicationTable, SubApplicationColumn.ID,
+ null, te.getId());
+ ColumnRWHelper.store(rowKey, subApplicationTable,
+ SubApplicationColumn.TYPE, null, te.getType());
+ ColumnRWHelper.store(rowKey, subApplicationTable,
+ SubApplicationColumn.CREATED_TIME, null, te.getCreatedTime());
+ ColumnRWHelper.store(rowKey, subApplicationTable,
+ SubApplicationColumn.FLOW_VERSION, null, flowVersion);
+ storeInfo(rowKey, te.getInfo(), flowVersion,
+ SubApplicationColumnPrefix.INFO, subApplicationTable);
+ storeMetrics(rowKey, te.getMetrics(), SubApplicationColumnPrefix.METRIC,
+ subApplicationTable);
+ storeEvents(rowKey, te.getEvents(), SubApplicationColumnPrefix.EVENT,
+ subApplicationTable);
+ storeConfig(rowKey, te.getConfigs(), SubApplicationColumnPrefix.CONFIG,
+ subApplicationTable);
+ storeRelations(rowKey, te.getIsRelatedToEntities(),
+ SubApplicationColumnPrefix.IS_RELATED_TO, subApplicationTable);
+ storeRelations(rowKey, te.getRelatesToEntities(),
+ SubApplicationColumnPrefix.RELATES_TO, subApplicationTable);
+ break;
+ default:
+ LOG.info("Invalid table name provided.");
+ break;
+ }
+ }
+
+ /**
+ * stores the info information from {@linkplain TimelineEntity}.
+ */
+ private <T extends BaseTable<T>> void storeInfo(byte[] rowKey,
+ Map<String, Object> info, String flowVersion,
+ ColumnPrefix<T> columnPrefix, TypedBufferedMutator<T > table)
+ throws IOException {
+ if (info != null) {
+ for (Map.Entry<String, Object> entry : info.entrySet()) {
+ ColumnRWHelper.store(rowKey, table, columnPrefix,
+ stringKeyConverter.encode(entry.getKey()), null, entry.getValue());
+ }
+ }
+ }
+
+ /**
+ * stores the config information from {@linkplain TimelineEntity}.
+ */
+ private <T extends BaseTable<T>> void storeConfig(
+ byte[] rowKey, Map<String, String> config,
+ ColumnPrefix<T> columnPrefix, TypedBufferedMutator<T> table)
+ throws IOException {
+ if (config != null) {
+ for (Map.Entry<String, String> entry : config.entrySet()) {
+ byte[] configKey = stringKeyConverter.encode(entry.getKey());
+ ColumnRWHelper.store(rowKey, table, columnPrefix, configKey,
+ null, entry.getValue());
+ }
+ }
+ }
+
+ /**
+ * stores the {@linkplain TimelineMetric} information from the
+ * {@linkplain TimelineEvent} object.
+ */
+ private <T extends BaseTable<T>> void storeMetrics(
+ byte[] rowKey, Set<TimelineMetric> metrics,
+ ColumnPrefix<T> columnPrefix, TypedBufferedMutator<T> table)
+ throws IOException {
+ if (metrics != null) {
+ for (TimelineMetric metric : metrics) {
+ byte[] metricColumnQualifier =
+ stringKeyConverter.encode(metric.getId());
+ Map<Long, Number> timeseries = metric.getValues();
+ for (Map.Entry<Long, Number> timeseriesEntry : timeseries.entrySet()) {
+ Long timestamp = timeseriesEntry.getKey();
+ ColumnRWHelper.store(rowKey, table, columnPrefix,
+ metricColumnQualifier, timestamp, timeseriesEntry.getValue());
+ }
+ }
+ }
+ }
+
+ /**
+ * Stores the events from the {@linkplain TimelineEvent} object.
+ */
+ private <T extends BaseTable<T>> void storeEvents(
+ byte[] rowKey, Set<TimelineEvent> events,
+ ColumnPrefix<T> columnPrefix, TypedBufferedMutator<T> table)
+ throws IOException {
+ if (events != null) {
+ for (TimelineEvent event : events) {
+ if (event != null) {
+ String eventId = event.getId();
+ if (eventId != null) {
+ long eventTimestamp = event.getTimestamp();
+ // if the timestamp is not set, use the current timestamp
+ if (eventTimestamp == TimelineEvent.INVALID_TIMESTAMP) {
+ LOG.warn("timestamp is not set for event " + eventId +
+ "! Using the current timestamp");
+ eventTimestamp = System.currentTimeMillis();
+ }
+ Map<String, Object> eventInfo = event.getInfo();
+ if ((eventInfo == null) || (eventInfo.size() == 0)) {
+ byte[] columnQualifierBytes =
+ new EventColumnName(eventId, eventTimestamp, null)
+ .getColumnQualifier();
+ ColumnRWHelper.store(rowKey, table, columnPrefix,
+ columnQualifierBytes, null, Separator.EMPTY_BYTES);
+ } else {
+ for (Map.Entry<String, Object> info : eventInfo.entrySet()) {
+ // eventId=infoKey
+ byte[] columnQualifierBytes =
+ new EventColumnName(eventId, eventTimestamp, info.getKey())
+ .getColumnQualifier();
+ ColumnRWHelper.store(rowKey, table, columnPrefix,
+ columnQualifierBytes, null, info.getValue());
+ } // for info: eventInfo
+ }
+ }
+ }
+ } // event : events
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage
+ * .TimelineWriter#aggregate
+ * (org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity,
+ * org.apache
+ * .hadoop.yarn.server.timelineservice.storage.TimelineAggregationTrack)
+ */
+ @Override
+ public TimelineWriteResponse aggregate(TimelineEntity data,
+ TimelineAggregationTrack track) throws IOException {
+ return null;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter#flush
+ * ()
+ */
+ @Override
+ public void flush() throws IOException {
+ // flush all buffered mutators
+ entityTable.flush();
+ appToFlowTable.flush();
+ applicationTable.flush();
+ flowRunTable.flush();
+ flowActivityTable.flush();
+ subApplicationTable.flush();
+ }
+
+ /**
+ * close the hbase connections The close APIs perform flushing and release any
+ * resources held.
+ */
+ @Override
+ protected void serviceStop() throws Exception {
+ if (entityTable != null) {
+ LOG.info("closing the entity table");
+ // The close API performs flushing and releases any resources held
+ entityTable.close();
+ }
+ if (appToFlowTable != null) {
+ LOG.info("closing the app_flow table");
+ // The close API performs flushing and releases any resources held
+ appToFlowTable.close();
+ }
+ if (applicationTable != null) {
+ LOG.info("closing the application table");
+ applicationTable.close();
+ }
+ if (flowRunTable != null) {
+ LOG.info("closing the flow run table");
+ // The close API performs flushing and releases any resources held
+ flowRunTable.close();
+ }
+ if (flowActivityTable != null) {
+ LOG.info("closing the flowActivityTable table");
+ // The close API performs flushing and releases any resources held
+ flowActivityTable.close();
+ }
+ if (subApplicationTable != null) {
+ subApplicationTable.close();
+ }
+ if (conn != null) {
+ LOG.info("closing the hbase Connection");
+ conn.close();
+ }
+ super.serviceStop();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
new file mode 100644
index 0000000..e9e4770
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
@@ -0,0 +1,368 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.PosixParser;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTableRW;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTableRW;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTableRW;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityTableRW;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTableRW;
+import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationTableRW;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This creates the schema for a hbase based backend for storing application
+ * timeline information.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public final class TimelineSchemaCreator {
+ private TimelineSchemaCreator() {
+ }
+
+ final static String NAME = TimelineSchemaCreator.class.getSimpleName();
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TimelineSchemaCreator.class);
+ private static final String SKIP_EXISTING_TABLE_OPTION_SHORT = "s";
+ private static final String APP_METRICS_TTL_OPTION_SHORT = "ma";
+ private static final String SUB_APP_METRICS_TTL_OPTION_SHORT = "msa";
+ private static final String APP_TABLE_NAME_SHORT = "a";
+ private static final String SUB_APP_TABLE_NAME_SHORT = "sa";
+ private static final String APP_TO_FLOW_TABLE_NAME_SHORT = "a2f";
+ private static final String ENTITY_METRICS_TTL_OPTION_SHORT = "me";
+ private static final String ENTITY_TABLE_NAME_SHORT = "e";
+ private static final String HELP_SHORT = "h";
+ private static final String CREATE_TABLES_SHORT = "c";
+
+ public static void main(String[] args) throws Exception {
+
+ LOG.info("Starting the schema creation");
+ Configuration hbaseConf =
+ HBaseTimelineStorageUtils.getTimelineServiceHBaseConf(
+ new YarnConfiguration());
+ // Grab input args and allow for -Dxyz style arguments
+ String[] otherArgs = new GenericOptionsParser(hbaseConf, args)
+ .getRemainingArgs();
+
+ // Grab the arguments we're looking for.
+ CommandLine commandLine = parseArgs(otherArgs);
+
+ if (commandLine.hasOption(HELP_SHORT)) {
+ // -help option has the highest precedence
+ printUsage();
+ } else if (commandLine.hasOption(CREATE_TABLES_SHORT)) {
+ // Grab the entityTableName argument
+ String entityTableName = commandLine.getOptionValue(
+ ENTITY_TABLE_NAME_SHORT);
+ if (StringUtils.isNotBlank(entityTableName)) {
+ hbaseConf.set(EntityTableRW.TABLE_NAME_CONF_NAME, entityTableName);
+ }
+ // Grab the entity metrics TTL
+ String entityTableMetricsTTL = commandLine.getOptionValue(
+ ENTITY_METRICS_TTL_OPTION_SHORT);
+ if (StringUtils.isNotBlank(entityTableMetricsTTL)) {
+ int entityMetricsTTL = Integer.parseInt(entityTableMetricsTTL);
+ new EntityTableRW().setMetricsTTL(entityMetricsTTL, hbaseConf);
+ }
+ // Grab the appToflowTableName argument
+ String appToflowTableName = commandLine.getOptionValue(
+ APP_TO_FLOW_TABLE_NAME_SHORT);
+ if (StringUtils.isNotBlank(appToflowTableName)) {
+ hbaseConf.set(
+ AppToFlowTableRW.TABLE_NAME_CONF_NAME, appToflowTableName);
+ }
+ // Grab the applicationTableName argument
+ String applicationTableName = commandLine.getOptionValue(
+ APP_TABLE_NAME_SHORT);
+ if (StringUtils.isNotBlank(applicationTableName)) {
+ hbaseConf.set(ApplicationTableRW.TABLE_NAME_CONF_NAME,
+ applicationTableName);
+ }
+ // Grab the application metrics TTL
+ String applicationTableMetricsTTL = commandLine.getOptionValue(
+ APP_METRICS_TTL_OPTION_SHORT);
+ if (StringUtils.isNotBlank(applicationTableMetricsTTL)) {
+ int appMetricsTTL = Integer.parseInt(applicationTableMetricsTTL);
+ new ApplicationTableRW().setMetricsTTL(appMetricsTTL, hbaseConf);
+ }
+
+ // Grab the subApplicationTableName argument
+ String subApplicationTableName = commandLine.getOptionValue(
+ SUB_APP_TABLE_NAME_SHORT);
+ if (StringUtils.isNotBlank(subApplicationTableName)) {
+ hbaseConf.set(SubApplicationTableRW.TABLE_NAME_CONF_NAME,
+ subApplicationTableName);
+ }
+ // Grab the subApplication metrics TTL
+ String subApplicationTableMetricsTTL = commandLine
+ .getOptionValue(SUB_APP_METRICS_TTL_OPTION_SHORT);
+ if (StringUtils.isNotBlank(subApplicationTableMetricsTTL)) {
+ int subAppMetricsTTL = Integer.parseInt(subApplicationTableMetricsTTL);
+ new SubApplicationTableRW().setMetricsTTL(subAppMetricsTTL, hbaseConf);
+ }
+
+ // create all table schemas in hbase
+ final boolean skipExisting = commandLine.hasOption(
+ SKIP_EXISTING_TABLE_OPTION_SHORT);
+ createAllSchemas(hbaseConf, skipExisting);
+ } else {
+ // print usage information if -create is not specified
+ printUsage();
+ }
+ }
+
+ /**
+ * Parse command-line arguments.
+ *
+ * @param args
+ * command line arguments passed to program.
+ * @return parsed command line.
+ * @throws ParseException
+ */
+ private static CommandLine parseArgs(String[] args) throws ParseException {
+ Options options = new Options();
+
+ // Input
+ Option o = new Option(HELP_SHORT, "help", false, "print help information");
+ o.setRequired(false);
+ options.addOption(o);
+
+ o = new Option(CREATE_TABLES_SHORT, "create", false,
+ "a mandatory option to create hbase tables");
+ o.setRequired(false);
+ options.addOption(o);
+
+ o = new Option(ENTITY_TABLE_NAME_SHORT, "entityTableName", true,
+ "entity table name");
+ o.setArgName("entityTableName");
+ o.setRequired(false);
+ options.addOption(o);
+
+ o = new Option(ENTITY_METRICS_TTL_OPTION_SHORT, "entityMetricsTTL", true,
+ "TTL for metrics column family");
+ o.setArgName("entityMetricsTTL");
+ o.setRequired(false);
+ options.addOption(o);
+
+ o = new Option(APP_TO_FLOW_TABLE_NAME_SHORT, "appToflowTableName", true,
+ "app to flow table name");
+ o.setArgName("appToflowTableName");
+ o.setRequired(false);
+ options.addOption(o);
+
+ o = new Option(APP_TABLE_NAME_SHORT, "applicationTableName", true,
+ "application table name");
+ o.setArgName("applicationTableName");
+ o.setRequired(false);
+ options.addOption(o);
+
+ o = new Option(APP_METRICS_TTL_OPTION_SHORT, "applicationMetricsTTL", true,
+ "TTL for metrics column family");
+ o.setArgName("applicationMetricsTTL");
+ o.setRequired(false);
+ options.addOption(o);
+
+ o = new Option(SUB_APP_TABLE_NAME_SHORT, "subApplicationTableName", true,
+ "subApplication table name");
+ o.setArgName("subApplicationTableName");
+ o.setRequired(false);
+ options.addOption(o);
+
+ o = new Option(SUB_APP_METRICS_TTL_OPTION_SHORT, "subApplicationMetricsTTL",
+ true, "TTL for metrics column family");
+ o.setArgName("subApplicationMetricsTTL");
+ o.setRequired(false);
+ options.addOption(o);
+
+ // Options without an argument
+ // No need to set arg name since we do not need an argument here
+ o = new Option(SKIP_EXISTING_TABLE_OPTION_SHORT, "skipExistingTable",
+ false, "skip existing Hbase tables and continue to create new tables");
+ o.setRequired(false);
+ options.addOption(o);
+
+ CommandLineParser parser = new PosixParser();
+ CommandLine commandLine = null;
+ try {
+ commandLine = parser.parse(options, args);
+ } catch (Exception e) {
+ LOG.error("ERROR: " + e.getMessage() + "\n");
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp(NAME + " ", options, true);
+ System.exit(-1);
+ }
+
+ return commandLine;
+ }
+
+ private static void printUsage() {
+ StringBuilder usage = new StringBuilder("Command Usage: \n");
+ usage.append("TimelineSchemaCreator [-help] Display help info" +
+ " for all commands. Or\n");
+ usage.append("TimelineSchemaCreator -create [OPTIONAL_OPTIONS]" +
+ " Create hbase tables.\n\n");
+ usage.append("The Optional options for creating tables include: \n");
+ usage.append("[-entityTableName <Entity Table Name>] " +
+ "The name of the Entity table\n");
+ usage.append("[-entityMetricsTTL <Entity Table Metrics TTL>]" +
+ " TTL for metrics in the Entity table\n");
+ usage.append("[-appToflowTableName <AppToflow Table Name>]" +
+ " The name of the AppToFlow table\n");
+ usage.append("[-applicationTableName <Application Table Name>]" +
+ " The name of the Application table\n");
+ usage.append("[-applicationMetricsTTL <Application Table Metrics TTL>]" +
+ " TTL for metrics in the Application table\n");
+ usage.append("[-subApplicationTableName <SubApplication Table Name>]" +
+ " The name of the SubApplication table\n");
+ usage.append("[-subApplicationMetricsTTL " +
+ " <SubApplication Table Metrics TTL>]" +
+ " TTL for metrics in the SubApplication table\n");
+ usage.append("[-skipExistingTable] Whether to skip existing" +
+ " hbase tables\n");
+ System.out.println(usage.toString());
+ }
+
+ /**
+ * Create all table schemas and log success or exception if failed.
+ * @param hbaseConf the hbase configuration to create tables with
+ * @param skipExisting whether to skip existing hbase tables
+ */
+ private static void createAllSchemas(Configuration hbaseConf,
+ boolean skipExisting) {
+ List<Exception> exceptions = new ArrayList<>();
+ try {
+ if (skipExisting) {
+ LOG.info("Will skip existing tables and continue on htable creation "
+ + "exceptions!");
+ }
+ createAllTables(hbaseConf, skipExisting);
+ LOG.info("Successfully created HBase schema. ");
+ } catch (IOException e) {
+ LOG.error("Error in creating hbase tables: ", e);
+ exceptions.add(e);
+ }
+
+ if (exceptions.size() > 0) {
+ LOG.warn("Schema creation finished with the following exceptions");
+ for (Exception e : exceptions) {
+ LOG.warn(e.getMessage());
+ }
+ System.exit(-1);
+ } else {
+ LOG.info("Schema creation finished successfully");
+ }
+ }
+
+ @VisibleForTesting
+ public static void createAllTables(Configuration hbaseConf,
+ boolean skipExisting) throws IOException {
+
+ Connection conn = null;
+ try {
+ conn = ConnectionFactory.createConnection(hbaseConf);
+ Admin admin = conn.getAdmin();
+ if (admin == null) {
+ throw new IOException("Cannot create table since admin is null");
+ }
+ try {
+ new EntityTableRW().createTable(admin, hbaseConf);
+ } catch (IOException e) {
+ if (skipExisting) {
+ LOG.warn("Skip and continue on: " + e.getMessage());
+ } else {
+ throw e;
+ }
+ }
+ try {
+ new AppToFlowTableRW().createTable(admin, hbaseConf);
+ } catch (IOException e) {
+ if (skipExisting) {
+ LOG.warn("Skip and continue on: " + e.getMessage());
+ } else {
+ throw e;
+ }
+ }
+ try {
+ new ApplicationTableRW().createTable(admin, hbaseConf);
+ } catch (IOException e) {
+ if (skipExisting) {
+ LOG.warn("Skip and continue on: " + e.getMessage());
+ } else {
+ throw e;
+ }
+ }
+ try {
+ new FlowRunTableRW().createTable(admin, hbaseConf);
+ } catch (IOException e) {
+ if (skipExisting) {
+ LOG.warn("Skip and continue on: " + e.getMessage());
+ } else {
+ throw e;
+ }
+ }
+ try {
+ new FlowActivityTableRW().createTable(admin, hbaseConf);
+ } catch (IOException e) {
+ if (skipExisting) {
+ LOG.warn("Skip and continue on: " + e.getMessage());
+ } else {
+ throw e;
+ }
+ }
+ try {
+ new SubApplicationTableRW().createTable(admin, hbaseConf);
+ } catch (IOException e) {
+ if (skipExisting) {
+ LOG.warn("Skip and continue on: " + e.getMessage());
+ } else {
+ throw e;
+ }
+ }
+ } finally {
+ if (conn != null) {
+ conn.close();
+ }
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTableRW.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTableRW.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTableRW.java
new file mode 100644
index 0000000..808994e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTableRW.java
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.application;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTableRW;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBaseSchemaConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Create, read and write to the Application Table.
+ */
+public class ApplicationTableRW extends BaseTableRW<ApplicationTable> {
+ /** application prefix. */
+ private static final String PREFIX =
+ YarnConfiguration.TIMELINE_SERVICE_PREFIX + "application";
+
+ /** config param name that specifies the application table name. */
+ public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
+
+ /**
+ * config param name that specifies the TTL for metrics column family in
+ * application table.
+ */
+ private static final String METRICS_TTL_CONF_NAME = PREFIX
+ + ".table.metrics.ttl";
+
+ /**
+ * config param name that specifies max-versions for metrics column family in
+ * entity table.
+ */
+ private static final String METRICS_MAX_VERSIONS =
+ PREFIX + ".table.metrics.max-versions";
+
+ /** default value for application table name. */
+ private static final String DEFAULT_TABLE_NAME =
+ "timelineservice.application";
+
+ /** default TTL is 30 days for metrics timeseries. */
+ private static final int DEFAULT_METRICS_TTL = 2592000;
+
+ /** default max number of versions. */
+ private static final int DEFAULT_METRICS_MAX_VERSIONS = 10000;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ApplicationTableRW.class);
+
+ public ApplicationTableRW() {
+ super(TABLE_NAME_CONF_NAME, DEFAULT_TABLE_NAME);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.BaseTableRW#
+ * createTable(org.apache.hadoop.hbase.client.Admin,
+ * org.apache.hadoop.conf.Configuration)
+ */
+ public void createTable(Admin admin, Configuration hbaseConf)
+ throws IOException {
+
+ TableName table = getTableName(hbaseConf);
+ if (admin.tableExists(table)) {
+ // do not disable / delete existing table
+ // similar to the approach taken by map-reduce jobs when
+ // output directory exists
+ throw new IOException("Table " + table.getNameAsString()
+ + " already exists.");
+ }
+
+ HTableDescriptor applicationTableDescp = new HTableDescriptor(table);
+ HColumnDescriptor infoCF =
+ new HColumnDescriptor(ApplicationColumnFamily.INFO.getBytes());
+ infoCF.setBloomFilterType(BloomType.ROWCOL);
+ applicationTableDescp.addFamily(infoCF);
+
+ HColumnDescriptor configCF =
+ new HColumnDescriptor(ApplicationColumnFamily.CONFIGS.getBytes());
+ configCF.setBloomFilterType(BloomType.ROWCOL);
+ configCF.setBlockCacheEnabled(true);
+ applicationTableDescp.addFamily(configCF);
+
+ HColumnDescriptor metricsCF =
+ new HColumnDescriptor(ApplicationColumnFamily.METRICS.getBytes());
+ applicationTableDescp.addFamily(metricsCF);
+ metricsCF.setBlockCacheEnabled(true);
+ // always keep 1 version (the latest)
+ metricsCF.setMinVersions(1);
+ metricsCF.setMaxVersions(
+ hbaseConf.getInt(METRICS_MAX_VERSIONS, DEFAULT_METRICS_MAX_VERSIONS));
+ metricsCF.setTimeToLive(hbaseConf.getInt(METRICS_TTL_CONF_NAME,
+ DEFAULT_METRICS_TTL));
+ applicationTableDescp.setRegionSplitPolicyClassName(
+ "org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy");
+ applicationTableDescp.setValue("KeyPrefixRegionSplitPolicy.prefix_length",
+ TimelineHBaseSchemaConstants.USERNAME_SPLIT_KEY_PREFIX_LENGTH);
+ admin.createTable(applicationTableDescp,
+ TimelineHBaseSchemaConstants.getUsernameSplits());
+ LOG.info("Status of table creation for " + table.getNameAsString() + "="
+ + admin.tableExists(table));
+ }
+
+ /**
+ * @param metricsTTL time to live parameter for the metrics in this table.
+ * @param hbaseConf configuration in which to set the metrics TTL config
+ * variable.
+ */
+ public void setMetricsTTL(int metricsTTL, Configuration hbaseConf) {
+ hbaseConf.setInt(METRICS_TTL_CONF_NAME, metricsTTL);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/package-info.java
new file mode 100644
index 0000000..03f508f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.storage.application
+ * contains classes related to implementation for application table.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.application;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTableRW.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTableRW.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTableRW.java
new file mode 100644
index 0000000..6460203
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTableRW.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow;
+
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTableRW;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBaseSchemaConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+/**
+ * Create, read and write to the AppToFlow Table.
+ */
+public class AppToFlowTableRW extends BaseTableRW<AppToFlowTable> {
+ /** app_flow prefix. */
+ private static final String PREFIX =
+ YarnConfiguration.TIMELINE_SERVICE_PREFIX + "app-flow";
+
+ /** config param name that specifies the app_flow table name. */
+ public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
+
+ /** default value for app_flow table name. */
+ private static final String DEFAULT_TABLE_NAME = "timelineservice.app_flow";
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AppToFlowTableRW.class);
+
+ public AppToFlowTableRW() {
+ super(TABLE_NAME_CONF_NAME, DEFAULT_TABLE_NAME);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.BaseTableRW#
+ * createTable(org.apache.hadoop.hbase.client.Admin,
+ * org.apache.hadoop.conf.Configuration)
+ */
+ public void createTable(Admin admin, Configuration hbaseConf)
+ throws IOException {
+
+ TableName table = getTableName(hbaseConf);
+ if (admin.tableExists(table)) {
+ // do not disable / delete existing table
+ // similar to the approach taken by map-reduce jobs when
+ // output directory exists
+ throw new IOException("Table " + table.getNameAsString()
+ + " already exists.");
+ }
+
+ HTableDescriptor appToFlowTableDescp = new HTableDescriptor(table);
+ HColumnDescriptor mappCF =
+ new HColumnDescriptor(AppToFlowColumnFamily.MAPPING.getBytes());
+ mappCF.setBloomFilterType(BloomType.ROWCOL);
+ appToFlowTableDescp.addFamily(mappCF);
+
+ appToFlowTableDescp
+ .setRegionSplitPolicyClassName(
+ "org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy");
+ appToFlowTableDescp.setValue("KeyPrefixRegionSplitPolicy.prefix_length",
+ TimelineHBaseSchemaConstants.USERNAME_SPLIT_KEY_PREFIX_LENGTH);
+ admin.createTable(appToFlowTableDescp,
+ TimelineHBaseSchemaConstants.getUsernameSplits());
+ LOG.info("Status of table creation for " + table.getNameAsString() + "="
+ + admin.tableExists(table));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/package-info.java
new file mode 100644
index 0000000..f01d982
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow
+ * contains classes related to implementation for app to flow table.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTableRW.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTableRW.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTableRW.java
new file mode 100644
index 0000000..12ebce4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTableRW.java
@@ -0,0 +1,167 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
+/**
+ * Implements behavior common to tables used in the timeline service storage. It
+ * is thread-safe, and can be used by multiple threads concurrently.
+ *
+ * @param <T> reference to the table instance class itself for type safety.
+ */
+public abstract class BaseTableRW<T extends BaseTable<T>> {
+
+ /**
+ * Name of config variable that is used to point to this table.
+ */
+ private final String tableNameConfName;
+
+ /**
+ * Unless the configuration overrides, this will be the default name for the
+ * table when it is created.
+ */
+ private final String defaultTableName;
+
+ /**
+ * @param tableNameConfName name of config variable that is used to point to
+ * this table.
+ * @param defaultTableName Default table name if table from config is not
+ * found.
+ */
+ protected BaseTableRW(String tableNameConfName, String defaultTableName) {
+ this.tableNameConfName = tableNameConfName;
+ this.defaultTableName = defaultTableName;
+ }
+
+ /**
+ * Used to create a type-safe mutator for this table.
+ *
+ * @param hbaseConf used to read table name.
+ * @param conn used to create a table from.
+ * @return a type safe {@link BufferedMutator} for the entity table.
+ * @throws IOException if any exception occurs while creating mutator for the
+ * table.
+ */
+ public TypedBufferedMutator<T> getTableMutator(Configuration hbaseConf,
+ Connection conn) throws IOException {
+
+ TableName tableName = this.getTableName(hbaseConf);
+
+ // Plain buffered mutator
+ BufferedMutator bufferedMutator = conn.getBufferedMutator(tableName);
+
+ // Now make this thing type safe.
+ // This is how service initialization should hang on to this variable, with
+ // the proper type
+ TypedBufferedMutator<T> table =
+ new TypedBufferedMutator<T>(bufferedMutator);
+
+ return table;
+ }
+
+ /**
+ * @param hbaseConf used to read settings that override defaults
+ * @param conn used to create table from
+ * @param scan that specifies what you want to read from this table.
+ * @return scanner for the table.
+ * @throws IOException if any exception occurs while getting the scanner.
+ */
+ public ResultScanner getResultScanner(Configuration hbaseConf,
+ Connection conn, Scan scan) throws IOException {
+ Table table = conn.getTable(getTableName(hbaseConf));
+ return table.getScanner(scan);
+ }
+
+ /**
+ *
+ * @param hbaseConf used to read settings that override defaults
+ * @param conn used to create table from
+ * @param get that specifies what single row you want to get from this table
+ * @return result of get operation
+ * @throws IOException if any exception occurs while getting the result.
+ */
+ public Result getResult(Configuration hbaseConf, Connection conn, Get get)
+ throws IOException {
+ Table table = conn.getTable(getTableName(hbaseConf));
+ return table.get(get);
+ }
+
+ /**
+ * Get the table name for the input table.
+ *
+ * @param conf HBase configuration from which table name will be fetched.
+ * @param tableName name of the table to be fetched
+ * @return A {@link TableName} object.
+ */
+ public static TableName getTableName(Configuration conf, String tableName) {
+ String tableSchemaPrefix = conf.get(
+ YarnConfiguration.TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX_NAME,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX);
+ return TableName.valueOf(tableSchemaPrefix + tableName);
+ }
+
+ /**
+ * Get the table name for this table.
+ *
+ * @param conf HBase configuration from which table name will be fetched.
+ * @return A {@link TableName} object.
+ */
+ public TableName getTableName(Configuration conf) {
+ String tableName = conf.get(tableNameConfName, defaultTableName);
+ return getTableName(conf, tableName);
+ }
+
+ /**
+ * Get the table name based on the input config parameters.
+ *
+ * @param conf HBase configuration from which table name will be fetched.
+ * @param tableNameInConf the table name parameter in conf.
+ * @param defaultTableName the default table name.
+ * @return A {@link TableName} object.
+ */
+ public static TableName getTableName(Configuration conf,
+ String tableNameInConf, String defaultTableName) {
+ String tableName = conf.get(tableNameInConf, defaultTableName);
+ return getTableName(conf, tableName);
+ }
+
+ /**
+ * Used to create the table in HBase. Should be called only once (per HBase
+ * instance).
+ *
+ * @param admin Used for doing HBase table operations.
+ * @param hbaseConf Hbase configuration.
+ * @throws IOException if any exception occurs while creating the table.
+ */
+ public abstract void createTable(Admin admin, Configuration hbaseConf)
+ throws IOException;
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/18] hadoop git commit: YARN-7919. Refactor timelineservice-hbase
module into submodules. Contributed by Haibo Chen.
Posted by ro...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java
deleted file mode 100644
index f521cd7..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.NavigableMap;
-
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
-
-/**
- * Identifies partially qualified columns for the {@link FlowRunTable}.
- */
-public enum FlowRunColumnPrefix implements ColumnPrefix<FlowRunTable> {
-
- /**
- * To store flow run info values.
- */
- METRIC(FlowRunColumnFamily.INFO, "m", null, new LongConverter());
-
- private final ColumnHelper<FlowRunTable> column;
- private final ColumnFamily<FlowRunTable> columnFamily;
-
- /**
- * Can be null for those cases where the provided column qualifier is the
- * entire column name.
- */
- private final String columnPrefix;
- private final byte[] columnPrefixBytes;
-
- private final AggregationOperation aggOp;
-
- /**
- * Private constructor, meant to be used by the enum definition.
- *
- * @param columnFamily that this column is stored in.
- * @param columnPrefix for this column.
- */
- private FlowRunColumnPrefix(ColumnFamily<FlowRunTable> columnFamily,
- String columnPrefix, AggregationOperation fra, ValueConverter converter) {
- this(columnFamily, columnPrefix, fra, converter, false);
- }
-
- private FlowRunColumnPrefix(ColumnFamily<FlowRunTable> columnFamily,
- String columnPrefix, AggregationOperation fra, ValueConverter converter,
- boolean compoundColQual) {
- column = new ColumnHelper<FlowRunTable>(columnFamily, converter, true);
- this.columnFamily = columnFamily;
- this.columnPrefix = columnPrefix;
- if (columnPrefix == null) {
- this.columnPrefixBytes = null;
- } else {
- // Future-proof by ensuring the right column prefix hygiene.
- this.columnPrefixBytes =
- Bytes.toBytes(Separator.SPACE.encode(columnPrefix));
- }
- this.aggOp = fra;
- }
-
- /**
- * @return the column name value
- */
- public String getColumnPrefix() {
- return columnPrefix;
- }
-
- public byte[] getColumnPrefixBytes() {
- return columnPrefixBytes.clone();
- }
-
- @Override
- public byte[] getColumnPrefixBytes(byte[] qualifierPrefix) {
- return ColumnHelper.getColumnQualifier(this.columnPrefixBytes,
- qualifierPrefix);
- }
-
- @Override
- public byte[] getColumnPrefixBytes(String qualifierPrefix) {
- return ColumnHelper.getColumnQualifier(this.columnPrefixBytes,
- qualifierPrefix);
- }
-
- @Override
- public byte[] getColumnFamilyBytes() {
- return columnFamily.getBytes();
- }
-
- public AggregationOperation getAttribute() {
- return aggOp;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
- * #store(byte[],
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.
- * TypedBufferedMutator, java.lang.String, java.lang.Long, java.lang.Object)
- */
- public void store(byte[] rowKey,
- TypedBufferedMutator<FlowRunTable> tableMutator, String qualifier,
- Long timestamp, Object inputValue, Attribute... attributes)
- throws IOException {
-
- // Null check
- if (qualifier == null) {
- throw new IOException("Cannot store column with null qualifier in "
- + tableMutator.getName().getNameAsString());
- }
-
- byte[] columnQualifier = getColumnPrefixBytes(qualifier);
- Attribute[] combinedAttributes =
- HBaseTimelineStorageUtils.combineAttributes(attributes, this.aggOp);
- column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue,
- combinedAttributes);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
- * #store(byte[],
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.
- * TypedBufferedMutator, java.lang.String, java.lang.Long, java.lang.Object)
- */
- public void store(byte[] rowKey,
- TypedBufferedMutator<FlowRunTable> tableMutator, byte[] qualifier,
- Long timestamp, Object inputValue, Attribute... attributes)
- throws IOException {
-
- // Null check
- if (qualifier == null) {
- throw new IOException("Cannot store column with null qualifier in "
- + tableMutator.getName().getNameAsString());
- }
-
- byte[] columnQualifier = getColumnPrefixBytes(qualifier);
- Attribute[] combinedAttributes =
- HBaseTimelineStorageUtils.combineAttributes(attributes, this.aggOp);
- column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue,
- combinedAttributes);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
- * #readResult(org.apache.hadoop.hbase.client.Result, java.lang.String)
- */
- public Object readResult(Result result, String qualifier) throws IOException {
- byte[] columnQualifier =
- ColumnHelper.getColumnQualifier(this.columnPrefixBytes, qualifier);
- return column.readResult(result, columnQualifier);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
- * #readResults(org.apache.hadoop.hbase.client.Result,
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter)
- */
- public <K> Map<K, Object> readResults(Result result,
- KeyConverter<K> keyConverter) throws IOException {
- return column.readResults(result, columnPrefixBytes, keyConverter);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
- * #readResultsWithTimestamps(org.apache.hadoop.hbase.client.Result,
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter)
- */
- public <K, V> NavigableMap<K, NavigableMap<Long, V>>
- readResultsWithTimestamps(Result result, KeyConverter<K> keyConverter)
- throws IOException {
- return column.readResultsWithTimestamps(result, columnPrefixBytes,
- keyConverter);
- }
-
- @Override
- public ValueConverter getValueConverter() {
- return column.getValueConverter();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
deleted file mode 100644
index 96a7cf3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
+++ /dev/null
@@ -1,277 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.TreeMap;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.CoprocessorEnvironment;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.Tag;
-import org.apache.hadoop.hbase.client.Durability;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
-import org.apache.hadoop.hbase.coprocessor.ObserverContext;
-import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.Region;
-import org.apache.hadoop.hbase.regionserver.InternalScanner;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScanType;
-import org.apache.hadoop.hbase.regionserver.Store;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
-import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
-import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimestampGenerator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Coprocessor for flow run table.
- */
-public class FlowRunCoprocessor extends BaseRegionObserver {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(FlowRunCoprocessor.class);
-
- private Region region;
- /**
- * generate a timestamp that is unique per row in a region this is per region.
- */
- private final TimestampGenerator timestampGenerator =
- new TimestampGenerator();
-
- @Override
- public void start(CoprocessorEnvironment e) throws IOException {
- if (e instanceof RegionCoprocessorEnvironment) {
- RegionCoprocessorEnvironment env = (RegionCoprocessorEnvironment) e;
- this.region = env.getRegion();
- }
- }
-
- /*
- * (non-Javadoc)
- *
- * This method adds the tags onto the cells in the Put. It is presumed that
- * all the cells in one Put have the same set of Tags. The existing cell
- * timestamp is overwritten for non-metric cells and each such cell gets a new
- * unique timestamp generated by {@link TimestampGenerator}
- *
- * @see
- * org.apache.hadoop.hbase.coprocessor.BaseRegionObserver#prePut(org.apache
- * .hadoop.hbase.coprocessor.ObserverContext,
- * org.apache.hadoop.hbase.client.Put,
- * org.apache.hadoop.hbase.regionserver.wal.WALEdit,
- * org.apache.hadoop.hbase.client.Durability)
- */
- @Override
- public void prePut(ObserverContext<RegionCoprocessorEnvironment> e, Put put,
- WALEdit edit, Durability durability) throws IOException {
- Map<String, byte[]> attributes = put.getAttributesMap();
- // Assumption is that all the cells in a put are the same operation.
- List<Tag> tags = new ArrayList<>();
- if ((attributes != null) && (attributes.size() > 0)) {
- for (Map.Entry<String, byte[]> attribute : attributes.entrySet()) {
- Tag t = HBaseTimelineStorageUtils.getTagFromAttribute(attribute);
- if (t != null) {
- tags.add(t);
- }
- }
- byte[] tagByteArray = Tag.fromList(tags);
- NavigableMap<byte[], List<Cell>> newFamilyMap = new TreeMap<>(
- Bytes.BYTES_COMPARATOR);
- for (Map.Entry<byte[], List<Cell>> entry : put.getFamilyCellMap()
- .entrySet()) {
- List<Cell> newCells = new ArrayList<>(entry.getValue().size());
- for (Cell cell : entry.getValue()) {
- // for each cell in the put add the tags
- // Assumption is that all the cells in
- // one put are the same operation
- // also, get a unique cell timestamp for non-metric cells
- // this way we don't inadvertently overwrite cell versions
- long cellTimestamp = getCellTimestamp(cell.getTimestamp(), tags);
- newCells.add(CellUtil.createCell(CellUtil.cloneRow(cell),
- CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell),
- cellTimestamp, KeyValue.Type.Put, CellUtil.cloneValue(cell),
- tagByteArray));
- }
- newFamilyMap.put(entry.getKey(), newCells);
- } // for each entry
- // Update the family map for the Put
- put.setFamilyCellMap(newFamilyMap);
- }
- }
-
- /**
- * Determines if the current cell's timestamp is to be used or a new unique
- * cell timestamp is to be used. The reason this is done is to inadvertently
- * overwrite cells when writes come in very fast. But for metric cells, the
- * cell timestamp signifies the metric timestamp. Hence we don't want to
- * overwrite it.
- *
- * @param timestamp
- * @param tags
- * @return cell timestamp
- */
- private long getCellTimestamp(long timestamp, List<Tag> tags) {
- // if ts not set (hbase sets to HConstants.LATEST_TIMESTAMP by default)
- // then use the generator
- if (timestamp == HConstants.LATEST_TIMESTAMP) {
- return timestampGenerator.getUniqueTimestamp();
- } else {
- return timestamp;
- }
- }
-
- /*
- * (non-Javadoc)
- *
- * Creates a {@link FlowScanner} Scan so that it can correctly process the
- * contents of {@link FlowRunTable}.
- *
- * @see
- * org.apache.hadoop.hbase.coprocessor.BaseRegionObserver#preGetOp(org.apache
- * .hadoop.hbase.coprocessor.ObserverContext,
- * org.apache.hadoop.hbase.client.Get, java.util.List)
- */
- @Override
- public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> e,
- Get get, List<Cell> results) throws IOException {
- Scan scan = new Scan(get);
- scan.setMaxVersions();
- RegionScanner scanner = null;
- try {
- scanner = new FlowScanner(e.getEnvironment(), scan,
- region.getScanner(scan), FlowScannerOperation.READ);
- scanner.next(results);
- e.bypass();
- } finally {
- if (scanner != null) {
- scanner.close();
- }
- }
- }
-
- /*
- * (non-Javadoc)
- *
- * Ensures that max versions are set for the Scan so that metrics can be
- * correctly aggregated and min/max can be correctly determined.
- *
- * @see
- * org.apache.hadoop.hbase.coprocessor.BaseRegionObserver#preScannerOpen(org
- * .apache.hadoop.hbase.coprocessor.ObserverContext,
- * org.apache.hadoop.hbase.client.Scan,
- * org.apache.hadoop.hbase.regionserver.RegionScanner)
- */
- @Override
- public RegionScanner preScannerOpen(
- ObserverContext<RegionCoprocessorEnvironment> e, Scan scan,
- RegionScanner scanner) throws IOException {
- // set max versions for scan to see all
- // versions to aggregate for metrics
- scan.setMaxVersions();
- return scanner;
- }
-
- /*
- * (non-Javadoc)
- *
- * Creates a {@link FlowScanner} Scan so that it can correctly process the
- * contents of {@link FlowRunTable}.
- *
- * @see
- * org.apache.hadoop.hbase.coprocessor.BaseRegionObserver#postScannerOpen(
- * org.apache.hadoop.hbase.coprocessor.ObserverContext,
- * org.apache.hadoop.hbase.client.Scan,
- * org.apache.hadoop.hbase.regionserver.RegionScanner)
- */
- @Override
- public RegionScanner postScannerOpen(
- ObserverContext<RegionCoprocessorEnvironment> e, Scan scan,
- RegionScanner scanner) throws IOException {
- return new FlowScanner(e.getEnvironment(), scan,
- scanner, FlowScannerOperation.READ);
- }
-
- @Override
- public InternalScanner preFlush(
- ObserverContext<RegionCoprocessorEnvironment> c, Store store,
- InternalScanner scanner) throws IOException {
- if (LOG.isDebugEnabled()) {
- if (store != null) {
- LOG.debug("preFlush store = " + store.getColumnFamilyName()
- + " flushableSize=" + store.getFlushableSize()
- + " flushedCellsCount=" + store.getFlushedCellsCount()
- + " compactedCellsCount=" + store.getCompactedCellsCount()
- + " majorCompactedCellsCount="
- + store.getMajorCompactedCellsCount() + " memstoreFlushSize="
- + store.getMemstoreFlushSize() + " memstoreSize="
- + store.getMemStoreSize() + " size=" + store.getSize()
- + " storeFilesCount=" + store.getStorefilesCount());
- }
- }
- return new FlowScanner(c.getEnvironment(), scanner,
- FlowScannerOperation.FLUSH);
- }
-
- @Override
- public void postFlush(ObserverContext<RegionCoprocessorEnvironment> c,
- Store store, StoreFile resultFile) {
- if (LOG.isDebugEnabled()) {
- if (store != null) {
- LOG.debug("postFlush store = " + store.getColumnFamilyName()
- + " flushableSize=" + store.getFlushableSize()
- + " flushedCellsCount=" + store.getFlushedCellsCount()
- + " compactedCellsCount=" + store.getCompactedCellsCount()
- + " majorCompactedCellsCount="
- + store.getMajorCompactedCellsCount() + " memstoreFlushSize="
- + store.getMemstoreFlushSize() + " memstoreSize="
- + store.getMemStoreSize() + " size=" + store.getSize()
- + " storeFilesCount=" + store.getStorefilesCount());
- }
- }
- }
-
- @Override
- public InternalScanner preCompact(
- ObserverContext<RegionCoprocessorEnvironment> e, Store store,
- InternalScanner scanner, ScanType scanType, CompactionRequest request)
- throws IOException {
-
- FlowScannerOperation requestOp = FlowScannerOperation.MINOR_COMPACTION;
- if (request != null) {
- requestOp = (request.isMajor() ? FlowScannerOperation.MAJOR_COMPACTION
- : FlowScannerOperation.MINOR_COMPACTION);
- LOG.info("Compactionrequest= " + request.toString() + " "
- + requestOp.toString() + " RegionName=" + e.getEnvironment()
- .getRegion().getRegionInfo().getRegionNameAsString());
- }
- return new FlowScanner(e.getEnvironment(), scanner, requestOp);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
deleted file mode 100644
index 7ce91cf..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
-
-import java.util.List;
-
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverterToString;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-
-/**
- * Represents a rowkey for the flow run table.
- */
-public class FlowRunRowKey {
- private final String clusterId;
- private final String userId;
- private final String flowName;
- private final Long flowRunId;
- private final FlowRunRowKeyConverter flowRunRowKeyConverter =
- new FlowRunRowKeyConverter();
-
- public FlowRunRowKey(String clusterId, String userId, String flowName,
- Long flowRunId) {
- this.clusterId = clusterId;
- this.userId = userId;
- this.flowName = flowName;
- this.flowRunId = flowRunId;
- }
-
- public String getClusterId() {
- return clusterId;
- }
-
- public String getUserId() {
- return userId;
- }
-
- public String getFlowName() {
- return flowName;
- }
-
- public Long getFlowRunId() {
- return flowRunId;
- }
-
- /**
- * Constructs a row key for the entity table as follows: {
- * clusterId!userId!flowName!Inverted Flow Run Id}.
- *
- * @return byte array with the row key
- */
- public byte[] getRowKey() {
- return flowRunRowKeyConverter.encode(this);
- }
-
-
- /**
- * Given the raw row key as bytes, returns the row key as an object.
- * @param rowKey Byte representation of row key.
- * @return A <cite>FlowRunRowKey</cite> object.
- */
- public static FlowRunRowKey parseRowKey(byte[] rowKey) {
- return new FlowRunRowKeyConverter().decode(rowKey);
- }
-
- /**
- * Constructs a row key for the flow run table as follows:
- * {@code clusterId!userId!flowName!Flow Run Id}.
- * @return String representation of row key
- */
- public String getRowKeyAsString() {
- return flowRunRowKeyConverter.encodeAsString(this);
- }
-
- /**
- * Given the encoded row key as string, returns the row key as an object.
- * @param encodedRowKey String representation of row key.
- * @return A <cite>FlowRunRowKey</cite> object.
- */
- public static FlowRunRowKey parseRowKeyFromString(String encodedRowKey) {
- return new FlowRunRowKeyConverter().decodeFromString(encodedRowKey);
- }
-
- /**
- * returns the Flow Key as a verbose String output.
- * @return String
- */
- @Override
- public String toString() {
- StringBuilder flowKeyStr = new StringBuilder();
- flowKeyStr.append("{clusterId=" + clusterId);
- flowKeyStr.append(" userId=" + userId);
- flowKeyStr.append(" flowName=" + flowName);
- flowKeyStr.append(" flowRunId=");
- flowKeyStr.append(flowRunId);
- flowKeyStr.append("}");
- return flowKeyStr.toString();
- }
-
- /**
- * Encodes and decodes row key for flow run table.
- * The row key is of the form : clusterId!userId!flowName!flowrunId.
- * flowrunId is a long and rest are strings.
- * <p>
- */
- final private static class FlowRunRowKeyConverter implements
- KeyConverter<FlowRunRowKey>, KeyConverterToString<FlowRunRowKey> {
-
- private FlowRunRowKeyConverter() {
- }
-
- /**
- * The flow run row key is of the form clusterId!userId!flowName!flowrunId
- * with each segment separated by !. The sizes below indicate sizes of each
- * one of these segments in sequence. clusterId, userId and flowName are
- * strings. flowrunId is a long hence 8 bytes in size. Strings are variable
- * in size (i.e. end whenever separator is encountered). This is used while
- * decoding and helps in determining where to split.
- */
- private static final int[] SEGMENT_SIZES = {Separator.VARIABLE_SIZE,
- Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG };
-
- /*
- * (non-Javadoc)
- *
- * Encodes FlowRunRowKey object into a byte array with each component/field
- * in FlowRunRowKey separated by Separator#QUALIFIERS. This leads to an flow
- * run row key of the form clusterId!userId!flowName!flowrunId If flowRunId
- * in passed FlowRunRowKey object is null (and the fields preceding it i.e.
- * clusterId, userId and flowName are not null), this returns a row key
- * prefix of the form clusterId!userName!flowName! flowRunId is inverted
- * while encoding as it helps maintain a descending order for flow keys in
- * flow run table.
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common
- * .KeyConverter#encode(java.lang.Object)
- */
- @Override
- public byte[] encode(FlowRunRowKey rowKey) {
- byte[] first =
- Separator.QUALIFIERS.join(Separator.encode(rowKey.getClusterId(),
- Separator.SPACE, Separator.TAB, Separator.QUALIFIERS), Separator
- .encode(rowKey.getUserId(), Separator.SPACE, Separator.TAB,
- Separator.QUALIFIERS), Separator.encode(rowKey.getFlowName(),
- Separator.SPACE, Separator.TAB, Separator.QUALIFIERS));
- if (rowKey.getFlowRunId() == null) {
- return Separator.QUALIFIERS.join(first, Separator.EMPTY_BYTES);
- } else {
- // Note that flowRunId is a long, so we can't encode them all at the
- // same
- // time.
- byte[] second =
- Bytes.toBytes(LongConverter.invertLong(rowKey.getFlowRunId()));
- return Separator.QUALIFIERS.join(first, second);
- }
- }
-
- /*
- * (non-Javadoc)
- *
- * Decodes an flow run row key of the form
- * clusterId!userId!flowName!flowrunId represented in byte format and
- * converts it into an FlowRunRowKey object. flowRunId is inverted while
- * decoding as it was inverted while encoding.
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common
- * .KeyConverter#decode(byte[])
- */
- @Override
- public FlowRunRowKey decode(byte[] rowKey) {
- byte[][] rowKeyComponents =
- Separator.QUALIFIERS.split(rowKey, SEGMENT_SIZES);
- if (rowKeyComponents.length != 4) {
- throw new IllegalArgumentException("the row key is not valid for "
- + "a flow run");
- }
- String clusterId =
- Separator.decode(Bytes.toString(rowKeyComponents[0]),
- Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
- String userId =
- Separator.decode(Bytes.toString(rowKeyComponents[1]),
- Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
- String flowName =
- Separator.decode(Bytes.toString(rowKeyComponents[2]),
- Separator.QUALIFIERS, Separator.TAB, Separator.SPACE);
- Long flowRunId =
- LongConverter.invertLong(Bytes.toLong(rowKeyComponents[3]));
- return new FlowRunRowKey(clusterId, userId, flowName, flowRunId);
- }
-
- @Override
- public String encodeAsString(FlowRunRowKey key) {
- if (key.clusterId == null || key.userId == null || key.flowName == null
- || key.flowRunId == null) {
- throw new IllegalArgumentException();
- }
- return TimelineReaderUtils.joinAndEscapeStrings(new String[] {
- key.clusterId, key.userId, key.flowName, key.flowRunId.toString()});
- }
-
- @Override
- public FlowRunRowKey decodeFromString(String encodedRowKey) {
- List<String> split = TimelineReaderUtils.split(encodedRowKey);
- if (split == null || split.size() != 4) {
- throw new IllegalArgumentException(
- "Invalid row key for flow run table.");
- }
- Long flowRunId = Long.valueOf(split.get(3));
- return new FlowRunRowKey(split.get(0), split.get(1), split.get(2),
- flowRunId);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKeyPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKeyPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKeyPrefix.java
deleted file mode 100644
index 23ebc66..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKeyPrefix.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
-
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.RowKeyPrefix;
-
-/**
- * Represents a partial rowkey (without the flowRunId) for the flow run table.
- */
-public class FlowRunRowKeyPrefix extends FlowRunRowKey implements
- RowKeyPrefix<FlowRunRowKey> {
-
- /**
- * Constructs a row key prefix for the flow run table as follows:
- * {@code clusterId!userI!flowName!}.
- *
- * @param clusterId identifying the cluster
- * @param userId identifying the user
- * @param flowName identifying the flow
- */
- public FlowRunRowKeyPrefix(String clusterId, String userId,
- String flowName) {
- super(clusterId, userId, flowName, null);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.application.
- * RowKeyPrefix#getRowKeyPrefix()
- */
- public byte[] getRowKeyPrefix() {
- // We know we're a FlowRunRowKey with null florRunId, so we can simply
- // delegate
- return super.getRowKey();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java
deleted file mode 100644
index a1d32ee..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.regionserver.BloomType;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Coprocessor;
-
-/**
- * The flow run table has column family info
- * Stores per flow run information
- * aggregated across applications.
- *
- * Metrics are also stored in the info column family.
- *
- * Example flow run table record:
- *
- * <pre>
- * flow_run table
- * |-------------------------------------------|
- * | Row key | Column Family |
- * | | info |
- * |-------------------------------------------|
- * | clusterId! | flow_version:version7 |
- * | userName! | |
- * | flowName! | running_apps:1 |
- * | flowRunId | |
- * | | min_start_time:1392995080000 |
- * | | #0:"" |
- * | | |
- * | | min_start_time:1392995081012 |
- * | | #0:appId2 |
- * | | |
- * | | min_start_time:1392993083210 |
- * | | #0:appId3 |
- * | | |
- * | | |
- * | | max_end_time:1392993084018 |
- * | | #0:"" |
- * | | |
- * | | |
- * | | m!mapInputRecords:127 |
- * | | #0:"" |
- * | | |
- * | | m!mapInputRecords:31 |
- * | | #2:appId2 |
- * | | |
- * | | m!mapInputRecords:37 |
- * | | #1:appId3 |
- * | | |
- * | | |
- * | | m!mapOutputRecords:181 |
- * | | #0:"" |
- * | | |
- * | | m!mapOutputRecords:37 |
- * | | #1:appId3 |
- * | | |
- * | | |
- * |-------------------------------------------|
- * </pre>
- */
-public class FlowRunTable extends BaseTable<FlowRunTable> {
- /** entity prefix. */
- private static final String PREFIX =
- YarnConfiguration.TIMELINE_SERVICE_PREFIX + ".flowrun";
-
- /** config param name that specifies the flowrun table name. */
- public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
-
- /** default value for flowrun table name. */
- public static final String DEFAULT_TABLE_NAME = "timelineservice.flowrun";
-
- private static final Logger LOG =
- LoggerFactory.getLogger(FlowRunTable.class);
-
- /** default max number of versions. */
- public static final int DEFAULT_METRICS_MAX_VERSIONS = Integer.MAX_VALUE;
-
- public FlowRunTable() {
- super(TABLE_NAME_CONF_NAME, DEFAULT_TABLE_NAME);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.BaseTable#createTable
- * (org.apache.hadoop.hbase.client.Admin,
- * org.apache.hadoop.conf.Configuration)
- */
- public void createTable(Admin admin, Configuration hbaseConf)
- throws IOException {
-
- TableName table = getTableName(hbaseConf);
- if (admin.tableExists(table)) {
- // do not disable / delete existing table
- // similar to the approach taken by map-reduce jobs when
- // output directory exists
- throw new IOException("Table " + table.getNameAsString()
- + " already exists.");
- }
-
- HTableDescriptor flowRunTableDescp = new HTableDescriptor(table);
- HColumnDescriptor infoCF =
- new HColumnDescriptor(FlowRunColumnFamily.INFO.getBytes());
- infoCF.setBloomFilterType(BloomType.ROWCOL);
- flowRunTableDescp.addFamily(infoCF);
- infoCF.setMinVersions(1);
- infoCF.setMaxVersions(DEFAULT_METRICS_MAX_VERSIONS);
-
- // TODO: figure the split policy
- String coprocessorJarPathStr = hbaseConf.get(
- YarnConfiguration.FLOW_RUN_COPROCESSOR_JAR_HDFS_LOCATION,
- YarnConfiguration.DEFAULT_HDFS_LOCATION_FLOW_RUN_COPROCESSOR_JAR);
-
- Path coprocessorJarPath = new Path(coprocessorJarPathStr);
- LOG.info("CoprocessorJarPath=" + coprocessorJarPath.toString());
- flowRunTableDescp.addCoprocessor(
- FlowRunCoprocessor.class.getCanonicalName(), coprocessorJarPath,
- Coprocessor.PRIORITY_USER, null);
- admin.createTable(flowRunTableDescp);
- LOG.info("Status of table creation for " + table.getNameAsString() + "="
- + admin.tableExists(table));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
deleted file mode 100644
index dbd0484..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
+++ /dev/null
@@ -1,729 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.Tag;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.InternalScanner;
-import org.apache.hadoop.hbase.regionserver.Region;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScannerContext;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.GenericConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.NumericValueConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimestampGenerator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Invoked via the coprocessor when a Get or a Scan is issued for flow run
- * table. Looks through the list of cells per row, checks their tags and does
- * operation on those cells as per the cell tags. Transforms reads of the stored
- * metrics into calculated sums for each column Also, finds the min and max for
- * start and end times in a flow run.
- */
-class FlowScanner implements RegionScanner, Closeable {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(FlowScanner.class);
-
- /**
- * use a special application id to represent the flow id this is needed since
- * TimestampGenerator parses the app id to generate a cell timestamp.
- */
- private static final String FLOW_APP_ID = "application_00000000000_0000";
-
- private final Region region;
- private final InternalScanner flowRunScanner;
- private final int batchSize;
- private final long appFinalValueRetentionThreshold;
- private RegionScanner regionScanner;
- private boolean hasMore;
- private byte[] currentRow;
- private List<Cell> availableCells = new ArrayList<>();
- private int currentIndex;
- private FlowScannerOperation action = FlowScannerOperation.READ;
-
- FlowScanner(RegionCoprocessorEnvironment env, InternalScanner internalScanner,
- FlowScannerOperation action) {
- this(env, null, internalScanner, action);
- }
-
- FlowScanner(RegionCoprocessorEnvironment env, Scan incomingScan,
- InternalScanner internalScanner, FlowScannerOperation action) {
- this.batchSize = incomingScan == null ? -1 : incomingScan.getBatch();
- // TODO initialize other scan attributes like Scan#maxResultSize
- this.flowRunScanner = internalScanner;
- if (internalScanner instanceof RegionScanner) {
- this.regionScanner = (RegionScanner) internalScanner;
- }
- this.action = action;
- if (env == null) {
- this.appFinalValueRetentionThreshold =
- YarnConfiguration.DEFAULT_APP_FINAL_VALUE_RETENTION_THRESHOLD;
- this.region = null;
- } else {
- this.region = env.getRegion();
- Configuration hbaseConf = env.getConfiguration();
- this.appFinalValueRetentionThreshold = hbaseConf.getLong(
- YarnConfiguration.APP_FINAL_VALUE_RETENTION_THRESHOLD,
- YarnConfiguration.DEFAULT_APP_FINAL_VALUE_RETENTION_THRESHOLD);
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug(" batch size=" + batchSize);
- }
- }
-
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.hadoop.hbase.regionserver.RegionScanner#getRegionInfo()
- */
- @Override
- public HRegionInfo getRegionInfo() {
- return region.getRegionInfo();
- }
-
- @Override
- public boolean nextRaw(List<Cell> cells) throws IOException {
- return nextRaw(cells, ScannerContext.newBuilder().build());
- }
-
- @Override
- public boolean nextRaw(List<Cell> cells, ScannerContext scannerContext)
- throws IOException {
- return nextInternal(cells, scannerContext);
- }
-
- @Override
- public boolean next(List<Cell> cells) throws IOException {
- return next(cells, ScannerContext.newBuilder().build());
- }
-
- @Override
- public boolean next(List<Cell> cells, ScannerContext scannerContext)
- throws IOException {
- return nextInternal(cells, scannerContext);
- }
-
- /**
- * Get value converter associated with a column or a column prefix. If nothing
- * matches, generic converter is returned.
- * @param colQualifierBytes
- * @return value converter implementation.
- */
- private static ValueConverter getValueConverter(byte[] colQualifierBytes) {
- // Iterate over all the column prefixes for flow run table and get the
- // appropriate converter for the column qualifier passed if prefix matches.
- for (FlowRunColumnPrefix colPrefix : FlowRunColumnPrefix.values()) {
- byte[] colPrefixBytes = colPrefix.getColumnPrefixBytes("");
- if (Bytes.compareTo(colPrefixBytes, 0, colPrefixBytes.length,
- colQualifierBytes, 0, colPrefixBytes.length) == 0) {
- return colPrefix.getValueConverter();
- }
- }
- // Iterate over all the columns for flow run table and get the
- // appropriate converter for the column qualifier passed if match occurs.
- for (FlowRunColumn column : FlowRunColumn.values()) {
- if (Bytes.compareTo(
- column.getColumnQualifierBytes(), colQualifierBytes) == 0) {
- return column.getValueConverter();
- }
- }
- // Return generic converter if nothing matches.
- return GenericConverter.getInstance();
- }
-
- /**
- * This method loops through the cells in a given row of the
- * {@link FlowRunTable}. It looks at the tags of each cell to figure out how
- * to process the contents. It then calculates the sum or min or max for each
- * column or returns the cell as is.
- *
- * @param cells
- * @param scannerContext
- * @return true if next row is available for the scanner, false otherwise
- * @throws IOException
- */
- private boolean nextInternal(List<Cell> cells, ScannerContext scannerContext)
- throws IOException {
- Cell cell = null;
- startNext();
- // Loop through all the cells in this row
- // For min/max/metrics we do need to scan the entire set of cells to get the
- // right one
- // But with flush/compaction, the number of cells being scanned will go down
- // cells are grouped per column qualifier then sorted by cell timestamp
- // (latest to oldest) per column qualifier
- // So all cells in one qualifier come one after the other before we see the
- // next column qualifier
- ByteArrayComparator comp = new ByteArrayComparator();
- byte[] previousColumnQualifier = Separator.EMPTY_BYTES;
- AggregationOperation currentAggOp = null;
- SortedSet<Cell> currentColumnCells = new TreeSet<>(KeyValue.COMPARATOR);
- Set<String> alreadySeenAggDim = new HashSet<>();
- int addedCnt = 0;
- long currentTimestamp = System.currentTimeMillis();
- ValueConverter converter = null;
- int limit = batchSize;
-
- while (limit <= 0 || addedCnt < limit) {
- cell = peekAtNextCell(scannerContext);
- if (cell == null) {
- break;
- }
- byte[] currentColumnQualifier = CellUtil.cloneQualifier(cell);
- if (previousColumnQualifier == null) {
- // first time in loop
- previousColumnQualifier = currentColumnQualifier;
- }
-
- converter = getValueConverter(currentColumnQualifier);
- if (comp.compare(previousColumnQualifier, currentColumnQualifier) != 0) {
- addedCnt += emitCells(cells, currentColumnCells, currentAggOp,
- converter, currentTimestamp);
- resetState(currentColumnCells, alreadySeenAggDim);
- previousColumnQualifier = currentColumnQualifier;
- currentAggOp = getCurrentAggOp(cell);
- converter = getValueConverter(currentColumnQualifier);
- }
- collectCells(currentColumnCells, currentAggOp, cell, alreadySeenAggDim,
- converter, scannerContext);
- nextCell(scannerContext);
- }
- if ((!currentColumnCells.isEmpty()) && ((limit <= 0 || addedCnt < limit))) {
- addedCnt += emitCells(cells, currentColumnCells, currentAggOp, converter,
- currentTimestamp);
- if (LOG.isDebugEnabled()) {
- if (addedCnt > 0) {
- LOG.debug("emitted cells. " + addedCnt + " for " + this.action
- + " rowKey="
- + FlowRunRowKey.parseRowKey(CellUtil.cloneRow(cells.get(0))));
- } else {
- LOG.debug("emitted no cells for " + this.action);
- }
- }
- }
- return hasMore();
- }
-
- private AggregationOperation getCurrentAggOp(Cell cell) {
- List<Tag> tags = Tag.asList(cell.getTagsArray(), cell.getTagsOffset(),
- cell.getTagsLength());
- // We assume that all the operations for a particular column are the same
- return HBaseTimelineStorageUtils.getAggregationOperationFromTagsList(tags);
- }
-
- /**
- * resets the parameters to an initialized state for next loop iteration.
- *
- * @param cell
- * @param currentAggOp
- * @param currentColumnCells
- * @param alreadySeenAggDim
- * @param collectedButNotEmitted
- */
- private void resetState(SortedSet<Cell> currentColumnCells,
- Set<String> alreadySeenAggDim) {
- currentColumnCells.clear();
- alreadySeenAggDim.clear();
- }
-
- private void collectCells(SortedSet<Cell> currentColumnCells,
- AggregationOperation currentAggOp, Cell cell,
- Set<String> alreadySeenAggDim, ValueConverter converter,
- ScannerContext scannerContext) throws IOException {
-
- if (currentAggOp == null) {
- // not a min/max/metric cell, so just return it as is
- currentColumnCells.add(cell);
- return;
- }
-
- switch (currentAggOp) {
- case GLOBAL_MIN:
- if (currentColumnCells.size() == 0) {
- currentColumnCells.add(cell);
- } else {
- Cell currentMinCell = currentColumnCells.first();
- Cell newMinCell = compareCellValues(currentMinCell, cell, currentAggOp,
- (NumericValueConverter) converter);
- if (!currentMinCell.equals(newMinCell)) {
- currentColumnCells.remove(currentMinCell);
- currentColumnCells.add(newMinCell);
- }
- }
- break;
- case GLOBAL_MAX:
- if (currentColumnCells.size() == 0) {
- currentColumnCells.add(cell);
- } else {
- Cell currentMaxCell = currentColumnCells.first();
- Cell newMaxCell = compareCellValues(currentMaxCell, cell, currentAggOp,
- (NumericValueConverter) converter);
- if (!currentMaxCell.equals(newMaxCell)) {
- currentColumnCells.remove(currentMaxCell);
- currentColumnCells.add(newMaxCell);
- }
- }
- break;
- case SUM:
- case SUM_FINAL:
- if (LOG.isTraceEnabled()) {
- LOG.trace("In collect cells "
- + " FlowSannerOperation="
- + this.action
- + " currentAggOp="
- + currentAggOp
- + " cell qualifier="
- + Bytes.toString(CellUtil.cloneQualifier(cell))
- + " cell value= "
- + converter.decodeValue(CellUtil.cloneValue(cell))
- + " timestamp=" + cell.getTimestamp());
- }
-
- // only if this app has not been seen yet, add to current column cells
- List<Tag> tags = Tag.asList(cell.getTagsArray(), cell.getTagsOffset(),
- cell.getTagsLength());
- String aggDim = HBaseTimelineStorageUtils
- .getAggregationCompactionDimension(tags);
- if (!alreadySeenAggDim.contains(aggDim)) {
- // if this agg dimension has already been seen,
- // since they show up in sorted order
- // we drop the rest which are older
- // in other words, this cell is older than previously seen cells
- // for that agg dim
- // but when this agg dim is not seen,
- // consider this cell in our working set
- currentColumnCells.add(cell);
- alreadySeenAggDim.add(aggDim);
- }
- break;
- default:
- break;
- } // end of switch case
- }
-
- /*
- * Processes the cells in input param currentColumnCells and populates
- * List<Cell> cells as the output based on the input AggregationOperation
- * parameter.
- */
- private int emitCells(List<Cell> cells, SortedSet<Cell> currentColumnCells,
- AggregationOperation currentAggOp, ValueConverter converter,
- long currentTimestamp) throws IOException {
- if ((currentColumnCells == null) || (currentColumnCells.size() == 0)) {
- return 0;
- }
- if (currentAggOp == null) {
- cells.addAll(currentColumnCells);
- return currentColumnCells.size();
- }
- if (LOG.isTraceEnabled()) {
- LOG.trace("In emitCells " + this.action + " currentColumnCells size= "
- + currentColumnCells.size() + " currentAggOp" + currentAggOp);
- }
-
- switch (currentAggOp) {
- case GLOBAL_MIN:
- case GLOBAL_MAX:
- cells.addAll(currentColumnCells);
- return currentColumnCells.size();
- case SUM:
- case SUM_FINAL:
- switch (action) {
- case FLUSH:
- case MINOR_COMPACTION:
- cells.addAll(currentColumnCells);
- return currentColumnCells.size();
- case READ:
- Cell sumCell = processSummation(currentColumnCells,
- (NumericValueConverter) converter);
- cells.add(sumCell);
- return 1;
- case MAJOR_COMPACTION:
- List<Cell> finalCells = processSummationMajorCompaction(
- currentColumnCells, (NumericValueConverter) converter,
- currentTimestamp);
- cells.addAll(finalCells);
- return finalCells.size();
- default:
- cells.addAll(currentColumnCells);
- return currentColumnCells.size();
- }
- default:
- cells.addAll(currentColumnCells);
- return currentColumnCells.size();
- }
- }
-
- /*
- * Returns a cell whose value is the sum of all cell values in the input set.
- * The new cell created has the timestamp of the most recent metric cell. The
- * sum of a metric for a flow run is the summation at the point of the last
- * metric update in that flow till that time.
- */
- private Cell processSummation(SortedSet<Cell> currentColumnCells,
- NumericValueConverter converter) throws IOException {
- Number sum = 0;
- Number currentValue = 0;
- long ts = 0L;
- long mostCurrentTimestamp = 0L;
- Cell mostRecentCell = null;
- for (Cell cell : currentColumnCells) {
- currentValue = (Number) converter.decodeValue(CellUtil.cloneValue(cell));
- ts = cell.getTimestamp();
- if (mostCurrentTimestamp < ts) {
- mostCurrentTimestamp = ts;
- mostRecentCell = cell;
- }
- sum = converter.add(sum, currentValue);
- }
- byte[] sumBytes = converter.encodeValue(sum);
- Cell sumCell =
- HBaseTimelineStorageUtils.createNewCell(mostRecentCell, sumBytes);
- return sumCell;
- }
-
-
- /**
- * Returns a list of cells that contains
- *
- * A) the latest cells for applications that haven't finished yet
- * B) summation
- * for the flow, based on applications that have completed and are older than
- * a certain time
- *
- * The new cell created has the timestamp of the most recent metric cell. The
- * sum of a metric for a flow run is the summation at the point of the last
- * metric update in that flow till that time.
- */
- @VisibleForTesting
- List<Cell> processSummationMajorCompaction(
- SortedSet<Cell> currentColumnCells, NumericValueConverter converter,
- long currentTimestamp)
- throws IOException {
- Number sum = 0;
- Number currentValue = 0;
- long ts = 0L;
- boolean summationDone = false;
- List<Cell> finalCells = new ArrayList<Cell>();
- if (currentColumnCells == null) {
- return finalCells;
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("In processSummationMajorCompaction,"
- + " will drop cells older than " + currentTimestamp
- + " CurrentColumnCells size=" + currentColumnCells.size());
- }
-
- for (Cell cell : currentColumnCells) {
- AggregationOperation cellAggOp = getCurrentAggOp(cell);
- // if this is the existing flow sum cell
- List<Tag> tags = Tag.asList(cell.getTagsArray(), cell.getTagsOffset(),
- cell.getTagsLength());
- String appId = HBaseTimelineStorageUtils
- .getAggregationCompactionDimension(tags);
- if (appId == FLOW_APP_ID) {
- sum = converter.add(sum, currentValue);
- summationDone = true;
- if (LOG.isTraceEnabled()) {
- LOG.trace("reading flow app id sum=" + sum);
- }
- } else {
- currentValue = (Number) converter.decodeValue(CellUtil
- .cloneValue(cell));
- // read the timestamp truncated by the generator
- ts = TimestampGenerator.getTruncatedTimestamp(cell.getTimestamp());
- if ((cellAggOp == AggregationOperation.SUM_FINAL)
- && ((ts + this.appFinalValueRetentionThreshold)
- < currentTimestamp)) {
- sum = converter.add(sum, currentValue);
- summationDone = true;
- if (LOG.isTraceEnabled()) {
- LOG.trace("MAJOR COMPACTION loop sum= " + sum
- + " discarding now: " + " qualifier="
- + Bytes.toString(CellUtil.cloneQualifier(cell)) + " value="
- + converter.decodeValue(CellUtil.cloneValue(cell))
- + " timestamp=" + cell.getTimestamp() + " " + this.action);
- }
- } else {
- // not a final value but it's the latest cell for this app
- // so include this cell in the list of cells to write back
- finalCells.add(cell);
- }
- }
- }
- if (summationDone) {
- Cell anyCell = currentColumnCells.first();
- List<Tag> tags = new ArrayList<Tag>();
- Tag t = new Tag(AggregationOperation.SUM_FINAL.getTagType(),
- Bytes.toBytes(FLOW_APP_ID));
- tags.add(t);
- t = new Tag(AggregationCompactionDimension.APPLICATION_ID.getTagType(),
- Bytes.toBytes(FLOW_APP_ID));
- tags.add(t);
- byte[] tagByteArray = Tag.fromList(tags);
- Cell sumCell = HBaseTimelineStorageUtils.createNewCell(
- CellUtil.cloneRow(anyCell),
- CellUtil.cloneFamily(anyCell),
- CellUtil.cloneQualifier(anyCell),
- TimestampGenerator.getSupplementedTimestamp(
- System.currentTimeMillis(), FLOW_APP_ID),
- converter.encodeValue(sum), tagByteArray);
- finalCells.add(sumCell);
- if (LOG.isTraceEnabled()) {
- LOG.trace("MAJOR COMPACTION final sum= " + sum + " for "
- + Bytes.toString(CellUtil.cloneQualifier(sumCell))
- + " " + this.action);
- }
- LOG.info("After major compaction for qualifier="
- + Bytes.toString(CellUtil.cloneQualifier(sumCell))
- + " with currentColumnCells.size="
- + currentColumnCells.size()
- + " returning finalCells.size=" + finalCells.size()
- + " with sum=" + sum.longValue()
- + " with cell timestamp " + sumCell.getTimestamp());
- } else {
- String qualifier = "";
- LOG.info("After major compaction for qualifier=" + qualifier
- + " with currentColumnCells.size="
- + currentColumnCells.size()
- + " returning finalCells.size=" + finalCells.size()
- + " with zero sum="
- + sum.longValue());
- }
- return finalCells;
- }
-
- /**
- * Determines which cell is to be returned based on the values in each cell
- * and the comparison operation MIN or MAX.
- *
- * @param previouslyChosenCell
- * @param currentCell
- * @param currentAggOp
- * @return the cell which is the min (or max) cell
- * @throws IOException
- */
- private Cell compareCellValues(Cell previouslyChosenCell, Cell currentCell,
- AggregationOperation currentAggOp, NumericValueConverter converter)
- throws IOException {
- if (previouslyChosenCell == null) {
- return currentCell;
- }
- try {
- Number previouslyChosenCellValue = (Number)converter.decodeValue(
- CellUtil.cloneValue(previouslyChosenCell));
- Number currentCellValue = (Number) converter.decodeValue(CellUtil
- .cloneValue(currentCell));
- switch (currentAggOp) {
- case GLOBAL_MIN:
- if (converter.compare(
- currentCellValue, previouslyChosenCellValue) < 0) {
- // new value is minimum, hence return this cell
- return currentCell;
- } else {
- // previously chosen value is miniumum, hence return previous min cell
- return previouslyChosenCell;
- }
- case GLOBAL_MAX:
- if (converter.compare(
- currentCellValue, previouslyChosenCellValue) > 0) {
- // new value is max, hence return this cell
- return currentCell;
- } else {
- // previously chosen value is max, hence return previous max cell
- return previouslyChosenCell;
- }
- default:
- return currentCell;
- }
- } catch (IllegalArgumentException iae) {
- LOG.error("caught iae during conversion to long ", iae);
- return currentCell;
- }
- }
-
- @Override
- public void close() throws IOException {
- if (flowRunScanner != null) {
- flowRunScanner.close();
- } else {
- LOG.warn("scanner close called but scanner is null");
- }
- }
-
- /**
- * Called to signal the start of the next() call by the scanner.
- */
- public void startNext() {
- currentRow = null;
- }
-
- /**
- * Returns whether or not the underlying scanner has more rows.
- */
- public boolean hasMore() {
- return currentIndex < availableCells.size() ? true : hasMore;
- }
-
- /**
- * Returns the next available cell for the current row and advances the
- * pointer to the next cell. This method can be called multiple times in a row
- * to advance through all the available cells.
- *
- * @param scannerContext
- * context information for the batch of cells under consideration
- * @return the next available cell or null if no more cells are available for
- * the current row
- * @throws IOException
- */
- public Cell nextCell(ScannerContext scannerContext) throws IOException {
- Cell cell = peekAtNextCell(scannerContext);
- if (cell != null) {
- currentIndex++;
- }
- return cell;
- }
-
- /**
- * Returns the next available cell for the current row, without advancing the
- * pointer. Calling this method multiple times in a row will continue to
- * return the same cell.
- *
- * @param scannerContext
- * context information for the batch of cells under consideration
- * @return the next available cell or null if no more cells are available for
- * the current row
- * @throws IOException if any problem is encountered while grabbing the next
- * cell.
- */
- public Cell peekAtNextCell(ScannerContext scannerContext) throws IOException {
- if (currentIndex >= availableCells.size()) {
- // done with current batch
- availableCells.clear();
- currentIndex = 0;
- hasMore = flowRunScanner.next(availableCells, scannerContext);
- }
- Cell cell = null;
- if (currentIndex < availableCells.size()) {
- cell = availableCells.get(currentIndex);
- if (currentRow == null) {
- currentRow = CellUtil.cloneRow(cell);
- } else if (!CellUtil.matchingRow(cell, currentRow)) {
- // moved on to the next row
- // don't use the current cell
- // also signal no more cells for this row
- return null;
- }
- }
- return cell;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.hadoop.hbase.regionserver.RegionScanner#getMaxResultSize()
- */
- @Override
- public long getMaxResultSize() {
- if (regionScanner == null) {
- throw new IllegalStateException(
- "RegionScanner.isFilterDone() called when the flow "
- + "scanner's scanner is not a RegionScanner");
- }
- return regionScanner.getMaxResultSize();
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.hadoop.hbase.regionserver.RegionScanner#getMvccReadPoint()
- */
- @Override
- public long getMvccReadPoint() {
- if (regionScanner == null) {
- throw new IllegalStateException(
- "RegionScanner.isFilterDone() called when the flow "
- + "scanner's internal scanner is not a RegionScanner");
- }
- return regionScanner.getMvccReadPoint();
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.hadoop.hbase.regionserver.RegionScanner#isFilterDone()
- */
- @Override
- public boolean isFilterDone() throws IOException {
- if (regionScanner == null) {
- throw new IllegalStateException(
- "RegionScanner.isFilterDone() called when the flow "
- + "scanner's internal scanner is not a RegionScanner");
- }
- return regionScanner.isFilterDone();
-
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.hadoop.hbase.regionserver.RegionScanner#reseek(byte[])
- */
- @Override
- public boolean reseek(byte[] bytes) throws IOException {
- if (regionScanner == null) {
- throw new IllegalStateException(
- "RegionScanner.reseek() called when the flow "
- + "scanner's internal scanner is not a RegionScanner");
- }
- return regionScanner.reseek(bytes);
- }
-
- @Override
- public int getBatch() {
- return batchSize;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScannerOperation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScannerOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScannerOperation.java
deleted file mode 100644
index 73c666f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScannerOperation.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
-
-
-/**
- * Identifies the scanner operation on the {@link FlowRunTable}.
- */
-public enum FlowScannerOperation {
-
- /**
- * If the scanner is opened for reading
- * during preGet or preScan.
- */
- READ,
-
- /**
- * If the scanner is opened during preFlush.
- */
- FLUSH,
-
- /**
- * If the scanner is opened during minor Compaction.
- */
- MINOR_COMPACTION,
-
- /**
- * If the scanner is opened during major Compaction.
- */
- MAJOR_COMPACTION
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/package-info.java
deleted file mode 100644
index 04963f3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/package-info.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Package org.apache.hadoop.yarn.server.timelineservice.storage.flow
- * contains classes related to implementation for flow related tables, viz. flow
- * run table and flow activity table.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java
deleted file mode 100644
index e78db2a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Package org.apache.hadoop.yarn.server.timelineservice.storage contains
- * classes which define and implement reading and writing to backend storage.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/AbstractTimelineStorageReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/AbstractTimelineStorageReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/AbstractTimelineStorageReader.java
deleted file mode 100644
index 5bacf66..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/AbstractTimelineStorageReader.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
-import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
-import org.apache.hadoop.yarn.webapp.NotFoundException;
-
-/**
- * The base class for reading timeline data from the HBase storage. This class
- * provides basic support to validate and augment reader context.
- */
-public abstract class AbstractTimelineStorageReader {
-
- private final TimelineReaderContext context;
- /**
- * Used to look up the flow context.
- */
- private final AppToFlowTable appToFlowTable = new AppToFlowTable();
-
- public AbstractTimelineStorageReader(TimelineReaderContext ctxt) {
- context = ctxt;
- }
-
- protected TimelineReaderContext getContext() {
- return context;
- }
-
- /**
- * Looks up flow context from AppToFlow table.
- *
- * @param appToFlowRowKey to identify Cluster and App Ids.
- * @param clusterId the cluster id.
- * @param hbaseConf HBase configuration.
- * @param conn HBase Connection.
- * @return flow context information.
- * @throws IOException if any problem occurs while fetching flow information.
- */
- protected FlowContext lookupFlowContext(AppToFlowRowKey appToFlowRowKey,
- String clusterId, Configuration hbaseConf, Connection conn)
- throws IOException {
- byte[] rowKey = appToFlowRowKey.getRowKey();
- Get get = new Get(rowKey);
- Result result = appToFlowTable.getResult(hbaseConf, conn, get);
- if (result != null && !result.isEmpty()) {
- Object flowName =
- AppToFlowColumnPrefix.FLOW_NAME.readResult(result, clusterId);
- Object flowRunId =
- AppToFlowColumnPrefix.FLOW_RUN_ID.readResult(result, clusterId);
- Object userId =
- AppToFlowColumnPrefix.USER_ID.readResult(result, clusterId);
- if (flowName == null || userId == null || flowRunId == null) {
- throw new NotFoundException(
- "Unable to find the context flow name, and flow run id, "
- + "and user id for clusterId=" + clusterId
- + ", appId=" + appToFlowRowKey.getAppId());
- }
- return new FlowContext((String)userId, (String)flowName,
- ((Number)flowRunId).longValue());
- } else {
- throw new NotFoundException(
- "Unable to find the context flow name, and flow run id, "
- + "and user id for clusterId=" + clusterId
- + ", appId=" + appToFlowRowKey.getAppId());
- }
- }
-
- /**
- * Sets certain parameters to defaults if the values are not provided.
- *
- * @param hbaseConf HBase Configuration.
- * @param conn HBase Connection.
- * @throws IOException if any exception is encountered while setting params.
- */
- protected void augmentParams(Configuration hbaseConf, Connection conn)
- throws IOException {
- defaultAugmentParams(hbaseConf, conn);
- }
-
- /**
- * Default behavior for all timeline readers to augment parameters.
- *
- * @param hbaseConf HBase Configuration.
- * @param conn HBase Connection.
- * @throws IOException if any exception is encountered while setting params.
- */
- final protected void defaultAugmentParams(Configuration hbaseConf,
- Connection conn) throws IOException {
- // In reality all three should be null or neither should be null
- if (context.getFlowName() == null || context.getFlowRunId() == null
- || context.getUserId() == null) {
- // Get flow context information from AppToFlow table.
- AppToFlowRowKey appToFlowRowKey =
- new AppToFlowRowKey(context.getAppId());
- FlowContext flowContext =
- lookupFlowContext(appToFlowRowKey, context.getClusterId(), hbaseConf,
- conn);
- context.setFlowName(flowContext.flowName);
- context.setFlowRunId(flowContext.flowRunId);
- context.setUserId(flowContext.userId);
- }
- }
-
- /**
- * Validates the required parameters to read the entities.
- */
- protected abstract void validateParams();
-
- /**
- * Encapsulates flow context information.
- */
- protected static class FlowContext {
- private final String userId;
- private final String flowName;
- private final Long flowRunId;
-
- public FlowContext(String user, String flowName, Long flowRunId) {
- this.userId = user;
- this.flowName = flowName;
- this.flowRunId = flowRunId;
- }
-
- protected String getUserId() {
- return userId;
- }
-
- protected String getFlowName() {
- return flowName;
- }
-
- protected Long getFlowRunId() {
- return flowRunId;
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/18] hadoop git commit: YARN-7919. Refactor timelineservice-hbase
module into submodules. Contributed by Haibo Chen.
Posted by ro...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java
deleted file mode 100644
index c115b18..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java
+++ /dev/null
@@ -1,354 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.Tag;
-import org.apache.hadoop.hbase.client.Query;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationCompactionDimension;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationOperation;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.text.NumberFormat;
-
-/**
- * A bunch of utility functions used in HBase TimelineService backend.
- */
-public final class HBaseTimelineStorageUtils {
- /** milliseconds in one day. */
- public static final long MILLIS_ONE_DAY = 86400000L;
- private static final Logger LOG =
- LoggerFactory.getLogger(HBaseTimelineStorageUtils.class);
-
- private HBaseTimelineStorageUtils() {
- }
-
-
- /**
- * Combines the input array of attributes and the input aggregation operation
- * into a new array of attributes.
- *
- * @param attributes Attributes to be combined.
- * @param aggOp Aggregation operation.
- * @return array of combined attributes.
- */
- public static Attribute[] combineAttributes(Attribute[] attributes,
- AggregationOperation aggOp) {
- int newLength = getNewLengthCombinedAttributes(attributes, aggOp);
- Attribute[] combinedAttributes = new Attribute[newLength];
-
- if (attributes != null) {
- System.arraycopy(attributes, 0, combinedAttributes, 0, attributes.length);
- }
-
- if (aggOp != null) {
- Attribute a2 = aggOp.getAttribute();
- combinedAttributes[newLength - 1] = a2;
- }
- return combinedAttributes;
- }
-
- /**
- * Returns a number for the new array size. The new array is the combination
- * of input array of attributes and the input aggregation operation.
- *
- * @param attributes Attributes.
- * @param aggOp Aggregation operation.
- * @return the size for the new array
- */
- private static int getNewLengthCombinedAttributes(Attribute[] attributes,
- AggregationOperation aggOp) {
- int oldLength = getAttributesLength(attributes);
- int aggLength = getAppOpLength(aggOp);
- return oldLength + aggLength;
- }
-
- private static int getAppOpLength(AggregationOperation aggOp) {
- if (aggOp != null) {
- return 1;
- }
- return 0;
- }
-
- private static int getAttributesLength(Attribute[] attributes) {
- if (attributes != null) {
- return attributes.length;
- }
- return 0;
- }
-
- /**
- * Returns the first seen aggregation operation as seen in the list of input
- * tags or null otherwise.
- *
- * @param tags list of HBase tags.
- * @return AggregationOperation
- */
- public static AggregationOperation getAggregationOperationFromTagsList(
- List<Tag> tags) {
- for (AggregationOperation aggOp : AggregationOperation.values()) {
- for (Tag tag : tags) {
- if (tag.getType() == aggOp.getTagType()) {
- return aggOp;
- }
- }
- }
- return null;
- }
-
- /**
- * Creates a {@link Tag} from the input attribute.
- *
- * @param attribute Attribute from which tag has to be fetched.
- * @return a HBase Tag.
- */
- public static Tag getTagFromAttribute(Map.Entry<String, byte[]> attribute) {
- // attribute could be either an Aggregation Operation or
- // an Aggregation Dimension
- // Get the Tag type from either
- AggregationOperation aggOp = AggregationOperation
- .getAggregationOperation(attribute.getKey());
- if (aggOp != null) {
- Tag t = new Tag(aggOp.getTagType(), attribute.getValue());
- return t;
- }
-
- AggregationCompactionDimension aggCompactDim =
- AggregationCompactionDimension.getAggregationCompactionDimension(
- attribute.getKey());
- if (aggCompactDim != null) {
- Tag t = new Tag(aggCompactDim.getTagType(), attribute.getValue());
- return t;
- }
- return null;
- }
-
- /**
- * creates a new cell based on the input cell but with the new value.
- *
- * @param origCell Original cell
- * @param newValue new cell value
- * @return cell
- * @throws IOException while creating new cell.
- */
- public static Cell createNewCell(Cell origCell, byte[] newValue)
- throws IOException {
- return CellUtil.createCell(CellUtil.cloneRow(origCell),
- CellUtil.cloneFamily(origCell), CellUtil.cloneQualifier(origCell),
- origCell.getTimestamp(), KeyValue.Type.Put.getCode(), newValue);
- }
-
- /**
- * creates a cell with the given inputs.
- *
- * @param row row of the cell to be created
- * @param family column family name of the new cell
- * @param qualifier qualifier for the new cell
- * @param ts timestamp of the new cell
- * @param newValue value of the new cell
- * @param tags tags in the new cell
- * @return cell
- * @throws IOException while creating the cell.
- */
- public static Cell createNewCell(byte[] row, byte[] family, byte[] qualifier,
- long ts, byte[] newValue, byte[] tags) throws IOException {
- return CellUtil.createCell(row, family, qualifier, ts, KeyValue.Type.Put,
- newValue, tags);
- }
-
- /**
- * returns app id from the list of tags.
- *
- * @param tags cell tags to be looked into
- * @return App Id as the AggregationCompactionDimension
- */
- public static String getAggregationCompactionDimension(List<Tag> tags) {
- String appId = null;
- for (Tag t : tags) {
- if (AggregationCompactionDimension.APPLICATION_ID.getTagType() == t
- .getType()) {
- appId = Bytes.toString(t.getValue());
- return appId;
- }
- }
- return appId;
- }
-
- /**
- * Converts an int into it's inverse int to be used in (row) keys
- * where we want to have the largest int value in the top of the table
- * (scans start at the largest int first).
- *
- * @param key value to be inverted so that the latest version will be first in
- * a scan.
- * @return inverted int
- */
- public static int invertInt(int key) {
- return Integer.MAX_VALUE - key;
- }
-
- /**
- * returns the timestamp of that day's start (which is midnight 00:00:00 AM)
- * for a given input timestamp.
- *
- * @param ts Timestamp.
- * @return timestamp of that day's beginning (midnight)
- */
- public static long getTopOfTheDayTimestamp(long ts) {
- long dayTimestamp = ts - (ts % MILLIS_ONE_DAY);
- return dayTimestamp;
- }
-
- private static final ThreadLocal<NumberFormat> APP_ID_FORMAT =
- new ThreadLocal<NumberFormat>() {
- @Override
- public NumberFormat initialValue() {
- NumberFormat fmt = NumberFormat.getInstance();
- fmt.setGroupingUsed(false);
- fmt.setMinimumIntegerDigits(4);
- return fmt;
- }
- };
-
- /**
- * A utility method that converts ApplicationId to string without using
- * FastNumberFormat in order to avoid the incompatibility issue caused
- * by mixing hadoop-common 2.5.1 and hadoop-yarn-api 3.0 in this module.
- * This is a work-around implementation as discussed in YARN-6905.
- *
- * @param appId application id
- * @return the string representation of the given application id
- *
- */
- public static String convertApplicationIdToString(ApplicationId appId) {
- StringBuilder sb = new StringBuilder(64);
- sb.append(ApplicationId.appIdStrPrefix);
- sb.append("_");
- sb.append(appId.getClusterTimestamp());
- sb.append('_');
- sb.append(APP_ID_FORMAT.get().format(appId.getId()));
- return sb.toString();
- }
-
- /**
- * @param conf YARN configuration. Used to see if there is an explicit config
- * pointing to the HBase config file to read. It should not be null
- * or a NullPointerException will be thrown.
- * @return a configuration with the HBase configuration from the classpath,
- * optionally overwritten by the timeline service configuration URL if
- * specified.
- * @throws MalformedURLException if a timeline service HBase configuration URL
- * is specified but is a malformed URL.
- */
- public static Configuration getTimelineServiceHBaseConf(Configuration conf)
- throws MalformedURLException {
- if (conf == null) {
- throw new NullPointerException();
- }
-
- Configuration hbaseConf;
- String timelineServiceHBaseConfFileURL =
- conf.get(YarnConfiguration.TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE);
- if (timelineServiceHBaseConfFileURL != null
- && timelineServiceHBaseConfFileURL.length() > 0) {
- LOG.info("Using hbase configuration at " +
- timelineServiceHBaseConfFileURL);
- // create a clone so that we don't mess with out input one
- hbaseConf = new Configuration(conf);
- Configuration plainHBaseConf = new Configuration(false);
- URL hbaseSiteXML = new URL(timelineServiceHBaseConfFileURL);
- plainHBaseConf.addResource(hbaseSiteXML);
- HBaseConfiguration.merge(hbaseConf, plainHBaseConf);
- } else {
- // default to what is on the classpath
- hbaseConf = HBaseConfiguration.create(conf);
- }
- return hbaseConf;
- }
-
- /**
- * Given a row key prefix stored in a byte array, return a byte array for its
- * immediate next row key.
- *
- * @param rowKeyPrefix The provided row key prefix, represented in an array.
- * @return the closest next row key of the provided row key.
- */
- public static byte[] calculateTheClosestNextRowKeyForPrefix(
- byte[] rowKeyPrefix) {
- // Essentially we are treating it like an 'unsigned very very long' and
- // doing +1 manually.
- // Search for the place where the trailing 0xFFs start
- int offset = rowKeyPrefix.length;
- while (offset > 0) {
- if (rowKeyPrefix[offset - 1] != (byte) 0xFF) {
- break;
- }
- offset--;
- }
-
- if (offset == 0) {
- // We got an 0xFFFF... (only FFs) stopRow value which is
- // the last possible prefix before the end of the table.
- // So set it to stop at the 'end of the table'
- return HConstants.EMPTY_END_ROW;
- }
-
- // Copy the right length of the original
- byte[] newStopRow = Arrays.copyOfRange(rowKeyPrefix, 0, offset);
- // And increment the last one
- newStopRow[newStopRow.length - 1]++;
- return newStopRow;
- }
-
- /**
- * Checks if passed object is of integral type(Short/Integer/Long).
- *
- * @param obj Object to be checked.
- * @return true if object passed is of type Short or Integer or Long, false
- * otherwise.
- */
- public static boolean isIntegralValue(Object obj) {
- return (obj instanceof Short) || (obj instanceof Integer) ||
- (obj instanceof Long);
- }
-
- public static void setMetricsTimeRange(Query query, byte[] metricsCf,
- long tsBegin, long tsEnd) {
- if (tsBegin != 0 || tsEnd != Long.MAX_VALUE) {
- query.setColumnFamilyTimeRange(metricsCf,
- tsBegin, ((tsEnd == Long.MAX_VALUE) ? Long.MAX_VALUE : (tsEnd + 1)));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/KeyConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/KeyConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/KeyConverter.java
deleted file mode 100644
index 4229e81..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/KeyConverter.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-/**
- * Interface which has to be implemented for encoding and decoding row keys and
- * columns.
- */
-public interface KeyConverter<T> {
- /**
- * Encodes a key as a byte array.
- *
- * @param key key to be encoded.
- * @return a byte array.
- */
- byte[] encode(T key);
-
- /**
- * Decodes a byte array and returns a key of type T.
- *
- * @param bytes byte representation
- * @return an object(key) of type T which has been constructed after decoding
- * the bytes.
- */
- T decode(byte[] bytes);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/KeyConverterToString.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/KeyConverterToString.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/KeyConverterToString.java
deleted file mode 100644
index 1f52a7b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/KeyConverterToString.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-/**
- * Interface which has to be implemented for encoding and decoding row keys or
- * column qualifiers as string.
- */
-public interface KeyConverterToString<T> {
- /**
- * Encode key as string.
- * @param key of type T to be encoded as string.
- * @return encoded value as string.
- */
- String encodeAsString(T key);
-
- /**
- * Decode row key from string to a key of type T.
- * @param encodedKey string representation of row key
- * @return type T which has been constructed after decoding string.
- */
- T decodeFromString(String encodedKey);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongConverter.java
deleted file mode 100644
index 6ab69f7..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongConverter.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import java.io.IOException;
-import java.io.Serializable;
-
-import org.apache.hadoop.hbase.util.Bytes;
-
-/**
- * Encodes a value by interpreting it as a Long and converting it to bytes and
- * decodes a set of bytes as a Long.
- */
-public final class LongConverter implements NumericValueConverter,
- Serializable {
-
- /**
- * Added because we implement Comparator<Number>.
- */
- private static final long serialVersionUID = 1L;
-
- public LongConverter() {
- }
-
- @Override
- public byte[] encodeValue(Object value) throws IOException {
- if (!HBaseTimelineStorageUtils.isIntegralValue(value)) {
- throw new IOException("Expected integral value");
- }
- return Bytes.toBytes(((Number)value).longValue());
- }
-
- @Override
- public Object decodeValue(byte[] bytes) throws IOException {
- if (bytes == null) {
- return null;
- }
- return Bytes.toLong(bytes);
- }
-
- /**
- * Compares two numbers as longs. If either number is null, it will be taken
- * as 0.
- *
- * @param num1 the first {@code Long} to compare.
- * @param num2 the second {@code Long} to compare.
- * @return -1 if num1 is less than num2, 0 if num1 is equal to num2 and 1 if
- * num1 is greater than num2.
- */
- @Override
- public int compare(Number num1, Number num2) {
- return Long.compare((num1 == null) ? 0L : num1.longValue(),
- (num2 == null) ? 0L : num2.longValue());
- }
-
- @Override
- public Number add(Number num1, Number num2, Number...numbers) {
- long sum = ((num1 == null) ? 0L : num1.longValue()) +
- ((num2 == null) ? 0L : num2.longValue());
- for (Number num : numbers) {
- sum = sum + ((num == null) ? 0L : num.longValue());
- }
- return sum;
- }
-
- /**
- * Converts a timestamp into it's inverse timestamp to be used in (row) keys
- * where we want to have the most recent timestamp in the top of the table
- * (scans start at the most recent timestamp first).
- *
- * @param key value to be inverted so that the latest version will be first in
- * a scan.
- * @return inverted long
- */
- public static long invertLong(long key) {
- return Long.MAX_VALUE - key;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongKeyConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongKeyConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongKeyConverter.java
deleted file mode 100644
index 4a724d6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongKeyConverter.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import java.io.IOException;
-/**
- * Encodes and decodes column names / row keys which are long.
- */
-public final class LongKeyConverter implements KeyConverter<Long> {
-
- /**
- * To delegate the actual work to.
- */
- private final LongConverter longConverter = new LongConverter();
-
- public LongKeyConverter() {
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter
- * #encode(java.lang.Object)
- */
- @Override
- public byte[] encode(Long key) {
- try {
- // IOException will not be thrown here as we are explicitly passing
- // Long.
- return longConverter.encodeValue(key);
- } catch (IOException e) {
- return null;
- }
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter
- * #decode(byte[])
- */
- @Override
- public Long decode(byte[] bytes) {
- try {
- return (Long) longConverter.decodeValue(bytes);
- } catch (IOException e) {
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/NumericValueConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/NumericValueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/NumericValueConverter.java
deleted file mode 100644
index 8fb6536..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/NumericValueConverter.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import java.util.Comparator;
-
-/**
- * Extends ValueConverter interface for numeric converters to support numerical
- * operations such as comparison, addition, etc.
- */
-public interface NumericValueConverter extends ValueConverter,
- Comparator<Number> {
- /**
- * Adds two or more numbers. If either of the numbers are null, it is taken as
- * 0.
- *
- * @param num1 the first number to add.
- * @param num2 the second number to add.
- * @param numbers Rest of the numbers to be added.
- * @return result after adding up the numbers.
- */
- Number add(Number num1, Number num2, Number...numbers);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Range.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Range.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Range.java
deleted file mode 100644
index 8a2e01a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Range.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * Encapsulates a range with start and end indices.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class Range {
- private final int startIdx;
- private final int endIdx;
-
- /**
- * Defines a range from start index (inclusive) to end index (exclusive).
- *
- * @param start
- * Starting index position
- * @param end
- * Ending index position (exclusive)
- */
- public Range(int start, int end) {
- if (start < 0 || end < start) {
- throw new IllegalArgumentException(
- "Invalid range, required that: 0 <= start <= end; start=" + start
- + ", end=" + end);
- }
-
- this.startIdx = start;
- this.endIdx = end;
- }
-
- public int start() {
- return startIdx;
- }
-
- public int end() {
- return endIdx;
- }
-
- public int length() {
- return endIdx - startIdx;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/RowKeyPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/RowKeyPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/RowKeyPrefix.java
deleted file mode 100644
index 6159dc7..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/RowKeyPrefix.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-/**
- * In queries where a single result is needed, an exact rowkey can be used
- * through the corresponding rowkey#getRowKey() method. For queries that need to
- * scan over a range of rowkeys, a partial (the initial part) of rowkeys are
- * used. Classes implementing RowKeyPrefix indicate that they are the initial
- * part of rowkeys, with different constructors with fewer number of argument to
- * form a partial rowkey, a prefix.
- *
- * @param <R> indicating the type of rowkey that a particular implementation is
- * a prefix for.
- */
-public interface RowKeyPrefix<R> {
-
- /**
- * Create a row key prefix, meaning a partial rowkey that can be used in range
- * scans. Which fields are included in the prefix will depend on the
- * constructor of the specific instance that was used. Output depends on which
- * constructor was used.
- * @return a prefix of the following form {@code fist!second!...!last!}
- */
- byte[] getRowKeyPrefix();
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
deleted file mode 100644
index 5090b4d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
+++ /dev/null
@@ -1,575 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.hadoop.hbase.util.Bytes;
-
-/**
- * Used to separate row qualifiers, column qualifiers and compound fields.
- */
-public enum Separator {
-
- /**
- * separator in key or column qualifier fields.
- */
- QUALIFIERS("!", "%0$"),
-
- /**
- * separator in values, and/or compound key/column qualifier fields.
- */
- VALUES("=", "%1$"),
-
- /**
- * separator in values, often used to avoid having these in qualifiers and
- * names. Note that if we use HTML form encoding through URLEncoder, we end up
- * getting a + for a space, which may already occur in strings, so we don't
- * want that.
- */
- SPACE(" ", "%2$"),
-
- /**
- * separator in values, often used to avoid having these in qualifiers and
- * names.
- */
- TAB("\t", "%3$");
-
- // a reserved character that starts each of the encoded values and is encoded
- // first in order to escape naturally occurring instances of encoded values
- // although it can be expressed as an enum instance, we define them as private
- // variables to hide it from callers
- private static final String PERCENT = "%";
- private static final String PERCENT_ENCODED = "%9$";
-
- private static final Pattern PERCENT_PATTERN =
- Pattern.compile(PERCENT, Pattern.LITERAL);
- private static final String PERCENT_REPLACEMENT =
- Matcher.quoteReplacement(PERCENT);
-
- private static final Pattern PERCENT_ENCODED_PATTERN =
- Pattern.compile(PERCENT_ENCODED, Pattern.LITERAL);
- private static final String PERCENT_ENCODED_REPLACEMENT =
- Matcher.quoteReplacement(PERCENT_ENCODED);
-
- /**
- * The string value of this separator.
- */
- private final String value;
-
- /**
- * The bye representation of value.
- */
- private final byte[] bytes;
-
- // pre-compiled patterns and quoted replacements for optimization
- private final Pattern valuePattern;
- private final String valueReplacement;
-
- private final Pattern encodedValuePattern;
- private final String encodedValueReplacement;
-
- /**
- * Indicator for variable size of an individual segment in a split. The
- * segment ends wherever separator is encountered.
- * Typically used for string.
- * Also used to indicate that there is no fixed number of splits which need to
- * be returned. If split limit is specified as this, all possible splits are
- * returned.
- */
- public static final int VARIABLE_SIZE = 0;
-
-
- /** empty string. */
- public static final String EMPTY_STRING = "";
-
- /** empty bytes. */
- public static final byte[] EMPTY_BYTES = new byte[0];
-
- /**
- * @param value of the separator to use. Cannot be null or empty string.
- * @param encodedValue choose something that isn't likely to occur in the data
- * itself. Cannot be null or empty string.
- */
- private Separator(String value, String encodedValue) {
- this.value = value;
-
- // validation
- if (value == null || value.length() == 0 || encodedValue == null
- || encodedValue.length() == 0) {
- throw new IllegalArgumentException(
- "Cannot create separator from null or empty string.");
- }
-
- this.bytes = Bytes.toBytes(value);
- this.valuePattern = Pattern.compile(value, Pattern.LITERAL);
- this.valueReplacement = Matcher.quoteReplacement(value);
-
- this.encodedValuePattern = Pattern.compile(encodedValue, Pattern.LITERAL);
- this.encodedValueReplacement = Matcher.quoteReplacement(encodedValue);
- }
-
- /**
- * @return the original value of the separator
- */
- public String getValue() {
- return value;
- }
-
- /**
- * Used to make token safe to be used with this separator without collisions.
- * It <em>must</em> be paired with {@link #decode(String)} for it to be
- * decoded correctly.
- * <p>
- * If you need to encode a given string for multiple separators,
- * {@link #encode(String, Separator...)} should be used over successive
- * invocations of this method. It will result in a more compact version of the
- * encoded value.
- *
- * @param token Token to be encoded.
- * @return the token with any occurrences of this separator URLEncoded.
- */
- public String encode(String token) {
- if (token == null || token.length() == 0) {
- // Nothing to replace
- return token;
- }
- // first encode the percent to escape naturally occurring encoded values
- String escaped = encodePercent(token);
- return encodeSingle(escaped, this);
- }
-
- private static String replace(String token, Pattern pattern,
- String replacement) {
- return pattern.matcher(token).replaceAll(replacement);
- }
-
- private static String encodeSingle(String token, Separator separator) {
- return replace(token, separator.valuePattern,
- separator.encodedValueReplacement);
- }
-
- private static String encodePercent(String token) {
- return replace(token, PERCENT_PATTERN, PERCENT_ENCODED_REPLACEMENT);
- }
-
- /**
- * Decode the token encoded using {@link #encode(String)}. It <em>must</em> be
- * used for the result encoded with {@link #encode(String)} to be able to
- * recover the original.
- *
- * @param token Token to be decoded.
- * @return the token with any occurrences of the encoded separator replaced by
- * the separator itself.
- */
- public String decode(String token) {
- if (token == null || token.length() == 0) {
- // Nothing to replace
- return token;
- }
- String escaped = decodeSingle(token, this);
- // decode percent to de-escape
- return decodePercent(escaped);
- }
-
- private static String decodeSingle(String token, Separator separator) {
- return replace(token, separator.encodedValuePattern,
- separator.valueReplacement);
- }
-
- private static String decodePercent(String token) {
- return replace(token, PERCENT_ENCODED_PATTERN, PERCENT_REPLACEMENT);
- }
-
- /**
- * Encode the given separators in the token with their encoding equivalents.
- * It <em>must</em> be paired with {@link #decode(byte[], Separator...)} or
- * {@link #decode(String, Separator...)} with the same separators for it to be
- * decoded correctly.
- * <p>
- * If you need to encode a given string for multiple separators, this form of
- * encoding should be used over successive invocations of
- * {@link #encode(String)}. It will result in a more compact version of the
- * encoded value.
- *
- * @param token containing possible separators that need to be encoded.
- * @param separators to be encoded in the token with their URLEncoding
- * equivalent.
- * @return non-null byte representation of the token with occurrences of the
- * separators encoded.
- */
- public static byte[] encode(String token, Separator... separators) {
- if (token == null || token.length() == 0) {
- return EMPTY_BYTES;
- }
- String result = token;
- // first encode the percent to escape naturally occurring encoded values
- result = encodePercent(token);
- for (Separator separator : separators) {
- if (separator != null) {
- result = encodeSingle(result, separator);
- }
- }
- return Bytes.toBytes(result);
- }
-
- /**
- * Decode the given separators in the token with their decoding equivalents.
- * It <em>must</em> be used for the result encoded with
- * {@link #encode(String, Separator...)} with the same separators to be able
- * to recover the original.
- *
- * @param token containing possible separators that need to be encoded.
- * @param separators to be encoded in the token with their URLEncoding
- * equivalent.
- * @return String representation of the token with occurrences of the URL
- * encoded separators decoded.
- */
- public static String decode(byte[] token, Separator... separators) {
- if (token == null) {
- return null;
- }
- return decode(Bytes.toString(token), separators);
- }
-
- /**
- * Decode the given separators in the token with their decoding equivalents.
- * It <em>must</em> be used for the result encoded with
- * {@link #encode(String, Separator...)} with the same separators to be able
- * to recover the original.
- *
- * @param token containing possible separators that need to be encoded.
- * @param separators to be encoded in the token with their URLEncoding
- * equivalent.
- * @return String representation of the token with occurrences of the URL
- * encoded separators decoded.
- */
- public static String decode(String token, Separator... separators) {
- if (token == null) {
- return null;
- }
- String result = token;
- for (Separator separator : separators) {
- if (separator != null) {
- result = decodeSingle(result, separator);
- }
- }
- // decode percent to de-escape
- return decodePercent(result);
- }
-
- /**
- * Returns a single byte array containing all of the individual arrays
- * components separated by this separator.
- *
- * @param components Byte array components to be joined together.
- * @return byte array after joining the components
- */
- public byte[] join(byte[]... components) {
- if (components == null || components.length == 0) {
- return EMPTY_BYTES;
- }
-
- int finalSize = 0;
- finalSize = this.value.length() * (components.length - 1);
- for (byte[] comp : components) {
- if (comp != null) {
- finalSize += comp.length;
- }
- }
-
- byte[] buf = new byte[finalSize];
- int offset = 0;
- for (int i = 0; i < components.length; i++) {
- if (components[i] != null) {
- System.arraycopy(components[i], 0, buf, offset, components[i].length);
- offset += components[i].length;
- }
- if (i < (components.length - 1)) {
- System.arraycopy(this.bytes, 0, buf, offset, this.value.length());
- offset += this.value.length();
- }
- }
- return buf;
- }
-
- /**
- * Concatenates items (as String), using this separator.
- *
- * @param items Items join, {@code toString()} will be called in each item.
- * Any occurrence of the separator in the individual strings will be
- * first encoded. Cannot be null.
- * @return non-null joined result. Note that when separator is {@literal null}
- * the result is simply all items concatenated and the process is not
- * reversible through {@link #splitEncoded(String)}
- */
- public String joinEncoded(String... items) {
- if (items == null || items.length == 0) {
- return "";
- }
-
- StringBuilder sb = new StringBuilder(encode(items[0].toString()));
- // Start at 1, we've already grabbed the first value at index 0
- for (int i = 1; i < items.length; i++) {
- sb.append(this.value);
- sb.append(encode(items[i].toString()));
- }
-
- return sb.toString();
- }
-
- /**
- * Concatenates items (as String), using this separator.
- *
- * @param items Items join, {@code toString()} will be called in each item.
- * Any occurrence of the separator in the individual strings will be
- * first encoded. Cannot be null.
- * @return non-null joined result. Note that when separator is {@literal null}
- * the result is simply all items concatenated and the process is not
- * reversible through {@link #splitEncoded(String)}
- */
- public String joinEncoded(Iterable<?> items) {
- if (items == null) {
- return "";
- }
- Iterator<?> i = items.iterator();
- if (!i.hasNext()) {
- return "";
- }
-
- StringBuilder sb = new StringBuilder(encode(i.next().toString()));
- while (i.hasNext()) {
- sb.append(this.value);
- sb.append(encode(i.next().toString()));
- }
-
- return sb.toString();
- }
-
- /**
- * @param compoundValue containing individual values separated by this
- * separator, which have that separator encoded.
- * @return non-null set of values from the compoundValue with the separator
- * decoded.
- */
- public Collection<String> splitEncoded(String compoundValue) {
- List<String> result = new ArrayList<String>();
- if (compoundValue != null) {
- for (String val : valuePattern.split(compoundValue)) {
- result.add(decode(val));
- }
- }
- return result;
- }
-
- /**
- * Splits the source array into multiple array segments using this separator,
- * up to a maximum of count items. This will naturally produce copied byte
- * arrays for each of the split segments.
- *
- * @param source to be split
- * @param limit on how many segments are supposed to be returned. A
- * non-positive value indicates no limit on number of segments.
- * @return source split by this separator.
- */
- public byte[][] split(byte[] source, int limit) {
- return split(source, this.bytes, limit);
- }
-
- /**
- * Splits the source array into multiple array segments using this separator.
- * The sizes indicate the sizes of the relative components/segments.
- * In case one of the segments contains this separator before the specified
- * size is reached, the separator will be considered part of that segment and
- * we will continue till size is reached.
- * Variable length strings cannot contain this separator and are indiced with
- * a size of {@value #VARIABLE_SIZE}. Such strings are encoded for this
- * separator and decoded after the results from split is returned.
- *
- * @param source byte array to be split.
- * @param sizes sizes of relative components/segments.
- * @return source split by this separator as per the sizes specified..
- */
- public byte[][] split(byte[] source, int[] sizes) {
- return split(source, this.bytes, sizes);
- }
-
- /**
- * Splits the source array into multiple array segments using this separator,
- * as many times as splits are found. This will naturally produce copied byte
- * arrays for each of the split segments.
- *
- * @param source byte array to be split
- * @return source split by this separator.
- */
- public byte[][] split(byte[] source) {
- return split(source, this.bytes);
- }
-
- /**
- * Returns a list of ranges identifying [start, end) -- closed, open --
- * positions within the source byte array that would be split using the
- * separator byte array.
- * The sizes indicate the sizes of the relative components/segments.
- * In case one of the segments contains this separator before the specified
- * size is reached, the separator will be considered part of that segment and
- * we will continue till size is reached.
- * Variable length strings cannot contain this separator and are indiced with
- * a size of {@value #VARIABLE_SIZE}. Such strings are encoded for this
- * separator and decoded after the results from split is returned.
- *
- * @param source the source data
- * @param separator the separator pattern to look for
- * @param sizes indicate the sizes of the relative components/segments.
- * @return a list of ranges.
- */
- private static List<Range> splitRanges(byte[] source, byte[] separator,
- int[] sizes) {
- List<Range> segments = new ArrayList<Range>();
- if (source == null || separator == null) {
- return segments;
- }
- // VARIABLE_SIZE here indicates that there is no limit to number of segments
- // to return.
- int limit = VARIABLE_SIZE;
- if (sizes != null && sizes.length > 0) {
- limit = sizes.length;
- }
- int start = 0;
- int currentSegment = 0;
- itersource: for (int i = 0; i < source.length; i++) {
- for (int j = 0; j < separator.length; j++) {
- if (source[i + j] != separator[j]) {
- continue itersource;
- }
- }
- // all separator elements matched
- if (limit > VARIABLE_SIZE) {
- if (segments.size() >= (limit - 1)) {
- // everything else goes in one final segment
- break;
- }
- if (sizes != null) {
- int currentSegExpectedSize = sizes[currentSegment];
- if (currentSegExpectedSize > VARIABLE_SIZE) {
- int currentSegSize = i - start;
- if (currentSegSize < currentSegExpectedSize) {
- // Segment not yet complete. More bytes to parse.
- continue itersource;
- } else if (currentSegSize > currentSegExpectedSize) {
- // Segment is not as per size.
- throw new IllegalArgumentException(
- "Segments not separated as per expected sizes");
- }
- }
- }
- }
- segments.add(new Range(start, i));
- start = i + separator.length;
- // i will be incremented again in outer for loop
- i += separator.length - 1;
- currentSegment++;
- }
- // add in remaining to a final range
- if (start <= source.length) {
- if (sizes != null) {
- // Check if final segment is as per size specified.
- if (sizes[currentSegment] > VARIABLE_SIZE &&
- source.length - start > sizes[currentSegment]) {
- // Segment is not as per size.
- throw new IllegalArgumentException(
- "Segments not separated as per expected sizes");
- }
- }
- segments.add(new Range(start, source.length));
- }
- return segments;
- }
-
- /**
- * Splits based on segments calculated based on limit/sizes specified for the
- * separator.
- *
- * @param source byte array to be split.
- * @param segments specifies the range for each segment.
- * @return a byte[][] split as per the segment ranges.
- */
- private static byte[][] split(byte[] source, List<Range> segments) {
- byte[][] splits = new byte[segments.size()][];
- for (int i = 0; i < segments.size(); i++) {
- Range r = segments.get(i);
- byte[] tmp = new byte[r.length()];
- if (tmp.length > 0) {
- System.arraycopy(source, r.start(), tmp, 0, r.length());
- }
- splits[i] = tmp;
- }
- return splits;
- }
-
- /**
- * Splits the source array into multiple array segments using the given
- * separator based on the sizes. This will naturally produce copied byte
- * arrays for each of the split segments.
- *
- * @param source source array.
- * @param separator separator represented as a byte array.
- * @param sizes sizes of relative components/segments.
- * @return byte[][] after splitting the source.
- */
- private static byte[][] split(byte[] source, byte[] separator, int[] sizes) {
- List<Range> segments = splitRanges(source, separator, sizes);
- return split(source, segments);
- }
-
- /**
- * Splits the source array into multiple array segments using the given
- * separator. This will naturally produce copied byte arrays for each of the
- * split segments.
- *
- * @param source Source array.
- * @param separator Separator represented as a byte array.
- * @return byte[][] after splitting the source.
- */
- private static byte[][] split(byte[] source, byte[] separator) {
- return split(source, separator, (int[]) null);
- }
-
- /**
- * Splits the source array into multiple array segments using the given
- * separator, up to a maximum of count items. This will naturally produce
- * copied byte arrays for each of the split segments.
- *
- * @param source Source array.
- * @param separator Separator represented as a byte array.
- * @param limit a non-positive value indicates no limit on number of segments.
- * @return byte[][] after splitting the input source.
- */
- private static byte[][] split(byte[] source, byte[] separator, int limit) {
- int[] sizes = null;
- if (limit > VARIABLE_SIZE) {
- sizes = new int[limit];
- }
- List<Range> segments = splitRanges(source, separator, sizes);
- return split(source, segments);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/StringKeyConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/StringKeyConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/StringKeyConverter.java
deleted file mode 100644
index 282848e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/StringKeyConverter.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-/**
- * Encodes and decodes column names / row keys which are merely strings.
- * Column prefixes are not part of the column name passed for encoding. It is
- * added later, if required in the associated ColumnPrefix implementations.
- */
-public final class StringKeyConverter implements KeyConverter<String> {
-
- public StringKeyConverter() {
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter
- * #encode(java.lang.Object)
- */
- @Override
- public byte[] encode(String key) {
- return Separator.encode(key, Separator.SPACE, Separator.TAB);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter
- * #decode(byte[])
- */
- @Override
- public String decode(byte[] bytes) {
- return Separator.decode(bytes, Separator.TAB, Separator.SPACE);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineHBaseSchemaConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineHBaseSchemaConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineHBaseSchemaConstants.java
deleted file mode 100644
index 8e6c259..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineHBaseSchemaConstants.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hbase.util.Bytes;
-
-/**
- * contains the constants used in the context of schema accesses for
- * {@link org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity}
- * information.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public final class TimelineHBaseSchemaConstants {
- private TimelineHBaseSchemaConstants() {
- }
-
- /**
- * Used to create a pre-split for tables starting with a username in the
- * prefix. TODO: this may have to become a config variable (string with
- * separators) so that different installations can presplit based on their own
- * commonly occurring names.
- */
- private final static byte[][] USERNAME_SPLITS = {
- Bytes.toBytes("a"), Bytes.toBytes("ad"), Bytes.toBytes("an"),
- Bytes.toBytes("b"), Bytes.toBytes("ca"), Bytes.toBytes("cl"),
- Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
- Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i"),
- Bytes.toBytes("j"), Bytes.toBytes("k"), Bytes.toBytes("l"),
- Bytes.toBytes("m"), Bytes.toBytes("n"), Bytes.toBytes("o"),
- Bytes.toBytes("q"), Bytes.toBytes("r"), Bytes.toBytes("s"),
- Bytes.toBytes("se"), Bytes.toBytes("t"), Bytes.toBytes("u"),
- Bytes.toBytes("v"), Bytes.toBytes("w"), Bytes.toBytes("x"),
- Bytes.toBytes("y"), Bytes.toBytes("z")
- };
-
- /**
- * The length at which keys auto-split.
- */
- public static final String USERNAME_SPLIT_KEY_PREFIX_LENGTH = "4";
-
- /**
- * @return splits for splits where a user is a prefix.
- */
- public static byte[][] getUsernameSplits() {
- byte[][] kloon = USERNAME_SPLITS.clone();
- // Deep copy.
- for (int row = 0; row < USERNAME_SPLITS.length; row++) {
- kloon[row] = Bytes.copy(USERNAME_SPLITS[row]);
- }
- return kloon;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimestampGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimestampGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimestampGenerator.java
deleted file mode 100644
index d03b37d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimestampGenerator.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-
-/**
- * Utility class that allows HBase coprocessors to interact with unique
- * timestamps.
- */
-public class TimestampGenerator {
-
- /*
- * if this is changed, then reading cell timestamps written with older
- * multiplier value will not work
- */
- public static final long TS_MULTIPLIER = 1000000L;
-
- private final AtomicLong lastTimestamp = new AtomicLong();
-
- /**
- * Returns the current wall clock time in milliseconds, multiplied by the
- * required precision.
- *
- * @return current timestamp.
- */
- public long currentTime() {
- // We want to align cell timestamps with current time.
- // cell timestamps are not be less than
- // System.currentTimeMillis() * TS_MULTIPLIER.
- return System.currentTimeMillis() * TS_MULTIPLIER;
- }
-
- /**
- * Returns a timestamp value unique within the scope of this
- * {@code TimestampGenerator} instance. For usage by HBase
- * {@code RegionObserver} coprocessors, this normally means unique within a
- * given region.
- *
- * Unlikely scenario of generating a non-unique timestamp: if there is a
- * sustained rate of more than 1M hbase writes per second AND if region fails
- * over within that time range of timestamps being generated then there may be
- * collisions writing to a cell version of the same column.
- *
- * @return unique timestamp.
- */
- public long getUniqueTimestamp() {
- long lastTs;
- long nextTs;
- do {
- lastTs = lastTimestamp.get();
- nextTs = Math.max(lastTs + 1, currentTime());
- } while (!lastTimestamp.compareAndSet(lastTs, nextTs));
- return nextTs;
- }
-
- /**
- * Returns a timestamp multiplied with TS_MULTIPLIER and last few digits of
- * application id.
- *
- * Unlikely scenario of generating a timestamp that is a duplicate: If more
- * than a 1M concurrent apps are running in one flow run AND write to same
- * column at the same time, then say appId of 1M and 1 will overlap
- * with appId of 001 and there may be collisions for that flow run's
- * specific column.
- *
- * @param incomingTS Timestamp to be converted.
- * @param appId Application Id.
- * @return a timestamp multiplied with TS_MULTIPLIER and last few digits of
- * application id
- */
- public static long getSupplementedTimestamp(long incomingTS, String appId) {
- long suffix = getAppIdSuffix(appId);
- long outgoingTS = incomingTS * TS_MULTIPLIER + suffix;
- return outgoingTS;
-
- }
-
- private static long getAppIdSuffix(String appIdStr) {
- if (appIdStr == null) {
- return 0L;
- }
- ApplicationId appId = ApplicationId.fromString(appIdStr);
- long id = appId.getId() % TS_MULTIPLIER;
- return id;
- }
-
- /**
- * truncates the last few digits of the timestamp which were supplemented by
- * the TimestampGenerator#getSupplementedTimestamp function.
- *
- * @param incomingTS Timestamp to be truncated.
- * @return a truncated timestamp value
- */
- public static long getTruncatedTimestamp(long incomingTS) {
- return incomingTS / TS_MULTIPLIER;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TypedBufferedMutator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TypedBufferedMutator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TypedBufferedMutator.java
deleted file mode 100644
index 64a11f8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TypedBufferedMutator.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import org.apache.hadoop.hbase.client.BufferedMutator;
-
-/**
- * Just a typed wrapper around {@link BufferedMutator} used to ensure that
- * columns can write only to the table mutator for the right table.
- */
-public interface TypedBufferedMutator<T> extends BufferedMutator {
- // This class is intentionally left (almost) blank
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ValueConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ValueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ValueConverter.java
deleted file mode 100644
index 757a6d3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ValueConverter.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import java.io.IOException;
-
-/**
- * Converter used to encode/decode value associated with a column prefix or a
- * column.
- */
-public interface ValueConverter {
-
- /**
- * Encode an object as a byte array depending on the converter implementation.
- *
- * @param value Value to be encoded.
- * @return a byte array
- * @throws IOException if any problem is encountered while encoding.
- */
- byte[] encodeValue(Object value) throws IOException;
-
- /**
- * Decode a byte array and convert it into an object depending on the
- * converter implementation.
- *
- * @param bytes Byte array to be decoded.
- * @return an object
- * @throws IOException if any problem is encountered while decoding.
- */
- Object decodeValue(byte[] bytes) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java
deleted file mode 100644
index 0df5b8a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Package org.apache.hadoop.yarn.server.timelineservice.storage.common contains
- * a set of utility classes used across backend storage reader and writer.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
deleted file mode 100644
index b228d84..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Column;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.GenericConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
-
-/**
- * Identifies fully qualified columns for the {@link EntityTable}.
- */
-public enum EntityColumn implements Column<EntityTable> {
-
- /**
- * Identifier for the entity.
- */
- ID(EntityColumnFamily.INFO, "id"),
-
- /**
- * The type of entity.
- */
- TYPE(EntityColumnFamily.INFO, "type"),
-
- /**
- * When the entity was created.
- */
- CREATED_TIME(EntityColumnFamily.INFO, "created_time", new LongConverter()),
-
- /**
- * The version of the flow that this entity belongs to.
- */
- FLOW_VERSION(EntityColumnFamily.INFO, "flow_version");
-
- private final ColumnHelper<EntityTable> column;
- private final ColumnFamily<EntityTable> columnFamily;
- private final String columnQualifier;
- private final byte[] columnQualifierBytes;
-
- EntityColumn(ColumnFamily<EntityTable> columnFamily,
- String columnQualifier) {
- this(columnFamily, columnQualifier, GenericConverter.getInstance());
- }
-
- EntityColumn(ColumnFamily<EntityTable> columnFamily,
- String columnQualifier, ValueConverter converter) {
- this.columnFamily = columnFamily;
- this.columnQualifier = columnQualifier;
- // Future-proof by ensuring the right column prefix hygiene.
- this.columnQualifierBytes =
- Bytes.toBytes(Separator.SPACE.encode(columnQualifier));
- this.column = new ColumnHelper<EntityTable>(columnFamily, converter);
- }
-
- /**
- * @return the column name value
- */
- private String getColumnQualifier() {
- return columnQualifier;
- }
-
- public void store(byte[] rowKey,
- TypedBufferedMutator<EntityTable> tableMutator, Long timestamp,
- Object inputValue, Attribute... attributes) throws IOException {
- column.store(rowKey, tableMutator, columnQualifierBytes, timestamp,
- inputValue, attributes);
- }
-
- public Object readResult(Result result) throws IOException {
- return column.readResult(result, columnQualifierBytes);
- }
-
- @Override
- public byte[] getColumnQualifierBytes() {
- return columnQualifierBytes.clone();
- }
-
- @Override
- public byte[] getColumnFamilyBytes() {
- return columnFamily.getBytes();
- }
-
- @Override
- public ValueConverter getValueConverter() {
- return column.getValueConverter();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java
deleted file mode 100644
index 7c63727..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
-
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-
-/**
- * Represents the entity table column families.
- */
-public enum EntityColumnFamily implements ColumnFamily<EntityTable> {
-
- /**
- * Info column family houses known columns, specifically ones included in
- * columnfamily filters.
- */
- INFO("i"),
-
- /**
- * Configurations are in a separate column family for two reasons: a) the size
- * of the config values can be very large and b) we expect that config values
- * are often separately accessed from other metrics and info columns.
- */
- CONFIGS("c"),
-
- /**
- * Metrics have a separate column family, because they have a separate TTL.
- */
- METRICS("m");
-
- /**
- * Byte representation of this column family.
- */
- private final byte[] bytes;
-
- /**
- * @param value create a column family with this name. Must be lower case and
- * without spaces.
- */
- EntityColumnFamily(String value) {
- // column families should be lower case and not contain any spaces.
- this.bytes = Bytes.toBytes(Separator.SPACE.encode(value));
- }
-
- public byte[] getBytes() {
- return Bytes.copy(bytes);
- }
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org