You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2018/02/23 19:44:43 UTC
[02/50] [abbrv] hadoop git commit: YARN-7919. Refactor
timelineservice-hbase module into submodules. Contributed by Haibo Chen.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationTable.java
deleted file mode 100644
index 785a243..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/SubApplicationTable.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.regionserver.BloomType;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBaseSchemaConstants;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * The sub application table has column families:
- * info, config and metrics.
- * Info stores information about a timeline entity object
- * config stores configuration data of a timeline entity object
- * metrics stores the metrics of a timeline entity object
- *
- * Example sub application table record:
- *
- * <pre>
- * |-------------------------------------------------------------------------|
- * | Row | Column Family | Column Family| Column Family|
- * | key | info | metrics | config |
- * |-------------------------------------------------------------------------|
- * | subAppUserId! | id:entityId | metricId1: | configKey1: |
- * | clusterId! | type:entityType | metricValue1 | configValue1 |
- * | entityType! | | @timestamp1 | |
- * | idPrefix!| | | | configKey2: |
- * | entityId! | created_time: | metricId1: | configValue2 |
- * | userId | 1392993084018 | metricValue2 | |
- * | | | @timestamp2 | |
- * | | i!infoKey: | | |
- * | | infoValue | metricId1: | |
- * | | | metricValue1 | |
- * | | | @timestamp2 | |
- * | | e!eventId=timestamp= | | |
- * | | infoKey: | | |
- * | | eventInfoValue | | |
- * | | | | |
- * | | r!relatesToKey: | | |
- * | | id3=id4=id5 | | |
- * | | | | |
- * | | s!isRelatedToKey | | |
- * | | id7=id9=id6 | | |
- * | | | | |
- * | | flowVersion: | | |
- * | | versionValue | | |
- * |-------------------------------------------------------------------------|
- * </pre>
- */
-public class SubApplicationTable extends BaseTable<SubApplicationTable> {
- /** sub app prefix. */
- private static final String PREFIX =
- YarnConfiguration.TIMELINE_SERVICE_PREFIX + "subapplication";
-
- /** config param name that specifies the subapplication table name. */
- public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
-
- /**
- * config param name that specifies the TTL for metrics column family in
- * subapplication table.
- */
- private static final String METRICS_TTL_CONF_NAME = PREFIX
- + ".table.metrics.ttl";
-
- /**
- * config param name that specifies max-versions for
- * metrics column family in subapplication table.
- */
- private static final String METRICS_MAX_VERSIONS =
- PREFIX + ".table.metrics.max-versions";
-
- /** default value for subapplication table name. */
- public static final String DEFAULT_TABLE_NAME =
- "timelineservice.subapplication";
-
- /** default TTL is 30 days for metrics timeseries. */
- private static final int DEFAULT_METRICS_TTL = 2592000;
-
- /** default max number of versions. */
- private static final int DEFAULT_METRICS_MAX_VERSIONS = 10000;
-
- private static final Logger LOG = LoggerFactory.getLogger(
- SubApplicationTable.class);
-
- public SubApplicationTable() {
- super(TABLE_NAME_CONF_NAME, DEFAULT_TABLE_NAME);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.yarn.server.timelineservice.storage.BaseTable#createTable
- * (org.apache.hadoop.hbase.client.Admin,
- * org.apache.hadoop.conf.Configuration)
- */
- public void createTable(Admin admin, Configuration hbaseConf)
- throws IOException {
-
- TableName table = getTableName(hbaseConf);
- if (admin.tableExists(table)) {
- // do not disable / delete existing table
- // similar to the approach taken by map-reduce jobs when
- // output directory exists
- throw new IOException("Table " + table.getNameAsString()
- + " already exists.");
- }
-
- HTableDescriptor subAppTableDescp = new HTableDescriptor(table);
- HColumnDescriptor infoCF =
- new HColumnDescriptor(SubApplicationColumnFamily.INFO.getBytes());
- infoCF.setBloomFilterType(BloomType.ROWCOL);
- subAppTableDescp.addFamily(infoCF);
-
- HColumnDescriptor configCF =
- new HColumnDescriptor(SubApplicationColumnFamily.CONFIGS.getBytes());
- configCF.setBloomFilterType(BloomType.ROWCOL);
- configCF.setBlockCacheEnabled(true);
- subAppTableDescp.addFamily(configCF);
-
- HColumnDescriptor metricsCF =
- new HColumnDescriptor(SubApplicationColumnFamily.METRICS.getBytes());
- subAppTableDescp.addFamily(metricsCF);
- metricsCF.setBlockCacheEnabled(true);
- // always keep 1 version (the latest)
- metricsCF.setMinVersions(1);
- metricsCF.setMaxVersions(
- hbaseConf.getInt(METRICS_MAX_VERSIONS, DEFAULT_METRICS_MAX_VERSIONS));
- metricsCF.setTimeToLive(hbaseConf.getInt(METRICS_TTL_CONF_NAME,
- DEFAULT_METRICS_TTL));
- subAppTableDescp.setRegionSplitPolicyClassName(
- "org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy");
- subAppTableDescp.setValue("KeyPrefixRegionSplitPolicy.prefix_length",
- TimelineHBaseSchemaConstants.USERNAME_SPLIT_KEY_PREFIX_LENGTH);
- admin.createTable(subAppTableDescp,
- TimelineHBaseSchemaConstants.getUsernameSplits());
- LOG.info("Status of table creation for " + table.getNameAsString() + "="
- + admin.tableExists(table));
- }
-
- /**
- * @param metricsTTL time to live parameter for the metricss in this table.
- * @param hbaseConf configururation in which to set the metrics TTL config
- * variable.
- */
- public void setMetricsTTL(int metricsTTL, Configuration hbaseConf) {
- hbaseConf.setInt(METRICS_TTL_CONF_NAME, metricsTTL);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/package-info.java
deleted file mode 100644
index 52cc399..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/subapplication/package-info.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication
- * contains classes related to implementation for subapplication table.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-package org.apache.hadoop.yarn.server.timelineservice.storage.subapplication;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestCustomApplicationIdConversion.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestCustomApplicationIdConversion.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestCustomApplicationIdConversion.java
deleted file mode 100644
index 73bc29e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestCustomApplicationIdConversion.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * Test for HBaseTimelineStorageUtils.convertApplicationIdToString(),
- * a custom conversion from ApplicationId to String that avoids the
- * incompatibility issue caused by mixing hadoop-common 2.5.1 and
- * hadoop-yarn-api 3.0. See YARN-6905.
- */
-public class TestCustomApplicationIdConversion {
- @Test
- public void testConvertAplicationIdToString() {
- ApplicationId applicationId = ApplicationId.newInstance(0, 1);
- String applicationIdStr =
- HBaseTimelineStorageUtils.convertApplicationIdToString(applicationId);
- Assert.assertEquals(applicationId,
- ApplicationId.fromString(applicationIdStr));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java
deleted file mode 100644
index 402a89b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import org.junit.Test;
-
-/**
- * Unit tests for HBaseTimelineStorageUtils static methos.
- */
-public class TestHBaseTimelineStorageUtils {
-
- @Test(expected=NullPointerException.class)
- public void testGetTimelineServiceHBaseConfNullArgument() throws Exception {
- HBaseTimelineStorageUtils.getTimelineServiceHBaseConf(null);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestKeyConverters.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestKeyConverters.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestKeyConverters.java
deleted file mode 100644
index 1bd363f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestKeyConverters.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.junit.Test;
-
-/**
- * Unit tests for key converters for various tables' row keys.
- *
- */
-public class TestKeyConverters {
-
- @Test
- public void testAppIdKeyConverter() {
- AppIdKeyConverter appIdKeyConverter = new AppIdKeyConverter();
- long currentTs = System.currentTimeMillis();
- ApplicationId appId1 = ApplicationId.newInstance(currentTs, 1);
- ApplicationId appId2 = ApplicationId.newInstance(currentTs, 2);
- ApplicationId appId3 = ApplicationId.newInstance(currentTs + 300, 1);
- String appIdStr1 = appId1.toString();
- String appIdStr2 = appId2.toString();
- String appIdStr3 = appId3.toString();
- byte[] appIdBytes1 = appIdKeyConverter.encode(appIdStr1);
- byte[] appIdBytes2 = appIdKeyConverter.encode(appIdStr2);
- byte[] appIdBytes3 = appIdKeyConverter.encode(appIdStr3);
- // App ids' should be encoded in a manner wherein descending order
- // is maintained.
- assertTrue(
- "Ordering of app ids' is incorrect",
- Bytes.compareTo(appIdBytes1, appIdBytes2) > 0
- && Bytes.compareTo(appIdBytes1, appIdBytes3) > 0
- && Bytes.compareTo(appIdBytes2, appIdBytes3) > 0);
- String decodedAppId1 = appIdKeyConverter.decode(appIdBytes1);
- String decodedAppId2 = appIdKeyConverter.decode(appIdBytes2);
- String decodedAppId3 = appIdKeyConverter.decode(appIdBytes3);
- assertTrue("Decoded app id is not same as the app id encoded",
- appIdStr1.equals(decodedAppId1));
- assertTrue("Decoded app id is not same as the app id encoded",
- appIdStr2.equals(decodedAppId2));
- assertTrue("Decoded app id is not same as the app id encoded",
- appIdStr3.equals(decodedAppId3));
- }
-
- @Test
- public void testEventColumnNameConverter() {
- String eventId = "=foo_=eve=nt=";
- byte[] valSepBytes = Bytes.toBytes(Separator.VALUES.getValue());
- byte[] maxByteArr =
- Bytes.createMaxByteArray(Bytes.SIZEOF_LONG - valSepBytes.length);
- byte[] ts = Bytes.add(valSepBytes, maxByteArr);
- Long eventTs = Bytes.toLong(ts);
- byte[] byteEventColName =
- new EventColumnName(eventId, eventTs, null).getColumnQualifier();
- KeyConverter<EventColumnName> eventColumnNameConverter =
- new EventColumnNameConverter();
- EventColumnName eventColName =
- eventColumnNameConverter.decode(byteEventColName);
- assertEquals(eventId, eventColName.getId());
- assertEquals(eventTs, eventColName.getTimestamp());
- assertNull(eventColName.getInfoKey());
-
- String infoKey = "f=oo_event_in=fo=_key";
- byteEventColName =
- new EventColumnName(eventId, eventTs, infoKey).getColumnQualifier();
- eventColName = eventColumnNameConverter.decode(byteEventColName);
- assertEquals(eventId, eventColName.getId());
- assertEquals(eventTs, eventColName.getTimestamp());
- assertEquals(infoKey, eventColName.getInfoKey());
- }
-
- @Test
- public void testLongKeyConverter() {
- LongKeyConverter longKeyConverter = new LongKeyConverter();
- confirmLongKeyConverter(longKeyConverter, Long.MIN_VALUE);
- confirmLongKeyConverter(longKeyConverter, -1234567890L);
- confirmLongKeyConverter(longKeyConverter, -128L);
- confirmLongKeyConverter(longKeyConverter, -127L);
- confirmLongKeyConverter(longKeyConverter, -1L);
- confirmLongKeyConverter(longKeyConverter, 0L);
- confirmLongKeyConverter(longKeyConverter, 1L);
- confirmLongKeyConverter(longKeyConverter, 127L);
- confirmLongKeyConverter(longKeyConverter, 128L);
- confirmLongKeyConverter(longKeyConverter, 1234567890L);
- confirmLongKeyConverter(longKeyConverter, Long.MAX_VALUE);
- }
-
- private void confirmLongKeyConverter(LongKeyConverter longKeyConverter,
- Long testValue) {
- Long decoded = longKeyConverter.decode(longKeyConverter.encode(testValue));
- assertEquals(testValue, decoded);
- }
-
- @Test
- public void testStringKeyConverter() {
- StringKeyConverter stringKeyConverter = new StringKeyConverter();
- String phrase = "QuackAttack now!";
-
- for (int i = 0; i < phrase.length(); i++) {
- String sub = phrase.substring(i, phrase.length());
- confirmStrignKeyConverter(stringKeyConverter, sub);
- confirmStrignKeyConverter(stringKeyConverter, sub + sub);
- }
- }
-
- private void confirmStrignKeyConverter(StringKeyConverter stringKeyConverter,
- String testValue) {
- String decoded =
- stringKeyConverter.decode(stringKeyConverter.encode(testValue));
- assertEquals(testValue, decoded);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java
deleted file mode 100644
index 4770238..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java
+++ /dev/null
@@ -1,276 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKeyPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKeyPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKeyPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationRowKey;
-import org.junit.Test;
-
-
-/**
- * Class to test the row key structures for various tables.
- *
- */
-public class TestRowKeys {
-
- private final static String QUALIFIER_SEP = Separator.QUALIFIERS.getValue();
- private final static byte[] QUALIFIER_SEP_BYTES = Bytes
- .toBytes(QUALIFIER_SEP);
- private final static String CLUSTER = "cl" + QUALIFIER_SEP + "uster";
- private final static String USER = QUALIFIER_SEP + "user";
- private final static String SUB_APP_USER = QUALIFIER_SEP + "subAppUser";
- private final static String FLOW_NAME = "dummy_" + QUALIFIER_SEP + "flow"
- + QUALIFIER_SEP;
- private final static Long FLOW_RUN_ID;
- private final static String APPLICATION_ID;
- static {
- long runid = Long.MAX_VALUE - 900L;
- byte[] longMaxByteArr = Bytes.toBytes(Long.MAX_VALUE);
- byte[] byteArr = Bytes.toBytes(runid);
- int sepByteLen = QUALIFIER_SEP_BYTES.length;
- if (sepByteLen <= byteArr.length) {
- for (int i = 0; i < sepByteLen; i++) {
- byteArr[i] = (byte) (longMaxByteArr[i] - QUALIFIER_SEP_BYTES[i]);
- }
- }
- FLOW_RUN_ID = Bytes.toLong(byteArr);
- long clusterTs = System.currentTimeMillis();
- byteArr = Bytes.toBytes(clusterTs);
- if (sepByteLen <= byteArr.length) {
- for (int i = 0; i < sepByteLen; i++) {
- byteArr[byteArr.length - sepByteLen + i] =
- (byte) (longMaxByteArr[byteArr.length - sepByteLen + i] -
- QUALIFIER_SEP_BYTES[i]);
- }
- }
- clusterTs = Bytes.toLong(byteArr);
- int seqId = 222;
- APPLICATION_ID = ApplicationId.newInstance(clusterTs, seqId).toString();
- }
-
- private static void verifyRowPrefixBytes(byte[] byteRowKeyPrefix) {
- int sepLen = QUALIFIER_SEP_BYTES.length;
- for (int i = 0; i < sepLen; i++) {
- assertTrue(
- "Row key prefix not encoded properly.",
- byteRowKeyPrefix[byteRowKeyPrefix.length - sepLen + i] ==
- QUALIFIER_SEP_BYTES[i]);
- }
- }
-
- @Test
- public void testApplicationRowKey() {
- byte[] byteRowKey =
- new ApplicationRowKey(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID,
- APPLICATION_ID).getRowKey();
- ApplicationRowKey rowKey = ApplicationRowKey.parseRowKey(byteRowKey);
- assertEquals(CLUSTER, rowKey.getClusterId());
- assertEquals(USER, rowKey.getUserId());
- assertEquals(FLOW_NAME, rowKey.getFlowName());
- assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId());
- assertEquals(APPLICATION_ID, rowKey.getAppId());
-
- byte[] byteRowKeyPrefix =
- new ApplicationRowKeyPrefix(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID)
- .getRowKeyPrefix();
- byte[][] splits =
- Separator.QUALIFIERS.split(byteRowKeyPrefix,
- new int[] {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
- Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
- Separator.VARIABLE_SIZE});
- assertEquals(5, splits.length);
- assertEquals(0, splits[4].length);
- assertEquals(FLOW_NAME,
- Separator.QUALIFIERS.decode(Bytes.toString(splits[2])));
- assertEquals(FLOW_RUN_ID,
- (Long) LongConverter.invertLong(Bytes.toLong(splits[3])));
- verifyRowPrefixBytes(byteRowKeyPrefix);
-
- byteRowKeyPrefix =
- new ApplicationRowKeyPrefix(CLUSTER, USER, FLOW_NAME).getRowKeyPrefix();
- splits =
- Separator.QUALIFIERS.split(byteRowKeyPrefix, new int[] {
- Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
- Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE });
- assertEquals(4, splits.length);
- assertEquals(0, splits[3].length);
- assertEquals(FLOW_NAME,
- Separator.QUALIFIERS.decode(Bytes.toString(splits[2])));
- verifyRowPrefixBytes(byteRowKeyPrefix);
- }
-
- /**
- * Tests the converters indirectly through the public methods of the
- * corresponding rowkey.
- */
- @Test
- public void testAppToFlowRowKey() {
- byte[] byteRowKey = new AppToFlowRowKey(APPLICATION_ID).getRowKey();
- AppToFlowRowKey rowKey = AppToFlowRowKey.parseRowKey(byteRowKey);
- assertEquals(APPLICATION_ID, rowKey.getAppId());
- }
-
- @Test
- public void testEntityRowKey() {
- TimelineEntity entity = new TimelineEntity();
- entity.setId("!ent!ity!!id!");
- entity.setType("entity!Type");
- entity.setIdPrefix(54321);
-
- byte[] byteRowKey =
- new EntityRowKey(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID, APPLICATION_ID,
- entity.getType(), entity.getIdPrefix(),
- entity.getId()).getRowKey();
- EntityRowKey rowKey = EntityRowKey.parseRowKey(byteRowKey);
- assertEquals(CLUSTER, rowKey.getClusterId());
- assertEquals(USER, rowKey.getUserId());
- assertEquals(FLOW_NAME, rowKey.getFlowName());
- assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId());
- assertEquals(APPLICATION_ID, rowKey.getAppId());
- assertEquals(entity.getType(), rowKey.getEntityType());
- assertEquals(entity.getIdPrefix(), rowKey.getEntityIdPrefix().longValue());
- assertEquals(entity.getId(), rowKey.getEntityId());
-
- byte[] byteRowKeyPrefix =
- new EntityRowKeyPrefix(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID,
- APPLICATION_ID, entity.getType(), null, null)
- .getRowKeyPrefix();
- byte[][] splits =
- Separator.QUALIFIERS.split(
- byteRowKeyPrefix,
- new int[] {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
- Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
- AppIdKeyConverter.getKeySize(), Separator.VARIABLE_SIZE,
- Bytes.SIZEOF_LONG, Separator.VARIABLE_SIZE });
- assertEquals(7, splits.length);
- assertEquals(APPLICATION_ID, new AppIdKeyConverter().decode(splits[4]));
- assertEquals(entity.getType(),
- Separator.QUALIFIERS.decode(Bytes.toString(splits[5])));
- verifyRowPrefixBytes(byteRowKeyPrefix);
-
- byteRowKeyPrefix =
- new EntityRowKeyPrefix(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID,
- APPLICATION_ID).getRowKeyPrefix();
- splits =
- Separator.QUALIFIERS.split(
- byteRowKeyPrefix,
- new int[] {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
- Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
- AppIdKeyConverter.getKeySize(), Separator.VARIABLE_SIZE});
- assertEquals(6, splits.length);
- assertEquals(0, splits[5].length);
- AppIdKeyConverter appIdKeyConverter = new AppIdKeyConverter();
- assertEquals(APPLICATION_ID, appIdKeyConverter.decode(splits[4]));
- verifyRowPrefixBytes(byteRowKeyPrefix);
- }
-
- @Test
- public void testFlowActivityRowKey() {
- Long ts = 1459900830000L;
- Long dayTimestamp = HBaseTimelineStorageUtils.getTopOfTheDayTimestamp(ts);
- byte[] byteRowKey =
- new FlowActivityRowKey(CLUSTER, ts, USER, FLOW_NAME).getRowKey();
- FlowActivityRowKey rowKey = FlowActivityRowKey.parseRowKey(byteRowKey);
- assertEquals(CLUSTER, rowKey.getClusterId());
- assertEquals(dayTimestamp, rowKey.getDayTimestamp());
- assertEquals(USER, rowKey.getUserId());
- assertEquals(FLOW_NAME, rowKey.getFlowName());
-
- byte[] byteRowKeyPrefix =
- new FlowActivityRowKeyPrefix(CLUSTER).getRowKeyPrefix();
- byte[][] splits =
- Separator.QUALIFIERS.split(byteRowKeyPrefix, new int[] {
- Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE });
- assertEquals(2, splits.length);
- assertEquals(0, splits[1].length);
- assertEquals(CLUSTER,
- Separator.QUALIFIERS.decode(Bytes.toString(splits[0])));
- verifyRowPrefixBytes(byteRowKeyPrefix);
-
- byteRowKeyPrefix =
- new FlowActivityRowKeyPrefix(CLUSTER, ts).getRowKeyPrefix();
- splits =
- Separator.QUALIFIERS.split(byteRowKeyPrefix,
- new int[] {Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
- Separator.VARIABLE_SIZE});
- assertEquals(3, splits.length);
- assertEquals(0, splits[2].length);
- assertEquals(CLUSTER,
- Separator.QUALIFIERS.decode(Bytes.toString(splits[0])));
- assertEquals(ts,
- (Long) LongConverter.invertLong(Bytes.toLong(splits[1])));
- verifyRowPrefixBytes(byteRowKeyPrefix);
- }
-
- @Test
- public void testFlowRunRowKey() {
- byte[] byteRowKey =
- new FlowRunRowKey(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID).getRowKey();
- FlowRunRowKey rowKey = FlowRunRowKey.parseRowKey(byteRowKey);
- assertEquals(CLUSTER, rowKey.getClusterId());
- assertEquals(USER, rowKey.getUserId());
- assertEquals(FLOW_NAME, rowKey.getFlowName());
- assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId());
-
- byte[] byteRowKeyPrefix =
- new FlowRunRowKey(CLUSTER, USER, FLOW_NAME, null).getRowKey();
- byte[][] splits =
- Separator.QUALIFIERS.split(byteRowKeyPrefix, new int[] {
- Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
- Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE });
- assertEquals(4, splits.length);
- assertEquals(0, splits[3].length);
- assertEquals(FLOW_NAME,
- Separator.QUALIFIERS.decode(Bytes.toString(splits[2])));
- verifyRowPrefixBytes(byteRowKeyPrefix);
- }
-
- @Test
- public void testSubAppRowKey() {
- TimelineEntity entity = new TimelineEntity();
- entity.setId("entity1");
- entity.setType("DAG");
- entity.setIdPrefix(54321);
-
- byte[] byteRowKey =
- new SubApplicationRowKey(SUB_APP_USER, CLUSTER,
- entity.getType(), entity.getIdPrefix(),
- entity.getId(), USER).getRowKey();
- SubApplicationRowKey rowKey = SubApplicationRowKey.parseRowKey(byteRowKey);
- assertEquals(CLUSTER, rowKey.getClusterId());
- assertEquals(SUB_APP_USER, rowKey.getSubAppUserId());
- assertEquals(entity.getType(), rowKey.getEntityType());
- assertEquals(entity.getIdPrefix(), rowKey.getEntityIdPrefix().longValue());
- assertEquals(entity.getId(), rowKey.getEntityId());
- assertEquals(USER, rowKey.getUserId());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeysAsString.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeysAsString.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeysAsString.java
deleted file mode 100644
index 148cf56..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeysAsString.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import static org.junit.Assert.assertEquals;
-
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationRowKey;
-import org.junit.Test;
-
-/**
- * Test for row key as string.
- */
-public class TestRowKeysAsString {
-
- private final static String CLUSTER =
- "cl" + TimelineReaderUtils.DEFAULT_DELIMITER_CHAR + "uster"
- + TimelineReaderUtils.DEFAULT_ESCAPE_CHAR;
- private final static String USER =
- TimelineReaderUtils.DEFAULT_ESCAPE_CHAR + "user";
- private final static String SUB_APP_USER =
- TimelineReaderUtils.DEFAULT_ESCAPE_CHAR + "subAppUser";
-
- private final static String FLOW_NAME =
- "dummy_" + TimelineReaderUtils.DEFAULT_DELIMITER_CHAR
- + TimelineReaderUtils.DEFAULT_ESCAPE_CHAR + "flow"
- + TimelineReaderUtils.DEFAULT_DELIMITER_CHAR;
- private final static Long FLOW_RUN_ID = System.currentTimeMillis();
- private final static String APPLICATION_ID =
- ApplicationId.newInstance(System.currentTimeMillis(), 1).toString();
-
- @Test(timeout = 10000)
- public void testApplicationRow() {
- String rowKeyAsString = new ApplicationRowKey(CLUSTER, USER, FLOW_NAME,
- FLOW_RUN_ID, APPLICATION_ID).getRowKeyAsString();
- ApplicationRowKey rowKey =
- ApplicationRowKey.parseRowKeyFromString(rowKeyAsString);
- assertEquals(CLUSTER, rowKey.getClusterId());
- assertEquals(USER, rowKey.getUserId());
- assertEquals(FLOW_NAME, rowKey.getFlowName());
- assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId());
- assertEquals(APPLICATION_ID, rowKey.getAppId());
- }
-
- @Test(timeout = 10000)
- public void testEntityRowKey() {
- char del = TimelineReaderUtils.DEFAULT_DELIMITER_CHAR;
- char esc = TimelineReaderUtils.DEFAULT_ESCAPE_CHAR;
- String id = del + esc + "ent" + esc + del + "ity" + esc + del + esc + "id"
- + esc + del + esc;
- String type = "entity" + esc + del + esc + "Type";
- TimelineEntity entity = new TimelineEntity();
- entity.setId(id);
- entity.setType(type);
- entity.setIdPrefix(54321);
-
- String rowKeyAsString =
- new EntityRowKey(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID, APPLICATION_ID,
- entity.getType(), entity.getIdPrefix(), entity.getId())
- .getRowKeyAsString();
- EntityRowKey rowKey = EntityRowKey.parseRowKeyFromString(rowKeyAsString);
- assertEquals(CLUSTER, rowKey.getClusterId());
- assertEquals(USER, rowKey.getUserId());
- assertEquals(FLOW_NAME, rowKey.getFlowName());
- assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId());
- assertEquals(APPLICATION_ID, rowKey.getAppId());
- assertEquals(entity.getType(), rowKey.getEntityType());
- assertEquals(entity.getIdPrefix(), rowKey.getEntityIdPrefix().longValue());
- assertEquals(entity.getId(), rowKey.getEntityId());
-
- }
-
- @Test(timeout = 10000)
- public void testFlowActivityRowKey() {
- Long ts = 1459900830000L;
- Long dayTimestamp = HBaseTimelineStorageUtils.getTopOfTheDayTimestamp(ts);
- String rowKeyAsString = new FlowActivityRowKey(CLUSTER, ts, USER, FLOW_NAME)
- .getRowKeyAsString();
- FlowActivityRowKey rowKey =
- FlowActivityRowKey.parseRowKeyFromString(rowKeyAsString);
- assertEquals(CLUSTER, rowKey.getClusterId());
- assertEquals(dayTimestamp, rowKey.getDayTimestamp());
- assertEquals(USER, rowKey.getUserId());
- assertEquals(FLOW_NAME, rowKey.getFlowName());
- }
-
- @Test(timeout = 10000)
- public void testFlowRunRowKey() {
- String rowKeyAsString =
- new FlowRunRowKey(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID)
- .getRowKeyAsString();
- FlowRunRowKey rowKey = FlowRunRowKey.parseRowKeyFromString(rowKeyAsString);
- assertEquals(CLUSTER, rowKey.getClusterId());
- assertEquals(USER, rowKey.getUserId());
- assertEquals(FLOW_NAME, rowKey.getFlowName());
- assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId());
- }
-
- @Test(timeout = 10000)
- public void testSubApplicationRowKey() {
- char del = TimelineReaderUtils.DEFAULT_DELIMITER_CHAR;
- char esc = TimelineReaderUtils.DEFAULT_ESCAPE_CHAR;
- String id = del + esc + "ent" + esc + del + "ity" + esc + del + esc + "id"
- + esc + del + esc;
- String type = "entity" + esc + del + esc + "Type";
- TimelineEntity entity = new TimelineEntity();
- entity.setId(id);
- entity.setType(type);
- entity.setIdPrefix(54321);
-
- String rowKeyAsString = new SubApplicationRowKey(SUB_APP_USER, CLUSTER,
- entity.getType(), entity.getIdPrefix(), entity.getId(), USER)
- .getRowKeyAsString();
- SubApplicationRowKey rowKey = SubApplicationRowKey
- .parseRowKeyFromString(rowKeyAsString);
- assertEquals(SUB_APP_USER, rowKey.getSubAppUserId());
- assertEquals(CLUSTER, rowKey.getClusterId());
- assertEquals(entity.getType(), rowKey.getEntityType());
- assertEquals(entity.getIdPrefix(), rowKey.getEntityIdPrefix().longValue());
- assertEquals(entity.getId(), rowKey.getEntityId());
- assertEquals(USER, rowKey.getUserId());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9af30d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestSeparator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestSeparator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestSeparator.java
deleted file mode 100644
index 7d37206..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestSeparator.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.Test;
-
-import com.google.common.collect.Iterables;
-
-public class TestSeparator {
-
- private static String villain = "Dr. Heinz Doofenshmirtz";
- private static String special =
- ". * | ? + \t ( ) [ ] { } ^ $ \\ \" %";
-
- /**
- *
- */
- @Test
- public void testEncodeDecodeString() {
-
- for (Separator separator : Separator.values()) {
- testEncodeDecode(separator, "");
- testEncodeDecode(separator, " ");
- testEncodeDecode(separator, "!");
- testEncodeDecode(separator, "?");
- testEncodeDecode(separator, "&");
- testEncodeDecode(separator, "+");
- testEncodeDecode(separator, "\t");
- testEncodeDecode(separator, "Dr.");
- testEncodeDecode(separator, "Heinz");
- testEncodeDecode(separator, "Doofenshmirtz");
- testEncodeDecode(separator, villain);
- testEncodeDecode(separator, special);
-
- assertNull(separator.encode(null));
-
- }
- }
-
- private void testEncodeDecode(Separator separator, String token) {
- String encoded = separator.encode(token);
- String decoded = separator.decode(encoded);
- String msg = "token:" + token + " separator:" + separator + ".";
- assertEquals(msg, token, decoded);
- }
-
- @Test
- public void testEncodeDecode() {
- testEncodeDecode("Dr.", Separator.QUALIFIERS);
- testEncodeDecode("Heinz", Separator.QUALIFIERS, Separator.QUALIFIERS);
- testEncodeDecode("Doofenshmirtz", Separator.QUALIFIERS, null,
- Separator.QUALIFIERS);
- testEncodeDecode("&Perry", Separator.QUALIFIERS, Separator.VALUES, null);
- testEncodeDecode("the ", Separator.QUALIFIERS, Separator.SPACE);
- testEncodeDecode("Platypus...", (Separator) null);
- testEncodeDecode("The what now ?!?", Separator.QUALIFIERS,
- Separator.VALUES, Separator.SPACE);
-
- }
- @Test
- public void testEncodedValues() {
- testEncodeDecode("Double-escape %2$ and %9$ or %%2$ or %%3$, nor %%%2$" +
- "= no problem!",
- Separator.QUALIFIERS, Separator.VALUES, Separator.SPACE, Separator.TAB);
- }
-
- @Test
- public void testSplits() {
- byte[] maxLongBytes = Bytes.toBytes(Long.MAX_VALUE);
- byte[] maxIntBytes = Bytes.toBytes(Integer.MAX_VALUE);
- for (Separator separator : Separator.values()) {
- String str1 = "cl" + separator.getValue() + "us";
- String str2 = separator.getValue() + "rst";
- byte[] sepByteArr = Bytes.toBytes(separator.getValue());
- byte[] longVal1Arr = Bytes.add(sepByteArr, Bytes.copy(maxLongBytes,
- sepByteArr.length, Bytes.SIZEOF_LONG - sepByteArr.length));
- byte[] intVal1Arr = Bytes.add(sepByteArr, Bytes.copy(maxIntBytes,
- sepByteArr.length, Bytes.SIZEOF_INT - sepByteArr.length));
- byte[] arr = separator.join(
- Bytes.toBytes(separator.encode(str1)), longVal1Arr,
- Bytes.toBytes(separator.encode(str2)), intVal1Arr);
- int[] sizes = {Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
- Separator.VARIABLE_SIZE, Bytes.SIZEOF_INT};
- byte[][] splits = separator.split(arr, sizes);
- assertEquals(4, splits.length);
- assertEquals(str1, separator.decode(Bytes.toString(splits[0])));
- assertEquals(Bytes.toLong(longVal1Arr), Bytes.toLong(splits[1]));
- assertEquals(str2, separator.decode(Bytes.toString(splits[2])));
- assertEquals(Bytes.toInt(intVal1Arr), Bytes.toInt(splits[3]));
-
- longVal1Arr = Bytes.add(Bytes.copy(maxLongBytes, 0, Bytes.SIZEOF_LONG -
- sepByteArr.length), sepByteArr);
- intVal1Arr = Bytes.add(Bytes.copy(maxIntBytes, 0, Bytes.SIZEOF_INT -
- sepByteArr.length), sepByteArr);
- arr = separator.join(Bytes.toBytes(separator.encode(str1)), longVal1Arr,
- Bytes.toBytes(separator.encode(str2)), intVal1Arr);
- splits = separator.split(arr, sizes);
- assertEquals(4, splits.length);
- assertEquals(str1, separator.decode(Bytes.toString(splits[0])));
- assertEquals(Bytes.toLong(longVal1Arr), Bytes.toLong(splits[1]));
- assertEquals(str2, separator.decode(Bytes.toString(splits[2])));
- assertEquals(Bytes.toInt(intVal1Arr), Bytes.toInt(splits[3]));
-
- longVal1Arr = Bytes.add(sepByteArr, Bytes.copy(maxLongBytes,
- sepByteArr.length, 4 - sepByteArr.length), sepByteArr);
- longVal1Arr = Bytes.add(longVal1Arr, Bytes.copy(maxLongBytes, 4, 3 -
- sepByteArr.length), sepByteArr);
- arr = separator.join(Bytes.toBytes(separator.encode(str1)), longVal1Arr,
- Bytes.toBytes(separator.encode(str2)), intVal1Arr);
- splits = separator.split(arr, sizes);
- assertEquals(4, splits.length);
- assertEquals(str1, separator.decode(Bytes.toString(splits[0])));
- assertEquals(Bytes.toLong(longVal1Arr), Bytes.toLong(splits[1]));
- assertEquals(str2, separator.decode(Bytes.toString(splits[2])));
- assertEquals(Bytes.toInt(intVal1Arr), Bytes.toInt(splits[3]));
-
- arr = separator.join(Bytes.toBytes(separator.encode(str1)),
- Bytes.toBytes(separator.encode(str2)), intVal1Arr, longVal1Arr);
- int[] sizes1 = {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
- Bytes.SIZEOF_INT, Bytes.SIZEOF_LONG};
- splits = separator.split(arr, sizes1);
- assertEquals(4, splits.length);
- assertEquals(str1, separator.decode(Bytes.toString(splits[0])));
- assertEquals(str2, separator.decode(Bytes.toString(splits[1])));
- assertEquals(Bytes.toInt(intVal1Arr), Bytes.toInt(splits[2]));
- assertEquals(Bytes.toLong(longVal1Arr), Bytes.toLong(splits[3]));
-
- try {
- int[] sizes2 = {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
- Bytes.SIZEOF_INT, 7};
- splits = separator.split(arr, sizes2);
- fail("Exception should have been thrown.");
- } catch (IllegalArgumentException e) {}
-
- try {
- int[] sizes2 = {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE, 2,
- Bytes.SIZEOF_LONG};
- splits = separator.split(arr, sizes2);
- fail("Exception should have been thrown.");
- } catch (IllegalArgumentException e) {}
- }
- }
-
- /**
- * Simple test to encode and decode using the same separators and confirm that
- * we end up with the same as what we started with.
- *
- * @param token
- * @param separators
- */
- private static void testEncodeDecode(String token, Separator... separators) {
- byte[] encoded = Separator.encode(token, separators);
- String decoded = Separator.decode(encoded, separators);
- assertEquals(token, decoded);
- }
-
- @Test
- public void testJoinStripped() {
- List<String> stringList = new ArrayList<String>(0);
- stringList.add("nothing");
-
- String joined = Separator.VALUES.joinEncoded(stringList);
- Iterable<String> split = Separator.VALUES.splitEncoded(joined);
- assertTrue(Iterables.elementsEqual(stringList, split));
-
- stringList = new ArrayList<String>(3);
- stringList.add("a");
- stringList.add("b?");
- stringList.add("c");
-
- joined = Separator.VALUES.joinEncoded(stringList);
- split = Separator.VALUES.splitEncoded(joined);
- assertTrue(Iterables.elementsEqual(stringList, split));
-
- String[] stringArray1 = {"else"};
- joined = Separator.VALUES.joinEncoded(stringArray1);
- split = Separator.VALUES.splitEncoded(joined);
- assertTrue(Iterables.elementsEqual(Arrays.asList(stringArray1), split));
-
- String[] stringArray2 = {"d", "e?", "f"};
- joined = Separator.VALUES.joinEncoded(stringArray2);
- split = Separator.VALUES.splitEncoded(joined);
- assertTrue(Iterables.elementsEqual(Arrays.asList(stringArray2), split));
-
- List<String> empty = new ArrayList<String>(0);
- split = Separator.VALUES.splitEncoded(null);
- assertTrue(Iterables.elementsEqual(empty, split));
-
- }
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org