You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by ha...@apache.org on 2016/11/07 06:52:34 UTC
incubator-eagle git commit: [EAGLE-411] Improve code coverage of
eagle-storage-hbase to 61%
Repository: incubator-eagle
Updated Branches:
refs/heads/master 0952732e6 -> 66baf258c
[EAGLE-411] Improve code coverage of eagle-storage-hbase to 61%
<!--
{% comment %}
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to you under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
{% endcomment %}
-->
Be sure to do all of the following to help us incorporate your contribution
quickly and easily:
- [ ] Make sure the PR title is formatted like:
`[EAGLE-<Jira issue #>] Description of pull request`
- [ ] Make sure tests pass via `mvn clean verify`. (Even better, enable
Travis-CI on your fork and ensure the whole test matrix passes).
- [ ] Replace `<Jira issue #>` in the title with the actual Jira issue
number, if there is one.
- [ ] If this contribution is large, please file an Apache
[Individual Contributor License Agreement](https://www.apache.org/licenses/icla.txt).
---
Author: chang chen <ba...@gmail.com>
Closes #608 from baibaichen/feature/hbase-ut.
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/66baf258
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/66baf258
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/66baf258
Branch: refs/heads/master
Commit: 66baf258c37fa035e57f7aa4de1bab16ee433fe6
Parents: 0952732
Author: chang chen <ba...@gmail.com>
Authored: Mon Nov 7 14:52:20 2016 +0800
Committer: Hao Chen <ha...@apache.org>
Committed: Mon Nov 7 14:52:20 2016 +0800
----------------------------------------------------------------------
.../eagle/service/hbase/EmbeddedHbase.java | 17 +-
.../eagle/service/hbase/TestHBaseBase.java | 13 +-
.../eagle/storage/hbase/HBaseStorage.java | 8 +-
.../coprocessor/TestGroupAggregateClient.java | 12 +-
.../TestGroupAggregateTimeSeriesClient.java | 27 ++-
.../storage/hbase/spi/TestHBaseStorage.java | 219 +++++++++++++++++++
...estHBaseStorageAggregateWithCoprocessor.java | 103 +++++++++
.../hbase/spi/TestHBaseStorageLoader.java | 20 +-
.../src/test/resources/application-co.conf | 33 +++
pom.xml | 5 +
10 files changed, 425 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/66baf258/eagle-core/eagle-embed/eagle-embed-hbase/src/main/java/org/apache/eagle/service/hbase/EmbeddedHbase.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-embed/eagle-embed-hbase/src/main/java/org/apache/eagle/service/hbase/EmbeddedHbase.java b/eagle-core/eagle-embed/eagle-embed-hbase/src/main/java/org/apache/eagle/service/hbase/EmbeddedHbase.java
index 5210cbd..be614d3 100644
--- a/eagle-core/eagle-embed/eagle-embed-hbase/src/main/java/org/apache/eagle/service/hbase/EmbeddedHbase.java
+++ b/eagle-core/eagle-embed/eagle-embed-hbase/src/main/java/org/apache/eagle/service/hbase/EmbeddedHbase.java
@@ -42,26 +42,37 @@ public class EmbeddedHbase {
this(port, DEFAULT_ZNODE);
}
- public static EmbeddedHbase getInstance() {
+ public static EmbeddedHbase getInstance(Configuration conf) {
if (hbase == null) {
synchronized (EmbeddedHbase.class) {
if (hbase == null) {
hbase = new EmbeddedHbase();
- hbase.start();
+ hbase.start(conf);
}
}
}
return hbase;
}
-
+
+ public static EmbeddedHbase getInstance() {
+ return getInstance(null);
+ }
+
private EmbeddedHbase() {
this(DEFAULT_PORT, DEFAULT_ZNODE);
}
public void start() {
+ start(null);
+ }
+
+ public void start(Configuration confMap) {
try {
util = new HBaseTestingUtility();
Configuration conf = util.getConfiguration();
+ if(confMap != null) {
+ conf.addResource(confMap);
+ }
conf.setInt("test.hbase.zookeeper.property.clientPort", port);
conf.set("zookeeper.znode.parent", znode);
conf.setInt("hbase.zookeeper.property.maxClientCnxns", 200);
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/66baf258/eagle-core/eagle-embed/eagle-embed-hbase/src/test/java/org/apache/eagle/service/hbase/TestHBaseBase.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-embed/eagle-embed-hbase/src/test/java/org/apache/eagle/service/hbase/TestHBaseBase.java b/eagle-core/eagle-embed/eagle-embed-hbase/src/test/java/org/apache/eagle/service/hbase/TestHBaseBase.java
index 9540147..bc4f68f 100644
--- a/eagle-core/eagle-embed/eagle-embed-hbase/src/test/java/org/apache/eagle/service/hbase/TestHBaseBase.java
+++ b/eagle-core/eagle-embed/eagle-embed-hbase/src/test/java/org/apache/eagle/service/hbase/TestHBaseBase.java
@@ -16,10 +16,10 @@
*/
package org.apache.eagle.service.hbase;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.*;
+
+
@Ignore
public class TestHBaseBase {
@@ -30,6 +30,11 @@ public class TestHBaseBase {
hbase = EmbeddedHbase.getInstance();
}
+ public static void setupHBaseWithConfig(Configuration config){
+ Assert.assertTrue("HBase test mini cluster should not start",null == hbase);
+ hbase = EmbeddedHbase.getInstance(config);
+ }
+
@AfterClass
public static void shutdownHBase() {
if (hbase != null) {
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/66baf258/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/HBaseStorage.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/HBaseStorage.java b/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/HBaseStorage.java
index 74c90bc..6287b7b 100644
--- a/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/HBaseStorage.java
+++ b/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/HBaseStorage.java
@@ -67,7 +67,9 @@ public class HBaseStorage extends DataStorageBase {
ModifyResult<String> result = new ModifyResult<>();
try {
GenericEntityWriter entityWriter = new GenericEntityWriter(entityDefinition);
- result.setIdentifiers(entityWriter.write(entities));
+ List<String> keys = entityWriter.write(entities);
+ result.setIdentifiers(keys);
+ result.setSize(keys.size());
result.setSuccess(true);
} catch (Exception e) {
LOG.error(e.getMessage(), e);
@@ -102,7 +104,9 @@ public class HBaseStorage extends DataStorageBase {
ModifyResult<String> result = new ModifyResult<String>();
try {
GenericDeleter deleter = new GenericDeleter(entityDefinition.getTable(), entityDefinition.getColumnFamily());
- result.setIdentifiers(deleter.delete(entities));
+ List<String> keys = deleter.delete(entities);
+ result.setIdentifiers(keys);
+ result.setSize(keys.size());
} catch (Exception ex) {
LOG.error(ex.getMessage(), ex);
result.setSuccess(false);
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/66baf258/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/aggregate/coprocessor/TestGroupAggregateClient.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/aggregate/coprocessor/TestGroupAggregateClient.java b/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/aggregate/coprocessor/TestGroupAggregateClient.java
index 86c08fd..be09f4e 100755
--- a/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/aggregate/coprocessor/TestGroupAggregateClient.java
+++ b/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/aggregate/coprocessor/TestGroupAggregateClient.java
@@ -23,12 +23,15 @@ import java.util.HashMap;
import java.util.List;
import org.apache.eagle.common.config.EagleConfigFactory;
+import org.apache.eagle.storage.hbase.query.coprocessor.AggregateProtocolEndPoint;
import org.apache.eagle.storage.hbase.query.coprocessor.impl.AggregateClientImpl;
import org.apache.eagle.storage.hbase.query.coprocessor.AggregateClient;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.HTableFactory;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.DoubleWritable;
import org.junit.*;
@@ -50,7 +53,6 @@ import org.apache.eagle.service.hbase.TestHBaseBase;
/**
* @since : 10/30/14,2014
*/
-@Ignore
public class TestGroupAggregateClient extends TestHBaseBase {
HTableInterface table;
long startTime;
@@ -62,6 +64,14 @@ public class TestGroupAggregateClient extends TestHBaseBase {
private final static Logger LOG = LoggerFactory.getLogger(TestGroupAggregateClient.class);
+ // This is Bad, It will hide TestHBaseBase.setUpHBase!!!!
+ @BeforeClass
+ public static void setUpHBase() {
+ Configuration conf = new Configuration();
+ conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,AggregateProtocolEndPoint.class.getName());
+ TestHBaseBase.setupHBaseWithConfig(conf);
+ }
+
@Before
public void setUp() {
hbase.createTable("unittest", "f");
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/66baf258/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/aggregate/coprocessor/TestGroupAggregateTimeSeriesClient.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/aggregate/coprocessor/TestGroupAggregateTimeSeriesClient.java b/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/aggregate/coprocessor/TestGroupAggregateTimeSeriesClient.java
index f93b68e..d258691 100755
--- a/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/aggregate/coprocessor/TestGroupAggregateTimeSeriesClient.java
+++ b/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/aggregate/coprocessor/TestGroupAggregateTimeSeriesClient.java
@@ -22,6 +22,9 @@ import java.util.HashMap;
import java.util.List;
import org.apache.eagle.common.config.EagleConfigFactory;
+import org.apache.eagle.storage.hbase.query.coprocessor.AggregateProtocolEndPoint;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.junit.Assert;
import org.apache.eagle.storage.hbase.query.coprocessor.AggregateClient;
@@ -30,7 +33,8 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.DoubleWritable;
import org.junit.Before;
-import org.junit.Ignore;
+import org.junit.BeforeClass;
+import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,7 +53,6 @@ import org.apache.eagle.storage.hbase.query.coprocessor.impl.AggregateClientImpl
/**
* @since : 11/10/14,2014
*/
-@Ignore
public class TestGroupAggregateTimeSeriesClient extends TestHBaseBase {
private final static Logger LOG = LoggerFactory.getLogger(TestGroupAggregateTimeSeriesClient.class);
@@ -62,6 +65,14 @@ public class TestGroupAggregateTimeSeriesClient extends TestHBaseBase {
Scan scan;
EntityDefinition ed;
+ // This is Bad, It will hide TestHBaseBase.setUpHBase!!!!
+ @BeforeClass
+ public static void setUpHBase() {
+ Configuration conf = new Configuration();
+ conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,AggregateProtocolEndPoint.class.getName());
+ TestHBaseBase.setupHBaseWithConfig(conf);
+ }
+
@Before
public void setUp() throws IllegalAccessException, InstantiationException {
ed = EntityDefinitionManager.getEntityDefinitionByEntityClass(TestTimeSeriesAPIEntity.class);
@@ -121,7 +132,7 @@ public class TestGroupAggregateTimeSeriesClient extends TestHBaseBase {
}
- //@Test
+ @Test
public void testGroupTimeSeriesAggCountClient() {
try {
List<GroupbyKeyValue> result = client.aggregate(table, ed, scan, Arrays.asList("cluster", "datacenter"), Arrays.asList(AggregateFunctionType.count), Arrays.asList("count"), true, startTime, System.currentTimeMillis(), 10).getKeyValues();
@@ -134,7 +145,7 @@ public class TestGroupAggregateTimeSeriesClient extends TestHBaseBase {
}
}
- //@Test
+ @Test
public void testGroupTimeSeriesAggMaxClient() {
try {
List<GroupbyKeyValue> result = client.aggregate(table, ed, scan, Arrays.asList("cluster", "datacenter"), Arrays.asList(AggregateFunctionType.max), Arrays.asList("field2"), true, startTime, System.currentTimeMillis(), 10).getKeyValues();
@@ -147,7 +158,7 @@ public class TestGroupAggregateTimeSeriesClient extends TestHBaseBase {
}
}
- //@Test
+ @Test
public void testGroupTimeSeriesAggMinClient() {
try {
List<GroupbyKeyValue> result = client.aggregate(table, ed, scan, Arrays.asList("cluster", "datacenter"), Arrays.asList(AggregateFunctionType.min), Arrays.asList("field2"), true, startTime, System.currentTimeMillis(), 10).getKeyValues();
@@ -160,7 +171,7 @@ public class TestGroupAggregateTimeSeriesClient extends TestHBaseBase {
}
}
- //@Test
+ @Test
public void testGroupTimeSeriesAggAvgClient() {
try {
List<GroupbyKeyValue> result = client.aggregate(table, ed, scan, Arrays.asList("cluster", "datacenter"), Arrays.asList(AggregateFunctionType.min), Arrays.asList("field2"), true, startTime, System.currentTimeMillis(), 10).getKeyValues();
@@ -173,7 +184,7 @@ public class TestGroupAggregateTimeSeriesClient extends TestHBaseBase {
}
}
- //@Test
+ @Test
public void testGroupTimeSeriesAggSumClient() {
try {
List<GroupbyKeyValue> result = client.aggregate(table, ed, scan, Arrays.asList("cluster", "datacenter"), Arrays.asList(AggregateFunctionType.sum), Arrays.asList("field2"), true, startTime, System.currentTimeMillis(), 10).getKeyValues();
@@ -186,7 +197,7 @@ public class TestGroupAggregateTimeSeriesClient extends TestHBaseBase {
}
}
- //@Test
+ @Test
public void testGroupTimeSeriesAggMultipleClient() {
try {
List<GroupbyKeyValue> result = client.aggregate(table, ed, scan,
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/66baf258/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/spi/TestHBaseStorage.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/spi/TestHBaseStorage.java b/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/spi/TestHBaseStorage.java
new file mode 100755
index 0000000..2a497d1
--- /dev/null
+++ b/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/spi/TestHBaseStorage.java
@@ -0,0 +1,219 @@
+package org.apache.eagle.storage.hbase.spi;
+
+import org.apache.eagle.common.DateTimeUtil;
+import org.apache.eagle.log.entity.meta.EntityDefinition;
+import org.apache.eagle.log.entity.meta.EntityDefinitionManager;
+import org.apache.eagle.log.entity.test.TestTimeSeriesAPIEntity;
+import org.apache.eagle.service.hbase.TestHBaseBase;
+import org.apache.eagle.storage.DataStorage;
+import org.apache.eagle.storage.DataStorageManager;
+import org.apache.eagle.storage.exception.QueryCompileException;
+import org.apache.eagle.storage.operation.CompiledQuery;
+import org.apache.eagle.storage.operation.RawQuery;
+import org.apache.eagle.storage.result.ModifyResult;
+import org.apache.eagle.storage.result.QueryResult;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.*;
+
+public class TestHBaseStorage extends TestHBaseBase {
+
+ static final Logger LOG = LoggerFactory.getLogger(TestHBaseStorage.class);
+ EntityDefinition entityDefinition;
+ DataStorage<String> storage;
+ long baseTimestamp;
+
+ private TestTimeSeriesAPIEntity newInstance() {
+ TestTimeSeriesAPIEntity instance = new TestTimeSeriesAPIEntity();
+ instance.setField1(123);
+ instance.setField2(234);
+ instance.setField3(1231312312L);
+ instance.setField4(12312312312L);
+ instance.setField5(123123.12312);
+ instance.setField6(-12312312.012);
+ instance.setField7(UUID.randomUUID().toString());
+ instance.setTags(new HashMap<String, String>() {
+ {
+ put("cluster", "c4ut");
+ put("datacenter", "d4ut");
+ put("random", UUID.randomUUID().toString());
+ }
+ });
+ instance.setTimestamp(System.currentTimeMillis());
+ return instance;
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ entityDefinition = EntityDefinitionManager.getEntityDefinitionByEntityClass(TestTimeSeriesAPIEntity.class);
+ entityDefinition.setTags(new String[] {"cluster", "datacenter", "random"});
+
+ storage = DataStorageManager.getDataStorageByEagleConfig();
+ storage.init();
+ GregorianCalendar gc = new GregorianCalendar();
+ gc.clear();
+ gc.set(2014, 1, 6, 1, 40, 12);
+ gc.setTimeZone(TimeZone.getTimeZone("UTC"));
+ baseTimestamp = gc.getTime().getTime();
+ LOG.info("timestamp: {}" , baseTimestamp);
+ }
+
+ @Test
+ public void testReadBySimpleQuery() throws QueryCompileException, IOException {
+ RawQuery rawQuery = new RawQuery();
+ rawQuery.setQuery("TestTimeSeriesAPIEntity[@cluster=\"c4ut\"]{*}");
+ System.out.println(DateTimeUtil.millisecondsToHumanDateWithSeconds(baseTimestamp));
+ rawQuery.setStartTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(baseTimestamp));
+ rawQuery.setEndTime(DateTimeUtil.millisecondsToHumanDateWithMilliseconds(baseTimestamp + 2000));
+ rawQuery.setPageSize(1000);
+ CompiledQuery query = new CompiledQuery(rawQuery);
+ QueryResult<TestTimeSeriesAPIEntity> result = storage.query(query, entityDefinition);
+ Assert.assertNotNull(result);
+ }
+
+ @Test
+ public void testReadByNotEqualCondition() throws QueryCompileException, IOException {
+ RawQuery rawQuery = new RawQuery();
+ rawQuery.setQuery("TestTimeSeriesAPIEntity[@cluster!=\"c4ut_not_found\" AND @field1 != 0]{*}");
+ System.out.println(DateTimeUtil.millisecondsToHumanDateWithSeconds(baseTimestamp));
+ rawQuery.setStartTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(baseTimestamp));
+ rawQuery.setEndTime(DateTimeUtil.millisecondsToHumanDateWithMilliseconds(baseTimestamp + 2000));
+ rawQuery.setPageSize(1000);
+ CompiledQuery query = new CompiledQuery(rawQuery);
+ QueryResult<TestTimeSeriesAPIEntity> result = storage.query(query, entityDefinition);
+ Assert.assertNotNull(result);
+ }
+
+ @Test
+ public void testReadByComplexQuery() throws QueryCompileException, IOException {
+ RawQuery rawQuery = new RawQuery();
+ rawQuery.setQuery("TestTimeSeriesAPIEntity[@cluster=\"c4ut\" AND @field4 > 1000 OR @datacenter =\"d4ut\" ]{@field1,@field2}");
+ rawQuery.setStartTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(baseTimestamp));
+ rawQuery.setEndTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(baseTimestamp + 2000));
+ rawQuery.setPageSize(1000);
+ CompiledQuery query = new CompiledQuery(rawQuery);
+ storage.query(query, entityDefinition);
+ }
+
+ @Test
+ public void testReadByComplexQueryWithLike() throws QueryCompileException, IOException {
+ RawQuery rawQuery = new RawQuery();
+ rawQuery.setQuery("TestTimeSeriesAPIEntity[@cluster=\"c4ut\" AND @field4 > 1000 AND @field7 CONTAINS \"99404f47e309\" OR @datacenter =\"d4ut\" ]{@field1,@field2}");
+ rawQuery.setStartTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(baseTimestamp));
+ rawQuery.setEndTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(baseTimestamp + 2000));
+ rawQuery.setPageSize(1000);
+ CompiledQuery query = new CompiledQuery(rawQuery);
+ storage.query(query, entityDefinition);
+ }
+
+ @Test
+ public void testWrite() throws IOException {
+ List<TestTimeSeriesAPIEntity> entityList = new ArrayList<TestTimeSeriesAPIEntity>();
+ int i = 0;
+ while (i++ < 5) {
+ TestTimeSeriesAPIEntity entity = newInstance();
+
+ entityList.add(entity);
+ }
+ ModifyResult<String> result = storage.create(entityList, entityDefinition);
+ Assert.assertTrue(result.getSize() > 0);
+ }
+
+ @Test
+ public void testWriteAndRead() throws IOException, QueryCompileException {
+ // record insert init time
+ final long startTime = System.currentTimeMillis();
+ // Write 1000 entities
+ List<TestTimeSeriesAPIEntity> entityList = new ArrayList<>();
+ int i = 0;
+ while (i++ < 5) {
+ entityList.add(newInstance())/**/;
+ }
+ ModifyResult<String> result = storage.create(entityList, entityDefinition);
+ Assert.assertTrue(result.getSize() >= 5);
+ // record insertion finish time
+ long endTime = System.currentTimeMillis();
+
+ // init read in time range [startTime, endTime)
+ RawQuery rawQuery = new RawQuery();
+ rawQuery.setQuery("TestTimeSeriesAPIEntity[]{*}");
+ rawQuery.setStartTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(startTime));
+ rawQuery.setEndTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(endTime + 1000));
+ rawQuery.setPageSize(10000);
+ CompiledQuery query = new CompiledQuery(rawQuery);
+ QueryResult queryResult = storage.query(query, entityDefinition);
+ Assert.assertTrue(queryResult.getSize() >= 5);
+ }
+
+ @Test
+ public void testWriteAndDelete() throws IOException, QueryCompileException {
+ // record insert init time
+ final long startTime = System.currentTimeMillis();
+ // Write 1000 entities
+ List<TestTimeSeriesAPIEntity> entityList = new ArrayList<>();
+ int i = 0;
+ while (i++ < 5) {
+ entityList.add(newInstance());
+ }
+ ModifyResult<String> result = storage.create(entityList, entityDefinition);
+ Assert.assertTrue(result.getSize() >= 5);
+ // record insertion finish time
+ long endTime = System.currentTimeMillis();
+
+ // delete in time range [startTime, endTime)
+ RawQuery rawQuery = new RawQuery();
+ rawQuery.setQuery("TestTimeSeriesAPIEntity[]{*}");
+ rawQuery.setStartTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(startTime - 1000));
+ rawQuery.setEndTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(endTime + 1000));
+ rawQuery.setPageSize(1000000);
+ CompiledQuery query = new CompiledQuery(rawQuery);
+ ModifyResult<String> queryResult = storage.delete(query, entityDefinition);
+ Assert.assertTrue(queryResult.getSize() >= 5);
+ }
+
+ @Test
+ public void testWriteAndUpdate() throws IOException, QueryCompileException {
+ // Write 5 entities
+ List<TestTimeSeriesAPIEntity> entityList = new ArrayList<>();
+ int i = 0;
+ while (i++ < 5) {
+ entityList.add(newInstance());
+ }
+ ModifyResult<String> result = storage.create(entityList, entityDefinition);
+ Assert.assertTrue(result.getSize() >= 5);
+
+ // record insertion finish time
+ ModifyResult<String> queryResult = storage.update(entityList, entityDefinition);
+ Assert.assertTrue(queryResult.getSize() >= 5);
+ }
+
+ @Test
+ public void testWriteAndAggregation() throws IOException, QueryCompileException {
+ // record insert init time
+ final long startTime = System.currentTimeMillis();
+ List<TestTimeSeriesAPIEntity> entityList = new ArrayList<>();
+ int i = 0;
+ while (i++ < 5) {
+ entityList.add(newInstance());
+ }
+ ModifyResult<String> result = storage.create(entityList, entityDefinition);
+ Assert.assertTrue(result.getSize() >= 5);
+ // record insertion finish time
+ long endTime = System.currentTimeMillis();
+
+ // init read in time range [startTime, endTime)
+ RawQuery rawQuery = new RawQuery();
+ rawQuery.setQuery("TestTimeSeriesAPIEntity[]<@c...@datacenter>{count,max(@field1),min(@field2),sum(@field3)}");
+ rawQuery.setStartTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(startTime));
+ rawQuery.setEndTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(endTime + 1000));
+ rawQuery.setPageSize(1000000);
+ CompiledQuery query = new CompiledQuery(rawQuery);
+ QueryResult queryResult = storage.query(query, entityDefinition);
+ Assert.assertTrue(queryResult.getSize() >= 1);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/66baf258/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/spi/TestHBaseStorageAggregateWithCoprocessor.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/spi/TestHBaseStorageAggregateWithCoprocessor.java b/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/spi/TestHBaseStorageAggregateWithCoprocessor.java
new file mode 100755
index 0000000..1f27a72
--- /dev/null
+++ b/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/spi/TestHBaseStorageAggregateWithCoprocessor.java
@@ -0,0 +1,103 @@
+package org.apache.eagle.storage.hbase.spi;
+
+import org.apache.eagle.common.DateTimeUtil;
+import org.apache.eagle.log.entity.meta.EntityDefinition;
+import org.apache.eagle.log.entity.meta.EntityDefinitionManager;
+import org.apache.eagle.log.entity.test.TestTimeSeriesAPIEntity;
+import org.apache.eagle.service.hbase.TestHBaseBase;
+import org.apache.eagle.storage.DataStorage;
+import org.apache.eagle.storage.DataStorageManager;
+import org.apache.eagle.storage.exception.QueryCompileException;
+import org.apache.eagle.storage.hbase.query.coprocessor.AggregateProtocolEndPoint;
+import org.apache.eagle.storage.operation.CompiledQuery;
+import org.apache.eagle.storage.operation.RawQuery;
+import org.apache.eagle.storage.result.ModifyResult;
+import org.apache.eagle.storage.result.QueryResult;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.*;
+
+public class TestHBaseStorageAggregateWithCoprocessor extends TestHBaseBase {
+ static final Logger LOG = LoggerFactory.getLogger(TestHBaseStorageAggregateWithCoprocessor.class);
+ EntityDefinition entityDefinition;
+ DataStorage<String> storage;
+ long baseTimestamp;
+
+ // This is Bad, It will hide TestHBaseBase.setUpHBase!!!!
+ @BeforeClass
+ public static void setUpHBase() {
+ System.setProperty("config.resource", "/application-co.conf");
+ Configuration conf = new Configuration();
+ conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, AggregateProtocolEndPoint.class.getName());
+ TestHBaseBase.setupHBaseWithConfig(conf);
+ }
+
+ private TestTimeSeriesAPIEntity newInstance() {
+ TestTimeSeriesAPIEntity instance = new TestTimeSeriesAPIEntity();
+ instance.setField1(123);
+ instance.setField2(234);
+ instance.setField3(1231312312L);
+ instance.setField4(12312312312L);
+ instance.setField5(123123.12312);
+ instance.setField6(-12312312.012);
+ instance.setField7(UUID.randomUUID().toString());
+ instance.setTags(new HashMap<String, String>() {
+ {
+ put("cluster", "c4ut");
+ put("datacenter", "d4ut");
+ put("random", UUID.randomUUID().toString());
+ }
+ });
+ instance.setTimestamp(System.currentTimeMillis());
+ return instance;
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ entityDefinition = EntityDefinitionManager.getEntityDefinitionByEntityClass(TestTimeSeriesAPIEntity.class);
+ entityDefinition.setTags(new String[] {"cluster", "datacenter", "random"});
+
+ storage = DataStorageManager.getDataStorageByEagleConfig();
+ storage.init();
+ GregorianCalendar gc = new GregorianCalendar();
+ gc.clear();
+ gc.set(2014, 1, 6, 1, 40, 12);
+ gc.setTimeZone(TimeZone.getTimeZone("UTC"));
+ baseTimestamp = gc.getTime().getTime();
+ LOG.info("timestamp: {}", baseTimestamp);
+ }
+
+
+ @Test
+ public void testWriteAndAggregation() throws IOException, QueryCompileException {
+ // record insert init time
+ final long startTime = System.currentTimeMillis();
+ List<TestTimeSeriesAPIEntity> entityList = new ArrayList<>();
+ int i = 0;
+ while (i++ < 5) {
+ entityList.add(newInstance());
+ }
+ ModifyResult<String> result = storage.create(entityList, entityDefinition);
+ Assert.assertTrue(result.getSize() >= 5);
+ // record insertion finish time
+ long endTime = System.currentTimeMillis();
+
+ // init read in time range [startTime, endTime)
+ RawQuery rawQuery = new RawQuery();
+ rawQuery.setQuery("TestTimeSeriesAPIEntity[]<@c...@datacenter>{count,max(@field1),min(@field2),sum(@field3)}");
+ rawQuery.setStartTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(startTime));
+ rawQuery.setEndTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(endTime + 1000));
+ rawQuery.setPageSize(1000000);
+ CompiledQuery query = new CompiledQuery(rawQuery);
+ QueryResult queryResult = storage.query(query, entityDefinition);
+ Assert.assertTrue(queryResult.getSize() >= 1);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/66baf258/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/spi/TestHBaseStorageLoader.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/spi/TestHBaseStorageLoader.java b/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/spi/TestHBaseStorageLoader.java
index a4c0e23..dd81eee 100644
--- a/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/spi/TestHBaseStorageLoader.java
+++ b/eagle-core/eagle-query/eagle-storage-hbase/src/test/java/org/apache/eagle/storage/hbase/spi/TestHBaseStorageLoader.java
@@ -16,30 +16,22 @@
*/
package org.apache.eagle.storage.hbase.spi;
+import org.apache.eagle.service.hbase.TestHBaseBase;
import org.apache.eagle.storage.DataStorageManager;
import org.apache.eagle.storage.exception.IllegalDataStorageTypeException;
import org.apache.eagle.storage.hbase.HBaseStorage;
import org.junit.Assert;
-import org.junit.Ignore;
import org.junit.Test;
/**
* @since 3/20/15
*/
-public class TestHBaseStorageLoader {
+public class TestHBaseStorageLoader extends TestHBaseBase {
@Test
- @Ignore("TODO: Add back after refactoring hbase related unit test cases")
- public void testHBaseStorageLoader() {
- try {
- assert DataStorageManager.getDataStorageByEagleConfig() instanceof HBaseStorage;
- } catch (IllegalDataStorageTypeException e) {
- Assert.fail(e.getMessage());
- }
+ public void testHBaseStorageLoader() throws IllegalDataStorageTypeException {
+
+ Assert.assertTrue(DataStorageManager.getDataStorageByEagleConfig() instanceof HBaseStorage);
+ Assert.assertTrue(DataStorageManager.newDataStorage("hbase") instanceof HBaseStorage);
- try {
- assert DataStorageManager.newDataStorage("hbase") instanceof HBaseStorage;
- } catch (IllegalDataStorageTypeException e) {
- Assert.fail(e.getMessage());
- }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/66baf258/eagle-core/eagle-query/eagle-storage-hbase/src/test/resources/application-co.conf
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-hbase/src/test/resources/application-co.conf b/eagle-core/eagle-query/eagle-storage-hbase/src/test/resources/application-co.conf
new file mode 100755
index 0000000..9765b17
--- /dev/null
+++ b/eagle-core/eagle-query/eagle-storage-hbase/src/test/resources/application-co.conf
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+service {
+ env = "dev"
+ host = "localhost"
+ port = 8080
+}
+
+storage {
+ type = "hbase"
+
+ hbase {
+ tableNamePrefixedWithEnvironment = false
+ coprocessorEnabled = true
+ zookeeperQuorum = "localhost"
+ zookeeperPropertyClientPort = 2181
+ zookeeperZnodeParent = "/hbase-unsecure"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/66baf258/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 6792e4a..869ee3f 100755
--- a/pom.xml
+++ b/pom.xml
@@ -1158,6 +1158,11 @@
<totalBranchRate>25</totalBranchRate>
<totalLineRate>25</totalLineRate>
</check>
+ <instrumentation>
+ <excludes>
+ <exclude>org/apache/eagle/storage/hbase/query/coprocessor/generated/*.class</exclude>
+ </excludes>
+ </instrumentation>
</configuration>
<dependencies>
<!--Use asm-5.0 to support Java 8-->