You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/07/21 11:05:24 UTC
incubator-kylin git commit: KYLIN-876 Drop BaseCuboidJobTest &
HiveToBaseCuboidMapperTest as HiveToBaseCuboidMapper now requires HCatRecord
input which is very hard to mockup in unit test. The mapper is still covered
by intergation test.
Repository: incubator-kylin
Updated Branches:
refs/heads/0.8 a4a2928d4 -> 4d015b823
KYLIN-876 Drop BaseCuboidJobTest & HiveToBaseCuboidMapperTest as HiveToBaseCuboidMapper now requires HCatRecord input which is very hard to mockup in unit test. The mapper is still covered by intergation test.
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/4d015b82
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/4d015b82
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/4d015b82
Branch: refs/heads/0.8
Commit: 4d015b8236bc343b204347a162019cfa40bfe41e
Parents: a4a2928
Author: Li, Yang <ya...@ebay.com>
Authored: Tue Jul 21 17:04:34 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Tue Jul 21 17:05:11 2015 +0800
----------------------------------------------------------------------
.../job/hadoop/cube/BaseCuboidJobTest.java | 73 ----------
.../hadoop/cube/HiveToBaseCuboidMapperTest.java | 145 -------------------
2 files changed, 218 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4d015b82/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJobTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJobTest.java
deleted file mode 100644
index a1d3569..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/BaseCuboidJobTest.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.*;
-
-import java.io.File;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.util.ToolRunner;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class BaseCuboidJobTest extends LocalFileMetadataTestCase {
-
- private Configuration conf;
-
- @Before
- public void setup() throws Exception {
- conf = new Configuration();
- conf.set("fs.default.name", "file:///");
- conf.set("mapred.job.tracker", "local");
-
- // for local runner out-of-memory issue
- conf.set("mapreduce.task.io.sort.mb", "10");
-
- createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- }
-
- @Test
- public void testJob() throws Exception {
- String input = "src/test/resources/data/flat_table/";
- String output = "target/test-output/base_cuboid/";
- String cubeName = "test_kylin_cube_with_slr_1_new_segment";
- String segmentName = "20130331080000_20131212080000";
- String jobname = "base_cuboid_job";
- String level = "0";
- FileUtil.fullyDelete(new File(output));
-
- String[] args = { "-input", input, "-cubename", cubeName, "-segmentname", segmentName, "-output", output, "-jobname", jobname, "-level", level };
- assertEquals("Job failed", 0, ToolRunner.run(conf, new BaseCuboidJob(), args));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4d015b82/job/src/test/java/org/apache/kylin/job/hadoop/cube/HiveToBaseCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/HiveToBaseCuboidMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/HiveToBaseCuboidMapperTest.java
deleted file mode 100644
index eba626a..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/HiveToBaseCuboidMapperTest.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.*;
-
-import java.io.File;
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.MapDriver;
-import org.apache.hadoop.mrunit.types.Pair;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.kv.RowKeyDecoder;
-import org.apache.kylin.metadata.measure.MeasureCodec;
-import org.apache.kylin.metadata.model.MeasureDesc;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class HiveToBaseCuboidMapperTest extends LocalFileMetadataTestCase {
-
- MapDriver<Text, Object, Text, Text> mapDriver;
- String localTempDir = System.getProperty("java.io.tmpdir") + File.separator;
-
- @Before
- public void setUp() throws Exception {
- createTestMetadata();
-
- // hack for distributed cache
- FileUtils.deleteDirectory(new File("../job/meta"));
- FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta"));
-
- HiveToBaseCuboidMapper<Text> mapper = new HiveToBaseCuboidMapper<Text>();
- mapDriver = MapDriver.newMapDriver(mapper);
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- FileUtils.deleteDirectory(new File("../job/meta"));
- }
-
- @Test
- public void testMapperWithHeader() throws Exception {
- String cubeName = "test_kylin_cube_with_slr_1_new_segment";
- String segmentName = "20130331080000_20131212080000";
- mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
- mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
- // mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL,
- // metadata);
- mapDriver.withInput(new Text("key"), new Text("2012-12-15118480Health & BeautyFragrancesWomenAuction15123456789132.33"));
- List<Pair<Text, Text>> result = mapDriver.run();
-
- CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
- CubeInstance cube = cubeMgr.getCube(cubeName);
-
- assertEquals(1, result.size());
- Text rowkey = result.get(0).getFirst();
- byte[] key = rowkey.getBytes();
- byte[] header = Bytes.head(key, 26);
- byte[] sellerId = Bytes.tail(header, 18);
- byte[] cuboidId = Bytes.head(header, 8);
- byte[] restKey = Bytes.tail(key, rowkey.getLength() - 26);
-
- RowKeyDecoder decoder = new RowKeyDecoder(cube.getFirstSegment());
- decoder.decode(key);
- assertEquals("[123456789, 2012-12-15, 11848, Health & Beauty, Fragrances, Women, Auction, 0, 15]", decoder.getValues().toString());
-
- assertTrue(Bytes.toString(sellerId).startsWith("123456789"));
- assertEquals(511, Bytes.toLong(cuboidId));
- assertEquals(22, restKey.length);
-
- verifyMeasures(cube.getDescriptor().getMeasures(), result.get(0).getSecond(), "132.33", "132.33", "132.33");
- }
-
- private void verifyMeasures(List<MeasureDesc> measures, Text valueBytes, String m1, String m2, String m3) {
- MeasureCodec codec = new MeasureCodec(measures);
- Object[] values = new Object[measures.size()];
- codec.decode(ByteBuffer.wrap(valueBytes.getBytes(), 0, valueBytes.getLength()), values);
- assertTrue(new BigDecimal(m1).equals(values[0]));
- assertTrue(new BigDecimal(m2).equals(values[1]));
- assertTrue(new BigDecimal(m3).equals(values[2]));
- }
-
- @Test
- public void testMapperWithNull() throws Exception {
- String cubeName = "test_kylin_cube_with_slr_1_new_segment";
- String segmentName = "20130331080000_20131212080000";
- mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
- mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
- // mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL,
- // metadata);
- mapDriver.withInput(new Text("key"), new Text("2012-12-15118480Health & BeautyFragrances\\NAuction15123456789\\N"));
- List<Pair<Text, Text>> result = mapDriver.run();
-
- CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
- CubeInstance cube = cubeMgr.getCube(cubeName);
-
- assertEquals(1, result.size());
- Text rowkey = result.get(0).getFirst();
- byte[] key = rowkey.getBytes();
- byte[] header = Bytes.head(key, 26);
- byte[] sellerId = Bytes.tail(header, 18);
- byte[] cuboidId = Bytes.head(header, 8);
- byte[] restKey = Bytes.tail(key, rowkey.getLength() - 26);
-
- RowKeyDecoder decoder = new RowKeyDecoder(cube.getFirstSegment());
- decoder.decode(key);
- assertEquals("[123456789, 2012-12-15, 11848, Health & Beauty, Fragrances, null, Auction, 0, 15]", decoder.getValues().toString());
-
- assertTrue(Bytes.toString(sellerId).startsWith("123456789"));
- assertEquals(511, Bytes.toLong(cuboidId));
- assertEquals(22, restKey.length);
-
- verifyMeasures(cube.getDescriptor().getMeasures(), result.get(0).getSecond(), "0", "0", "0");
- }
-}