You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/07/24 01:20:15 UTC
[07/28] incubator-kylin git commit: KYLIN-875 Split job module into
'core-job', 'engine-mr', 'source-hive',
'storage-hbase'. The old job remains as an assembly project.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/job/src/main/java/org/apache/kylin/storage/hbase/MergeGCStep.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/storage/hbase/MergeGCStep.java b/job/src/main/java/org/apache/kylin/storage/hbase/MergeGCStep.java
deleted file mode 100644
index 5fc57e3..0000000
--- a/job/src/main/java/org/apache/kylin/storage/hbase/MergeGCStep.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.job.exception.ExecuteException;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableContext;
-import org.apache.kylin.job.execution.ExecuteResult;
-import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-
-/**
- * Drop HBase tables that is no longer needed
- */
-public class MergeGCStep extends AbstractExecutable {
-
- private static final String OLD_HTABLES = "oldHTables";
-
- private static final Logger logger = LoggerFactory.getLogger(MergeGCStep.class);
-
- public MergeGCStep() {
- super();
- }
-
- @Override
- protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
-
- StringBuffer output = new StringBuffer();
-
- List<String> oldTables = getOldHTables();
- if (oldTables != null && oldTables.size() > 0) {
- String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- Configuration conf = HBaseConfiguration.create();
- HBaseAdmin admin = null;
- try {
- admin = new HBaseAdmin(conf);
- for (String table : oldTables) {
- if (admin.tableExists(table)) {
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table));
- String host = tableDescriptor.getValue(IRealizationConstants.HTableTag);
- if (metadataUrlPrefix.equalsIgnoreCase(host)) {
- if (admin.isTableEnabled(table)) {
- admin.disableTable(table);
- }
- admin.deleteTable(table);
- logger.debug("Dropped htable: " + table);
- output.append("HBase table " + table + " is dropped. \n");
- } else {
- logger.debug("Skip htable: " + table);
- output.append("Skip htable: " + table + ". \n");
- }
- }
- }
-
- } catch (IOException e) {
- output.append("Got error when drop HBase table, exiting... \n");
- // This should not block the merge job; Orphans should be cleaned up in StorageCleanupJob
- return new ExecuteResult(ExecuteResult.State.ERROR, output.append(e.getLocalizedMessage()).toString());
- } finally {
- if (admin != null)
- try {
- admin.close();
- } catch (IOException e) {
- logger.error(e.getLocalizedMessage());
- }
- }
- }
-
- return new ExecuteResult(ExecuteResult.State.SUCCEED, output.toString());
- }
-
- public void setOldHTables(List<String> ids) {
- setParam(OLD_HTABLES, StringUtils.join(ids, ","));
- }
-
- private List<String> getOldHTables() {
- final String ids = getParam(OLD_HTABLES);
- if (ids != null) {
- final String[] splitted = StringUtils.split(ids, ",");
- ArrayList<String> result = Lists.newArrayListWithExpectedSize(splitted.length);
- for (String id : splitted) {
- result.add(id);
- }
- return result;
- } else {
- return Collections.emptyList();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/job/src/test/java/org/apache/kylin/job/BaseTestExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BaseTestExecutable.java b/job/src/test/java/org/apache/kylin/job/BaseTestExecutable.java
deleted file mode 100644
index 5ab97c9..0000000
--- a/job/src/test/java/org/apache/kylin/job/BaseTestExecutable.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job;
-
-import org.apache.kylin.job.execution.AbstractExecutable;
-
-/**
- */
-public abstract class BaseTestExecutable extends AbstractExecutable {
-
-
- public BaseTestExecutable() {
- super();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/job/src/test/java/org/apache/kylin/job/BasicLocalMetaTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BasicLocalMetaTest.java b/job/src/test/java/org/apache/kylin/job/BasicLocalMetaTest.java
deleted file mode 100644
index d034a73..0000000
--- a/job/src/test/java/org/apache/kylin/job/BasicLocalMetaTest.java
+++ /dev/null
@@ -1,24 +0,0 @@
-package org.apache.kylin.job;
-
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- */
-public class BasicLocalMetaTest extends LocalFileMetadataTestCase {
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void basicTest() {
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
index 1b626d2..8966742 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -49,7 +49,7 @@ import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
import org.apache.kylin.job.manager.ExecutableManager;
import org.apache.kylin.storage.hbase.HBaseMetadataTestCase;
-import org.apache.kylin.storage.hbase.ZookeeperJobLock;
+import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
index 41457ed..4c335e6 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
@@ -39,7 +39,7 @@ import org.apache.kylin.job.invertedindex.IIJobBuilder;
import org.apache.kylin.job.manager.ExecutableManager;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.storage.hbase.HBaseMetadataTestCase;
-import org.apache.kylin.storage.hbase.ZookeeperJobLock;
+import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
import org.junit.*;
import java.io.File;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/job/src/test/java/org/apache/kylin/job/DeployUtil.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/DeployUtil.java b/job/src/test/java/org/apache/kylin/job/DeployUtil.java
index 7ebfac0..ab172c0 100644
--- a/job/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/job/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -18,7 +18,13 @@
package org.apache.kylin.job;
-import com.google.common.collect.Lists;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
@@ -29,11 +35,10 @@ import org.apache.kylin.common.persistence.ResourceTool;
import org.apache.kylin.common.util.AbstractKylinTestCase;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.cube.CubeUpdate;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeUpdate;
import org.apache.kylin.job.dataGen.FactTableGenerator;
-import org.apache.kylin.job.hadoop.hive.SqlHiveDataTypeMapping;
import org.apache.kylin.job.streaming.KafkaDataLoader;
import org.apache.kylin.job.streaming.StreamingTableDataGenerator;
import org.apache.kylin.metadata.MetadataManager;
@@ -49,8 +54,7 @@ import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.*;
-import java.util.List;
+import com.google.common.collect.Lists;
public class DeployUtil {
@SuppressWarnings("unused")
@@ -262,7 +266,7 @@ public class DeployUtil {
if (i > 0) {
ddl.append(",");
}
- ddl.append(col.getName() + " " + SqlHiveDataTypeMapping.getHiveDataType((col.getDatatype())) + "\n");
+ ddl.append(col.getName() + " " + getHiveDataType((col.getDatatype())) + "\n");
}
ddl.append(")" + "\n");
@@ -272,4 +276,10 @@ public class DeployUtil {
return new String[] { dropsql, ddl.toString() };
}
+ private static String getHiveDataType(String javaDataType) {
+ String hiveDataType = javaDataType.toLowerCase().startsWith("varchar") ? "string" : javaDataType;
+ hiveDataType = javaDataType.toLowerCase().startsWith("integer") ? "int" : hiveDataType;
+
+ return hiveDataType.toLowerCase();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/job/src/test/java/org/apache/kylin/job/ErrorTestExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ErrorTestExecutable.java b/job/src/test/java/org/apache/kylin/job/ErrorTestExecutable.java
deleted file mode 100644
index ac4828e..0000000
--- a/job/src/test/java/org/apache/kylin/job/ErrorTestExecutable.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job;
-
-import org.apache.kylin.job.exception.ExecuteException;
-import org.apache.kylin.job.execution.ExecutableContext;
-import org.apache.kylin.job.execution.ExecuteResult;
-
-/**
- */
-public class ErrorTestExecutable extends BaseTestExecutable {
-
- public ErrorTestExecutable() {
- super();
- }
-
- @Override
- protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- }
- throw new RuntimeException("test error");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/job/src/test/java/org/apache/kylin/job/FailedTestExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/FailedTestExecutable.java b/job/src/test/java/org/apache/kylin/job/FailedTestExecutable.java
deleted file mode 100644
index e18cc0d..0000000
--- a/job/src/test/java/org/apache/kylin/job/FailedTestExecutable.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job;
-
-import org.apache.kylin.job.exception.ExecuteException;
-import org.apache.kylin.job.execution.ExecutableContext;
-import org.apache.kylin.job.execution.ExecuteResult;
-
-/**
- */
-public class FailedTestExecutable extends BaseTestExecutable {
-
- public FailedTestExecutable() {
- super();
- }
-
- @Override
- protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- }
- return new ExecuteResult(ExecuteResult.State.FAILED, "failed");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java b/job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
deleted file mode 100644
index b4f6a98..0000000
--- a/job/src/test/java/org/apache/kylin/job/SelfStopExecutable.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job;
-
-import org.apache.kylin.job.exception.ExecuteException;
-import org.apache.kylin.job.execution.ExecutableContext;
-import org.apache.kylin.job.execution.ExecuteResult;
-
-/**
- */
-public class SelfStopExecutable extends BaseTestExecutable {
-
- public SelfStopExecutable() {
- super();
- }
-
- @Override
- protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- }
- if (isDiscarded()) {
- return new ExecuteResult(ExecuteResult.State.STOPPED, "stopped");
- } else {
- return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/job/src/test/java/org/apache/kylin/job/SucceedTestExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/SucceedTestExecutable.java b/job/src/test/java/org/apache/kylin/job/SucceedTestExecutable.java
deleted file mode 100644
index 1421f10..0000000
--- a/job/src/test/java/org/apache/kylin/job/SucceedTestExecutable.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job;
-
-import org.apache.kylin.job.exception.ExecuteException;
-import org.apache.kylin.job.execution.ExecutableContext;
-import org.apache.kylin.job.execution.ExecuteResult;
-
-/**
- */
-public class SucceedTestExecutable extends BaseTestExecutable {
-
- public SucceedTestExecutable() {
- super();
- }
-
- @Override
- protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- }
- return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/job/src/test/java/org/apache/kylin/job/hadoop/cube/CopySeq.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CopySeq.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/CopySeq.java
deleted file mode 100644
index 505cdf8..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CopySeq.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.hadoop.cube;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.SequenceFile.Reader;
-import org.apache.hadoop.io.SequenceFile.Writer;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.hadoop.io.compress.CompressionCodecFactory;
-import org.apache.hadoop.util.ReflectionUtils;
-
-/**
- * @author yangli9
- *
- */
-public class CopySeq {
-
- public static void main(String[] args) throws IOException {
- copyTo64MB(args[0], args[1]);
- }
-
- public static void copyTo64MB(String src, String dst) throws IOException {
- Configuration hconf = new Configuration();
- Path srcPath = new Path(src);
- Path dstPath = new Path(dst);
-
- FileSystem fs = FileSystem.get(hconf);
- long srcSize = fs.getFileStatus(srcPath).getLen();
- int copyTimes = (int) (67108864 / srcSize); // 64 MB
- System.out.println("Copy " + copyTimes + " times");
-
- Reader reader = new Reader(hconf, SequenceFile.Reader.file(srcPath));
- Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf);
- Text value = new Text();
-
- Writer writer = SequenceFile.createWriter(hconf, Writer.file(dstPath), Writer.keyClass(key.getClass()), Writer.valueClass(Text.class), Writer.compression(CompressionType.BLOCK, getLZOCodec(hconf)));
-
- int count = 0;
- while (reader.next(key, value)) {
- for (int i = 0; i < copyTimes; i++) {
- writer.append(key, value);
- count++;
- }
- }
-
- System.out.println("Len: " + writer.getLength());
- System.out.println("Rows: " + count);
-
- reader.close();
- writer.close();
- }
-
- static CompressionCodec getLZOCodec(Configuration hconf) {
- CompressionCodecFactory factory = new CompressionCodecFactory(hconf);
- return factory.getCodecByClassName("org.apache.hadoop.io.compress.LzoCodec");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper2Test.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper2Test.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper2Test.java
deleted file mode 100644
index 5262936..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapper2Test.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.*;
-
-import java.io.File;
-import java.nio.ByteBuffer;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Mapper.Context;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.metadata.measure.MeasureCodec;
-import org.apache.kylin.cube.model.CubeDesc;
-
-/**
- * @author yangli9
- *
- */
-@SuppressWarnings({ "rawtypes", "unchecked" })
-public class CubeHFileMapper2Test extends LocalFileMetadataTestCase {
-
- String cubeName = "test_kylin_cube_with_slr_ready";
-
- MeasureCodec codec;
- ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
- Object[] outKV = new Object[2];
-
- @Before
- public void setup() throws Exception {
- this.createTestMetadata();
- // hack for distributed cache
- FileUtils.deleteDirectory(new File("../job/meta"));
- FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta"));
- CubeDesc desc = CubeManager.getInstance(getTestConfig()).getCube(cubeName).getDescriptor();
- codec = new MeasureCodec(desc.getMeasures());
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- FileUtils.deleteDirectory(new File("../job/meta"));
- }
-
- @Test
- public void testBasic() throws Exception {
-
- Configuration hconf = new Configuration();
- Context context = MockupMapContext.create(hconf, getTestConfig().getMetadataUrl(), cubeName, outKV);
-
- CubeHFileMapper mapper = new CubeHFileMapper();
- mapper.setup(context);
-
- Text key = new Text("not important");
- Text value = new Text(new byte[] { 2, 2, 51, -79, 1 });
-
- mapper.map(key, value, context);
-
- ImmutableBytesWritable outKey = (ImmutableBytesWritable) outKV[0];
- KeyValue outValue = (KeyValue) outKV[1];
-
- assertTrue(Bytes.compareTo(key.getBytes(), 0, key.getLength(), outKey.get(), outKey.getOffset(), outKey.getLength()) == 0);
-
- assertTrue(Bytes.compareTo(value.getBytes(), 0, value.getLength(), outValue.getValueArray(), outValue.getValueOffset(), outValue.getValueLength()) == 0);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapperTest.java
deleted file mode 100644
index 7b86f1b..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeHFileMapperTest.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.*;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.MapDriver;
-import org.apache.hadoop.mrunit.types.Pair;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import org.apache.kylin.job.constant.BatchConstants;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class CubeHFileMapperTest {
-
- MapDriver<Text, Text, ImmutableBytesWritable, KeyValue> mapDriver;
-
- private String cube_name = "FLAT_ITEM_CUBE";
-
- @Before
- public void setUp() {
- CubeHFileMapper mapper = new CubeHFileMapper();
- mapDriver = MapDriver.newMapDriver(mapper);
- }
-
- @SuppressWarnings("deprecation")
- @Test
- @Ignore("not maintaining")
- public void testMapper2() throws IOException {
- mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cube_name);
-
- mapDriver.addInput(new Text("52010tech"), new Text("35.432"));
-
- List<Pair<ImmutableBytesWritable, KeyValue>> result = mapDriver.run();
-
- assertEquals(2, result.size());
-
- byte[] bytes = { 0, 0, 0, 0, 0, 0, 0, 119, 33, 0, 22, 1, 0, 121, 7 };
- ImmutableBytesWritable key = new ImmutableBytesWritable(bytes);
-
- Pair<ImmutableBytesWritable, KeyValue> p1 = result.get(0);
- Pair<ImmutableBytesWritable, KeyValue> p2 = result.get(1);
-
- assertEquals(key, p1.getFirst());
- assertEquals("cf1", new String(p1.getSecond().getFamily()));
- assertEquals("usd_amt", new String(p1.getSecond().getQualifier()));
- assertEquals("35.43", new String(p1.getSecond().getValue()));
-
- assertEquals(key, p2.getFirst());
- assertEquals("cf1", new String(p2.getSecond().getFamily()));
- assertEquals("item_count", new String(p2.getSecond().getQualifier()));
- assertEquals("2", new String(p2.getSecond().getValue()));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeReducerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeReducerTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeReducerTest.java
deleted file mode 100644
index 5369284..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/CubeReducerTest.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.*;
-
-import java.io.File;
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
-import org.apache.hadoop.mrunit.types.Pair;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.apache.kylin.metadata.measure.LongMutable;
-import org.apache.kylin.metadata.measure.MeasureCodec;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- */
-public class CubeReducerTest extends LocalFileMetadataTestCase {
-
- ReduceDriver<Text, Text, Text, Text> reduceDriver;
- String localTempDir = System.getProperty("java.io.tmpdir") + File.separator;
-
- ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
-
- @Before
- public void setUp() throws Exception {
- createTestMetadata();
-
- // hack for distributed cache
- FileUtils.deleteDirectory(new File("../job/meta"));
- FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta"));
-
- CuboidReducer reducer = new CuboidReducer();
- reduceDriver = ReduceDriver.newReduceDriver(reducer);
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- FileUtils.deleteDirectory(new File("../job/meta"));
- }
-
- @Test
- public void testReducer() throws Exception {
-
- reduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr_ready");
-
- CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
- MeasureCodec codec = new MeasureCodec(cubeDesc.getMeasures());
-
- Text key1 = new Text("72010ustech");
- List<Text> values1 = new ArrayList<Text>();
- values1.add(newValueText(codec, "15.09", "15.09", "15.09", 1));
- values1.add(newValueText(codec, "20.34", "20.34", "20.34", 1));
- values1.add(newValueText(codec, "10", "10", "10", 1));
-
- Text key2 = new Text("1tech");
- List<Text> values2 = new ArrayList<Text>();
- values2.add(newValueText(codec, "15.09", "15.09", "15.09", 1));
- values2.add(newValueText(codec, "20.34", "20.34", "20.34", 1));
-
- Text key3 = new Text("0");
- List<Text> values3 = new ArrayList<Text>();
- values3.add(newValueText(codec, "146.52", "146.52", "146.52", 4));
-
- reduceDriver.withInput(key1, values1);
- reduceDriver.withInput(key2, values2);
- reduceDriver.withInput(key3, values3);
-
- List<Pair<Text, Text>> result = reduceDriver.run();
-
- Pair<Text, Text> p1 = new Pair<Text, Text>(new Text("72010ustech"), newValueText(codec, "45.43", "10", "20.34", 3));
- Pair<Text, Text> p2 = new Pair<Text, Text>(new Text("1tech"), newValueText(codec, "35.43", "15.09", "20.34", 2));
- Pair<Text, Text> p3 = new Pair<Text, Text>(new Text("0"), newValueText(codec, "146.52", "146.52", "146.52", 4));
-
- assertEquals(3, result.size());
-
- assertTrue(result.contains(p1));
- assertTrue(result.contains(p2));
- assertTrue(result.contains(p3));
- }
-
- private Text newValueText(MeasureCodec codec, String sum, String min, String max, int count) {
- Object[] values = new Object[] { new BigDecimal(sum), new BigDecimal(min), new BigDecimal(max), new LongMutable(count) };
-
- buf.clear();
- codec.encode(values, buf);
-
- Text t = new Text();
- t.set(buf.array(), 0, buf.position());
- return t;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/job/src/test/java/org/apache/kylin/job/hadoop/cube/HiveToBaseCuboidMapperPerformanceTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/HiveToBaseCuboidMapperPerformanceTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/HiveToBaseCuboidMapperPerformanceTest.java
deleted file mode 100644
index cf9cfe0..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/HiveToBaseCuboidMapperPerformanceTest.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.hadoop.cube;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.SequenceFile.Reader;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapreduce.Mapper.Context;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * @author yangli9
- *
- */
-@SuppressWarnings({ "rawtypes", "unchecked" })
-public class HiveToBaseCuboidMapperPerformanceTest {
-
- String metadataUrl = "hbase:yadesk00:2181:/hbase-unsecure";
- String cubeName = "test_kylin_cube_with_slr";
- Path srcPath = new Path("/download/test_kylin_cube_with_slr_intermediate_table_64mb.seq");
-
- @Ignore("convenient trial tool for dev")
- @Test
- public void test() throws IOException, InterruptedException {
- Configuration hconf = new Configuration();
- HiveToBaseCuboidMapper mapper = new HiveToBaseCuboidMapper();
- Context context = MockupMapContext.create(hconf, metadataUrl, cubeName, null);
-
- mapper.setup(context);
-
- Reader reader = new Reader(hconf, SequenceFile.Reader.file(srcPath));
- Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf);
- Text value = new Text();
-
- while (reader.next(key, value)) {
- mapper.map(key, value, context);
- }
-
- reader.close();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionJobTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionJobTest.java
deleted file mode 100644
index 932670f..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionJobTest.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.hadoop.cube;
-
-///*
-// * Copyright 2013-2014 eBay Software Foundation
-// *
-// * Licensed under the Apache License, Version 2.0 (the "License");
-// * you may not use this file except in compliance with the License.
-// * You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-// */
-//
-//package org.apache.kylin.index.cube;
-//
-//import static org.junit.Assert.*;
-//
-//import java.io.File;
-//import java.io.IOException;
-//
-//import org.apache.hadoop.conf.Configuration;
-//import org.apache.hadoop.fs.FileUtil;
-//import org.apache.hadoop.util.ToolRunner;
-//import org.junit.Before;
-//import org.junit.Test;
-//
-//import org.apache.kylin.metadata.MetadataManager;
-//
-///**
-// * @author xjiang
-// *
-// */
-//public class KeyDistributionJobTest {
-//
-// private Configuration conf;
-//
-// @Before
-// public void setup() throws IOException {
-// conf = new Configuration();
-// conf.set("fs.default.name", "file:///");
-// conf.set("mapred.job.tracker", "local");
-// }
-//
-// @Test
-// public void testJob() throws Exception {
-// final String input = "src/test/resources/data/base_cuboid/,src/test/resources/data/6d_cuboid/";
-// final String output = "target/test-output/key_distribution/";
-// final String cubeName = "test_kylin_cube_with_slr";
-// final String metadata = MetadataManager.getMetadataUrlFromEnv();
-//
-// FileUtil.fullyDelete(new File(output));
-//
-// String[] args =
-// { "-input", input, "-cubename", cubeName, "-output", output, "-metadata", metadata,
-// "-columnpercentage", "50", "-splitnumber", "10" };
-// assertEquals("Job failed", 0, ToolRunner.run(new KeyDistributionJob(), args));
-// }
-//
-// }
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionMapperTest.java
deleted file mode 100644
index 001953f..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/KeyDistributionMapperTest.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.hadoop.cube;
-
-///*
-// * Copyright 2013-2014 eBay Software Foundation
-// *
-// * Licensed under the Apache License, Version 2.0 (the "License");
-// * you may not use this file except in compliance with the License.
-// * You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-// */
-//
-//package org.apache.kylin.index.cube;
-//
-//import static org.junit.Assert.*;
-//
-//import java.io.File;
-//import java.io.IOException;
-//import java.util.List;
-//
-//import org.apache.hadoop.io.LongWritable;
-//import org.apache.hadoop.io.Text;
-//import org.apache.hadoop.mrunit.mapreduce.MapDriver;
-//import org.apache.hadoop.mrunit.types.Pair;
-//import org.junit.Before;
-//import org.junit.Test;
-//
-//import org.apache.kylin.index.BatchConstants;
-//import org.apache.kylin.metadata.MetadataManager;
-//
-///**
-// * @author ysong1
-// *
-// */
-//public class KeyDistributionMapperTest {
-// @SuppressWarnings("rawtypes")
-// MapDriver mapDriver;
-// String localTempDir = System.getProperty("java.io.tmpdir") + File.separator;
-//
-// @Before
-// public void setUp() {
-// KeyDistributionMapper mapper = new KeyDistributionMapper();
-// mapDriver = MapDriver.newMapDriver(mapper);
-// }
-//
-// @SuppressWarnings("unchecked")
-// @Test
-// public void testMapperWithoutHeader() throws IOException {
-// String matadata = MetadataManager.getMetadataUrlFromEnv();
-// mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_without_slr");
-// mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL, matadata);
-// mapDriver.getConfiguration().set(KeyDistributionJob.KEY_COLUMN_PERCENTAGE, "7");
-// mapDriver.getConfiguration().set(KeyDistributionJob.KEY_HEADER_LENGTH, "8");
-//
-// Text inputKey1 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey2 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 122, 1, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey3 =
-// new Text(new byte[] { 2, 2, 2, 2, 2, 2, 2, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey4 =
-// new Text(new byte[] { 3, 3, 3, 3, 3, 3, 3, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey5 =
-// new Text(new byte[] { 4, 4, 4, 4, 4, 4, 4, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey6 =
-// new Text(new byte[] { 5, 5, 5, 5, 5, 5, 5, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey7 =
-// new Text(new byte[] { 6, 6, 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-//
-// mapDriver.addInput(inputKey1, new Text("abc"));
-// mapDriver.addInput(inputKey2, new Text("abc"));
-// mapDriver.addInput(inputKey3, new Text("abc"));
-// mapDriver.addInput(inputKey4, new Text("abc"));
-// mapDriver.addInput(inputKey5, new Text("abc"));
-// mapDriver.addInput(inputKey6, new Text("abc"));
-// mapDriver.addInput(inputKey7, new Text("abc"));
-//
-// List<Pair<Text, LongWritable>> result = mapDriver.run();
-//
-// assertEquals(7, result.size());
-//
-// byte[] key1 = result.get(0).getFirst().getBytes();
-// LongWritable value1 = result.get(0).getSecond();
-// assertArrayEquals(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11 }, key1);
-// assertEquals(2, value1.get());
-//
-// byte[] key7 = result.get(6).getFirst().getBytes();
-// LongWritable value7 = result.get(6).getSecond();
-// assertArrayEquals(new byte[] { 0 }, key7);
-// assertEquals(7, value7.get());
-// }
-//
-// @SuppressWarnings("unchecked")
-// @Test
-// public void testMapperWithHeader() throws IOException {
-// String matadata = MetadataManager.getMetadataUrlFromEnv();
-// mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr");
-// mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL, matadata);
-// mapDriver.getConfiguration().set(KeyDistributionJob.KEY_COLUMN_PERCENTAGE, "7");
-// mapDriver.getConfiguration().set(KeyDistributionJob.KEY_HEADER_LENGTH, "26");
-//
-// Text inputKey1 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0,
-// 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey2 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0,
-// 0, 0, 0, 0, 0, 127, 11, 122, 1, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey3 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 2, 2,
-// 2, 2, 2, 2, 2, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey4 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 3, 3,
-// 3, 3, 3, 3, 3, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey5 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 4, 4,
-// 4, 4, 4, 4, 4, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey6 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 5, 5,
-// 5, 5, 5, 5, 5, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-// Text inputKey7 =
-// new Text(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 6, 6,
-// 6, 6, 6, 6, 6, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7 });
-//
-// mapDriver.addInput(inputKey1, new Text("abc"));
-// mapDriver.addInput(inputKey2, new Text("abc"));
-// mapDriver.addInput(inputKey3, new Text("abc"));
-// mapDriver.addInput(inputKey4, new Text("abc"));
-// mapDriver.addInput(inputKey5, new Text("abc"));
-// mapDriver.addInput(inputKey6, new Text("abc"));
-// mapDriver.addInput(inputKey7, new Text("abc"));
-//
-// List<Pair<Text, LongWritable>> result = mapDriver.run();
-//
-// assertEquals(7, result.size());
-//
-// byte[] key1 = result.get(0).getFirst().getBytes();
-// LongWritable value1 = result.get(0).getSecond();
-// assertArrayEquals(new byte[] { 0, 0, 0, 0, 0, 0, 0, 127, 11, 56, -23, 0, 22, 98, 1, 0, 121, 7, 0, 0,
-// 0, 0, 0, 0, 0, 127, 11 }, key1);
-// assertEquals(2, value1.get());
-//
-// byte[] key7 = result.get(6).getFirst().getBytes();
-// LongWritable value7 = result.get(6).getSecond();
-// assertArrayEquals(new byte[] { 0 }, key7);
-// assertEquals(7, value7.get());
-// }
-// }
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java
deleted file mode 100644
index aae54b4..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.*;
-
-import java.io.File;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.util.ToolRunner;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-
-/**
- * @author ysong1
- */
-public class MergeCuboidJobTest extends LocalFileMetadataTestCase {
-
- private Configuration conf;
-
- @Before
- public void setup() throws Exception {
- conf = new Configuration();
- conf.set("fs.default.name", "file:///");
- conf.set("mapred.job.tracker", "local");
-
- // for local runner out-of-memory issue
- conf.set("mapreduce.task.io.sort.mb", "10");
- createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- }
-
- @Test
- public void test() throws Exception {
- // String input =
- // "src/test/resources/data/base_cuboid,src/test/resources/data/6d_cuboid";
- String output = "target/test-output/merged_cuboid";
- String cubeName = "test_kylin_cube_with_slr_ready";
- String jobname = "merge_cuboid";
-
- File baseFolder = File.createTempFile("kylin-f24668f6-dcff-4cb6-a89b-77f1119df8fa-", "base");
- baseFolder.delete();
- baseFolder.mkdir();
- FileUtils.copyDirectory(new File("src/test/resources/data/base_cuboid"), baseFolder);
- baseFolder.deleteOnExit();
-
- File sixDFolder = File.createTempFile("kylin-f24668f6-dcff-4cb6-a89b-77f1119df8fa-", "6d");
- sixDFolder.delete();
- sixDFolder.mkdir();
- FileUtils.copyDirectory(new File("src/test/resources/data/base_cuboid"), sixDFolder);
- sixDFolder.deleteOnExit();
-
- FileUtil.fullyDelete(new File(output));
-
- // CubeManager cubeManager =
- // CubeManager.getInstanceFromEnv(getTestConfig());
-
- String[] args = { "-input", baseFolder.getAbsolutePath() + "," + sixDFolder.getAbsolutePath(), "-cubename", cubeName, "-segmentname", "20130331080000_20131212080000", "-output", output, "-jobname", jobname };
- assertEquals("Job failed", 0, ToolRunner.run(conf, new MergeCuboidJob(), args));
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
deleted file mode 100644
index 4c782e7..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.*;
-
-import java.io.File;
-import java.io.IOException;
-import java.text.ParseException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.MapDriver;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.CubeUpdate;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.dict.DictionaryGenerator;
-import org.apache.kylin.dict.DictionaryInfo;
-import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.dict.TrieDictionary;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.DataType;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.project.ProjectManager;
-import org.apache.kylin.source.ReadableTable.TableSignature;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author honma
- */
-@SuppressWarnings("rawtypes")
-public class MergeCuboidMapperTest extends LocalFileMetadataTestCase {
-
- private static final Logger logger = LoggerFactory.getLogger(MergeCuboidMapperTest.class);
-
- MapDriver<Text, Text, Text, Text> mapDriver;
- CubeManager cubeManager;
- CubeInstance cube;
- DictionaryManager dictionaryManager;
-
- TblColRef lfn;
- TblColRef lsi;
- TblColRef ssc;
-
- private DictionaryInfo makeSharedDict() throws IOException {
- TableSignature signature = new TableSignature();
- signature.setSize(100);
- signature.setLastModifiedTime(System.currentTimeMillis());
- signature.setPath("fake_common_dict");
-
- DictionaryInfo newDictInfo = new DictionaryInfo("", "", 0, "string", signature);
-
- List<byte[]> values = new ArrayList<byte[]>();
- values.add(new byte[] { 101, 101, 101 });
- values.add(new byte[] { 102, 102, 102 });
- Dictionary<?> dict = DictionaryGenerator.buildDictionaryFromValueList(DataType.getInstance(newDictInfo.getDataType()), values);
- newDictInfo.setCardinality(dict.getSize());
- dictionaryManager.trySaveNewDict(dict, newDictInfo);
- ((TrieDictionary) dict).dump(System.out);
-
- return newDictInfo;
- }
-
- @Before
- public void setUp() throws Exception {
-
- createTestMetadata();
-
- logger.info("The metadataUrl is : " + getTestConfig());
-
- MetadataManager.clearCache();
- CubeManager.clearCache();
- ProjectManager.clearCache();
- DictionaryManager.clearCache();
-
- // hack for distributed cache
- // CubeManager.removeInstance(KylinConfig.createInstanceFromUri("../job/meta"));//to
- // make sure the following mapper could get latest CubeManger
- FileUtils.deleteDirectory(new File("../job/meta"));
-
- MergeCuboidMapper mapper = new MergeCuboidMapper();
- mapDriver = MapDriver.newMapDriver(mapper);
-
- cubeManager = CubeManager.getInstance(getTestConfig());
- cube = cubeManager.getCube("test_kylin_cube_without_slr_left_join_ready_2_segments");
- dictionaryManager = DictionaryManager.getInstance(getTestConfig());
- lfn = cube.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "LSTG_FORMAT_NAME");
- lsi = cube.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "CAL_DT");
- ssc = cube.getDescriptor().findColumnRef("DEFAULT.TEST_CATEGORY_GROUPINGS", "META_CATEG_NAME");
-
- DictionaryInfo sharedDict = makeSharedDict();
-
- boolean isFirstSegment = true;
- for (CubeSegment segment : cube.getSegments()) {
-
- TableSignature signature = new TableSignature();
- signature.setSize(100);
- signature.setLastModifiedTime(System.currentTimeMillis());
- signature.setPath("fake_dict_for" + lfn.getName() + segment.getName());
-
- DictionaryInfo newDictInfo = new DictionaryInfo(lfn.getTable(), lfn.getColumnDesc().getName(), lfn.getColumnDesc().getZeroBasedIndex(), "string", signature);
-
- List<byte[]> values = new ArrayList<byte[]>();
- values.add(new byte[] { 97, 97, 97 });
- if (isFirstSegment)
- values.add(new byte[] { 99, 99, 99 });
- else
- values.add(new byte[] { 98, 98, 98 });
- Dictionary<?> dict = DictionaryGenerator.buildDictionaryFromValueList(DataType.getInstance(newDictInfo.getDataType()), values);
- newDictInfo.setCardinality(dict.getSize());
- dictionaryManager.trySaveNewDict(dict, newDictInfo);
- ((TrieDictionary) dict).dump(System.out);
-
- segment.putDictResPath(lfn, newDictInfo.getResourcePath());
- segment.putDictResPath(lsi, sharedDict.getResourcePath());
- segment.putDictResPath(ssc, sharedDict.getResourcePath());
-
- // cubeManager.saveResource(segment.getCubeInstance());
- // cubeManager.afterCubeUpdated(segment.getCubeInstance());
-
- isFirstSegment = false;
- }
-
- CubeUpdate cubeBuilder = new CubeUpdate(cube);
- cubeBuilder.setToUpdateSegs(cube.getSegments().toArray(new CubeSegment[cube.getSegments().size()]));
- cube = cubeManager.updateCube(cubeBuilder);
-
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- FileUtils.deleteDirectory(new File("../job/meta"));
- }
-
- @Test
- public void test() throws IOException, ParseException {
-
- // String cubeName = "test_kylin_cube_without_slr_left_join_ready_2_segments";
-
- CubeSegment newSeg = cubeManager.mergeSegments(cube, 0L, 1386835200000L, false);
- // String segmentName = newSeg.getName();
-
- final Dictionary<?> dictionary = cubeManager.getDictionary(newSeg, lfn);
- assertTrue(dictionary == null);
- // ((TrieDictionary) dictionary).dump(System.out);
-
- // hack for distributed cache
- // File metaDir = new File("../job/meta");
- // FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), metaDir);
- //
- // mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
- // mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
- // // mapDriver.getConfiguration().set(KylinConfig.KYLIN_METADATA_URL,
- // // "../job/meta");
- //
- // byte[] key = new byte[] { 0, 0, 0, 0, 0, 0, 0, -92, 1, 1, 1 };
- // byte[] value = new byte[] { 1, 2, 3 };
- // byte[] newkey = new byte[] { 0, 0, 0, 0, 0, 0, 0, -92, 1, 1, 2 };
- // byte[] newvalue = new byte[] { 1, 2, 3 };
- //
- // mapDriver.withInput(new Text(key), new Text(value));
- // mapDriver.withOutput(new Text(newkey), new Text(newvalue));
- // mapDriver.setMapInputPath(new Path("/apps/hdmi-prod/b_kylin/prod/kylin-f24668f6-dcff-4cb6-a89b-77f1119df8fa/vac_sw_cube_v4/cuboid/15d_cuboid"));
- //
- // mapDriver.runTest();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/job/src/test/java/org/apache/kylin/job/hadoop/cube/MockupMapContext.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MockupMapContext.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MockupMapContext.java
deleted file mode 100644
index debcf3e..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MockupMapContext.java
+++ /dev/null
@@ -1,314 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.hadoop.cube;
-
-import java.io.IOException;
-import java.net.URI;
-
-import org.apache.commons.lang.NotImplementedException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configuration.IntegerRanges;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.RawComparator;
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.JobID;
-import org.apache.hadoop.mapreduce.MapContext;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Mapper.Context;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.hadoop.mapreduce.Partitioner;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
-import org.apache.hadoop.mapreduce.lib.map.WrappedMapper;
-import org.apache.hadoop.security.Credentials;
-
-import org.apache.kylin.job.constant.BatchConstants;
-
-/**
- * @author yangli9
- *
- */
-@SuppressWarnings({ "rawtypes", "unchecked" })
-public class MockupMapContext {
-
- public static Context create(final Configuration hconf, String metadataUrl, String cubeName, final Object[] outKV) {
-
- hconf.set(BatchConstants.CFG_CUBE_NAME, cubeName);
-
- return new WrappedMapper().getMapContext(new MapContext() {
-
- @Override
- public boolean nextKeyValue() throws IOException, InterruptedException {
- throw new NotImplementedException();
- }
-
- @Override
- public Object getCurrentKey() throws IOException, InterruptedException {
- throw new NotImplementedException();
- }
-
- @Override
- public Object getCurrentValue() throws IOException, InterruptedException {
- throw new NotImplementedException();
- }
-
- @Override
- public void write(Object key, Object value) throws IOException, InterruptedException {
- System.out.println("Write -- k:" + key + ", v:" + value);
- if (outKV != null) {
- outKV[0] = key;
- outKV[1] = value;
- }
- }
-
- @Override
- public OutputCommitter getOutputCommitter() {
- throw new NotImplementedException();
- }
-
- @Override
- public TaskAttemptID getTaskAttemptID() {
- throw new NotImplementedException();
- }
-
- @Override
- public void setStatus(String msg) {
- throw new NotImplementedException();
- }
-
- @Override
- public String getStatus() {
- throw new NotImplementedException();
- }
-
- @Override
- public float getProgress() {
- throw new NotImplementedException();
- }
-
- @Override
- public Counter getCounter(Enum<?> counterName) {
- throw new NotImplementedException();
- }
-
- @Override
- public Counter getCounter(String groupName, String counterName) {
- throw new NotImplementedException();
- }
-
- @Override
- public Configuration getConfiguration() {
- return hconf;
- }
-
- @Override
- public Credentials getCredentials() {
- throw new NotImplementedException();
- }
-
- @Override
- public JobID getJobID() {
- throw new NotImplementedException();
- }
-
- @Override
- public int getNumReduceTasks() {
- throw new NotImplementedException();
- }
-
- @Override
- public Path getWorkingDirectory() throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<?> getOutputKeyClass() {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<?> getOutputValueClass() {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<?> getMapOutputKeyClass() {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<?> getMapOutputValueClass() {
- throw new NotImplementedException();
- }
-
- @Override
- public String getJobName() {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends InputFormat<?, ?>> getInputFormatClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends Mapper<?, ?, ?, ?>> getMapperClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends Reducer<?, ?, ?, ?>> getCombinerClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends Reducer<?, ?, ?, ?>> getReducerClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends OutputFormat<?, ?>> getOutputFormatClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public Class<? extends Partitioner<?, ?>> getPartitionerClass() throws ClassNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public RawComparator<?> getSortComparator() {
- throw new NotImplementedException();
- }
-
- @Override
- public String getJar() {
- throw new NotImplementedException();
- }
-
- @Override
- public RawComparator<?> getGroupingComparator() {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean getJobSetupCleanupNeeded() {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean getTaskCleanupNeeded() {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean getProfileEnabled() {
- throw new NotImplementedException();
- }
-
- @Override
- public String getProfileParams() {
- throw new NotImplementedException();
- }
-
- @Override
- public IntegerRanges getProfileTaskRange(boolean isMap) {
- throw new NotImplementedException();
- }
-
- @Override
- public String getUser() {
- throw new NotImplementedException();
- }
-
- @Override
- public boolean getSymlink() {
- throw new NotImplementedException();
- }
-
- @Override
- public Path[] getArchiveClassPaths() {
- throw new NotImplementedException();
- }
-
- @Override
- public URI[] getCacheArchives() throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public URI[] getCacheFiles() throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public Path[] getLocalCacheArchives() throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public Path[] getLocalCacheFiles() throws IOException {
- throw new NotImplementedException();
- }
-
- @Override
- public Path[] getFileClassPaths() {
- throw new NotImplementedException();
- }
-
- @Override
- public String[] getArchiveTimestamps() {
- throw new NotImplementedException();
- }
-
- @Override
- public String[] getFileTimestamps() {
- throw new NotImplementedException();
- }
-
- @Override
- public int getMaxMapAttempts() {
- throw new NotImplementedException();
- }
-
- @Override
- public int getMaxReduceAttempts() {
- throw new NotImplementedException();
- }
-
- @Override
- public void progress() {
- throw new NotImplementedException();
- }
-
- @Override
- public InputSplit getInputSplit() {
- throw new NotImplementedException();
- }
-
- @Override
- public RawComparator<?> getCombinerKeyGroupingComparator() {
- throw new NotImplementedException();
- }
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidJobTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidJobTest.java
deleted file mode 100644
index c8a3d51..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidJobTest.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.*;
-
-import java.io.File;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.util.ToolRunner;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class NDCuboidJobTest extends LocalFileMetadataTestCase {
-
- private Configuration conf;
-
- @Before
- public void setup() throws Exception {
- conf = new Configuration();
- conf.set("fs.default.name", "file:///");
- conf.set("mapred.job.tracker", "local");
-
- // for local runner out-of-memory issue
- conf.set("mapreduce.task.io.sort.mb", "10");
-
- createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- }
-
- @Test
- public void testJob6D() throws Exception {
- String input = "src/test/resources/data/base_cuboid/";
- String output = "target/test-output/6d_cuboid";
- String cubeName = "test_kylin_cube_with_slr_1_new_segment";
- String segmentName = "20130331080000_20131212080000";
- String jobname = "6d_cuboid";
- String level = "1";
-
- FileUtil.fullyDelete(new File(output));
-
- String[] args = { "-input", input, "-cubename", cubeName, "-segmentname", segmentName, "-output", output, "-jobname", jobname, "-level", level };
- assertEquals("Job failed", 0, ToolRunner.run(conf, new NDCuboidJob(), args));
- }
-
- @Test
- public void testJob5D() throws Exception {
- final String input = "src/test/resources/data/6d_cuboid/";
- final String output = "target/test-output/5d_cuboid";
- final String cubeName = "test_kylin_cube_with_slr_1_new_segment";
- String segmentName = "20130331080000_20131212080000";
- String jobname = "5d_cuboid";
- String level = "2";
-
- FileUtil.fullyDelete(new File(output));
-
- String[] args = { "-input", input, "-cubename", cubeName, "-segmentname", segmentName, "-output", output, "-jobname", jobname, "-level", level };
- assertEquals("Job failed", 0, ToolRunner.run(conf, new NDCuboidJob(), args));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapperTest.java
deleted file mode 100644
index a6b0372..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapperTest.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.hadoop.cube;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
-import org.apache.hadoop.mrunit.types.Pair;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.job.constant.BatchConstants;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import javax.annotation.Nullable;
-import java.io.File;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-
-import static org.junit.Assert.*;
-
-/**
- * @author George Song (ysong1)
- */
-public class NDCuboidMapperTest extends LocalFileMetadataTestCase {
- MapReduceDriver<Text, Text, Text, Text, Text, Text> mapReduceDriver;
- String localTempDir = System.getProperty("java.io.tmpdir") + File.separator;
-
- @Before
- public void setUp() throws Exception {
- createTestMetadata();
-
- // hack for distributed cache
- FileUtils.deleteDirectory(new File("../job/meta"));
- FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta"));
-
- NDCuboidMapper mapper = new NDCuboidMapper();
- CuboidReducer reducer = new CuboidReducer();
- mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- FileUtils.deleteDirectory(new File("../job/meta"));
- }
-
- @Test
- public void testMapReduceWithSlr() throws IOException {
-
- String cubeName = "test_kylin_cube_with_slr_1_new_segment";
- String segmentName = "20130331080000_20131212080000";
- mapReduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
- mapReduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
-
- byte[] key = { 0, 0, 0, 0, 0, 0, 1, -1, 49, 48, 48, 48, 48, 48, 48, 48, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 11, 54, -105, 55, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 };
- byte[] value = { 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 1 };
- Pair<Text, Text> input1 = new Pair<Text, Text>(new Text(key), new Text(value));
-
- mapReduceDriver.addInput(input1);
-
- List<Pair<Text, Text>> result = mapReduceDriver.run();
-
- assertEquals(4, result.size());
-
- byte[] resultKey = { 0, 0, 0, 0, 0, 0, 1, 127, 49, 48, 48, 48, 48, 48, 48, 48, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 55, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 };
- byte[] resultValue = { 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 14, 7, 23, -16, 56, 92, 114, -80, 118, 1 };
- Pair<Text, Text> output1 = new Pair<Text, Text>(new Text(resultKey), new Text(resultValue));
-
- //As we will truncate decimal(KYLIN-766), value will no longer equals to resultValue
- Collection<Text> keys = Collections2.transform(result, new Function<Pair<Text, Text>, Text>() {
- @Nullable
- @Override
- public Text apply(Pair<Text, Text> input) {
- return input.getFirst();
- }
- });
- assertTrue(keys.contains(output1.getFirst()));
- assertTrue(!result.contains(output1));
-
- long[] keySet = new long[result.size()];
-
- System.out.println(Bytes.toLong(new byte[] { 0, 0, 0, 0, 0, 0, 1, -1 }));
- for (int i = 0; i < result.size(); i++) {
- byte[] bytes = new byte[result.get(i).getFirst().getLength()];
- System.arraycopy(result.get(i).getFirst().getBytes(), 0, bytes, 0, result.get(i).getFirst().getLength());
- System.out.println(Bytes.toLong(bytes));
- keySet[i] = Bytes.toLong(bytes);
- }
-
- // refer to CuboidSchedulerTest.testGetSpanningCuboid()
- assertArrayEquals(new long[] { 383, 447, 503, 504 }, keySet);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2ef9831e/job/src/test/java/org/apache/kylin/job/hadoop/cube/RandomKeyDistributionMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/RandomKeyDistributionMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/RandomKeyDistributionMapperTest.java
deleted file mode 100644
index 4aae185..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/RandomKeyDistributionMapperTest.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.hadoop.cube;
-
-import static org.junit.Assert.*;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mrunit.mapreduce.MapDriver;
-import org.apache.hadoop.mrunit.types.Pair;
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.kylin.job.constant.BatchConstants;
-import org.apache.kylin.job.hadoop.invertedindex.RandomKeyDistributionMapper;
-
-/**
- * @author ysong1
- *
- */
-public class RandomKeyDistributionMapperTest {
-
- MapDriver<Text, Text, Text, LongWritable> mapDriver;
-
- @Before
- @SuppressWarnings({ "rawtypes", "unchecked" })
- public void setUp() {
- RandomKeyDistributionMapper mapper = new RandomKeyDistributionMapper();
- mapDriver = MapDriver.newMapDriver(mapper);
- }
-
- @Test
- public void test() throws IOException {
- List<Text> data = new ArrayList<Text>();
- for (int i = 0; i < 1001; i++) {
- data.add(new Text(String.valueOf(i)));
- }
-
- for (Text t : data) {
- mapDriver.addInput(t, new Text("abc"));
- }
-
- mapDriver.getConfiguration().set(BatchConstants.MAPPER_SAMPLE_NUMBER, "100");
- List<Pair<Text, LongWritable>> result = mapDriver.run();
- assertEquals(100, result.size());
-
- for (Pair<Text, LongWritable> p : result) {
- System.out.println(p.getFirst());
- }
- }
-
-}