You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/05/25 08:11:33 UTC
[01/11] incubator-kylin git commit: fix ci: clean broadcase module
Repository: incubator-kylin
Updated Branches:
refs/heads/new697 [created] ada6b61e2
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index d168ddd..ba62bb2 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -63,7 +63,6 @@ import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Component;
import java.io.IOException;
-import java.net.UnknownHostException;
import java.util.*;
/**
@@ -81,7 +80,7 @@ public class CubeService extends BasicService {
private AccessService accessService;
@PostFilter(Constant.ACCESS_POST_FILTER_READ)
- public List<CubeInstance> listAllCubes(final String cubeName, final String projectName,final String modelName) {
+ public List<CubeInstance> listAllCubes(final String cubeName, final String projectName, final String modelName) {
List<CubeInstance> cubeInstances = null;
ProjectInstance project = (null != projectName) ? getProjectManager().getProject(projectName) : null;
@@ -93,14 +92,14 @@ public class CubeService extends BasicService {
List<CubeInstance> filterModelCubes = new ArrayList<CubeInstance>();
- if(modelName!=null){
+ if (modelName != null) {
for (CubeInstance cubeInstance : cubeInstances) {
boolean isCubeMatch = cubeInstance.getDescriptor().getModelName().toLowerCase().equals(modelName.toLowerCase());
if (isCubeMatch) {
filterModelCubes.add(cubeInstance);
}
}
- }else{
+ } else {
filterModelCubes = cubeInstances;
}
@@ -116,12 +115,12 @@ public class CubeService extends BasicService {
return filterCubes;
}
- public List<CubeInstance> getCubes(final String cubeName, final String projectName,final String modelName ,final Integer limit, final Integer offset) {
+ public List<CubeInstance> getCubes(final String cubeName, final String projectName, final String modelName, final Integer limit, final Integer offset) {
List<CubeInstance> cubes;
- cubes = listAllCubes(cubeName, projectName,modelName);
+ cubes = listAllCubes(cubeName, projectName, modelName);
- if(limit==null||offset==null){
+ if (limit == null || offset == null) {
return cubes;
}
@@ -154,7 +153,7 @@ public class CubeService extends BasicService {
String owner = SecurityContextHolder.getContext().getAuthentication().getName();
cube.setOwner(owner);
- return getCubeManager().updateCube(cube);
+ return getCubeManager().updateCube(cube, true);
}
public CubeInstance createCubeAndDesc(String cubeName, String projectName, CubeDesc desc) throws IOException {
@@ -174,7 +173,6 @@ public class CubeService extends BasicService {
createdDesc = getCubeDescManager().updateCubeDesc(desc);
}
-
if (!createdDesc.getError().isEmpty()) {
if (isNew) {
getCubeDescManager().removeCubeDesc(createdDesc);
@@ -235,7 +233,7 @@ public class CubeService extends BasicService {
}
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
- public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newProjectName) throws IOException, JobException {
+ public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newProjectName) throws IOException, JobException {
final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
if (!cubingJobs.isEmpty()) {
throw new JobException("Cube schema shouldn't be changed with running job.");
@@ -276,20 +274,14 @@ public class CubeService extends BasicService {
accessService.clean(cube, true);
}
- public boolean isCubeEditable(CubeInstance ci) {
- return ci.getStatus() == RealizationStatusEnum.DISABLED;
- }
-
public boolean isCubeDescEditable(CubeDesc cd) {
- List<CubeInstance> list = getCubeManager().getCubesByDesc(cd.getName());
- if (list.isEmpty()) {
+ String cubeName = getCubeNameFromDesc(cd.getName());
+ CubeInstance cube = getCubeManager().getCube(cubeName);
+ if (cube == null) {
return true;
}
- Iterator<CubeInstance> it = list.iterator();
- while (it.hasNext()) {
- if (!isCubeEditable(it.next())) {
- return false;
- }
+ if (cube.getSegments().size() != 0) {
+ return false;
}
return true;
}
@@ -306,14 +298,6 @@ public class CubeService extends BasicService {
}
}
- public void reloadCubeCache(String cubeName) {
- CubeManager.getInstance(this.getConfig()).loadCubeCache(cubeName);
- }
-
- public void removeCubeCache(String cubeName) {
- CubeManager.getInstance(this.getConfig()).removeCubeCacheLocal(cubeName);
- }
-
/**
* Stop all jobs belonging to this cube and clean out all segments
*
@@ -323,7 +307,7 @@ public class CubeService extends BasicService {
* @throws JobException
*/
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
- @Caching(evict = {@CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true)})
+ @Caching(evict = { @CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true) })
public CubeInstance purgeCube(CubeInstance cube) throws IOException, JobException {
String cubeName = cube.getName();
@@ -349,7 +333,7 @@ public class CubeService extends BasicService {
* @throws JobException
*/
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
- @Caching(evict = {@CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true)})
+ @Caching(evict = { @CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true) })
public CubeInstance disableCube(CubeInstance cube) throws IOException, JobException {
String cubeName = cube.getName();
@@ -361,7 +345,7 @@ public class CubeService extends BasicService {
cube.setStatus(RealizationStatusEnum.DISABLED);
try {
- return getCubeManager().updateCube(cube);
+ return getCubeManager().updateCube(cube, true);
} catch (IOException e) {
cube.setStatus(ostatus);
throw e;
@@ -398,7 +382,7 @@ public class CubeService extends BasicService {
cube.setStatus(RealizationStatusEnum.READY);
try {
- return getCubeManager().updateCube(cube);
+ return getCubeManager().updateCube(cube, true);
} catch (IOException e) {
cube.setStatus(ostatus);
throw e;
@@ -511,7 +495,6 @@ public class CubeService extends BasicService {
getExecutableManager().addJob(job);
}
-
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
public void updateCubeNotifyList(CubeInstance cube, List<String> notifyList) throws IOException {
CubeDesc desc = cube.getDescriptor();
@@ -543,7 +526,7 @@ public class CubeService extends BasicService {
}
}
cube.getSegments().clear();
- CubeManager.getInstance(getConfig()).updateCube(cube);
+ CubeManager.getInstance(getConfig()).updateCube(cube, true);
}
@PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER + " or " + Constant.ACCESS_HAS_ROLE_ADMIN)
@@ -557,7 +540,6 @@ public class CubeService extends BasicService {
getProjectManager().addTableDescToProject(tables, project);
}
-
@PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER + " or " + Constant.ACCESS_HAS_ROLE_ADMIN)
public void calculateCardinalityIfNotPresent(String[] tables, String submitter) throws IOException {
MetadataManager metaMgr = getMetadataManager();
@@ -569,5 +551,4 @@ public class CubeService extends BasicService {
}
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/server/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/JobService.java b/server/src/main/java/org/apache/kylin/rest/service/JobService.java
index e370a27..7c96801 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -100,20 +100,20 @@ public class JobService extends BasicService {
private ExecutableState parseToExecutableState(JobStatusEnum status) {
switch (status) {
- case DISCARDED:
- return ExecutableState.DISCARDED;
- case ERROR:
- return ExecutableState.ERROR;
- case FINISHED:
- return ExecutableState.SUCCEED;
- case NEW:
- return ExecutableState.READY;
- case PENDING:
- return ExecutableState.READY;
- case RUNNING:
- return ExecutableState.RUNNING;
- default:
- throw new RuntimeException("illegal status:" + status);
+ case DISCARDED:
+ return ExecutableState.DISCARDED;
+ case ERROR:
+ return ExecutableState.ERROR;
+ case FINISHED:
+ return ExecutableState.SUCCEED;
+ case NEW:
+ return ExecutableState.READY;
+ case PENDING:
+ return ExecutableState.READY;
+ case RUNNING:
+ return ExecutableState.RUNNING;
+ default:
+ throw new RuntimeException("illegal status:" + status);
}
}
@@ -213,37 +213,37 @@ public class JobService extends BasicService {
private JobStatusEnum parseToJobStatus(ExecutableState state) {
switch (state) {
- case READY:
- return JobStatusEnum.PENDING;
- case RUNNING:
- return JobStatusEnum.RUNNING;
- case ERROR:
- return JobStatusEnum.ERROR;
- case DISCARDED:
- return JobStatusEnum.DISCARDED;
- case SUCCEED:
- return JobStatusEnum.FINISHED;
- case STOPPED:
- default:
- throw new RuntimeException("invalid state:" + state);
+ case READY:
+ return JobStatusEnum.PENDING;
+ case RUNNING:
+ return JobStatusEnum.RUNNING;
+ case ERROR:
+ return JobStatusEnum.ERROR;
+ case DISCARDED:
+ return JobStatusEnum.DISCARDED;
+ case SUCCEED:
+ return JobStatusEnum.FINISHED;
+ case STOPPED:
+ default:
+ throw new RuntimeException("invalid state:" + state);
}
}
private JobStepStatusEnum parseToJobStepStatus(ExecutableState state) {
switch (state) {
- case READY:
- return JobStepStatusEnum.PENDING;
- case RUNNING:
- return JobStepStatusEnum.RUNNING;
- case ERROR:
- return JobStepStatusEnum.ERROR;
- case DISCARDED:
- return JobStepStatusEnum.DISCARDED;
- case SUCCEED:
- return JobStepStatusEnum.FINISHED;
- case STOPPED:
- default:
- throw new RuntimeException("invalid state:" + state);
+ case READY:
+ return JobStepStatusEnum.PENDING;
+ case RUNNING:
+ return JobStepStatusEnum.RUNNING;
+ case ERROR:
+ return JobStepStatusEnum.ERROR;
+ case DISCARDED:
+ return JobStepStatusEnum.DISCARDED;
+ case SUCCEED:
+ return JobStepStatusEnum.FINISHED;
+ case STOPPED:
+ default:
+ throw new RuntimeException("invalid state:" + state);
}
}
@@ -264,7 +264,7 @@ public class JobService extends BasicService {
final CubeSegment segment = cubeInstance.getSegmentById(segmentId);
if (segment != null && segment.getStatus() == SegmentStatusEnum.NEW) {
cubeInstance.getSegments().remove(segment);
- getCubeManager().updateCube(cubeInstance);
+ getCubeManager().updateCube(cubeInstance, true);
}
getExecutableManager().discardJob(jobId);
return jobInstance;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/server/src/main/java/org/apache/kylin/rest/service/ProjectService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/ProjectService.java b/server/src/main/java/org/apache/kylin/rest/service/ProjectService.java
index 2013a9f..abaecd2 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/ProjectService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/ProjectService.java
@@ -18,12 +18,12 @@
package org.apache.kylin.rest.service;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.InternalErrorException;
+import org.apache.kylin.rest.request.CreateProjectRequest;
import org.apache.kylin.rest.request.UpdateProjectRequest;
+import org.apache.kylin.rest.security.AclPermission;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -31,11 +31,9 @@ import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Component;
-import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.metadata.project.ProjectManager;
-import org.apache.kylin.rest.constant.Constant;
-import org.apache.kylin.rest.request.CreateProjectRequest;
-import org.apache.kylin.rest.security.AclPermission;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
/**
* @author xduo
@@ -107,12 +105,5 @@ public class ProjectService extends BasicService {
accessService.clean(project, true);
}
- public void reloadProjectCache(String name) throws IOException {
- getProjectManager().reloadProject(name);
- }
-
- public void removeProjectCache(String name) {
- ProjectManager.clearCache();
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
index 7a88ba9..559ec89 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
@@ -77,7 +77,13 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
context.addServlet(new ServletHolder(new BroadcasterReceiveServlet(new BroadcasterReceiveServlet.BroadcasterHandler() {
@Override
public void handle(String type, String name, String event) {
- final CacheService cacheService = new CacheService() {
+ final CacheService serviceA = new CacheService() {
+ @Override
+ public KylinConfig getConfig() {
+ return configA;
+ }
+ };
+ final CacheService serviceB = new CacheService() {
@Override
public KylinConfig getConfig() {
return configB;
@@ -87,20 +93,26 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
Broadcaster.EVENT wipeEvent = Broadcaster.EVENT.getEvent(event);
final String log = "wipe cache type: " + wipeType + " event:" + wipeEvent + " name:" + name;
logger.info(log);
- counter.incrementAndGet();
- switch (wipeEvent) {
+ try {
+ switch (wipeEvent) {
case CREATE:
case UPDATE:
- cacheService.rebuildCache(wipeType, name);
+ serviceA.rebuildCache(wipeType, name);
+ serviceB.rebuildCache(wipeType, name);
break;
case DROP:
- cacheService.removeCache(wipeType, name);
+ serviceA.removeCache(wipeType, name);
+ serviceB.removeCache(wipeType, name);
break;
default:
throw new RuntimeException("invalid type:" + wipeEvent);
+ }
+ } finally {
+ counter.incrementAndGet();
}
}
})), "/");
+
server.start();
}
@@ -146,12 +158,15 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
private static CubeManager getCubeManager(KylinConfig config) throws Exception {
return CubeManager.getInstance(config);
}
+
private static ProjectManager getProjectManager(KylinConfig config) throws Exception {
return ProjectManager.getInstance(config);
}
+
private static CubeDescManager getCubeDescManager(KylinConfig config) throws Exception {
return CubeDescManager.getInstance(config);
}
+
private static MetadataManager getMetadataManager(KylinConfig config) throws Exception {
return MetadataManager.getInstance(config);
}
@@ -200,10 +215,11 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
assertTrue(!containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME), RealizationType.CUBE, cubeName));
assertTrue(!containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME), RealizationType.CUBE, cubeName));
cubeManager.createCube(cubeName, ProjectInstance.DEFAULT_PROJECT_NAME, cubeDesc, null);
- assertNotNull(cubeManager.getCube(cubeName));
//one for cube update, one for project update
assertEquals(2, broadcaster.getCounterAndClear());
waitForCounterAndClear(2);
+
+ assertNotNull(cubeManager.getCube(cubeName));
assertNotNull(cubeManagerB.getCube(cubeName));
assertTrue(containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME), RealizationType.CUBE, cubeName));
assertTrue(containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME), RealizationType.CUBE, cubeName));
@@ -215,24 +231,24 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
CubeSegment segment = new CubeSegment();
segment.setName("test_segment");
cube.getSegments().add(segment);
- cubeManager.updateCube(cube);
- //only one for update cube
- assertEquals(1, broadcaster.getCounterAndClear());
- waitForCounterAndClear(1);
+ cubeManager.updateCube(cube, true);
+ //one for cube update, one for project update
+ assertEquals(2, broadcaster.getCounterAndClear());
+ waitForCounterAndClear(2);
assertEquals(1, cubeManagerB.getCube(cubeName).getSegments().size());
assertEquals(segment.getName(), cubeManagerB.getCube(cubeName).getSegments().get(0).getName());
//delete cube
cubeManager.dropCube(cubeName, false);
- assertTrue(cubeManager.getCube(cubeName) == null);
- assertTrue(!containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME), RealizationType.CUBE, cubeName));
//one for cube update, one for project update
assertEquals(2, broadcaster.getCounterAndClear());
waitForCounterAndClear(2);
+
+ assertTrue(cubeManager.getCube(cubeName) == null);
+ assertTrue(!containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME), RealizationType.CUBE, cubeName));
assertTrue(cubeManagerB.getCube(cubeName) == null);
assertTrue(!containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME), RealizationType.CUBE, cubeName));
-
final String cubeDescName = "test_cube_desc";
cubeDesc.setName(cubeDescName);
cubeDesc.setLastModified(0);
@@ -245,7 +261,6 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
assertNotNull(cubeDescManager.getCubeDesc(cubeDescName));
assertNotNull(cubeDescManagerB.getCubeDesc(cubeDescName));
-
cubeDesc.setNotifyList(Arrays.asList("test@email", "test@email", "test@email"));
cubeDescManager.updateCubeDesc(cubeDesc);
assertEquals(1, broadcaster.getCounterAndClear());
@@ -259,8 +274,6 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
assertTrue(cubeDescManager.getCubeDesc(cubeDescName) == null);
assertTrue(cubeDescManagerB.getCubeDesc(cubeDescName) == null);
-
-
getStore().deleteResource("/cube/a_whole_new_cube.json");
}
@@ -273,7 +286,6 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
return tableDesc;
}
-
@Test
public void testMetaCRUD() throws Exception {
final MetadataManager metadataManager = MetadataManager.getInstance(configA);
@@ -291,9 +303,6 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
assertNotNull(metadataManager.getTableDesc(tableDesc.getIdentity()));
assertNotNull(metadataManagerB.getTableDesc(tableDesc.getIdentity()));
-
-
-
final String dataModelName = "test_data_model";
DataModelDesc dataModelDesc = metadataManager.getDataModelDesc("test_kylin_left_join_model_desc");
dataModelDesc.setName(dataModelName);
@@ -302,7 +311,7 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
assertTrue(metadataManagerB.getDataModelDesc(dataModelName) == null);
dataModelDesc.setName(dataModelName);
- metadataManager.createDataModelDesc(dataModelDesc,"default","ADMIN");
+ metadataManager.createDataModelDesc(dataModelDesc, "default", "ADMIN");
//one for data model creation, one for project meta update
assertEquals(2, broadcaster.getCounterAndClear());
waitForCounterAndClear(2);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
index 88e31f4..4cf3952 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
@@ -18,17 +18,16 @@
package org.apache.kylin.rest.service;
-import java.net.UnknownHostException;
-import java.util.List;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-
import com.fasterxml.jackson.core.JsonProcessingException;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.job.exception.JobException;
import org.apache.kylin.metadata.project.ProjectInstance;
+import org.junit.Assert;
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.net.UnknownHostException;
+import java.util.List;
/**
* @author xduo
@@ -52,7 +51,6 @@ public class CubeServiceTest extends ServiceTestBase {
Assert.assertNotNull(cubes);
CubeInstance cube = cubes.get(0);
cubeService.isCubeDescEditable(cube.getDescriptor());
- cubeService.isCubeEditable(cube);
cubes = cubeService.getCubes(null, null,null, 1, 0);
Assert.assertTrue(cubes.size() == 1);
[10/11] incubator-kylin git commit: KYLIN-693 character all build
cube/ii test cases
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ada6b61e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index eb50a90..0a1e594 100644
--- a/pom.xml
+++ b/pom.xml
@@ -25,7 +25,7 @@
<groupId>org.apache</groupId>
<artifactId>apache</artifactId>
<version>16</version>
- </parent>
+ </parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
@@ -99,8 +99,10 @@
<sonar.dynamicAnalysis>reuseReports</sonar.dynamicAnalysis>
<sonar.jacoco.reportPath>${project.basedir}/../target/jacoco.exec</sonar.jacoco.reportPath>
<sonar.language>java</sonar.language>
- <sonar.jacoco.excludes>org/apache/kylin/**/tools/**:**/*CLI.java:net/hydromatic/optiq/**:org/eigenbase/sql2rel/**</sonar.jacoco.excludes>
-
+ <sonar.jacoco.excludes>
+ org/apache/kylin/**/tools/**:**/*CLI.java:net/hydromatic/optiq/**:org/eigenbase/sql2rel/**
+ </sonar.jacoco.excludes>
+
</properties>
<licenses>
@@ -596,6 +598,8 @@
<reportsDirectory>${project.basedir}/../target/surefire-reports</reportsDirectory>
<excludes>
<exclude>**/IT*.java</exclude>
+ <exclude>**/BuildCube*.java</exclude>
+ <exclude>**/BuildII*.java</exclude>
</excludes>
<systemProperties>
<property>
@@ -613,73 +617,97 @@
</systemProperties>
</configuration>
</plugin>
-
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>integration-tests</id>
+ <phase>integration-test</phase>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <excludes>
+ <exclude>**/ITKafka*.java</exclude>
+ </excludes>
+ <systemProperties>
+ <property>
+ <name>useSandbox</name>
+ <value>true</value>
+ </property>
+ </systemProperties>
+ </configuration>
+ </plugin>
</plugins>
</build>
</profile>
<profile>
- <!-- This profile adds/overrides few features of the 'apache-release'
- profile in the parent pom. -->
- <id>apache-release</id>
- <build>
- <plugins>
- <!-- Apache-RAT checks for files without headers.
- If run on a messy developer's sandbox, it will fail.
- This serves as a reminder to only build a release in a clean
- sandbox! -->
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <configuration>
- <!-- Exclude files/folders for apache release -->
- <excludes>
- <exclude>*.md</exclude>
- <exclude>docs/website</exclude>
- <exclude>docs/*.md</exclude>
- </excludes>
- </configuration>
- <executions>
- <execution>
- <phase>verify</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- <dependencies>
- <dependency>
- <groupId>org.apache.maven.doxia</groupId>
- <artifactId>doxia-core</artifactId>
- <version>1.6</version>
- <exclusions>
- <exclusion>
- <groupId>xerces</groupId>
- <artifactId>xercesImpl</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <groupId>net.ju-n.maven.plugins</groupId>
- <artifactId>checksum-maven-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>artifacts</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <algorithms>
- <algorithm>MD5</algorithm>
- <algorithm>SHA-1</algorithm>
- </algorithms>
- <failOnError>false</failOnError>
- </configuration>
- </plugin>
- </plugins>
- </build>
- </profile>
+ <!-- This profile adds/overrides few features of the 'apache-release'
+ profile in the parent pom. -->
+ <id>apache-release</id>
+ <build>
+ <plugins>
+ <!-- Apache-RAT checks for files without headers.
+ If run on a messy developer's sandbox, it will fail.
+ This serves as a reminder to only build a release in a clean
+ sandbox! -->
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <!-- Exclude files/folders for apache release -->
+ <excludes>
+ <exclude>*.md</exclude>
+ <exclude>docs/website</exclude>
+ <exclude>docs/*.md</exclude>
+ </excludes>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>verify</phase>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.maven.doxia</groupId>
+ <artifactId>doxia-core</artifactId>
+ <version>1.6</version>
+ <exclusions>
+ <exclusion>
+ <groupId>xerces</groupId>
+ <artifactId>xercesImpl</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ </plugin>
+ <plugin>
+ <groupId>net.ju-n.maven.plugins</groupId>
+ <artifactId>checksum-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>artifacts</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <algorithms>
+ <algorithm>MD5</algorithm>
+ <algorithm>SHA-1</algorithm>
+ </algorithms>
+ <failOnError>false</failOnError>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
</project>
[04/11] incubator-kylin git commit: KYLIN-697 non-integration tests
all passed
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
deleted file mode 100644
index 6bceab7..0000000
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- *
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *
- * contributor license agreements. See the NOTICE file distributed with
- *
- * this work for additional information regarding copyright ownership.
- *
- * The ASF licenses this file to You under the Apache License, Version 2.0
- *
- * (the "License"); you may not use this file except in compliance with
- *
- * the License. You may obtain a copy of the License at
- *
- *
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- *
- *
- * Unless required by applicable law or agreed to in writing, software
- *
- * distributed under the License is distributed on an "AS IS" BASIS,
- *
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *
- * See the License for the specific language governing permissions and
- *
- * limitations under the License.
- *
- * /
- */
-
-package org.apache.kylin.job;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hive.hcatalog.data.schema.HCatSchema;
-import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.dict.lookup.HiveTableReader;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.invertedindex.model.IIJoinedFlatTableDesc;
-import org.apache.kylin.job.common.ShellExecutable;
-import org.apache.kylin.job.constant.ExecutableConstants;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
-import org.apache.kylin.job.hadoop.invertedindex.IICreateHTableJob;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.apache.kylin.streaming.StreamMessage;
-import org.apache.kylin.streaming.invertedindex.IIStreamBuilder;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingDeque;
-
-import static org.junit.Assert.fail;
-
-/**
- */
-public class BuildIIWithStreamTest {
-
- private static final Logger logger = LoggerFactory.getLogger(BuildIIWithStreamTest.class);
-
- private static final String[] II_NAME = new String[] { "test_kylin_ii_left_join", "test_kylin_ii_inner_join" };
- private IIManager iiManager;
- private KylinConfig kylinConfig;
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
- }
-
- @Before
- public void before() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
- DeployUtil.overrideJobJarLocations();
-
- kylinConfig = KylinConfig.getInstanceFromEnv();
- iiManager = IIManager.getInstance(kylinConfig);
- iiManager = IIManager.getInstance(kylinConfig);
- for (String iiInstance : II_NAME) {
-
- IIInstance ii = iiManager.getII(iiInstance);
- if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
- ii.setStatus(RealizationStatusEnum.DISABLED);
- iiManager.updateII(ii,true);
- }
- }
- }
-
- @AfterClass
- public static void afterClass() throws Exception {
- backup();
- }
-
- private static int cleanupOldStorage() throws Exception {
- String[] args = { "--delete", "true" };
- int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
- return exitCode;
- }
-
- private static void backup() throws Exception {
- int exitCode = cleanupOldStorage();
- if (exitCode == 0) {
- exportHBaseData();
- }
- }
-
- private static void exportHBaseData() throws IOException {
- ExportHBaseData export = new ExportHBaseData();
- export.exportTables();
- }
-
- private String createIntermediateTable(IIDesc desc, KylinConfig kylinConfig) throws IOException {
- IIJoinedFlatTableDesc intermediateTableDesc = new IIJoinedFlatTableDesc(desc);
- JobEngineConfig jobEngineConfig = new JobEngineConfig(kylinConfig);
- final String uuid = UUID.randomUUID().toString();
- final String dropTableHql = JoinedFlatTable.generateDropTableStatement(intermediateTableDesc, uuid);
- final String createTableHql = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, jobEngineConfig.getHdfsWorkingDirectory() + "/kylin-" + uuid, uuid);
- String insertDataHqls;
- try {
- insertDataHqls = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, uuid, jobEngineConfig);
- } catch (IOException e1) {
- e1.printStackTrace();
- throw new RuntimeException("Failed to generate insert data SQL for intermediate table.");
- }
-
- ShellExecutable step = new ShellExecutable();
- StringBuffer buf = new StringBuffer();
- buf.append("hive -e \"");
- buf.append(dropTableHql + "\n");
- buf.append(createTableHql + "\n");
- buf.append(insertDataHqls + "\n");
- buf.append("\"");
-
- step.setCmd(buf.toString());
- logger.info(step.getCmd());
- step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);
- kylinConfig.getCliCommandExecutor().execute(step.getCmd(), null);
- return intermediateTableDesc.getTableName(uuid);
- }
-
- private void clearSegment(String iiName) throws Exception {
- IIInstance ii = iiManager.getII(iiName);
- ii.getSegments().clear();
- iiManager.updateII(ii,true);
- }
-
- private IISegment createSegment(String iiName) throws Exception {
- clearSegment(iiName);
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- long date1 = 0;
- long date2 = f.parse("2015-01-01").getTime();
- return buildSegment(iiName, date1, date2);
- }
-
- private IISegment buildSegment(String iiName, long startDate, long endDate) throws Exception {
- IIInstance iiInstance = iiManager.getII(iiName);
- IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
- iiInstance.getSegments().add(segment);
- iiManager.updateII(iiInstance,true);
- return segment;
- }
-
- private void buildII(String iiName) throws Exception {
- final IIDesc desc = iiManager.getII(iiName).getDescriptor();
- final String tableName = createIntermediateTable(desc, kylinConfig);
- logger.info("intermediate table name:" + tableName);
- final Configuration conf = new Configuration();
- HCatInputFormat.setInput(conf, "default", tableName);
- final HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
- logger.info(StringUtils.join(tableSchema.getFieldNames(), "\n"));
- HiveTableReader reader = new HiveTableReader("default", tableName);
- final List<TblColRef> tblColRefs = desc.listAllColumns();
- for (TblColRef tblColRef : tblColRefs) {
- if (desc.isMetricsCol(tblColRef)) {
- logger.info("matrix:" + tblColRef.getName());
- } else {
- logger.info("measure:" + tblColRef.getName());
- }
- }
- LinkedBlockingDeque<StreamMessage> queue = new LinkedBlockingDeque<StreamMessage>();
- final IISegment segment = createSegment(iiName);
- String[] args = new String[] { "-iiname", iiName, "-htablename", segment.getStorageLocationIdentifier() };
- ToolRunner.run(new IICreateHTableJob(), args);
-
- ExecutorService executorService = Executors.newSingleThreadExecutor();
- final IIStreamBuilder streamBuilder = new IIStreamBuilder(queue, iiName, segment.getStorageLocationIdentifier(), segment.getIIDesc(), 0);
-
- List<String[]> sorted = getSortedRows(reader, desc.getTimestampColumn());
- int count = sorted.size();
- for (String[] row : sorted) {
- logger.info("another row: " + StringUtils.join(row, ","));
- queue.put(parse(row));
- }
-
- reader.close();
- logger.info("total record count:" + count + " htable:" + segment.getStorageLocationIdentifier());
- queue.put(StreamMessage.EOF);
- final Future<?> future = executorService.submit(streamBuilder);
- try {
- future.get();
- } catch (Exception e) {
- logger.error("stream build failed", e);
- fail("stream build failed");
- }
-
- logger.info("stream build finished, htable name:" + segment.getStorageLocationIdentifier());
- }
-
- @Test
- public void test() throws Exception {
- for (String iiName : II_NAME) {
- buildII(iiName);
- IIInstance ii = iiManager.getII(iiName);
- if (ii.getStatus() != RealizationStatusEnum.READY) {
- ii.setStatus(RealizationStatusEnum.READY);
- iiManager.updateII(ii,true);
- }
- }
- }
-
- private StreamMessage parse(String[] row) {
- return new StreamMessage(System.currentTimeMillis(), StringUtils.join(row, ",").getBytes());
- }
-
- private List<String[]> getSortedRows(HiveTableReader reader, final int tsCol) throws IOException {
- List<String[]> unsorted = Lists.newArrayList();
- while (reader.next()) {
- unsorted.add(reader.getRow());
- }
- Collections.sort(unsorted, new Comparator<String[]>() {
- @Override
- public int compare(String[] o1, String[] o2) {
- long t1 = DateFormat.stringToMillis(o1[tsCol]);
- long t2 = DateFormat.stringToMillis(o2[tsCol]);
- return Long.compare(t1, t2);
- }
- });
- return unsorted;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/job/src/test/java/org/apache/kylin/job/IIStreamBuilderTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/IIStreamBuilderTest.java b/job/src/test/java/org/apache/kylin/job/IIStreamBuilderTest.java
deleted file mode 100644
index 6aaeaa2..0000000
--- a/job/src/test/java/org/apache/kylin/job/IIStreamBuilderTest.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- *
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *
- * contributor license agreements. See the NOTICE file distributed with
- *
- * this work for additional information regarding copyright ownership.
- *
- * The ASF licenses this file to You under the Apache License, Version 2.0
- *
- * (the "License"); you may not use this file except in compliance with
- *
- * the License. You may obtain a copy of the License at
- *
- *
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- *
- *
- * Unless required by applicable law or agreed to in writing, software
- *
- * distributed under the License is distributed on an "AS IS" BASIS,
- *
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *
- * See the License for the specific language governing permissions and
- *
- * limitations under the License.
- *
- * /
- */
-
-package org.apache.kylin.job;
-
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
-import org.apache.kylin.job.streaming.StreamingBootstrap;
-import org.junit.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-
-/**
- */
-public class IIStreamBuilderTest {
-
- private static final Logger logger = LoggerFactory.getLogger(IIStreamBuilderTest.class);
-
- private KylinConfig kylinConfig;
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
- }
-
- @AfterClass
- public static void afterClass() throws Exception {
-// backup();
- }
-
- private static void backup() throws Exception {
- int exitCode = cleanupOldStorage();
- if (exitCode == 0) {
- exportHBaseData();
- }
- }
-
- private static int cleanupOldStorage() throws Exception {
- String[] args = {"--delete", "true"};
-
- int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
- return exitCode;
- }
-
- private static void exportHBaseData() throws IOException {
- ExportHBaseData export = new ExportHBaseData();
- export.exportTables();
- }
-
- @Before
- public void before() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
-
- kylinConfig = KylinConfig.getInstanceFromEnv();
- DeployUtil.initCliWorkDir();
- DeployUtil.deployMetadata();
- DeployUtil.overrideJobJarLocations();
- }
-
- @Test
- public void test() throws Exception {
- final StreamingBootstrap bootstrap = StreamingBootstrap.getInstance(kylinConfig);
- bootstrap.start("eagle", 0);
- Thread.sleep(30 * 60 * 1000);
- logger.info("time is up, stop streaming");
- bootstrap.stop();
- Thread.sleep(5 * 1000);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithEngineTest.java
new file mode 100644
index 0000000..e99bd01
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithEngineTest.java
@@ -0,0 +1,279 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.job.cube.CubingJob;
+import org.apache.kylin.job.cube.CubingJobBuilder;
+import org.apache.kylin.job.engine.JobEngineConfig;
+import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.execution.ExecutableState;
+import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
+import org.apache.kylin.job.manager.ExecutableManager;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.lang.reflect.Method;
+import java.text.SimpleDateFormat;
+import java.util.List;
+import java.util.TimeZone;
+import java.util.concurrent.*;
+
+import static org.junit.Assert.assertEquals;
+
+public class ITBuildCubeWithEngineTest {
+
+ private JobEngineConfig jobEngineConfig;
+
+ private CubeManager cubeManager;
+
+ private DefaultScheduler scheduler;
+
+ protected ExecutableManager jobService;
+
+ private static final Log logger = LogFactory.getLog(ITBuildCubeWithEngineTest.class);
+
+ protected void waitForJob(String jobId) {
+ while (true) {
+ AbstractExecutable job = jobService.getJob(jobId);
+ if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) {
+ break;
+ } else {
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
+ System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ }
+
+ @Before
+ public void before() throws Exception {
+ HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+
+ DeployUtil.initCliWorkDir();
+ DeployUtil.deployMetadata();
+ DeployUtil.overrideJobJarLocations();
+
+
+ final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ jobService = ExecutableManager.getInstance(kylinConfig);
+ scheduler = DefaultScheduler.getInstance();
+ scheduler.init(new JobEngineConfig(kylinConfig));
+ if (!scheduler.hasStarted()) {
+ throw new RuntimeException("scheduler has not been started");
+ }
+ cubeManager = CubeManager.getInstance(kylinConfig);
+ jobEngineConfig = new JobEngineConfig(kylinConfig);
+ for (String jobId : jobService.getAllJobIds()) {
+ if(jobService.getJob(jobId) instanceof CubingJob){
+ jobService.deleteJob(jobId);
+ }
+ }
+
+ }
+
+ @After
+ public void after() {
+ HBaseMetadataTestCase.staticCleanupTestMetadata();
+ }
+
+ @Test
+ public void test() throws Exception {
+ DeployUtil.prepareTestData("left", "test_kylin_cube_with_slr_left_join_empty");
+ testInner();
+ testLeft();
+ }
+
+ private void testInner() throws Exception {
+ String[] testCase = new String[]{
+ "testInnerJoinCube",
+ "testInnerJoinCube2",
+ };
+ runTestAndAssertSucceed(testCase);
+ }
+
+ private void testLeft() throws Exception {
+ String[] testCase = new String[]{
+ "testLeftJoinCube",
+ "testLeftJoinCube2",
+ };
+ runTestAndAssertSucceed(testCase);
+ }
+
+ private void runTestAndAssertSucceed(String[] testCase) throws Exception {
+ ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
+ final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
+ List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
+ for (int i = 0; i < testCase.length; i++) {
+ tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch)));
+ }
+ countDownLatch.await();
+ try {
+ for (int i = 0; i < tasks.size(); ++i) {
+ Future<List<String>> task = tasks.get(i);
+ final List<String> jobIds = task.get();
+ for (String jobId : jobIds) {
+ assertJobSucceed(jobId);
+ }
+ }
+ } catch (Exception ex) {
+ logger.error(ex);
+ throw ex;
+ }
+ }
+
+ private void assertJobSucceed(String jobId) {
+ assertEquals("The job '" + jobId + "' is failed.", ExecutableState.SUCCEED, jobService.getOutput(jobId).getState());
+ }
+
+ private class TestCallable implements Callable<List<String>> {
+
+ private final String methodName;
+ private final CountDownLatch countDownLatch;
+
+ public TestCallable(String methodName, CountDownLatch countDownLatch) {
+ this.methodName = methodName;
+ this.countDownLatch = countDownLatch;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public List<String> call() throws Exception {
+ try {
+ final Method method = ITBuildCubeWithEngineTest.class.getDeclaredMethod(methodName);
+ method.setAccessible(true);
+ return (List<String>) method.invoke(ITBuildCubeWithEngineTest.this);
+ } finally {
+ countDownLatch.countDown();
+ }
+ }
+ }
+
+ @SuppressWarnings("unused") // called by reflection
+ private List<String> testInnerJoinCube2() throws Exception {
+ clearSegment("test_kylin_cube_with_slr_empty");
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+ long date1 = 0;
+ long date2 = f.parse("2013-01-01").getTime();
+ long date3 = f.parse("2022-01-01").getTime();
+ List<String> result = Lists.newArrayList();
+ result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date2));
+ result.add(buildSegment("test_kylin_cube_with_slr_empty", date2, date3));
+ return result;
+ }
+
+ @SuppressWarnings("unused") // called by reflection
+ private List<String> testInnerJoinCube() throws Exception {
+ clearSegment("test_kylin_cube_without_slr_empty");
+
+
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+
+ // this cube's start date is 0, end date is 20501112000000
+ long date1 = 0;
+ long date2 = f.parse("2013-01-01").getTime();
+
+
+ // this cube doesn't support incremental build, always do full build
+
+ List<String> result = Lists.newArrayList();
+ result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date2));
+ return result;
+ }
+
+ @SuppressWarnings("unused") // called by reflection
+ private List<String> testLeftJoinCube2() throws Exception {
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+ List<String> result = Lists.newArrayList();
+ final String cubeName = "test_kylin_cube_without_slr_left_join_empty";
+ // this cube's start date is 0, end date is 20120601000000
+ long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
+ long dateEnd = f.parse("2012-06-01").getTime();
+
+ clearSegment(cubeName);
+ result.add(buildSegment(cubeName, dateStart, dateEnd));
+
+ // then submit an append job, start date is 20120601000000, end
+ // date is 20220101000000
+ dateStart = f.parse("2012-06-01").getTime();
+ dateEnd = f.parse("2022-01-01").getTime();
+ result.add(buildSegment(cubeName, dateStart, dateEnd));
+ return result;
+
+ }
+
+ @SuppressWarnings("unused") // called by reflection
+ private List<String> testLeftJoinCube() throws Exception {
+ String cubeName = "test_kylin_cube_with_slr_left_join_empty";
+ clearSegment(cubeName);
+
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+ long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
+ long dateEnd = f.parse("2050-11-12").getTime();
+
+ // this cube's start date is 0, end date is 20501112000000
+ List<String> result = Lists.newArrayList();
+ result.add(buildSegment(cubeName, dateStart, dateEnd));
+ return result;
+
+ }
+
+ private void clearSegment(String cubeName) throws Exception {
+ CubeInstance cube = cubeManager.getCube(cubeName);
+ cube.getSegments().clear();
+ cubeManager.updateCube(cube,true);
+ }
+
+
+ private String buildSegment(String cubeName, long startDate, long endDate) throws Exception {
+ CubeSegment segment = cubeManager.appendSegments(cubeManager.getCube(cubeName), endDate);
+ CubingJobBuilder cubingJobBuilder = new CubingJobBuilder(jobEngineConfig);
+ CubingJob job = cubingJobBuilder.buildJob(segment);
+ jobService.addJob(job);
+ waitForJob(job.getId());
+ return job.getId();
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithStreamTest.java
new file mode 100644
index 0000000..8a26397
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithStreamTest.java
@@ -0,0 +1,224 @@
+/*
+ *
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ * contributor license agreements. See the NOTICE file distributed with
+ *
+ * this work for additional information regarding copyright ownership.
+ *
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ * (the "License"); you may not use this file except in compliance with
+ *
+ * the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ *
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ *
+ * limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.job;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Maps;
+import com.google.common.collect.SetMultimap;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hive.hcatalog.data.schema.HCatSchema;
+import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.DimensionDesc;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.dict.DictionaryGenerator;
+import org.apache.kylin.dict.lookup.HiveTableReader;
+import org.apache.kylin.job.hadoop.cubev2.IGTRecordWriter;
+import org.apache.kylin.job.hadoop.cubev2.InMemCubeBuilder;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.gridtable.GTRecord;
+import org.junit.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+/**
+ *
+ * This class is going to be deleted
+ */
+@Ignore("For dev testing")
+public class ITBuildCubeWithStreamTest {
+
+ private static final Logger logger = LoggerFactory.getLogger(ITBuildCubeWithStreamTest.class);
+
+ private KylinConfig kylinConfig;
+ private CubeManager cubeManager;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ }
+
+ @Before
+ public void before() throws Exception {
+ HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+ DeployUtil.overrideJobJarLocations();
+
+ kylinConfig = KylinConfig.getInstanceFromEnv();
+ cubeManager = CubeManager.getInstance(kylinConfig);
+
+ }
+
+ @After
+ public void after() {
+ HBaseMetadataTestCase.staticCleanupTestMetadata();
+ }
+
+ @Test
+ public void test() throws Exception {
+ CubeInstance cube = cubeManager.getCube("test_kylin_cube_without_slr_left_join_empty");
+ final CubeDesc desc = cube.getDescriptor();
+ // cube.getSegments().clear();
+ // cubeManager.updateCube(cube);
+
+ CubeSegment cubeSegment = cube.getSegment("19700101000000_20150401000000", SegmentStatusEnum.NEW);
+ Map<TblColRef, Dictionary<?>> dictionaryMap = Maps.newHashMap();
+
+//
+ for (DimensionDesc dim : desc.getDimensions()) {
+ // dictionary
+ for (TblColRef col : dim.getColumnRefs()) {
+ if (desc.getRowkey().isUseDictionary(col)) {
+ Dictionary dict = cubeSegment.getDictionary(col);
+ if (dict == null) {
+ throw new IllegalArgumentException("Dictionary for " + col + " was not found.");
+ }
+ logger.info("Dictionary for " + col + " was put into dictionary map.");
+ dictionaryMap.put(col, cubeSegment.getDictionary(col));
+ }
+ }
+ }
+
+// final String tableName = createIntermediateTable(desc, kylinConfig, null);
+ String tableName = "kylin_intermediate_test_kylin_cube_without_slr_desc_19700101000000_20130112000000_a24dec89_efbd_425f_9a5f_8b78dd1412af"; // has 3089 records;
+// tableName = "kylin_intermediate_test_kylin_cube_without_slr_desc_19700101000000_20130112000000_a5e1eb5d_da6b_475d_9807_be0b61f03215"; // only 20 rows;
+// tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150302000000_0a183367_f245_43d1_8850_1c138c8514c3";
+// tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150301000000_ce061464_7962_4642_bd7d_7c3d8fbe9389";
+ tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150401000000_fb7ae579_d987_4900_a3b7_c60c731cd269"; // 2 million records
+ logger.info("intermediate table name:" + tableName);
+
+
+ ArrayBlockingQueue queue = new ArrayBlockingQueue<List<String>>(10000);
+
+ InMemCubeBuilder cubeBuilder = new InMemCubeBuilder(queue, cube, dictionaryMap, new ConsoleGTRecordWriter());
+ ExecutorService executorService = Executors.newSingleThreadExecutor();
+ Future<?> future = executorService.submit(cubeBuilder);
+
+ final Configuration conf = new Configuration();
+ HCatInputFormat.setInput(conf, "default", tableName);
+ final HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
+ logger.info(StringUtils.join(tableSchema.getFieldNames(), "\n"));
+ HiveTableReader reader = new HiveTableReader("default", tableName);
+ List<String> row;
+ int counter = 0;
+ while (reader.next()) {
+ row = reader.getRowAsList();
+ queue.put(row);
+ counter++;
+ if(counter == 200000)
+ break;
+ }
+ queue.put(new ArrayList<String>(0));
+ reader.close();
+
+ try {
+ future.get();
+ } catch (Exception e) {
+ logger.error("stream build failed", e);
+ throw new IOException("Failed to build cube ", e);
+ }
+
+ logger.info("stream build finished");
+ }
+
+
+ private void buildDictionary(List<List<String>> table, CubeDesc desc, Map<TblColRef, Dictionary<?>> dictionaryMap) {
+ SetMultimap<TblColRef, String> valueMap = HashMultimap.create();
+
+ List<TblColRef> dimColumns = desc.listDimensionColumnsExcludingDerived();
+ for (List<String> row : table) {
+ for (int i = 0; i < dimColumns.size(); i++) {
+ String cell = row.get(i);
+ valueMap.put(dimColumns.get(i), cell);
+ }
+ }
+
+ for (DimensionDesc dim : desc.getDimensions()) {
+ // dictionary
+ for (TblColRef col : dim.getColumnRefs()) {
+ if (desc.getRowkey().isUseDictionary(col)) {
+ Dictionary dict = DictionaryGenerator.buildDictionaryFromValueList(col.getType(), Collections2.transform(valueMap.get(col), new Function<String, byte[]>() {
+ @Nullable
+ @Override
+ public byte[] apply(String input) {
+ if (input == null)
+ return null;
+ return input.getBytes();
+ }
+ }));
+
+ logger.info("Building dictionary for " + col);
+ dictionaryMap.put(col, dict);
+ }
+ }
+ }
+
+ }
+
+
+ class ConsoleGTRecordWriter implements IGTRecordWriter {
+
+ boolean verbose = false;
+
+ @Override
+ public void write(Long cuboidId, GTRecord record) throws IOException {
+ if (verbose)
+ System.out.println(record.toString());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/job/src/test/java/org/apache/kylin/job/ITBuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITBuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/ITBuildIIWithEngineTest.java
new file mode 100644
index 0000000..23f97d0
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/ITBuildIIWithEngineTest.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.invertedindex.IISegment;
+import org.apache.kylin.job.engine.JobEngineConfig;
+import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.execution.ExecutableState;
+import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
+import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
+import org.apache.kylin.job.invertedindex.IIJob;
+import org.apache.kylin.job.invertedindex.IIJobBuilder;
+import org.apache.kylin.job.manager.ExecutableManager;
+import org.apache.kylin.metadata.realization.RealizationStatusEnum;
+import org.junit.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.text.SimpleDateFormat;
+import java.util.List;
+import java.util.TimeZone;
+import java.util.concurrent.*;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author shaoshi
+ */
+public class ITBuildIIWithEngineTest {
+
+ private JobEngineConfig jobEngineConfig;
+ private IIManager iiManager;
+
+ private DefaultScheduler scheduler;
+ protected ExecutableManager jobService;
+
+ protected static final String[] TEST_II_INSTANCES = new String[] { "test_kylin_ii_inner_join", "test_kylin_ii_left_join" };
+
+ private static final Log logger = LogFactory.getLog(ITBuildIIWithEngineTest.class);
+
+ protected void waitForJob(String jobId) {
+ while (true) {
+ AbstractExecutable job = jobService.getJob(jobId);
+ if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) {
+ break;
+ } else {
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ }
+
+ @Before
+ public void before() throws Exception {
+ HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+
+ //DeployUtil.initCliWorkDir();
+ // DeployUtil.deployMetadata();
+ DeployUtil.overrideJobJarLocations();
+
+ final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ jobService = ExecutableManager.getInstance(kylinConfig);
+ scheduler = DefaultScheduler.getInstance();
+ scheduler.init(new JobEngineConfig(kylinConfig));
+ if (!scheduler.hasStarted()) {
+ throw new RuntimeException("scheduler has not been started");
+ }
+ jobEngineConfig = new JobEngineConfig(kylinConfig);
+ for (String jobId : jobService.getAllJobIds()) {
+ if (jobService.getJob(jobId) instanceof IIJob) {
+ jobService.deleteJob(jobId);
+ }
+ }
+
+ iiManager = IIManager.getInstance(kylinConfig);
+ for (String iiInstance : TEST_II_INSTANCES) {
+
+ IIInstance ii = iiManager.getII(iiInstance);
+ if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
+ ii.setStatus(RealizationStatusEnum.DISABLED);
+ iiManager.updateII(ii, true);
+ }
+ }
+ }
+
+ @After
+ public void after() throws Exception {
+
+ for (String iiInstance : TEST_II_INSTANCES) {
+ IIInstance ii = iiManager.getII(iiInstance);
+ if (ii.getStatus() != RealizationStatusEnum.READY) {
+ ii.setStatus(RealizationStatusEnum.READY);
+ iiManager.updateII(ii, true);
+ }
+ }
+ backup();
+ }
+
+ @Test
+ @Ignore
+ public void testBuildII() throws Exception {
+
+ String[] testCase = new String[] { "buildIIInnerJoin", "buildIILeftJoin" };
+ ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
+ final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
+ List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
+ for (int i = 0; i < testCase.length; i++) {
+ tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch)));
+ }
+ countDownLatch.await();
+ for (int i = 0; i < tasks.size(); ++i) {
+ Future<List<String>> task = tasks.get(i);
+ final List<String> jobIds = task.get();
+ for (String jobId : jobIds) {
+ assertJobSucceed(jobId);
+ }
+ }
+
+ }
+
+ private void assertJobSucceed(String jobId) {
+ assertEquals(ExecutableState.SUCCEED, jobService.getOutput(jobId).getState());
+ }
+
+ private class TestCallable implements Callable<List<String>> {
+
+ private final String methodName;
+ private final CountDownLatch countDownLatch;
+
+ public TestCallable(String methodName, CountDownLatch countDownLatch) {
+ this.methodName = methodName;
+ this.countDownLatch = countDownLatch;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public List<String> call() throws Exception {
+ try {
+ final Method method = ITBuildIIWithEngineTest.class.getDeclaredMethod(methodName);
+ method.setAccessible(true);
+ return (List<String>) method.invoke(ITBuildIIWithEngineTest.this);
+ } finally {
+ countDownLatch.countDown();
+ }
+ }
+ }
+
+ protected List<String> buildIIInnerJoin() throws Exception {
+ return buildII(TEST_II_INSTANCES[0]);
+ }
+
+ protected List<String> buildIILeftJoin() throws Exception {
+ return buildII(TEST_II_INSTANCES[1]);
+ }
+
+ protected List<String> buildII(String iiName) throws Exception {
+ clearSegment(iiName);
+
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+
+ long date1 = 0;
+ long date2 = f.parse("2015-01-01").getTime();
+
+ List<String> result = Lists.newArrayList();
+ result.add(buildSegment(iiName, date1, date2));
+ return result;
+ }
+
+ private void clearSegment(String iiName) throws Exception {
+ IIInstance ii = iiManager.getII(iiName);
+ ii.getSegments().clear();
+ iiManager.updateII(ii,true);
+ }
+
+ private String buildSegment(String iiName, long startDate, long endDate) throws Exception {
+ IIInstance iiInstance = iiManager.getII(iiName);
+ IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
+ iiInstance.getSegments().add(segment);
+ iiManager.updateII(iiInstance, true);
+ IIJobBuilder iiJobBuilder = new IIJobBuilder(jobEngineConfig);
+ IIJob job = iiJobBuilder.buildJob(segment);
+ jobService.addJob(job);
+ waitForJob(job.getId());
+ return job.getId();
+ }
+
+ private int cleanupOldStorage() throws Exception {
+ String[] args = { "--delete", "true" };
+
+ int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
+ return exitCode;
+ }
+
+ private void backup() throws Exception {
+ int exitCode = cleanupOldStorage();
+ if (exitCode == 0) {
+ exportHBaseData();
+ }
+ }
+
+ private void exportHBaseData() throws IOException {
+ ExportHBaseData export = new ExportHBaseData();
+ export.exportTables();
+ }
+
+ public static void main(String[] args) throws Exception {
+ ITBuildIIWithEngineTest instance = new ITBuildIIWithEngineTest();
+
+ ITBuildIIWithEngineTest.beforeClass();
+ instance.before();
+ instance.testBuildII();
+ instance.after();
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/job/src/test/java/org/apache/kylin/job/ITBuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITBuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/ITBuildIIWithStreamTest.java
new file mode 100644
index 0000000..cfb7bdb
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/ITBuildIIWithStreamTest.java
@@ -0,0 +1,270 @@
+/*
+ *
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ * contributor license agreements. See the NOTICE file distributed with
+ *
+ * this work for additional information regarding copyright ownership.
+ *
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ * (the "License"); you may not use this file except in compliance with
+ *
+ * the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ *
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ *
+ * limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.job;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hive.hcatalog.data.schema.HCatSchema;
+import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.dict.lookup.HiveTableReader;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.invertedindex.IISegment;
+import org.apache.kylin.invertedindex.model.IIDesc;
+import org.apache.kylin.invertedindex.model.IIJoinedFlatTableDesc;
+import org.apache.kylin.job.common.ShellExecutable;
+import org.apache.kylin.job.constant.ExecutableConstants;
+import org.apache.kylin.job.engine.JobEngineConfig;
+import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
+import org.apache.kylin.job.hadoop.invertedindex.IICreateHTableJob;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.RealizationStatusEnum;
+import org.apache.kylin.streaming.StreamMessage;
+import org.apache.kylin.streaming.invertedindex.IIStreamBuilder;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingDeque;
+
+import static org.junit.Assert.fail;
+
+/**
+ */
+public class ITBuildIIWithStreamTest {
+
+ private static final Logger logger = LoggerFactory.getLogger(ITBuildIIWithStreamTest.class);
+
+ private static final String[] II_NAME = new String[] { "test_kylin_ii_left_join", "test_kylin_ii_inner_join" };
+ private IIManager iiManager;
+ private KylinConfig kylinConfig;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ }
+
+ @Before
+ public void before() throws Exception {
+ HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+ DeployUtil.overrideJobJarLocations();
+
+ kylinConfig = KylinConfig.getInstanceFromEnv();
+ iiManager = IIManager.getInstance(kylinConfig);
+ iiManager = IIManager.getInstance(kylinConfig);
+ for (String iiInstance : II_NAME) {
+
+ IIInstance ii = iiManager.getII(iiInstance);
+ if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
+ ii.setStatus(RealizationStatusEnum.DISABLED);
+ iiManager.updateII(ii,true);
+ }
+ }
+ }
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+ backup();
+ }
+
+ private static int cleanupOldStorage() throws Exception {
+ String[] args = { "--delete", "true" };
+ int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
+ return exitCode;
+ }
+
+ private static void backup() throws Exception {
+ int exitCode = cleanupOldStorage();
+ if (exitCode == 0) {
+ exportHBaseData();
+ }
+ }
+
+ private static void exportHBaseData() throws IOException {
+ ExportHBaseData export = new ExportHBaseData();
+ export.exportTables();
+ }
+
+ private String createIntermediateTable(IIDesc desc, KylinConfig kylinConfig) throws IOException {
+ IIJoinedFlatTableDesc intermediateTableDesc = new IIJoinedFlatTableDesc(desc);
+ JobEngineConfig jobEngineConfig = new JobEngineConfig(kylinConfig);
+ final String uuid = UUID.randomUUID().toString();
+ final String dropTableHql = JoinedFlatTable.generateDropTableStatement(intermediateTableDesc, uuid);
+ final String createTableHql = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, jobEngineConfig.getHdfsWorkingDirectory() + "/kylin-" + uuid, uuid);
+ String insertDataHqls;
+ try {
+ insertDataHqls = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, uuid, jobEngineConfig);
+ } catch (IOException e1) {
+ e1.printStackTrace();
+ throw new RuntimeException("Failed to generate insert data SQL for intermediate table.");
+ }
+
+ ShellExecutable step = new ShellExecutable();
+ StringBuffer buf = new StringBuffer();
+ buf.append("hive -e \"");
+ buf.append(dropTableHql + "\n");
+ buf.append(createTableHql + "\n");
+ buf.append(insertDataHqls + "\n");
+ buf.append("\"");
+
+ step.setCmd(buf.toString());
+ logger.info(step.getCmd());
+ step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);
+ kylinConfig.getCliCommandExecutor().execute(step.getCmd(), null);
+ return intermediateTableDesc.getTableName(uuid);
+ }
+
+ private void clearSegment(String iiName) throws Exception {
+ IIInstance ii = iiManager.getII(iiName);
+ ii.getSegments().clear();
+ iiManager.updateII(ii,true);
+ }
+
+ private IISegment createSegment(String iiName) throws Exception {
+ clearSegment(iiName);
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+
+ long date1 = 0;
+ long date2 = f.parse("2015-01-01").getTime();
+ return buildSegment(iiName, date1, date2);
+ }
+
+ private IISegment buildSegment(String iiName, long startDate, long endDate) throws Exception {
+ IIInstance iiInstance = iiManager.getII(iiName);
+ IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
+ iiInstance.getSegments().add(segment);
+ iiManager.updateII(iiInstance,true);
+ return segment;
+ }
+
+ private void buildII(String iiName) throws Exception {
+ final IIDesc desc = iiManager.getII(iiName).getDescriptor();
+ final String tableName = createIntermediateTable(desc, kylinConfig);
+ logger.info("intermediate table name:" + tableName);
+ final Configuration conf = new Configuration();
+ HCatInputFormat.setInput(conf, "default", tableName);
+ final HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
+ logger.info(StringUtils.join(tableSchema.getFieldNames(), "\n"));
+ HiveTableReader reader = new HiveTableReader("default", tableName);
+ final List<TblColRef> tblColRefs = desc.listAllColumns();
+ for (TblColRef tblColRef : tblColRefs) {
+ if (desc.isMetricsCol(tblColRef)) {
+ logger.info("matrix:" + tblColRef.getName());
+ } else {
+ logger.info("measure:" + tblColRef.getName());
+ }
+ }
+ LinkedBlockingDeque<StreamMessage> queue = new LinkedBlockingDeque<StreamMessage>();
+ final IISegment segment = createSegment(iiName);
+ String[] args = new String[] { "-iiname", iiName, "-htablename", segment.getStorageLocationIdentifier() };
+ ToolRunner.run(new IICreateHTableJob(), args);
+
+ ExecutorService executorService = Executors.newSingleThreadExecutor();
+ final IIStreamBuilder streamBuilder = new IIStreamBuilder(queue, iiName, segment.getStorageLocationIdentifier(), segment.getIIDesc(), 0);
+
+ List<String[]> sorted = getSortedRows(reader, desc.getTimestampColumn());
+ int count = sorted.size();
+ for (String[] row : sorted) {
+ logger.info("another row: " + StringUtils.join(row, ","));
+ queue.put(parse(row));
+ }
+
+ reader.close();
+ logger.info("total record count:" + count + " htable:" + segment.getStorageLocationIdentifier());
+ queue.put(StreamMessage.EOF);
+ final Future<?> future = executorService.submit(streamBuilder);
+ try {
+ future.get();
+ } catch (Exception e) {
+ logger.error("stream build failed", e);
+ fail("stream build failed");
+ }
+
+ logger.info("stream build finished, htable name:" + segment.getStorageLocationIdentifier());
+ }
+
+ @Test
+ public void test() throws Exception {
+ for (String iiName : II_NAME) {
+ buildII(iiName);
+ IIInstance ii = iiManager.getII(iiName);
+ if (ii.getStatus() != RealizationStatusEnum.READY) {
+ ii.setStatus(RealizationStatusEnum.READY);
+ iiManager.updateII(ii,true);
+ }
+ }
+ }
+
+ private StreamMessage parse(String[] row) {
+ return new StreamMessage(System.currentTimeMillis(), StringUtils.join(row, ",").getBytes());
+ }
+
+ private List<String[]> getSortedRows(HiveTableReader reader, final int tsCol) throws IOException {
+ List<String[]> unsorted = Lists.newArrayList();
+ while (reader.next()) {
+ unsorted.add(reader.getRow());
+ }
+ Collections.sort(unsorted, new Comparator<String[]>() {
+ @Override
+ public int compare(String[] o1, String[] o2) {
+ long t1 = DateFormat.stringToMillis(o1[tsCol]);
+ long t2 = DateFormat.stringToMillis(o2[tsCol]);
+ return Long.compare(t1, t2);
+ }
+ });
+ return unsorted;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/job/src/test/java/org/apache/kylin/job/ITIIStreamBuilderTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITIIStreamBuilderTest.java b/job/src/test/java/org/apache/kylin/job/ITIIStreamBuilderTest.java
new file mode 100644
index 0000000..2edf457
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/ITIIStreamBuilderTest.java
@@ -0,0 +1,108 @@
+/*
+ *
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ * contributor license agreements. See the NOTICE file distributed with
+ *
+ * this work for additional information regarding copyright ownership.
+ *
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ * (the "License"); you may not use this file except in compliance with
+ *
+ * the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ *
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ *
+ * limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.job;
+
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
+import org.apache.kylin.job.streaming.StreamingBootstrap;
+import org.junit.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ */
+public class ITIIStreamBuilderTest {
+
+ private static final Logger logger = LoggerFactory.getLogger(ITIIStreamBuilderTest.class);
+
+ private KylinConfig kylinConfig;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ }
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+// backup();
+ }
+
+ private static void backup() throws Exception {
+ int exitCode = cleanupOldStorage();
+ if (exitCode == 0) {
+ exportHBaseData();
+ }
+ }
+
+ private static int cleanupOldStorage() throws Exception {
+ String[] args = {"--delete", "true"};
+
+ int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
+ return exitCode;
+ }
+
+ private static void exportHBaseData() throws IOException {
+ ExportHBaseData export = new ExportHBaseData();
+ export.exportTables();
+ }
+
+ @Before
+ public void before() throws Exception {
+ HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+
+ kylinConfig = KylinConfig.getInstanceFromEnv();
+ DeployUtil.initCliWorkDir();
+ DeployUtil.deployMetadata();
+ DeployUtil.overrideJobJarLocations();
+ }
+
+ @Test
+ public void test() throws Exception {
+ final StreamingBootstrap bootstrap = StreamingBootstrap.getInstance(kylinConfig);
+ bootstrap.start("eagle", 0);
+ Thread.sleep(30 * 60 * 1000);
+ logger.info("time is up, stop streaming");
+ bootstrap.stop();
+ Thread.sleep(5 * 1000);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java b/job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
deleted file mode 100644
index 22afb1d..0000000
--- a/job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.impl.threadpool;
-
-import static org.junit.Assert.*;
-
-import org.apache.kylin.job.SelfStopExecutable;
-import org.junit.Test;
-
-import org.apache.kylin.job.BaseTestExecutable;
-import org.apache.kylin.job.ErrorTestExecutable;
-import org.apache.kylin.job.FailedTestExecutable;
-import org.apache.kylin.job.SucceedTestExecutable;
-import org.apache.kylin.job.execution.DefaultChainedExecutable;
-import org.apache.kylin.job.execution.ExecutableState;
-
-/**
- */
-public class DefaultSchedulerTest extends BaseSchedulerTest {
-
- @Test
- public void testSingleTaskJob() throws Exception {
- DefaultChainedExecutable job = new DefaultChainedExecutable();
- BaseTestExecutable task1 = new SucceedTestExecutable();
- job.addTask(task1);
- jobService.addJob(job);
- waitForJobFinish(job.getId());
- assertEquals(ExecutableState.SUCCEED, jobService.getOutput(job.getId()).getState());
- assertEquals(ExecutableState.SUCCEED, jobService.getOutput(task1.getId()).getState());
- }
-
- @Test
- public void testSucceed() throws Exception {
- DefaultChainedExecutable job = new DefaultChainedExecutable();
- BaseTestExecutable task1 = new SucceedTestExecutable();
- BaseTestExecutable task2 = new SucceedTestExecutable();
- job.addTask(task1);
- job.addTask(task2);
- jobService.addJob(job);
- waitForJobFinish(job.getId());
- assertEquals(ExecutableState.SUCCEED, jobService.getOutput(job.getId()).getState());
- assertEquals(ExecutableState.SUCCEED, jobService.getOutput(task1.getId()).getState());
- assertEquals(ExecutableState.SUCCEED, jobService.getOutput(task2.getId()).getState());
- }
- @Test
- public void testSucceedAndFailed() throws Exception {
- DefaultChainedExecutable job = new DefaultChainedExecutable();
- BaseTestExecutable task1 = new SucceedTestExecutable();
- BaseTestExecutable task2 = new FailedTestExecutable();
- job.addTask(task1);
- job.addTask(task2);
- jobService.addJob(job);
- waitForJobFinish(job.getId());
- assertEquals(ExecutableState.ERROR, jobService.getOutput(job.getId()).getState());
- assertEquals(ExecutableState.SUCCEED, jobService.getOutput(task1.getId()).getState());
- assertEquals(ExecutableState.ERROR, jobService.getOutput(task2.getId()).getState());
- }
- @Test
- public void testSucceedAndError() throws Exception {
- DefaultChainedExecutable job = new DefaultChainedExecutable();
- BaseTestExecutable task1 = new ErrorTestExecutable();
- BaseTestExecutable task2 = new SucceedTestExecutable();
- job.addTask(task1);
- job.addTask(task2);
- jobService.addJob(job);
- waitForJobFinish(job.getId());
- assertEquals(ExecutableState.ERROR, jobService.getOutput(job.getId()).getState());
- assertEquals(ExecutableState.ERROR, jobService.getOutput(task1.getId()).getState());
- assertEquals(ExecutableState.READY, jobService.getOutput(task2.getId()).getState());
- }
-
- @Test
- public void testDiscard() throws Exception {
- DefaultChainedExecutable job = new DefaultChainedExecutable();
- BaseTestExecutable task1 = new SelfStopExecutable();
- job.addTask(task1);
- jobService.addJob(job);
- waitForJobStatus(job.getId(), ExecutableState.RUNNING, 500);
- jobService.discardJob(job.getId());
- waitForJobFinish(job.getId());
- assertEquals(ExecutableState.DISCARDED, jobService.getOutput(job.getId()).getState());
- assertEquals(ExecutableState.DISCARDED, jobService.getOutput(task1.getId()).getState());
- Thread.sleep(5000);
- System.out.println(job);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/job/src/test/java/org/apache/kylin/job/impl/threadpool/ITDefaultSchedulerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/impl/threadpool/ITDefaultSchedulerTest.java b/job/src/test/java/org/apache/kylin/job/impl/threadpool/ITDefaultSchedulerTest.java
new file mode 100644
index 0000000..bf4a1dc
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/impl/threadpool/ITDefaultSchedulerTest.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job.impl.threadpool;
+
+import org.apache.kylin.job.*;
+import org.apache.kylin.job.execution.DefaultChainedExecutable;
+import org.apache.kylin.job.execution.ExecutableState;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ */
+public class ITDefaultSchedulerTest extends BaseSchedulerTest {
+
+ @Test
+ public void testSingleTaskJob() throws Exception {
+ DefaultChainedExecutable job = new DefaultChainedExecutable();
+ BaseTestExecutable task1 = new SucceedTestExecutable();
+ job.addTask(task1);
+ jobService.addJob(job);
+ waitForJobFinish(job.getId());
+ assertEquals(ExecutableState.SUCCEED, jobService.getOutput(job.getId()).getState());
+ assertEquals(ExecutableState.SUCCEED, jobService.getOutput(task1.getId()).getState());
+ }
+
+ @Test
+ public void testSucceed() throws Exception {
+ DefaultChainedExecutable job = new DefaultChainedExecutable();
+ BaseTestExecutable task1 = new SucceedTestExecutable();
+ BaseTestExecutable task2 = new SucceedTestExecutable();
+ job.addTask(task1);
+ job.addTask(task2);
+ jobService.addJob(job);
+ waitForJobFinish(job.getId());
+ assertEquals(ExecutableState.SUCCEED, jobService.getOutput(job.getId()).getState());
+ assertEquals(ExecutableState.SUCCEED, jobService.getOutput(task1.getId()).getState());
+ assertEquals(ExecutableState.SUCCEED, jobService.getOutput(task2.getId()).getState());
+ }
+ @Test
+ public void testSucceedAndFailed() throws Exception {
+ DefaultChainedExecutable job = new DefaultChainedExecutable();
+ BaseTestExecutable task1 = new SucceedTestExecutable();
+ BaseTestExecutable task2 = new FailedTestExecutable();
+ job.addTask(task1);
+ job.addTask(task2);
+ jobService.addJob(job);
+ waitForJobFinish(job.getId());
+ assertEquals(ExecutableState.ERROR, jobService.getOutput(job.getId()).getState());
+ assertEquals(ExecutableState.SUCCEED, jobService.getOutput(task1.getId()).getState());
+ assertEquals(ExecutableState.ERROR, jobService.getOutput(task2.getId()).getState());
+ }
+ @Test
+ public void testSucceedAndError() throws Exception {
+ DefaultChainedExecutable job = new DefaultChainedExecutable();
+ BaseTestExecutable task1 = new ErrorTestExecutable();
+ BaseTestExecutable task2 = new SucceedTestExecutable();
+ job.addTask(task1);
+ job.addTask(task2);
+ jobService.addJob(job);
+ waitForJobFinish(job.getId());
+ assertEquals(ExecutableState.ERROR, jobService.getOutput(job.getId()).getState());
+ assertEquals(ExecutableState.ERROR, jobService.getOutput(task1.getId()).getState());
+ assertEquals(ExecutableState.READY, jobService.getOutput(task2.getId()).getState());
+ }
+
+ @Test
+ public void testDiscard() throws Exception {
+ DefaultChainedExecutable job = new DefaultChainedExecutable();
+ BaseTestExecutable task1 = new SelfStopExecutable();
+ job.addTask(task1);
+ jobService.addJob(job);
+ waitForJobStatus(job.getId(), ExecutableState.RUNNING, 500);
+ jobService.discardJob(job.getId());
+ waitForJobFinish(job.getId());
+ assertEquals(ExecutableState.DISCARDED, jobService.getOutput(job.getId()).getState());
+ assertEquals(ExecutableState.DISCARDED, jobService.getOutput(task1.getId()).getState());
+ Thread.sleep(5000);
+ System.out.println(job);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/metadata/src/test/java/org/apache/kylin/metadata/tool/HiveSourceTableLoaderTest.java
----------------------------------------------------------------------
diff --git a/metadata/src/test/java/org/apache/kylin/metadata/tool/HiveSourceTableLoaderTest.java b/metadata/src/test/java/org/apache/kylin/metadata/tool/HiveSourceTableLoaderTest.java
deleted file mode 100644
index 0534f0f..0000000
--- a/metadata/src/test/java/org/apache/kylin/metadata/tool/HiveSourceTableLoaderTest.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.tool;
-
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.Set;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.metadata.util.HiveSourceTableLoader;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class HiveSourceTableLoaderTest extends HBaseMetadataTestCase {
-
- @Before
- public void setup() throws Exception {
- super.createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- super.cleanupTestMetadata();
- }
-
- @Test
- public void test() throws IOException {
- if (!useSandbox())
- return;
-
- KylinConfig config = getTestConfig();
- String[] toLoad = new String[] { "DEFAULT.TEST_KYLIN_FACT", "EDW.TEST_CAL_DT" };
- Set<String> loaded = HiveSourceTableLoader.reloadHiveTables(toLoad, config);
-
- assertTrue(loaded.size() == toLoad.length);
- for (String str : toLoad)
- assertTrue(loaded.contains(str));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/metadata/src/test/java/org/apache/kylin/metadata/tool/ITHiveSourceTableLoaderTest.java
----------------------------------------------------------------------
diff --git a/metadata/src/test/java/org/apache/kylin/metadata/tool/ITHiveSourceTableLoaderTest.java b/metadata/src/test/java/org/apache/kylin/metadata/tool/ITHiveSourceTableLoaderTest.java
new file mode 100644
index 0000000..f357855
--- /dev/null
+++ b/metadata/src/test/java/org/apache/kylin/metadata/tool/ITHiveSourceTableLoaderTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.tool;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.metadata.util.HiveSourceTableLoader;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ITHiveSourceTableLoaderTest extends HBaseMetadataTestCase {
+
+ @Before
+ public void setup() throws Exception {
+ super.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ super.cleanupTestMetadata();
+ }
+
+ @Test
+ public void test() throws IOException {
+ if (!useSandbox())
+ return;
+
+ KylinConfig config = getTestConfig();
+ String[] toLoad = new String[] { "DEFAULT.TEST_KYLIN_FACT", "EDW.TEST_CAL_DT" };
+ Set<String> loaded = HiveSourceTableLoader.reloadHiveTables(toLoad, config);
+
+ assertTrue(loaded.size() == toLoad.length);
+ for (String str : toLoad)
+ assertTrue(loaded.contains(str));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index cb091a9..eb50a90 100644
--- a/pom.xml
+++ b/pom.xml
@@ -555,19 +555,13 @@
<configuration>
<reportsDirectory>${project.basedir}/../target/surefire-reports</reportsDirectory>
<excludes>
+ <exclude>**/IT*.java</exclude>
<!--Build cube/II need to be run separately-->
- <exclude>**/BuildCubeWithEngineTest.java</exclude>
- <exclude>**/BuildIIWithEngineTest.java</exclude>
- <exclude>**/BuildIIWithStreamTest.java</exclude>
- <exclude>**/IIStreamBuilderTest.java</exclude>
<!--minicluster does not have hive-->
- <exclude>**/SnapshotManagerTest.java</exclude>
- <exclude>**/HiveTableReaderTest.java</exclude>
- <exclude>**/TableControllerTest.java</exclude>
-
- <exclude>**/Kafka*Test.java</exclude>
- <exclude>**/RequesterTest.java</exclude>
+ <exclude>**/ITSnapshotManagerTest.java</exclude>
+ <exclude>**/ITHiveTableReaderTest.java</exclude>
+ <exclude>**/ITTableControllerTest.java</exclude>
</excludes>
<systemProperties>
<property>
@@ -601,14 +595,7 @@
<configuration>
<reportsDirectory>${project.basedir}/../target/surefire-reports</reportsDirectory>
<excludes>
- <!--Build cube/II need to be run separately-->
- <exclude>**/BuildCubeWithEngineTest.java</exclude>
- <exclude>**/BuildIIWithEngineTest.java</exclude>
- <exclude>**/BuildIIWithStreamTest.java</exclude>
- <exclude>**/IIStreamBuilderTest.java</exclude>
-
- <exclude>**/Kafka*Test.java</exclude>
- <exclude>**/RequesterTest.java</exclude>
+ <exclude>**/IT*.java</exclude>
</excludes>
<systemProperties>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/query/src/test/java/org/apache/kylin/query/test/CombinationTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/CombinationTest.java b/query/src/test/java/org/apache/kylin/query/test/CombinationTest.java
deleted file mode 100644
index 9e8e92b..0000000
--- a/query/src/test/java/org/apache/kylin/query/test/CombinationTest.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.query.test;
-
-import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverEnabler;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.sql.SQLException;
-import java.util.Arrays;
-import java.util.Collection;
-
-/**
- */
-@RunWith(Parameterized.class)
-public class CombinationTest extends KylinQueryTest {
-
- @BeforeClass
- public static void setUp() throws SQLException {
- System.out.println("setUp in CombinationTest");
- }
-
- @AfterClass
- public static void tearDown() {
- clean();
- }
-
- /**
- * return all config combinations, where first setting specifies join type
- * (inner or left), and the second setting specifies whether to force using
- * coprocessors(on, off or unset).
- */
- @Parameterized.Parameters
- public static Collection<Object[]> configs() {
-// return Arrays.asList(new Object[][] { { "inner", "unset" }, { "left", "unset" }, { "inner", "off" }, { "left", "off" }, { "inner", "on" }, { "left", "on" }, });
- return Arrays.asList(new Object[][]{{"inner", "on"}, {"left", "on"}});
- }
-
- public CombinationTest(String joinType, String coprocessorToggle) throws Exception {
-
- KylinQueryTest.clean();
-
- KylinQueryTest.joinType = joinType;
- KylinQueryTest.setupAll();
-
- if (coprocessorToggle.equals("on")) {
- ObserverEnabler.forceCoprocessorOn();
- } else if (coprocessorToggle.equals("off")) {
- ObserverEnabler.forceCoprocessorOff();
- } else if (coprocessorToggle.equals("unset")) {
- // unset
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
deleted file mode 100644
index 185b22e..0000000
--- a/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.query.test;
-
-import com.google.common.collect.Maps;
-import org.apache.kylin.metadata.realization.RealizationType;
-import org.apache.kylin.query.routing.RoutingRules.RealizationPriorityRule;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Map;
-
-/**
- */
-@RunWith(Parameterized.class)
-public class IIQueryTest extends KylinQueryTest {
- @BeforeClass
- public static void setUp() throws Exception {
-
- // give II higher priority than other realizations
- Map<RealizationType, Integer> priorities = Maps.newHashMap();
- priorities.put(RealizationType.INVERTED_INDEX, 0);
- priorities.put(RealizationType.CUBE, 1);
- priorities.put(RealizationType.HYBRID, 1);
- RealizationPriorityRule.setPriorities(priorities);
-
- }
-
- @AfterClass
- public static void tearDown() throws Exception {
- KylinQueryTest.tearDown();//invoke super class
-
- Map<RealizationType, Integer> priorities = Maps.newHashMap();
- priorities.put(RealizationType.INVERTED_INDEX, 1);
- priorities.put(RealizationType.CUBE, 0);
- priorities.put(RealizationType.HYBRID, 0);
- RealizationPriorityRule.setPriorities(priorities);
- }
-
- @Parameterized.Parameters
- public static Collection<Object[]> configs() {
- return Arrays.asList(new Object[][] { { "inner" }, { "left" } });
- }
-
- public IIQueryTest(String joinType) throws Exception {
-
- KylinQueryTest.clean();
-
- KylinQueryTest.joinType = joinType;
- KylinQueryTest.setupAll();
-
- }
-
- @Test
- public void testSingleRunQuery() throws Exception {
- super.testSingleRunQuery();
- }
-
- @Test
- public void testDetailedQuery() throws Exception {
- execAndCompQuery("src/test/resources/query/sql_ii", null, true);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/query/src/test/java/org/apache/kylin/query/test/ITCombinationTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/ITCombinationTest.java b/query/src/test/java/org/apache/kylin/query/test/ITCombinationTest.java
new file mode 100644
index 0000000..a38e85c
--- /dev/null
+++ b/query/src/test/java/org/apache/kylin/query/test/ITCombinationTest.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.query.test;
+
+import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverEnabler;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.sql.SQLException;
+import java.util.Arrays;
+import java.util.Collection;
+
+/**
+ */
+@RunWith(Parameterized.class)
+public class ITCombinationTest extends ITKylinQueryTest {
+
+ @BeforeClass
+ public static void setUp() throws SQLException {
+ System.out.println("setUp in ITCombinationTest");
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ clean();
+ }
+
+ /**
+ * return all config combinations, where first setting specifies join type
+ * (inner or left), and the second setting specifies whether to force using
+ * coprocessors(on, off or unset).
+ */
+ @Parameterized.Parameters
+ public static Collection<Object[]> configs() {
+// return Arrays.asList(new Object[][] { { "inner", "unset" }, { "left", "unset" }, { "inner", "off" }, { "left", "off" }, { "inner", "on" }, { "left", "on" }, });
+ return Arrays.asList(new Object[][]{{"inner", "on"}, {"left", "on"}});
+ }
+
+ public ITCombinationTest(String joinType, String coprocessorToggle) throws Exception {
+
+ ITKylinQueryTest.clean();
+
+ ITKylinQueryTest.joinType = joinType;
+ ITKylinQueryTest.setupAll();
+
+ if (coprocessorToggle.equals("on")) {
+ ObserverEnabler.forceCoprocessorOn();
+ } else if (coprocessorToggle.equals("off")) {
+ ObserverEnabler.forceCoprocessorOff();
+ } else if (coprocessorToggle.equals("unset")) {
+ // unset
+ }
+ }
+}
[02/11] incubator-kylin git commit: fix ci: clean broadcase module
Posted by ma...@apache.org.
fix ci: clean broadcase module
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/f5695644
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/f5695644
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/f5695644
Branch: refs/heads/new697
Commit: f56956443e456b497a3aaf3a2cc4626f58e72f5c
Parents: cd7b918
Author: honma <ho...@ebay.com>
Authored: Fri May 15 17:32:18 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon May 25 11:22:58 2015 +0800
----------------------------------------------------------------------
.../kylin/common/restclient/Broadcaster.java | 11 +-
.../common/restclient/MultiValueCache.java | 88 ----------
.../common/restclient/SingleValueCache.java | 9 +-
.../org/apache/kylin/cube/CubeDescManager.java | 25 ++-
.../java/org/apache/kylin/cube/CubeManager.java | 161 +++++++++----------
.../apache/kylin/cube/CubeManagerCacheTest.java | 11 +-
.../kylin/invertedindex/IIDescManager.java | 8 +-
.../apache/kylin/invertedindex/IIManager.java | 101 ++++++------
.../invertedindex/IIDescManagerTest.java | 2 +-
.../kylin/job/cube/MergeDictionaryStep.java | 15 +-
.../job/cube/UpdateCubeInfoAfterBuildStep.java | 9 +-
.../job/hadoop/invertedindex/IIBulkLoadJob.java | 2 +-
.../kylin/job/BuildCubeWithEngineTest.java | 2 +-
.../apache/kylin/job/BuildIIWithEngineTest.java | 19 ++-
.../apache/kylin/job/BuildIIWithStreamTest.java | 31 ++--
.../java/org/apache/kylin/job/DeployUtil.java | 27 ++--
.../job/hadoop/cube/MergeCuboidMapperTest.java | 31 ++--
.../apache/kylin/metadata/MetadataManager.java | 33 ++--
.../kylin/metadata/project/ProjectManager.java | 77 ++++-----
.../kylin/rest/controller/CubeController.java | 60 +++----
.../apache/kylin/rest/service/BasicService.java | 15 +-
.../apache/kylin/rest/service/CacheService.java | 132 +++++++--------
.../apache/kylin/rest/service/CubeService.java | 55 +++----
.../apache/kylin/rest/service/JobService.java | 82 +++++-----
.../kylin/rest/service/ProjectService.java | 23 +--
.../kylin/rest/service/CacheServiceTest.java | 51 +++---
.../kylin/rest/service/CubeServiceTest.java | 14 +-
27 files changed, 465 insertions(+), 629 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java b/common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java
index 6fbe413..f745410 100644
--- a/common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java
+++ b/common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java
@@ -35,9 +35,6 @@ import java.util.concurrent.atomic.AtomicLong;
/**
* Broadcast kylin event out
- *
- * @author jianliu
- *
*/
public class Broadcaster {
@@ -113,11 +110,11 @@ public class Broadcaster {
return counter.getAndSet(0);
}
- public static enum EVENT {
+ public enum EVENT {
CREATE("create"), UPDATE("update"), DROP("drop");
private String text;
- private EVENT(String text) {
+ EVENT(String text) {
this.text = text;
}
@@ -136,11 +133,11 @@ public class Broadcaster {
}
}
- public static enum TYPE {
+ public enum TYPE {
ALL("all"), CUBE("cube"), CUBE_DESC("cube_desc"), PROJECT("project"), INVERTED_INDEX("inverted_index"), INVERTED_INDEX_DESC("ii_desc"), TABLE("table"), DATA_MODEL("data_model"), HYBRID("hybrid");
private String text;
- private TYPE(String text) {
+ TYPE(String text) {
this.text = text;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/common/src/main/java/org/apache/kylin/common/restclient/MultiValueCache.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/restclient/MultiValueCache.java b/common/src/main/java/org/apache/kylin/common/restclient/MultiValueCache.java
deleted file mode 100644
index b479969..0000000
--- a/common/src/main/java/org/apache/kylin/common/restclient/MultiValueCache.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.restclient;
-
-import java.util.Set;
-
-import com.google.common.collect.HashMultimap;
-
-/**
- * @author xjiang
- *
- */
-public class MultiValueCache<K, V> extends AbstractRestCache<K, V> {
-
- private final HashMultimap<K, V> innerCache;
-
- public MultiValueCache(Broadcaster.TYPE syncType) {
- super(syncType);
- innerCache = HashMultimap.create();
- }
-
- public void put(K key, V value) {
- Broadcaster.EVENT eventType = innerCache.containsKey(key) ? Broadcaster.EVENT.UPDATE : Broadcaster.EVENT.CREATE;
- synchronized (this) {
- innerCache.put(key, value);
- }
- syncRemote(key, eventType);
- }
-
- public void putLocal(K key, V value) {
- synchronized (this) {
- innerCache.put(key, value);
- }
- }
-
- public void remove(K key) {
- if (innerCache.containsKey(key)) {
- innerCache.removeAll(key);
- syncRemote(key, Broadcaster.EVENT.DROP);
- }
- }
-
- public void removeLocal(K key) {
- if (innerCache.containsKey(key)) {
- innerCache.removeAll(key);
- }
- }
-
- public void clear() {
- innerCache.clear();
- }
-
- public int size() {
- return innerCache.size();
- }
-
- public Set<V> get(K key) {
- return innerCache.get(key);
- }
-
- public Set<K> keySet() {
- return innerCache.keySet();
- }
-
- public boolean containsKey(Object key) {
- return innerCache.containsKey(key);
- }
-
- public boolean containsEntry(Object key, Object value) {
- return innerCache.containsEntry(key, value);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java b/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
index 9848201..fb44206 100644
--- a/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
+++ b/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
@@ -43,8 +43,10 @@ public abstract class SingleValueCache<K, V> extends AbstractRestCache<K, V> {
}
public void put(K key, V value) {
- final V result = innerCache.put(key, value);
- if (result == null) {
+ //enforce all cache changes coming from REST
+ //final V result = innerCache.put(key, value);
+
+ if (!innerCache.containsKey(key)) {
syncRemote(key, Broadcaster.EVENT.CREATE);
} else {
syncRemote(key, Broadcaster.EVENT.UPDATE);
@@ -57,7 +59,8 @@ public abstract class SingleValueCache<K, V> extends AbstractRestCache<K, V> {
public void remove(K key) {
if (innerCache.containsKey(key)) {
- innerCache.remove(key);
+ //enforce all cache changes coming from REST
+ //innerCache.remove(key);
syncRemote(key, Broadcaster.EVENT.DROP);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java b/cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index ac5fc2b..d406da1 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -18,26 +18,25 @@
package org.apache.kylin.cube;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.kylin.common.restclient.CaseInsensitiveStringCache;
import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.validation.CubeMetadataValidator;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.Serializer;
import org.apache.kylin.common.restclient.Broadcaster;
+import org.apache.kylin.common.restclient.CaseInsensitiveStringCache;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.validation.CubeMetadataValidator;
+import org.apache.kylin.cube.model.validation.ValidateContext;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.MetadataManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
/**
* Manager class for CubeDesc; extracted from #CubeManager
@@ -108,7 +107,7 @@ public class CubeDescManager {
* @param name
* @throws IOException
*/
- public CubeDesc reloadCubeDesc(String name) throws IOException {
+ public CubeDesc reloadCubeDescLocal(String name) throws IOException {
// Save Source
String path = CubeDesc.getCubeDescResourcePath(name);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 76a6c6d..b986c36 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -52,7 +52,6 @@ import java.io.IOException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
-
/**
* @author yangli9
*/
@@ -138,7 +137,7 @@ public class CubeManager implements IRealizationProvider {
Iterator<CubeInstance> it = list.iterator();
while (it.hasNext()) {
CubeInstance ci = it.next();
- if (descName.equals(ci.getDescName())) {
+ if (descName.equalsIgnoreCase(ci.getDescName())) {
result.add(ci);
}
}
@@ -154,7 +153,7 @@ public class CubeManager implements IRealizationProvider {
DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, factColumnsPath);
cubeSeg.putDictResPath(col, dictInfo.getResourcePath());
- saveResource(cubeSeg.getCubeInstance());
+ updateCube(cubeSeg.getCubeInstance(), false);
return dictInfo;
}
@@ -192,7 +191,7 @@ public class CubeManager implements IRealizationProvider {
cubeSeg.putSnapshotResPath(lookupTable, snapshot.getResourcePath());
- saveResource(cubeSeg.getCubeInstance());
+ updateCube(cubeSeg.getCubeInstance(), false);
return snapshot;
}
@@ -202,25 +201,26 @@ public class CubeManager implements IRealizationProvider {
logger.info("Dropping cube '" + cubeName + "'");
// load projects before remove cube from project
- ResourceStore store = getStore();
-
// delete cube instance and cube desc
CubeInstance cube = getCube(cubeName);
- if (deleteDesc && cube.getDescriptor() != null)
- store.deleteResource(cube.getDescriptor().getResourcePath());
-
- store.deleteResource(cube.getResourcePath());
+ if (deleteDesc && cube.getDescriptor() != null) {
+ CubeDescManager.getInstance(config).removeCubeDesc(cube.getDescriptor());
+ }
+ removeCube(cube);
// delete cube from project
ProjectManager.getInstance(config).removeRealizationsFromProjects(RealizationType.CUBE, cubeName);
- // clean cube cache
- this.afterCubeDropped(cube);
-
return cube;
}
+ private void removeCube(CubeInstance cube) throws IOException {
+ // remove cube and update cache
+ getStore().deleteResource(cube.getResourcePath());
+ cubeMap.remove(cube.getName());
+ }
+
// sync on update
public CubeInstance createCube(String cubeName, String projectName, CubeDesc desc, String owner) throws IOException {
logger.info("Creating cube '" + projectName + "-->" + cubeName + "' from desc '" + desc.getName() + "'");
@@ -228,16 +228,27 @@ public class CubeManager implements IRealizationProvider {
// save cube resource
CubeInstance cube = CubeInstance.create(cubeName, projectName, desc);
cube.setOwner(owner);
- saveResource(cube);
+ updateCube(cube, false);
ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.CUBE, cubeName, projectName, owner);
return cube;
}
- public CubeInstance updateCube(CubeInstance cube) throws IOException {
+ /**
+ * if not sure whether to enable updateProject, just use it
+ */
+ public CubeInstance updateCube(CubeInstance cube, boolean updateProject) throws IOException {
+
logger.info("Updating cube instance '" + cube.getName());
- saveResource(cube);
+ getStore().putResource(cube.getResourcePath(), cube, CUBE_SERIALIZER);
+ cubeMap.put(cube.getName(), cube);
+
+ if (updateProject) {
+ logger.info("Updating project instance for cube:'" + cube.getName());
+ ProjectManager.getInstance(config).updateProject(RealizationType.CUBE, cube.getName());
+ }
+
return cube;
}
@@ -258,7 +269,7 @@ public class CubeManager implements IRealizationProvider {
cube.getSegments().add(appendSegment);
cube.getSegments().add(mergeSegment);
Collections.sort(cube.getSegments());
- updateCube(cube);
+ updateCube(cube, false);
return new Pair<CubeSegment, CubeSegment>(appendSegment, mergeSegment);
}
@@ -277,19 +288,18 @@ public class CubeManager implements IRealizationProvider {
validateNewSegments(cube, newSegment);
cube.getSegments().add(newSegment);
Collections.sort(cube.getSegments());
- updateCube(cube);
+ updateCube(cube, false);
return newSegment;
}
-
public CubeSegment refreshSegment(CubeInstance cube, long startDate, long endDate) throws IOException {
checkNoBuildingSegment(cube);
CubeSegment newSegment = newSegment(cube, startDate, endDate);
cube.getSegments().add(newSegment);
Collections.sort(cube.getSegments());
- updateCube(cube);
+ updateCube(cube, false);
return newSegment;
}
@@ -304,7 +314,7 @@ public class CubeManager implements IRealizationProvider {
validateNewSegments(cube, newSegment);
cube.getSegments().add(newSegment);
Collections.sort(cube.getSegments());
- updateCube(cube);
+ updateCube(cube, false);
return newSegment;
}
@@ -371,28 +381,17 @@ public class CubeManager implements IRealizationProvider {
*
* @param cubeName
*/
- public void loadCubeCache(String cubeName) {
+ public void reloadCubeLocal(String cubeName) {
try {
- loadCubeInstance(CubeInstance.concatResourcePath(cubeName));
+ reloadCubeLocalAt(CubeInstance.concatResourcePath(cubeName));
} catch (IOException e) {
logger.error(e.getLocalizedMessage(), e);
}
}
- /**
- * After cube deletion, remove cube related cache
- *
- * @param cube
- */
- public void removeCubeCache(CubeInstance cube) {
- final String cubeName = cube.getName().toUpperCase();
- cubeMap.remove(cubeName);
- usedStorageLocation.removeAll(cubeName);
- }
-
- public void removeCubeCacheLocal(String cubeName) {
+ public void removeCubeLocal(String cubeName) {
cubeMap.removeLocal(cubeName);
- usedStorageLocation.removeAll(cubeName);
+ usedStorageLocation.removeAll(cubeName.toUpperCase());
}
public LookupStringTable getLookupTable(CubeSegment cubeSegment, DimensionDesc dim) {
@@ -412,18 +411,41 @@ public class CubeManager implements IRealizationProvider {
}
}
- private void saveResource(CubeInstance cube) throws IOException {
- ResourceStore store = getStore();
- store.putResource(cube.getResourcePath(), cube, CUBE_SERIALIZER);
- this.afterCubeUpdated(cube);
- }
+ private CubeSegment newSegment(CubeInstance cubeInstance, long startDate, long endDate) {
+ if (startDate >= endDate)
+ throw new IllegalArgumentException("New segment range invalid, start date must be earlier than end date, " + startDate + " < " + endDate);
- private void afterCubeUpdated(CubeInstance updatedCube) {
- cubeMap.put(updatedCube.getName(), updatedCube);
+ CubeSegment segment = new CubeSegment();
+ String incrementalSegName = CubeSegment.getSegmentName(startDate, endDate);
+ segment.setUuid(UUID.randomUUID().toString());
+ segment.setName(incrementalSegName);
+ Date creatTime = new Date();
+ segment.setCreateTimeUTC(creatTime.getTime());
+ segment.setDateRangeStart(startDate);
+ segment.setDateRangeEnd(endDate);
+ segment.setStatus(SegmentStatusEnum.NEW);
+ segment.setStorageLocationIdentifier(generateStorageLocation());
+
+ segment.setCubeInstance(cubeInstance);
+
+ segment.validate();
+ return segment;
}
- private void afterCubeDropped(CubeInstance droppedCube) {
- removeCubeCache(droppedCube);
+ private String generateStorageLocation() {
+ String namePrefix = IRealizationConstants.CubeHbaseStorageLocationPrefix;
+ String tableName = "";
+ do {
+ StringBuffer sb = new StringBuffer();
+ sb.append(namePrefix);
+ Random ran = new Random();
+ for (int i = 0; i < HBASE_TABLE_LENGTH; i++) {
+ sb.append(ALPHA_NUM.charAt(ran.nextInt(ALPHA_NUM.length())));
+ }
+ tableName = sb.toString();
+ } while (this.usedStorageLocation.containsValue(tableName));
+
+ return tableName;
}
public CubeSegment autoMergeCubeSegments(CubeInstance cube) throws IOException {
@@ -480,43 +502,6 @@ public class CubeManager implements IRealizationProvider {
return null;
}
- private CubeSegment newSegment(CubeInstance cubeInstance, long startDate, long endDate) {
- if (startDate >= endDate)
- throw new IllegalArgumentException("New segment range invalid, start date must be earlier than end date, " + startDate + " < " + endDate);
-
- CubeSegment segment = new CubeSegment();
- String incrementalSegName = CubeSegment.getSegmentName(startDate, endDate);
- segment.setUuid(UUID.randomUUID().toString());
- segment.setName(incrementalSegName);
- Date creatTime = new Date();
- segment.setCreateTimeUTC(creatTime.getTime());
- segment.setDateRangeStart(startDate);
- segment.setDateRangeEnd(endDate);
- segment.setStatus(SegmentStatusEnum.NEW);
- segment.setStorageLocationIdentifier(generateStorageLocation());
-
- segment.setCubeInstance(cubeInstance);
-
- segment.validate();
- return segment;
- }
-
- private String generateStorageLocation() {
- String namePrefix = IRealizationConstants.CubeHbaseStorageLocationPrefix;
- String tableName = "";
- do {
- StringBuffer sb = new StringBuffer();
- sb.append(namePrefix);
- Random ran = new Random();
- for (int i = 0; i < HBASE_TABLE_LENGTH; i++) {
- sb.append(ALPHA_NUM.charAt(ran.nextInt(ALPHA_NUM.length())));
- }
- tableName = sb.toString();
- } while (this.usedStorageLocation.containsValue(tableName));
-
- return tableName;
- }
-
public void promoteNewlyBuiltSegments(CubeInstance cube, CubeSegment... newSegments) throws IOException {
List<CubeSegment> tobe = calculateToBeSegments(cube);
@@ -542,7 +527,7 @@ public class CubeManager implements IRealizationProvider {
cube.setStatus(RealizationStatusEnum.READY);
logger.info("Promoting cube " + cube + ", new segments " + newSegments);
- saveResource(cube);
+ updateCube(cube, true);
}
private void validateNewSegments(CubeInstance cube, CubeSegment... newSegments) {
@@ -579,7 +564,7 @@ public class CubeManager implements IRealizationProvider {
}
firstSeg.validate();
- for (int i = 0, j = 1; j < tobe.size(); ) {
+ for (int i = 0, j = 1; j < tobe.size();) {
CubeSegment is = tobe.get(i);
CubeSegment js = tobe.get(j);
js.validate();
@@ -644,16 +629,16 @@ public class CubeManager implements IRealizationProvider {
logger.debug("Loading Cube from folder " + store.getReadableResourcePath(ResourceStore.CUBE_RESOURCE_ROOT));
for (String path : paths) {
- loadCubeInstance(path);
+ reloadCubeLocalAt(path);
}
logger.debug("Loaded " + paths.size() + " Cube(s)");
}
- private synchronized CubeInstance loadCubeInstance(String path) throws IOException {
+ private synchronized CubeInstance reloadCubeLocalAt(String path) throws IOException {
ResourceStore store = getStore();
- CubeInstance cubeInstance = null;
+ CubeInstance cubeInstance;
try {
cubeInstance = store.getResource(path, CubeInstance.class, CUBE_SERIALIZER);
cubeInstance.setConfig(config);
@@ -668,7 +653,7 @@ public class CubeManager implements IRealizationProvider {
cubeMap.putLocal(cubeName, cubeInstance);
for (CubeSegment segment : cubeInstance.getSegments()) {
- usedStorageLocation.put(cubeName, segment.getStorageLocationIdentifier());
+ usedStorageLocation.put(cubeName.toUpperCase(), segment.getStorageLocationIdentifier());
}
return cubeInstance;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java b/cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
index 5e64e01..7c74993 100644
--- a/cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
+++ b/cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
@@ -18,18 +18,17 @@
package org.apache.kylin.cube;
-import static org.junit.Assert.*;
-
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.metadata.MetadataManager;
+import static org.junit.Assert.assertEquals;
/**
* @author yangli9
@@ -68,7 +67,7 @@ public class CubeManagerCacheTest extends LocalFileMetadataTestCase {
assertEquals(RealizationStatusEnum.DISABLED, createdCube.getStatus());
createdCube.setStatus(RealizationStatusEnum.DESCBROKEN);
- cubeManager.updateCube(createdCube);
+ cubeManager.updateCube(createdCube,true);
assertEquals(RealizationStatusEnum.DESCBROKEN, cubeManager.getCube("a_whole_new_cube").getStatus());
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIDescManager.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIDescManager.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIDescManager.java
index ae38955..cf0af86 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIDescManager.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIDescManager.java
@@ -18,6 +18,7 @@
package org.apache.kylin.invertedindex;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -27,7 +28,6 @@ import org.apache.kylin.common.restclient.CaseInsensitiveStringCache;
import org.apache.kylin.invertedindex.model.IIDesc;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.MetadataManager;
-import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -90,7 +90,7 @@ public class IIDescManager {
reloadAllIIDesc();
}
- public List<IIDesc> listAllDesc(){
+ public List<IIDesc> listAllDesc() {
return new ArrayList<IIDesc>(iiDescMap.values());
}
@@ -105,7 +105,7 @@ public class IIDescManager {
* @param name
* @throws IOException
*/
- public IIDesc reloadIIDesc(String name) throws IOException {
+ public IIDesc reloadIIDescLocal(String name) throws IOException {
// Save Source
String path = IIDesc.getIIDescResourcePath(name);
@@ -170,7 +170,7 @@ public class IIDescManager {
}
public void removeIIDescLocal(String name) throws IOException {
- iiDescMap.remove(name);
+ iiDescMap.removeLocal(name);
}
private void reloadAllIIDesc() throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java
index c1a0765..ea40ebd 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java
@@ -18,6 +18,8 @@
package org.apache.kylin.invertedindex;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
@@ -31,6 +33,8 @@ import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.invertedindex.model.IIDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.metadata.realization.IRealizationProvider;
@@ -39,7 +43,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
/**
@@ -92,7 +99,7 @@ public class IIManager implements IRealizationProvider {
private CaseInsensitiveStringCache<IIInstance> iiMap = new CaseInsensitiveStringCache<IIInstance>(Broadcaster.TYPE.INVERTED_INDEX);
// for generation hbase table name of a new segment
- private HashSet<String> usedStorageLocation = new HashSet<String>();
+ private Multimap<String, String> usedStorageLocation = HashMultimap.create();
private IIManager(KylinConfig config) throws IOException {
logger.info("Initializing IIManager with config " + config);
@@ -106,7 +113,6 @@ public class IIManager implements IRealizationProvider {
}
public IIInstance getII(String iiName) {
- iiName = iiName.toUpperCase();
return iiMap.get(iiName);
}
@@ -137,7 +143,7 @@ public class IIManager implements IRealizationProvider {
DictionaryInfo dict = dictMgr.buildDictionary(iiDesc.getModel(), "true", column, factColumnsPath);
iiSeg.putDictResPath(column, dict.getResourcePath());
}
- saveResource(iiSeg.getIIInstance());
+ updateII(iiSeg.getIIInstance(), false);
}
/**
@@ -168,58 +174,55 @@ public class IIManager implements IRealizationProvider {
if (this.getII(ii.getName()) != null)
throw new IllegalArgumentException("The II name '" + ii.getName() + "' already exists.");
- // other logic is the same as update.
- return updateII(ii);
- }
-
- public void updateIIStreamingOffset(String iiName, int partition, long offset) throws IOException {
-
- }
-
-
- public IIInstance updateII(IIInstance ii) throws IOException {
- logger.info("Updating II instance '" + ii.getName());
-
- // save resource
- saveResource(ii);
-
- logger.info("II with " + ii.getSegments().size() + " segments is saved");
+ this.updateII(ii, false);
+ String projectName = (null == ii.getProjectName()) ? ProjectInstance.DEFAULT_PROJECT_NAME : ii.getProjectName();
+ ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.INVERTED_INDEX, ii.getName(), projectName, ii.getOwner());
return ii;
}
- public void loadIICache(String iiName) {
+ public void reloadIILocal(String iiName) {
try {
- loadIIInstance(IIInstance.concatResourcePath(iiName));
+ reloadIILocalAt(IIInstance.concatResourcePath(iiName));
} catch (IOException e) {
logger.error(e.getLocalizedMessage(), e);
}
}
- public void removeIICache(IIInstance ii) {
- iiMap.remove(ii.getName());
+ public IIInstance dropII(String iiName, boolean deleteDesc) throws IOException {
+ logger.info("Dropping II '" + iiName + "'");
+
+ IIInstance ii = getII(iiName);
- for (IISegment segment : ii.getSegments()) {
- usedStorageLocation.remove(segment.getName());
+ if (deleteDesc && ii.getDescriptor() != null) {
+ IIDescManager.getInstance(config).removeIIDesc(ii.getDescriptor());
}
+
+ removeII(ii);
+ ProjectManager.getInstance(config).removeRealizationsFromProjects(RealizationType.INVERTED_INDEX, iiName);
+
+ return ii;
}
- public void removeIILocalCache(String name) {
- iiMap.removeLocal(name);
- //TODO
- // for (IISegment segment : ii.getSegments()) {
- // usedStorageLocation.remove(segment.getName());
- // }
+ private void removeII(IIInstance ii) throws IOException {
+ getStore().deleteResource(ii.getResourcePath());
+ iiMap.remove(ii.getName());
}
- private void saveResource(IIInstance ii) throws IOException {
- ResourceStore store = getStore();
- store.putResource(ii.getResourcePath(), ii, II_SERIALIZER);
- this.afterIIUpdated(ii);
+ public void removeIILocal(String name) {
+ iiMap.removeLocal(name);
+ usedStorageLocation.removeAll(name.toUpperCase());
}
- private void afterIIUpdated(IIInstance updatedII) {
- iiMap.put(updatedII.getName(), updatedII);
+ public void updateII(IIInstance ii, boolean updateProject) throws IOException {
+ logger.info("Updating II instance : " + ii.getName());
+ getStore().putResource(ii.getResourcePath(), ii, II_SERIALIZER);
+ iiMap.put(ii.getName(), ii);
+
+ if (updateProject) {
+ logger.info("Updating project instance for ii: " + ii.getName());
+ ProjectManager.getInstance(config).updateProject(RealizationType.INVERTED_INDEX, ii.getName());
+ }
}
/**
@@ -251,7 +254,7 @@ public class IIManager implements IRealizationProvider {
int idx = (int) (Math.random() * ALPHA_NUM.length());
sb.append(ALPHA_NUM.charAt(idx));
}
- if (usedStorageLocation.contains(sb.toString())) {
+ if (usedStorageLocation.containsValue(sb.toString())) {
continue;
} else {
return sb.toString();
@@ -266,31 +269,31 @@ public class IIManager implements IRealizationProvider {
logger.debug("Loading II from folder " + store.getReadableResourcePath(ResourceStore.II_RESOURCE_ROOT));
for (String path : paths) {
- loadIIInstance(path);
+ reloadIILocalAt(path);
}
logger.debug("Loaded " + paths.size() + " II(s)");
}
- private synchronized IIInstance loadIIInstance(String path) throws IOException {
+ private synchronized IIInstance reloadIILocalAt(String path) throws IOException {
ResourceStore store = getStore();
logger.debug("Loading IIInstance " + store.getReadableResourcePath(path));
- IIInstance IIInstance = null;
+ IIInstance ii = null;
try {
- IIInstance = store.getResource(path, IIInstance.class, II_SERIALIZER);
- IIInstance.setConfig(config);
+ ii = store.getResource(path, IIInstance.class, II_SERIALIZER);
+ ii.setConfig(config);
- if (StringUtils.isBlank(IIInstance.getName()))
+ if (StringUtils.isBlank(ii.getName()))
throw new IllegalStateException("IIInstance name must not be blank");
- iiMap.putLocal(IIInstance.getName(), IIInstance);
+ iiMap.putLocal(ii.getName(), ii);
- for (IISegment segment : IIInstance.getSegments()) {
- usedStorageLocation.add(segment.getName());
+ for (IISegment segment : ii.getSegments()) {
+ usedStorageLocation.put(ii.getName().toUpperCase(), segment.getStorageLocationIdentifier());
}
- return IIInstance;
+ return ii;
} catch (Exception e) {
logger.error("Error during load ii instance " + path, e);
return null;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java
index a13a0d1..a456dcd 100644
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java
+++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java
@@ -95,7 +95,7 @@ public class IIDescManagerTest extends LocalFileMetadataTestCase {
Assert.assertEquals(desc, newDesc);
// reload the cache
- mgr.reloadIIDesc(TEST_II_DESC_NAME);
+ mgr.reloadIIDescLocal(TEST_II_DESC_NAME);
newDesc = mgr.getIIDesc(TEST_II_DESC_NAME);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java b/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java
index b9aa7a4..d0a7db3 100644
--- a/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java
+++ b/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java
@@ -18,16 +18,8 @@
package org.apache.kylin.job.cube;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang.StringUtils;
-
import com.google.common.collect.Lists;
+import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -42,6 +34,9 @@ import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
import org.apache.kylin.metadata.model.TblColRef;
+import java.io.IOException;
+import java.util.*;
+
public class MergeDictionaryStep extends AbstractExecutable {
private static final String CUBE_NAME = "cubeName";
@@ -68,7 +63,7 @@ public class MergeDictionaryStep extends AbstractExecutable {
makeDictForNewSegment(conf, cube, newSegment, mergingSegments);
makeSnapshotForNewSegment(cube, newSegment, mergingSegments);
- mgr.updateCube(cube);
+ mgr.updateCube(cube,false);
return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
} catch (IOException e) {
logger.error("fail to merge dictionary or lookup snapshots", e);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java b/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java
index e9c2a2c..8140003 100644
--- a/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java
+++ b/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java
@@ -18,11 +18,8 @@
package org.apache.kylin.job.cube;
-import java.io.IOException;
-
-import org.apache.commons.lang.StringUtils;
-
import com.google.common.base.Preconditions;
+import org.apache.commons.lang.StringUtils;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
@@ -33,6 +30,8 @@ import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
import org.apache.kylin.job.execution.Output;
+import java.io.IOException;
+
/**
*/
public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
@@ -128,7 +127,7 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
if (segmentReady) {
cubeManager.promoteNewlyBuiltSegments(cube, segment);
} else {
- cubeManager.updateCube(cube);
+ cubeManager.updateCube(cube, true);
}
return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java
index f636af4..145a3c8 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java
@@ -64,7 +64,7 @@ public class IIBulkLoadJob extends AbstractHadoopJob {
IISegment seg = ii.getFirstSegment();
seg.setStorageLocationIdentifier(tableName);
seg.setStatus(SegmentStatusEnum.READY);
- mgr.updateII(ii);
+ mgr.updateII(ii,true);
return hbaseExitCode;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
index 812eb9d..547a6e7 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -263,7 +263,7 @@ public class BuildCubeWithEngineTest {
private void clearSegment(String cubeName) throws Exception {
CubeInstance cube = cubeManager.getCube(cubeName);
cube.getSegments().clear();
- cubeManager.updateCube(cube);
+ cubeManager.updateCube(cube,true);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
index af8682e..9c0219a 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
@@ -61,7 +61,7 @@ public class BuildIIWithEngineTest {
private DefaultScheduler scheduler;
protected ExecutableManager jobService;
- protected static final String[] TEST_II_INSTANCES = new String[]{ "test_kylin_ii_inner_join", "test_kylin_ii_left_join"};
+ protected static final String[] TEST_II_INSTANCES = new String[] { "test_kylin_ii_inner_join", "test_kylin_ii_left_join" };
private static final Log logger = LogFactory.getLog(BuildIIWithEngineTest.class);
@@ -104,7 +104,7 @@ public class BuildIIWithEngineTest {
}
jobEngineConfig = new JobEngineConfig(kylinConfig);
for (String jobId : jobService.getAllJobIds()) {
- if(jobService.getJob(jobId) instanceof IIJob){
+ if (jobService.getJob(jobId) instanceof IIJob) {
jobService.deleteJob(jobId);
}
}
@@ -115,7 +115,7 @@ public class BuildIIWithEngineTest {
IIInstance ii = iiManager.getII(iiInstance);
if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
ii.setStatus(RealizationStatusEnum.DISABLED);
- iiManager.updateII(ii);
+ iiManager.updateII(ii, true);
}
}
}
@@ -127,7 +127,7 @@ public class BuildIIWithEngineTest {
IIInstance ii = iiManager.getII(iiInstance);
if (ii.getStatus() != RealizationStatusEnum.READY) {
ii.setStatus(RealizationStatusEnum.READY);
- iiManager.updateII(ii);
+ iiManager.updateII(ii, true);
}
}
backup();
@@ -137,7 +137,7 @@ public class BuildIIWithEngineTest {
@Ignore
public void testBuildII() throws Exception {
- String[] testCase = new String[]{"buildIIInnerJoin", "buildIILeftJoin"};
+ String[] testCase = new String[] { "buildIIInnerJoin", "buildIILeftJoin" };
ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
@@ -183,10 +183,9 @@ public class BuildIIWithEngineTest {
}
protected List<String> buildIIInnerJoin() throws Exception {
- return buildII(TEST_II_INSTANCES[0]);
+ return buildII(TEST_II_INSTANCES[0]);
}
-
protected List<String> buildIILeftJoin() throws Exception {
return buildII(TEST_II_INSTANCES[1]);
}
@@ -208,14 +207,14 @@ public class BuildIIWithEngineTest {
private void clearSegment(String iiName) throws Exception {
IIInstance ii = iiManager.getII(iiName);
ii.getSegments().clear();
- iiManager.updateII(ii);
+ iiManager.updateII(ii,true);
}
private String buildSegment(String iiName, long startDate, long endDate) throws Exception {
IIInstance iiInstance = iiManager.getII(iiName);
IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
iiInstance.getSegments().add(segment);
- iiManager.updateII(iiInstance);
+ iiManager.updateII(iiInstance, true);
IIJobBuilder iiJobBuilder = new IIJobBuilder(jobEngineConfig);
IIJob job = iiJobBuilder.buildJob(segment);
jobService.addJob(job);
@@ -224,7 +223,7 @@ public class BuildIIWithEngineTest {
}
private int cleanupOldStorage() throws Exception {
- String[] args = {"--delete", "true"};
+ String[] args = { "--delete", "true" };
int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
return exitCode;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
index 65faad7..6bceab7 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -34,17 +34,7 @@
package org.apache.kylin.job;
-import static org.junit.Assert.fail;
-
-import java.io.File;
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingDeque;
-
+import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ToolRunner;
@@ -77,7 +67,16 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Lists;
+import java.io.File;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingDeque;
+
+import static org.junit.Assert.fail;
/**
*/
@@ -109,7 +108,7 @@ public class BuildIIWithStreamTest {
IIInstance ii = iiManager.getII(iiInstance);
if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
ii.setStatus(RealizationStatusEnum.DISABLED);
- iiManager.updateII(ii);
+ iiManager.updateII(ii,true);
}
}
}
@@ -169,7 +168,7 @@ public class BuildIIWithStreamTest {
private void clearSegment(String iiName) throws Exception {
IIInstance ii = iiManager.getII(iiName);
ii.getSegments().clear();
- iiManager.updateII(ii);
+ iiManager.updateII(ii,true);
}
private IISegment createSegment(String iiName) throws Exception {
@@ -186,7 +185,7 @@ public class BuildIIWithStreamTest {
IIInstance iiInstance = iiManager.getII(iiName);
IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
iiInstance.getSegments().add(segment);
- iiManager.updateII(iiInstance);
+ iiManager.updateII(iiInstance,true);
return segment;
}
@@ -243,7 +242,7 @@ public class BuildIIWithStreamTest {
IIInstance ii = iiManager.getII(iiName);
if (ii.getStatus() != RealizationStatusEnum.READY) {
ii.setStatus(RealizationStatusEnum.READY);
- iiManager.updateII(ii);
+ iiManager.updateII(ii,true);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/job/src/test/java/org/apache/kylin/job/DeployUtil.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/DeployUtil.java b/job/src/test/java/org/apache/kylin/job/DeployUtil.java
index 329eef1..6d348bd 100644
--- a/job/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/job/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -18,35 +18,30 @@
package org.apache.kylin.job;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.InputStream;
-
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.job.dataGen.FactTableGenerator;
-import org.apache.maven.model.Model;
-import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
-import org.apache.tools.ant.filters.StringInputStream;
-import org.codehaus.plexus.util.FileUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.ResourceTool;
import org.apache.kylin.common.util.AbstractKylinTestCase;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.HiveClient;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.job.dataGen.FactTableGenerator;
import org.apache.kylin.job.hadoop.hive.SqlHiveDataTypeMapping;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.maven.model.Model;
+import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
+import org.apache.tools.ant.filters.StringInputStream;
+import org.codehaus.plexus.util.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
public class DeployUtil {
@SuppressWarnings("unused")
@@ -65,7 +60,7 @@ public class DeployUtil {
// update cube desc signature.
for (CubeInstance cube : CubeManager.getInstance(config()).listAllCubes()) {
cube.getDescriptor().setSignature(cube.getDescriptor().calculateSignature());
- CubeManager.getInstance(config()).updateCube(cube);
+ CubeManager.getInstance(config()).updateCube(cube,true);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
index 1d0b16b..9f7e5f9 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
@@ -18,34 +18,29 @@
package org.apache.kylin.job.hadoop.cube;
-import java.io.File;
-import java.io.IOException;
-import java.text.ParseException;
-import java.util.ArrayList;
-import java.util.List;
-
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mrunit.mapreduce.MapDriver;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.dict.DictionaryGenerator;
-import org.apache.kylin.dict.DictionaryInfo;
-import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.dict.TrieDictionary;
+import org.apache.kylin.dict.*;
import org.apache.kylin.dict.lookup.TableSignature;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.ProjectManager;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.List;
import static org.junit.Assert.assertTrue;
@@ -139,7 +134,7 @@ public class MergeCuboidMapperTest extends LocalFileMetadataTestCase {
// cubeManager.saveResource(segment.getCubeInstance());
// cubeManager.afterCubeUpdated(segment.getCubeInstance());
- cubeManager.updateCube(cube);
+ cubeManager.updateCube(cube,true);
isFirstSegment = false;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 331bd83..2833218 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -18,23 +18,6 @@
package org.apache.kylin.metadata;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.*;
-import java.util.Map.Entry;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.metadata.project.ProjectManager;
-import org.apache.kylin.metadata.project.RealizationEntry;
-import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.metadata.realization.RealizationRegistry;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.kylin.common.KylinConfig;
@@ -45,8 +28,22 @@ import org.apache.kylin.common.restclient.Broadcaster;
import org.apache.kylin.common.restclient.CaseInsensitiveStringCache;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.project.RealizationEntry;
+import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.metadata.realization.RealizationRegistry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import javax.xml.crypto.Data;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.*;
+import java.util.Map.Entry;
+import java.util.concurrent.ConcurrentHashMap;
/**
* Serves (and caches) metadata for Kylin instance.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java b/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
index 029179f..76b1c0f 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
@@ -102,20 +102,19 @@ public class ProjectManager {
logger.debug("Loading Project from folder " + store.getReadableResourcePath(ResourceStore.PROJECT_RESOURCE_ROOT));
for (String path : paths) {
- reloadProjectAt(path);
+ reloadProjectLocalAt(path);
}
wireProjectAndRealizations(projectMap.values());
logger.debug("Loaded " + projectMap.size() + " Project(s)");
}
- public ProjectInstance reloadProject(String project) throws IOException {
- return reloadProjectAt(ProjectInstance.concatResourcePath(project));
+ public ProjectInstance reloadProjectLocal(String project) throws IOException {
+ return reloadProjectLocalAt(ProjectInstance.concatResourcePath(project));
}
- private ProjectInstance reloadProjectAt(String path) throws IOException {
- ResourceStore store = getStore();
+ private ProjectInstance reloadProjectLocalAt(String path) throws IOException {
- ProjectInstance projectInstance = store.getResource(path, ProjectInstance.class, PROJECT_SERIALIZER);
+ ProjectInstance projectInstance = getStore().getResource(path, ProjectInstance.class, PROJECT_SERIALIZER);
if (projectInstance == null) {
logger.warn("reload project at path:" + path + " not found, this:" + this.toString());
return null;
@@ -159,16 +158,16 @@ public class ProjectManager {
}
public ProjectInstance createProject(String projectName, String owner, String description) throws IOException {
- logger.info("Creating project '" + projectName);
+ logger.info("Creating project " + projectName);
ProjectInstance currentProject = getProject(projectName);
if (currentProject == null) {
- currentProject = ProjectInstance.create(projectName, owner, description, null,null);
+ currentProject = ProjectInstance.create(projectName, owner, description, null, null);
} else {
throw new IllegalStateException("The project named " + projectName + "already exists");
}
- saveResource(currentProject);
+ updateProject(currentProject);
return currentProject;
}
@@ -189,21 +188,30 @@ public class ProjectManager {
logger.info("Dropping project '" + projectInstance.getName() + "'");
- deleteResource(projectInstance);
+ removeProject(projectInstance);
return projectInstance;
}
+ //passive update due to underlying realization update
+ public void updateProject(RealizationType type, String realizationName) throws IOException {
+ for (ProjectInstance proj : findProjects(type, realizationName)) {
+ updateProject(proj);
+ }
+ }
+
+ //update project itself
public ProjectInstance updateProject(ProjectInstance project, String newName, String newDesc) throws IOException {
if (!project.getName().equals(newName)) {
ProjectInstance newProject = this.createProject(newName, project.getOwner(), newDesc);
- // FIXME table lost??
+
newProject.setCreateTimeUTC(project.getCreateTimeUTC());
newProject.recordUpdateTime(System.currentTimeMillis());
newProject.setRealizationEntries(project.getRealizationEntries());
+ newProject.setTables(project.getTables());
- deleteResource(project);
- saveResource(newProject);
+ removeProject(project);
+ updateProject(newProject);
return newProject;
} else {
@@ -213,12 +221,24 @@ public class ProjectManager {
if (project.getUuid() == null)
project.updateRandomUuid();
- saveResource(project);
+ updateProject(project);
return project;
}
}
+ private void updateProject(ProjectInstance prj) throws IOException {
+ getStore().putResource(prj.getResourcePath(), prj, PROJECT_SERIALIZER);
+ projectMap.put(norm(prj.getName()), prj); // triggers update broadcast
+ clearL2Cache();
+ }
+
+ private void removeProject(ProjectInstance proj) throws IOException {
+ getStore().deleteResource(proj.getResourcePath());
+ projectMap.remove(norm(proj.getName()));
+ clearL2Cache();
+ }
+
public boolean isModelInProject(String projectName, String modelName) {
return this.getProject(projectName).containsModel(modelName);
}
@@ -231,7 +251,7 @@ public class ProjectManager {
public void removeModelFromProjects(String modelName) throws IOException {
for (ProjectInstance projectInstance : findProjects(modelName)) {
projectInstance.removeModel(modelName);
- saveResource(projectInstance);
+ updateProject(projectInstance);
}
}
@@ -239,10 +259,10 @@ public class ProjectManager {
String newProjectName = ProjectInstance.getNormalizedProjectName(project);
ProjectInstance newProject = getProject(newProjectName);
if (newProject == null) {
- throw new IllegalArgumentException("Project "+newProjectName+" does not exist.");
+ throw new IllegalArgumentException("Project " + newProjectName + " does not exist.");
}
newProject.addModel(modelName);
- saveResource(newProject);
+ updateProject(newProject);
return newProject;
}
@@ -252,7 +272,6 @@ public class ProjectManager {
return addRealizationToProject(type, realizationName, newProjectName, owner);
}
-
private ProjectInstance addRealizationToProject(RealizationType type, String realizationName, String project, String user) throws IOException {
String newProjectName = norm(project);
ProjectInstance newProject = getProject(newProjectName);
@@ -260,7 +279,7 @@ public class ProjectManager {
newProject = this.createProject(newProjectName, user, "This is a project automatically added when adding realization " + realizationName + "(" + type + ")");
}
newProject.addRealizationEntry(type, realizationName);
- saveResource(newProject);
+ updateProject(newProject);
return newProject;
}
@@ -268,7 +287,7 @@ public class ProjectManager {
public void removeRealizationsFromProjects(RealizationType type, String realizationName) throws IOException {
for (ProjectInstance projectInstance : findProjects(type, realizationName)) {
projectInstance.removeRealization(type, realizationName);
- saveResource(projectInstance);
+ updateProject(projectInstance);
}
}
@@ -283,26 +302,10 @@ public class ProjectManager {
projectInstance.addTable(table.getIdentity());
}
- saveResource(projectInstance);
+ updateProject(projectInstance);
return projectInstance;
}
- private void saveResource(ProjectInstance prj) throws IOException {
- ResourceStore store = getStore();
- store.putResource(prj.getResourcePath(), prj, PROJECT_SERIALIZER);
-
- prj = reloadProjectAt(prj.getResourcePath());
- projectMap.put(norm(prj.getName()), prj); // triggers update broadcast
- clearL2Cache();
- }
-
- private void deleteResource(ProjectInstance proj) throws IOException {
- ResourceStore store = getStore();
- store.deleteResource(proj.getResourcePath());
- projectMap.remove(norm(proj.getName()));
- clearL2Cache();
- }
-
public List<ProjectInstance> findProjects(RealizationType type, String realizationName) {
List<ProjectInstance> result = Lists.newArrayList();
for (ProjectInstance prj : projectMap.values()) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index b1f9b4a..466a007 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -20,25 +20,18 @@ package org.apache.kylin.rest.controller;
import java.io.IOException;
import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
+import java.util.*;
import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.CubeBuildTypeEnum;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
import org.apache.kylin.job.JobInstance;
import org.apache.kylin.job.JoinedFlatTable;
import org.apache.kylin.job.exception.JobException;
-import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.rest.exception.BadRequestException;
@@ -58,12 +51,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.AccessDeniedException;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Controller;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestMethod;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.*;
import com.codahale.metrics.annotation.Metered;
import com.fasterxml.jackson.core.JsonParseException;
@@ -86,10 +74,10 @@ public class CubeController extends BasicController {
@Autowired
private JobService jobService;
- @RequestMapping(value = "", method = {RequestMethod.GET})
+ @RequestMapping(value = "", method = { RequestMethod.GET })
@ResponseBody
@Metered(name = "listCubes")
- public List<CubeInstance> getCubes(@RequestParam(value = "cubeName", required = false) String cubeName,@RequestParam(value = "modelName", required = false) String modelName, @RequestParam(value = "projectName", required = false) String projectName, @RequestParam(value="limit",required = false) Integer limit, @RequestParam(value = "offset" ,required = false) Integer offset) {
+ public List<CubeInstance> getCubes(@RequestParam(value = "cubeName", required = false) String cubeName, @RequestParam(value = "modelName", required = false) String modelName, @RequestParam(value = "projectName", required = false) String projectName, @RequestParam(value = "limit", required = false) Integer limit, @RequestParam(value = "offset", required = false) Integer offset) {
return cubeService.getCubes(cubeName, projectName, modelName, limit, offset);
}
@@ -101,7 +89,7 @@ public class CubeController extends BasicController {
* @throws UnknownHostException
* @throws IOException
*/
- @RequestMapping(value = "/{cubeName}/segs/{segmentName}/sql", method = {RequestMethod.GET})
+ @RequestMapping(value = "/{cubeName}/segs/{segmentName}/sql", method = { RequestMethod.GET })
@ResponseBody
public GeneralResponse getSql(@PathVariable String cubeName, @PathVariable String segmentName) {
CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
@@ -123,7 +111,7 @@ public class CubeController extends BasicController {
* @param notifyList
* @throws IOException
*/
- @RequestMapping(value = "/{cubeName}/notify_list", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/notify_list", method = { RequestMethod.PUT })
@ResponseBody
public void updateNotifyList(@PathVariable String cubeName, @RequestBody List<String> notifyList) {
CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
@@ -141,7 +129,7 @@ public class CubeController extends BasicController {
}
- @RequestMapping(value = "/{cubeName}/cost", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/cost", method = { RequestMethod.PUT })
@ResponseBody
@Metered(name = "updateCubeCost")
public CubeInstance updateCubeCost(@PathVariable String cubeName, @RequestParam(value = "cost") int cost) {
@@ -154,7 +142,7 @@ public class CubeController extends BasicController {
}
}
- @RequestMapping(value = "/{cubeName}/coprocessor", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/coprocessor", method = { RequestMethod.PUT })
@ResponseBody
public Map<String, Boolean> updateCubeCoprocessor(@PathVariable String cubeName, @RequestParam(value = "force") String force) {
try {
@@ -172,7 +160,7 @@ public class CubeController extends BasicController {
*
* @throws IOException
*/
- @RequestMapping(value = "/{cubeName}/segs/{segmentName}/refresh_lookup", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/segs/{segmentName}/refresh_lookup", method = { RequestMethod.PUT })
@ResponseBody
public CubeInstance rebuildLookupSnapshot(@PathVariable String cubeName, @PathVariable String segmentName, @RequestParam(value = "lookupTable") String lookupTable) {
try {
@@ -191,7 +179,7 @@ public class CubeController extends BasicController {
* @throws SchedulerException
* @throws IOException
*/
- @RequestMapping(value = "/{cubeName}/rebuild", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/rebuild", method = { RequestMethod.PUT })
@ResponseBody
public JobInstance rebuild(@PathVariable String cubeName, @RequestBody JobBuildRequest jobBuildRequest) {
try {
@@ -208,7 +196,7 @@ public class CubeController extends BasicController {
}
}
- @RequestMapping(value = "/{cubeName}/disable", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/disable", method = { RequestMethod.PUT })
@ResponseBody
@Metered(name = "disableCube")
public CubeInstance disableCube(@PathVariable String cubeName) {
@@ -227,7 +215,7 @@ public class CubeController extends BasicController {
}
}
- @RequestMapping(value = "/{cubeName}/purge", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/purge", method = { RequestMethod.PUT })
@ResponseBody
@Metered(name = "purgeCube")
public CubeInstance purgeCube(@PathVariable String cubeName) {
@@ -246,7 +234,7 @@ public class CubeController extends BasicController {
}
}
- @RequestMapping(value = "/{cubeName}/enable", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/enable", method = { RequestMethod.PUT })
@ResponseBody
@Metered(name = "enableCube")
public CubeInstance enableCube(@PathVariable String cubeName) {
@@ -264,7 +252,7 @@ public class CubeController extends BasicController {
}
}
- @RequestMapping(value = "/{cubeName}", method = {RequestMethod.DELETE})
+ @RequestMapping(value = "/{cubeName}", method = { RequestMethod.DELETE })
@ResponseBody
@Metered(name = "deleteCube")
public void deleteCube(@PathVariable String cubeName) {
@@ -287,7 +275,7 @@ public class CubeController extends BasicController {
* @return Table metadata array
* @throws IOException
*/
- @RequestMapping(value = "", method = {RequestMethod.POST})
+ @RequestMapping(value = "", method = { RequestMethod.POST })
@ResponseBody
@Metered(name = "saveCube")
public CubeRequest saveCubeDesc(@RequestBody CubeRequest cubeRequest) {
@@ -324,7 +312,7 @@ public class CubeController extends BasicController {
* @throws JsonProcessingException
* @throws IOException
*/
- @RequestMapping(value = "", method = {RequestMethod.PUT})
+ @RequestMapping(value = "", method = { RequestMethod.PUT })
@ResponseBody
@Metered(name = "updateCube")
public CubeRequest updateCubeDesc(@RequestBody CubeRequest cubeRequest) throws JsonProcessingException {
@@ -338,11 +326,17 @@ public class CubeController extends BasicController {
// Check if the cube is editable
if (!cubeService.isCubeDescEditable(desc)) {
- String error = "Cube desc " + desc.getName().toUpperCase() + " is not editable.";
+ String error = "Purge the related cube before editing its desc. Desc name: " + desc.getName();
updateRequest(cubeRequest, false, error);
return cubeRequest;
}
+ //cube renaming:
+ if (!cubeRequest.getCubeName().equalsIgnoreCase(CubeService.getCubeNameFromDesc(desc.getName()))) {
+ deleteCube(cubeRequest.getCubeName());
+ saveCubeDesc(cubeRequest);
+ }
+
try {
CubeInstance cube = cubeService.getCubeManager().getCube(cubeRequest.getCubeName());
String projectName = (null == cubeRequest.getProject()) ? ProjectInstance.DEFAULT_PROJECT_NAME : cubeRequest.getProject();
@@ -373,7 +367,7 @@ public class CubeController extends BasicController {
* @return true
* @throws IOException
*/
- @RequestMapping(value = "/{cubeName}/hbase", method = {RequestMethod.GET})
+ @RequestMapping(value = "/{cubeName}/hbase", method = { RequestMethod.GET })
@ResponseBody
@Metered(name = "getHBaseInfo")
public List<HBaseResponse> getHBaseInfo(@PathVariable String cubeName) {
@@ -429,14 +423,12 @@ public class CubeController extends BasicController {
return desc;
}
-
-
/**
* @return
*/
private String omitMessage(List<String> errors) {
StringBuffer buffer = new StringBuffer();
- for (Iterator<String> iterator = errors.iterator(); iterator.hasNext(); ) {
+ for (Iterator<String> iterator = errors.iterator(); iterator.hasNext();) {
String string = (String) iterator.next();
buffer.append(string);
buffer.append("\n");
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/BasicService.java b/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
index c0f366d..0ae9376 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
@@ -23,15 +23,15 @@ import com.google.common.base.Predicate;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.Lists;
import com.google.common.io.Files;
-
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.invertedindex.IIDescManager;
import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.job.cube.CubingJob;
-import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.manager.ExecutableManager;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.project.ProjectInstance;
@@ -41,7 +41,6 @@ import org.apache.kylin.query.enumerator.OLAPQuery;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.schema.OLAPSchemaFactory;
import org.apache.kylin.rest.controller.QueryController;
-import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.cache.annotation.CacheEvict;
@@ -49,7 +48,6 @@ import org.springframework.cache.annotation.Caching;
import org.springframework.jdbc.datasource.DriverManagerDataSource;
import javax.sql.DataSource;
-
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
@@ -78,6 +76,7 @@ public abstract class BasicService {
}
public void removeOLAPDataSource(String project) {
+ logger.info("removeOLAPDataSource is called for project " + project);
if (StringUtils.isEmpty(project))
throw new IllegalArgumentException("removeOLAPDataSource: project name not given");
@@ -85,9 +84,9 @@ public abstract class BasicService {
olapDataSources.remove(project);
}
- public static void resetOLAPDataSources() {
+ public static void removeAllOLAPDataSources() {
// brutal, yet simplest way
- logger.info("resetOLAPDataSources is called.");
+ logger.info("removeAllOLAPDataSources is called.");
olapDataSources.clear();
}
@@ -135,7 +134,7 @@ public abstract class BasicService {
public void cleanDataCache() {
CubeManager.clearCache();
ProjectManager.clearCache();
- BasicService.resetOLAPDataSources();
+ removeAllOLAPDataSources();
}
public final KylinConfig getKylinConfig() {
@@ -184,7 +183,7 @@ public abstract class BasicService {
if (cubeName == null) {
return true;
}
- return ((CubingJob) executable).getCubeName().equalsIgnoreCase(cubeName);
+ return ((CubingJob) executable).getCubeName().equalsIgnoreCase(cubeName);
} else {
return false;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index 6ea563f..9b89932 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -24,16 +24,13 @@ import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.model.CubeBuildTypeEnum;
import org.apache.kylin.invertedindex.IIDescManager;
import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.job.cube.CubingJob;
import org.apache.kylin.job.cube.CubingJobBuilder;
import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.exception.JobException;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
-import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.controller.QueryController;
import org.slf4j.Logger;
@@ -44,7 +41,6 @@ import org.springframework.cache.annotation.Caching;
import org.springframework.stereotype.Component;
import java.io.IOException;
-import java.util.List;
/**
*/
@@ -61,58 +57,48 @@ public class CacheService extends BasicService {
final String log = "rebuild cache type: " + cacheType + " name:" + cacheKey;
try {
switch (cacheType) {
- case CUBE:
- getCubeManager().loadCubeCache(cacheKey);
- cleanProjectCacheByRealization(RealizationType.CUBE, cacheKey);
- mergeCubeOnNewSegmentReady(cacheKey);
- break;
- case CUBE_DESC:
- getCubeDescManager().reloadCubeDesc(cacheKey);
- break;
- case PROJECT:
- getProjectManager().reloadProject(cacheKey);
- break;
- case INVERTED_INDEX:
- getIIManager().loadIICache(cacheKey);
- cleanProjectCacheByRealization(RealizationType.INVERTED_INDEX, cacheKey);
- break;
- case INVERTED_INDEX_DESC:
- getIIDescManager().reloadIIDesc(cacheKey);
- break;
- case TABLE:
- getMetadataManager().reloadTableCache(cacheKey);
- IIDescManager.clearCache();
- CubeDescManager.clearCache();
- break;
- case DATA_MODEL:
- getMetadataManager().reloadDataModelDesc(cacheKey);
- IIDescManager.clearCache();
- CubeDescManager.clearCache();
- break;
- case ALL:
- getMetadataManager().reload();
- CubeDescManager.clearCache();
- CubeManager.clearCache();
- IIDescManager.clearCache();
- IIManager.clearCache();
- ProjectManager.clearCache();
- BasicService.resetOLAPDataSources();
- break;
- default:
- throw new RuntimeException("invalid cacheType:" + cacheType);
+ case CUBE:
+ getCubeManager().reloadCubeLocal(cacheKey);
+ mergeCubeOnNewSegmentReady(cacheKey);
+ break;
+ case CUBE_DESC:
+ getCubeDescManager().reloadCubeDescLocal(cacheKey);
+ break;
+ case PROJECT:
+ ProjectInstance projectInstance = getProjectManager().reloadProjectLocal(cacheKey);
+ removeOLAPDataSource(projectInstance.getName());
+ break;
+ case INVERTED_INDEX:
+ getIIManager().reloadIILocal(cacheKey);
+ break;
+ case INVERTED_INDEX_DESC:
+ getIIDescManager().reloadIIDescLocal(cacheKey);
+ break;
+ case TABLE:
+ getMetadataManager().reloadTableCache(cacheKey);
+ IIDescManager.clearCache();
+ CubeDescManager.clearCache();
+ break;
+ case DATA_MODEL:
+ getMetadataManager().reloadDataModelDesc(cacheKey);
+ IIDescManager.clearCache();
+ CubeDescManager.clearCache();
+ break;
+ case ALL:
+ getMetadataManager().reload();
+ CubeDescManager.clearCache();
+ CubeManager.clearCache();
+ IIDescManager.clearCache();
+ IIManager.clearCache();
+ ProjectManager.clearCache();
+ removeAllOLAPDataSources();
+ break;
+ default:
+ throw new RuntimeException("invalid cacheType:" + cacheType);
}
} catch (IOException e) {
throw new RuntimeException("error " + log, e);
}
-
- }
-
- private void cleanProjectCacheByRealization(RealizationType type, String realizationName) throws IOException {
- List<ProjectInstance> projectInstances = getProjectManager().findProjects(type, realizationName);
- for (ProjectInstance pi : projectInstances) {
- getProjectManager().reloadProject(pi.getName());
- removeOLAPDataSource(pi.getName());
- }
}
@Caching(evict = { @CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true) })
@@ -120,27 +106,27 @@ public class CacheService extends BasicService {
final String log = "remove cache type: " + cacheType + " name:" + cacheKey;
try {
switch (cacheType) {
- case CUBE:
- getCubeManager().removeCubeCacheLocal(cacheKey);
- break;
- case CUBE_DESC:
- getCubeDescManager().removeLocalCubeDesc(cacheKey);
- break;
- case PROJECT:
- ProjectManager.clearCache();
- break;
- case INVERTED_INDEX:
- getIIManager().removeIILocalCache(cacheKey);
- break;
- case INVERTED_INDEX_DESC:
- getIIDescManager().removeIIDescLocal(cacheKey);
- break;
- case TABLE:
- throw new UnsupportedOperationException(log);
- case DATA_MODEL:
- throw new UnsupportedOperationException(log);
- default:
- throw new RuntimeException("invalid cacheType:" + cacheType);
+ case CUBE:
+ getCubeManager().removeCubeLocal(cacheKey);
+ break;
+ case CUBE_DESC:
+ getCubeDescManager().removeLocalCubeDesc(cacheKey);
+ break;
+ case PROJECT:
+ ProjectManager.clearCache();
+ break;
+ case INVERTED_INDEX:
+ getIIManager().removeIILocal(cacheKey);
+ break;
+ case INVERTED_INDEX_DESC:
+ getIIDescManager().removeIIDescLocal(cacheKey);
+ break;
+ case TABLE:
+ throw new UnsupportedOperationException(log);
+ case DATA_MODEL:
+ throw new UnsupportedOperationException(log);
+ default:
+ throw new RuntimeException("invalid cacheType:" + cacheType);
}
} catch (IOException e) {
throw new RuntimeException("error " + log, e);
[05/11] incubator-kylin git commit: KYLIN-697 non-integration tests
all passed
Posted by ma...@apache.org.
KYLIN-697 non-integration tests all passed
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/42f124fb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/42f124fb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/42f124fb
Branch: refs/heads/new697
Commit: 42f124fbbc249683f65d152078ae24ecd364bfa4
Parents: 08eedde
Author: honma <ho...@ebay.com>
Authored: Mon May 25 11:13:46 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon May 25 11:24:16 2015 +0800
----------------------------------------------------------------------
.../persistence/HBaseResourceStoreTest.java | 208 --------------
.../persistence/ITHBaseResourceStoreTest.java | 208 ++++++++++++++
.../kylin/common/util/ITMockHTableTest.java | 158 +++++++++++
.../kylin/common/util/MockHTableTest.java | 158 -----------
.../apache/kylin/dict/HiveTableReaderTest.java | 51 ----
.../kylin/dict/ITHiveTableReaderTest.java | 51 ++++
.../kylin/dict/ITSnapshotManagerTest.java | 82 ++++++
.../apache/kylin/dict/SnapshotManagerTest.java | 82 ------
.../Installation/Off Hadoop CLI Installation.md | 2 +-
.../MISC/FAQ on Kylin Installation and Usage.md | 2 +-
...n Kylin test case with HBase Mini Cluster.md | 2 +-
jdbc/kylin_jdbc.log.2014-12-22 | 18 ++
.../kylin/job/BuildCubeWithEngineTest.java | 279 -------------------
.../kylin/job/BuildCubeWithStreamTest.java | 224 ---------------
.../apache/kylin/job/BuildIIWithEngineTest.java | 254 -----------------
.../apache/kylin/job/BuildIIWithStreamTest.java | 270 ------------------
.../apache/kylin/job/IIStreamBuilderTest.java | 108 -------
.../kylin/job/ITBuildCubeWithEngineTest.java | 279 +++++++++++++++++++
.../kylin/job/ITBuildCubeWithStreamTest.java | 224 +++++++++++++++
.../kylin/job/ITBuildIIWithEngineTest.java | 254 +++++++++++++++++
.../kylin/job/ITBuildIIWithStreamTest.java | 270 ++++++++++++++++++
.../apache/kylin/job/ITIIStreamBuilderTest.java | 108 +++++++
.../impl/threadpool/DefaultSchedulerTest.java | 102 -------
.../impl/threadpool/ITDefaultSchedulerTest.java | 97 +++++++
.../tool/HiveSourceTableLoaderTest.java | 59 ----
.../tool/ITHiveSourceTableLoaderTest.java | 59 ++++
pom.xml | 23 +-
.../kylin/query/test/CombinationTest.java | 72 -----
.../apache/kylin/query/test/IIQueryTest.java | 85 ------
.../kylin/query/test/ITCombinationTest.java | 72 +++++
.../apache/kylin/query/test/ITIIQueryTest.java | 85 ++++++
.../kylin/query/test/ITKylinQueryTest.java | 234 ++++++++++++++++
.../apache/kylin/query/test/KylinQueryTest.java | 234 ----------------
.../org/apache/kylin/jdbc/ITJDBCDriverTest.java | 245 ++++++++++++++++
.../org/apache/kylin/jdbc/JDBCDriverTest.java | 245 ----------------
.../rest/controller/ITTableControllerTest.java | 74 +++++
.../rest/controller/TableControllerTest.java | 74 -----
.../kylin/rest/service/CacheServiceTest.java | 11 +-
.../storage/hbase/ITInvertedIndexHBaseTest.java | 114 ++++++++
.../storage/hbase/InvertedIndexHBaseTest.java | 114 --------
.../kylin/storage/test/ITStorageTest.java | 154 ++++++++++
.../apache/kylin/storage/test/StorageTest.java | 154 ----------
.../kylin/streaming/ITKafkaConsumerTest.java | 108 +++++++
.../kylin/streaming/ITKafkaRequesterTest.java | 76 +++++
.../kylin/streaming/KafkaConsumerTest.java | 108 -------
.../kylin/streaming/KafkaRequesterTest.java | 76 -----
46 files changed, 2979 insertions(+), 2988 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/common/src/test/java/org/apache/kylin/common/persistence/HBaseResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/persistence/HBaseResourceStoreTest.java b/common/src/test/java/org/apache/kylin/common/persistence/HBaseResourceStoreTest.java
deleted file mode 100644
index f4788f4..0000000
--- a/common/src/test/java/org/apache/kylin/common/persistence/HBaseResourceStoreTest.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.persistence;
-
-import static org.junit.Assert.*;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class HBaseResourceStoreTest extends HBaseMetadataTestCase {
-
- @Before
- public void setup() throws Exception {
- this.createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testHBaseStore() throws Exception {
- testAStore(ResourceStore.getStore(KylinConfig.getInstanceFromEnv()));
- }
-
- @Test
- public void testHBaseStoreWithLargeCell() throws Exception {
- String path = "/cube/_test_large_cell.json";
- String largeContent = "THIS_IS_A_LARGE_CELL";
- StringEntity content = new StringEntity(largeContent);
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- int origSize = config.getHBaseKeyValueSize();
- ResourceStore store = ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
-
- try {
- config.setProperty("kylin.hbase.client.keyvalue.maxsize", String.valueOf(largeContent.length() - 1));
-
- store.deleteResource(path);
-
- store.putResource(path, content, StringEntity.serializer);
- assertTrue(store.exists(path));
- StringEntity t = store.getResource(path, StringEntity.class, StringEntity.serializer);
- assertEquals(content, t);
-
- Path redirectPath = ((HBaseResourceStore) store).bigCellHDFSPath(path);
- Configuration hconf = HadoopUtil.getCurrentConfiguration();
- FileSystem fileSystem = FileSystem.get(hconf);
- assertTrue(fileSystem.exists(redirectPath));
-
- FSDataInputStream in = fileSystem.open(redirectPath);
- assertEquals(largeContent, in.readUTF());
- in.close();
-
- store.deleteResource(path);
- } finally {
- config.setProperty("kylin.hbase.client.keyvalue.maxsize", "" + origSize);
- store.deleteResource(path);
- }
- }
-
- void testAStore(ResourceStore store) throws IOException {
- String dir1 = "/cube";
- String path1 = "/cube/_test.json";
- StringEntity content1 = new StringEntity("anything");
- String dir2 = "/table";
- String path2 = "/table/_test.json";
- StringEntity content2 = new StringEntity("something");
-
- // cleanup legacy if any
- store.deleteResource(path1);
- store.deleteResource(path2);
-
- StringEntity t;
-
- // put/get
- store.putResource(path1, content1, StringEntity.serializer);
- assertTrue(store.exists(path1));
- t = store.getResource(path1, StringEntity.class, StringEntity.serializer);
- assertEquals(content1, t);
-
- store.putResource(path2, content2, StringEntity.serializer);
- assertTrue(store.exists(path2));
- t = store.getResource(path2, StringEntity.class, StringEntity.serializer);
- assertEquals(content2, t);
-
- // overwrite
- t.str = "new string";
- store.putResource(path2, t, StringEntity.serializer);
-
- // write conflict
- try {
- t.setLastModified(t.lastModified - 1);
- store.putResource(path2, t, StringEntity.serializer);
- fail("write conflict should trigger IllegalStateException");
- } catch (IllegalStateException e) {
- // expected
- }
-
- // list
- ArrayList<String> list;
-
- list = store.listResources(dir1);
- assertTrue(list.contains(path1));
- assertTrue(list.contains(path2) == false);
-
- list = store.listResources(dir2);
- assertTrue(list.contains(path2));
- assertTrue(list.contains(path1) == false);
-
- list = store.listResources("/");
- assertTrue(list.contains(dir1));
- assertTrue(list.contains(dir2));
- assertTrue(list.contains(path1) == false);
- assertTrue(list.contains(path2) == false);
-
- list = store.listResources(path1);
- assertNull(list);
- list = store.listResources(path2);
- assertNull(list);
-
- // delete/exist
- store.deleteResource(path1);
- assertTrue(store.exists(path1) == false);
- list = store.listResources(dir1);
- assertTrue(list == null || list.contains(path1) == false);
-
- store.deleteResource(path2);
- assertTrue(store.exists(path2) == false);
- list = store.listResources(dir2);
- assertTrue(list == null || list.contains(path2) == false);
- }
-
- public static class StringEntity extends RootPersistentEntity {
-
- static final Serializer<StringEntity> serializer = new Serializer<StringEntity>() {
- @Override
- public void serialize(StringEntity obj, DataOutputStream out) throws IOException {
- out.writeUTF(obj.str);
- }
-
- @Override
- public StringEntity deserialize(DataInputStream in) throws IOException {
- String str = in.readUTF();
- return new StringEntity(str);
- }
- };
-
- String str;
-
- public StringEntity(String str) {
- this.str = str;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = super.hashCode();
- result = prime * result + ((str == null) ? 0 : str.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == this)
- return true;
- if (!(obj instanceof StringEntity))
- return false;
- return StringUtils.equals(this.str, ((StringEntity) obj).str);
- }
-
- @Override
- public String toString() {
- return str;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/common/src/test/java/org/apache/kylin/common/persistence/ITHBaseResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/persistence/ITHBaseResourceStoreTest.java b/common/src/test/java/org/apache/kylin/common/persistence/ITHBaseResourceStoreTest.java
new file mode 100644
index 0000000..6195423
--- /dev/null
+++ b/common/src/test/java/org/apache/kylin/common/persistence/ITHBaseResourceStoreTest.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.persistence;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.common.util.HadoopUtil;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+
+import static org.junit.Assert.*;
+
+public class ITHBaseResourceStoreTest extends HBaseMetadataTestCase {
+
+ @Before
+ public void setup() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testHBaseStore() throws Exception {
+ testAStore(ResourceStore.getStore(KylinConfig.getInstanceFromEnv()));
+ }
+
+ @Test
+ public void testHBaseStoreWithLargeCell() throws Exception {
+ String path = "/cube/_test_large_cell.json";
+ String largeContent = "THIS_IS_A_LARGE_CELL";
+ StringEntity content = new StringEntity(largeContent);
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ int origSize = config.getHBaseKeyValueSize();
+ ResourceStore store = ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
+
+ try {
+ config.setProperty("kylin.hbase.client.keyvalue.maxsize", String.valueOf(largeContent.length() - 1));
+
+ store.deleteResource(path);
+
+ store.putResource(path, content, StringEntity.serializer);
+ assertTrue(store.exists(path));
+ StringEntity t = store.getResource(path, StringEntity.class, StringEntity.serializer);
+ assertEquals(content, t);
+
+ Path redirectPath = ((HBaseResourceStore) store).bigCellHDFSPath(path);
+ Configuration hconf = HadoopUtil.getCurrentConfiguration();
+ FileSystem fileSystem = FileSystem.get(hconf);
+ assertTrue(fileSystem.exists(redirectPath));
+
+ FSDataInputStream in = fileSystem.open(redirectPath);
+ assertEquals(largeContent, in.readUTF());
+ in.close();
+
+ store.deleteResource(path);
+ } finally {
+ config.setProperty("kylin.hbase.client.keyvalue.maxsize", "" + origSize);
+ store.deleteResource(path);
+ }
+ }
+
+ void testAStore(ResourceStore store) throws IOException {
+ String dir1 = "/cube";
+ String path1 = "/cube/_test.json";
+ StringEntity content1 = new StringEntity("anything");
+ String dir2 = "/table";
+ String path2 = "/table/_test.json";
+ StringEntity content2 = new StringEntity("something");
+
+ // cleanup legacy if any
+ store.deleteResource(path1);
+ store.deleteResource(path2);
+
+ StringEntity t;
+
+ // put/get
+ store.putResource(path1, content1, StringEntity.serializer);
+ assertTrue(store.exists(path1));
+ t = store.getResource(path1, StringEntity.class, StringEntity.serializer);
+ assertEquals(content1, t);
+
+ store.putResource(path2, content2, StringEntity.serializer);
+ assertTrue(store.exists(path2));
+ t = store.getResource(path2, StringEntity.class, StringEntity.serializer);
+ assertEquals(content2, t);
+
+ // overwrite
+ t.str = "new string";
+ store.putResource(path2, t, StringEntity.serializer);
+
+ // write conflict
+ try {
+ t.setLastModified(t.lastModified - 1);
+ store.putResource(path2, t, StringEntity.serializer);
+ fail("write conflict should trigger IllegalStateException");
+ } catch (IllegalStateException e) {
+ // expected
+ }
+
+ // list
+ ArrayList<String> list;
+
+ list = store.listResources(dir1);
+ assertTrue(list.contains(path1));
+ assertTrue(list.contains(path2) == false);
+
+ list = store.listResources(dir2);
+ assertTrue(list.contains(path2));
+ assertTrue(list.contains(path1) == false);
+
+ list = store.listResources("/");
+ assertTrue(list.contains(dir1));
+ assertTrue(list.contains(dir2));
+ assertTrue(list.contains(path1) == false);
+ assertTrue(list.contains(path2) == false);
+
+ list = store.listResources(path1);
+ assertNull(list);
+ list = store.listResources(path2);
+ assertNull(list);
+
+ // delete/exist
+ store.deleteResource(path1);
+ assertTrue(store.exists(path1) == false);
+ list = store.listResources(dir1);
+ assertTrue(list == null || list.contains(path1) == false);
+
+ store.deleteResource(path2);
+ assertTrue(store.exists(path2) == false);
+ list = store.listResources(dir2);
+ assertTrue(list == null || list.contains(path2) == false);
+ }
+
+ public static class StringEntity extends RootPersistentEntity {
+
+ static final Serializer<StringEntity> serializer = new Serializer<StringEntity>() {
+ @Override
+ public void serialize(StringEntity obj, DataOutputStream out) throws IOException {
+ out.writeUTF(obj.str);
+ }
+
+ @Override
+ public StringEntity deserialize(DataInputStream in) throws IOException {
+ String str = in.readUTF();
+ return new StringEntity(str);
+ }
+ };
+
+ String str;
+
+ public StringEntity(String str) {
+ this.str = str;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = super.hashCode();
+ result = prime * result + ((str == null) ? 0 : str.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this)
+ return true;
+ if (!(obj instanceof StringEntity))
+ return false;
+ return StringUtils.equals(this.str, ((StringEntity) obj).str);
+ }
+
+ @Override
+ public String toString() {
+ return str;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/common/src/test/java/org/apache/kylin/common/util/ITMockHTableTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/ITMockHTableTest.java b/common/src/test/java/org/apache/kylin/common/util/ITMockHTableTest.java
new file mode 100644
index 0000000..0913ad7
--- /dev/null
+++ b/common/src/test/java/org/apache/kylin/common/util/ITMockHTableTest.java
@@ -0,0 +1,158 @@
+package org.apache.kylin.common.util;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.filter.CompareFilter;
+import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.HBaseConnection;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+
+/**
+ * MockHTable Test.
+ *
+ * compare to real HTable.
+ */
+
+public class ITMockHTableTest extends HBaseMetadataTestCase {
+ private final Logger logger = LoggerFactory.getLogger(this.getClass());
+
+ private static HTableInterface mock;
+ private static HTableInterface real;
+ private static String emailCF = "EMAIL";
+ private static String addressCF = "ADDRESS";
+ private static String emailCQ = "email";
+ private static String addressCQ = "address";
+ private static String tableName = "MockHTable-TEST";
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+
+ staticCreateTestMetadata();
+
+ HBaseConnection.deleteTable(KylinConfig.getInstanceFromEnv().getStorageUrl(), tableName);
+ HBaseConnection.createHTableIfNeeded(KylinConfig.getInstanceFromEnv().getStorageUrl(), tableName, emailCF, addressCF);
+ real = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(tableName);
+ mock = new MockHTable(tableName, emailCF, addressCF);
+
+ setupDefaultData();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ HBaseConnection.deleteTable(KylinConfig.getInstanceFromEnv().getStorageUrl(), tableName);
+ staticCleanupTestMetadata();
+ }
+
+ private static void setupDefaultData() throws IOException, InterruptedException {
+ Put john = createPutForPerson("John Doe", "john.doe@email.com", "US");
+ Put jane = createPutForPerson("Jane Doe", "jane.doe@other.email.com", "US");
+ Put me = createPutForPerson("Yunsang Choi", "oddpoet@gmail", "jeju");
+ mutateAll(john, jane, me);
+ }
+
+ private static Put createPutForPerson(String name, String email, String address) {
+ Put person = new Put(name.getBytes());
+ person.add(emailCF.getBytes(), emailCQ.getBytes(), email.getBytes());
+ person.add(addressCF.getBytes(), addressCQ.getBytes(), address.getBytes());
+ return person;
+ }
+
+ private static <R extends Row> void mutateAll(R... rows) throws IOException, InterruptedException, IOException {
+ real.batch(Lists.newArrayList(rows));
+ mock.batch(Lists.newArrayList(rows));
+ }
+
+ @Test
+ public void test_get() throws Exception {
+ Get get = new Get("John Doe".getBytes());
+
+ Result realResult = real.get(get);
+ Result mockResult = mock.get(get);
+
+ assertThat(realResult.isEmpty(), is(false));
+ assertThat(mockResult.isEmpty(), is(realResult.isEmpty()));
+ }
+
+ @Test
+ public void test_get_with_filter() throws Exception {
+ Get get = new Get("John Doe".getBytes());
+ get.setFilter(new SingleColumnValueFilter(emailCF.getBytes(), emailCQ.getBytes(), CompareFilter.CompareOp.EQUAL, "WRONG EMAIL".getBytes()
+
+ ));
+
+ Result realResult = real.get(get);
+ Result mockResult = mock.get(get);
+
+ assertThat(realResult.isEmpty(), is(true));
+ assertThat(mockResult.isEmpty(), is(realResult.isEmpty()));
+ }
+
+ @Test
+ public void test_exists() throws IOException {
+ Get get = new Get("John Doe".getBytes());
+ boolean realResult = real.exists(get);
+ boolean mockResult = mock.exists(get);
+
+ assertThat(realResult, is(true));
+ assertThat(realResult, is(mockResult));
+ }
+
+ @Test
+ public void test_exists_include_not_exist_column() throws IOException {
+ Get get = new Get("John Doe".getBytes());
+ get.addColumn(emailCF.getBytes(), emailCQ.getBytes());
+ get.addColumn(emailCF.getBytes(), "NOT_EXIST_COLUMN".getBytes());
+ boolean realResult = real.exists(get);
+ boolean mockResult = mock.exists(get);
+
+ assertThat(realResult, is(true));
+ assertThat(realResult, is(mockResult));
+ }
+
+ @Test
+ public void test_exists_with_only_not_exist_column() throws IOException {
+ Get get = new Get("John Doe".getBytes());
+ get.addColumn(emailCF.getBytes(), "NOT_EXIST_COLUMN".getBytes());
+ boolean realResult = real.exists(get);
+ boolean mockResult = mock.exists(get);
+
+ assertThat(realResult, is(false));
+ assertThat(realResult, is(mockResult));
+ }
+
+ @Test
+ public void test_scan_with_filter() throws Exception {
+ Scan scan = new Scan();
+ scan.setFilter(new SingleColumnValueFilter(addressCF.getBytes(), addressCQ.getBytes(), CompareFilter.CompareOp.EQUAL, "US".getBytes()));
+
+ ResultScanner realResult = real.getScanner(scan);
+ ResultScanner mockResult = mock.getScanner(scan);
+
+ logger.debug("mock : {}", mockResult);
+
+ assertThat(Iterables.size(realResult), is(2));
+ assertThat(Iterables.size(mockResult), is(2));
+ }
+
+ @Test
+ public void test_scan_for_pre_match() throws Exception {
+ Scan scan = new Scan("J".getBytes(), "K".getBytes()); // start with 'J' only
+
+ ResultScanner realResult = real.getScanner(scan);
+ ResultScanner mockResult = mock.getScanner(scan);
+
+ assertThat(Iterables.size(realResult), is(2));
+ assertThat(Iterables.size(mockResult), is(2));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/common/src/test/java/org/apache/kylin/common/util/MockHTableTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/MockHTableTest.java b/common/src/test/java/org/apache/kylin/common/util/MockHTableTest.java
deleted file mode 100644
index 825fafc..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/MockHTableTest.java
+++ /dev/null
@@ -1,158 +0,0 @@
-package org.apache.kylin.common.util;
-
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import org.apache.hadoop.hbase.client.*;
-import org.apache.hadoop.hbase.filter.CompareFilter;
-import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.HBaseConnection;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.is;
-
-/**
- * MockHTable Test.
- *
- * compare to real HTable.
- */
-
-public class MockHTableTest extends HBaseMetadataTestCase {
- private final Logger logger = LoggerFactory.getLogger(this.getClass());
-
- private static HTableInterface mock;
- private static HTableInterface real;
- private static String emailCF = "EMAIL";
- private static String addressCF = "ADDRESS";
- private static String emailCQ = "email";
- private static String addressCQ = "address";
- private static String tableName = "MockHTable-TEST";
-
- @BeforeClass
- public static void setUp() throws Exception {
-
- staticCreateTestMetadata();
-
- HBaseConnection.deleteTable(KylinConfig.getInstanceFromEnv().getStorageUrl(), tableName);
- HBaseConnection.createHTableIfNeeded(KylinConfig.getInstanceFromEnv().getStorageUrl(), tableName, emailCF, addressCF);
- real = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(tableName);
- mock = new MockHTable(tableName, emailCF, addressCF);
-
- setupDefaultData();
- }
-
- @AfterClass
- public static void tearDown() throws Exception {
- HBaseConnection.deleteTable(KylinConfig.getInstanceFromEnv().getStorageUrl(), tableName);
- staticCleanupTestMetadata();
- }
-
- private static void setupDefaultData() throws IOException, InterruptedException {
- Put john = createPutForPerson("John Doe", "john.doe@email.com", "US");
- Put jane = createPutForPerson("Jane Doe", "jane.doe@other.email.com", "US");
- Put me = createPutForPerson("Yunsang Choi", "oddpoet@gmail", "jeju");
- mutateAll(john, jane, me);
- }
-
- private static Put createPutForPerson(String name, String email, String address) {
- Put person = new Put(name.getBytes());
- person.add(emailCF.getBytes(), emailCQ.getBytes(), email.getBytes());
- person.add(addressCF.getBytes(), addressCQ.getBytes(), address.getBytes());
- return person;
- }
-
- private static <R extends Row> void mutateAll(R... rows) throws IOException, InterruptedException, IOException {
- real.batch(Lists.newArrayList(rows));
- mock.batch(Lists.newArrayList(rows));
- }
-
- @Test
- public void test_get() throws Exception {
- Get get = new Get("John Doe".getBytes());
-
- Result realResult = real.get(get);
- Result mockResult = mock.get(get);
-
- assertThat(realResult.isEmpty(), is(false));
- assertThat(mockResult.isEmpty(), is(realResult.isEmpty()));
- }
-
- @Test
- public void test_get_with_filter() throws Exception {
- Get get = new Get("John Doe".getBytes());
- get.setFilter(new SingleColumnValueFilter(emailCF.getBytes(), emailCQ.getBytes(), CompareFilter.CompareOp.EQUAL, "WRONG EMAIL".getBytes()
-
- ));
-
- Result realResult = real.get(get);
- Result mockResult = mock.get(get);
-
- assertThat(realResult.isEmpty(), is(true));
- assertThat(mockResult.isEmpty(), is(realResult.isEmpty()));
- }
-
- @Test
- public void test_exists() throws IOException {
- Get get = new Get("John Doe".getBytes());
- boolean realResult = real.exists(get);
- boolean mockResult = mock.exists(get);
-
- assertThat(realResult, is(true));
- assertThat(realResult, is(mockResult));
- }
-
- @Test
- public void test_exists_include_not_exist_column() throws IOException {
- Get get = new Get("John Doe".getBytes());
- get.addColumn(emailCF.getBytes(), emailCQ.getBytes());
- get.addColumn(emailCF.getBytes(), "NOT_EXIST_COLUMN".getBytes());
- boolean realResult = real.exists(get);
- boolean mockResult = mock.exists(get);
-
- assertThat(realResult, is(true));
- assertThat(realResult, is(mockResult));
- }
-
- @Test
- public void test_exists_with_only_not_exist_column() throws IOException {
- Get get = new Get("John Doe".getBytes());
- get.addColumn(emailCF.getBytes(), "NOT_EXIST_COLUMN".getBytes());
- boolean realResult = real.exists(get);
- boolean mockResult = mock.exists(get);
-
- assertThat(realResult, is(false));
- assertThat(realResult, is(mockResult));
- }
-
- @Test
- public void test_scan_with_filter() throws Exception {
- Scan scan = new Scan();
- scan.setFilter(new SingleColumnValueFilter(addressCF.getBytes(), addressCQ.getBytes(), CompareFilter.CompareOp.EQUAL, "US".getBytes()));
-
- ResultScanner realResult = real.getScanner(scan);
- ResultScanner mockResult = mock.getScanner(scan);
-
- logger.debug("mock : {}", mockResult);
-
- assertThat(Iterables.size(realResult), is(2));
- assertThat(Iterables.size(mockResult), is(2));
- }
-
- @Test
- public void test_scan_for_pre_match() throws Exception {
- Scan scan = new Scan("J".getBytes(), "K".getBytes()); // start with 'J' only
-
- ResultScanner realResult = real.getScanner(scan);
- ResultScanner mockResult = mock.getScanner(scan);
-
- assertThat(Iterables.size(realResult), is(2));
- assertThat(Iterables.size(mockResult), is(2));
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/dictionary/src/test/java/org/apache/kylin/dict/HiveTableReaderTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/HiveTableReaderTest.java b/dictionary/src/test/java/org/apache/kylin/dict/HiveTableReaderTest.java
deleted file mode 100644
index 25279c6..0000000
--- a/dictionary/src/test/java/org/apache/kylin/dict/HiveTableReaderTest.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.dict;
-
-import java.io.IOException;
-
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.dict.lookup.HiveTableReader;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * This test case need the hive runtime; Please run it with sandbox;
- * @author shaoshi
- *
- * It is in the exclude list of default profile in pom.xml
- */
-public class HiveTableReaderTest extends HBaseMetadataTestCase {
-
-
- @Test
- public void test() throws IOException {
- HiveTableReader reader = new HiveTableReader("default", "test_kylin_fact");
- int rowNumber = 0;
- while (reader.next()) {
- String[] row = reader.getRow();
- Assert.assertEquals(9, row.length);
- //System.out.println(ArrayUtils.toString(row));
- rowNumber++;
- }
-
- reader.close();
- Assert.assertEquals(10000, rowNumber);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/dictionary/src/test/java/org/apache/kylin/dict/ITHiveTableReaderTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/ITHiveTableReaderTest.java b/dictionary/src/test/java/org/apache/kylin/dict/ITHiveTableReaderTest.java
new file mode 100644
index 0000000..8559f8b
--- /dev/null
+++ b/dictionary/src/test/java/org/apache/kylin/dict/ITHiveTableReaderTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.dict;
+
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.dict.lookup.HiveTableReader;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+
+/**
+ * This test case need the hive runtime; Please run it with sandbox;
+ * @author shaoshi
+ *
+ * It is in the exclude list of default profile in pom.xml
+ */
+public class ITHiveTableReaderTest extends HBaseMetadataTestCase {
+
+
+ @Test
+ public void test() throws IOException {
+ HiveTableReader reader = new HiveTableReader("default", "test_kylin_fact");
+ int rowNumber = 0;
+ while (reader.next()) {
+ String[] row = reader.getRow();
+ Assert.assertEquals(9, row.length);
+ //System.out.println(ArrayUtils.toString(row));
+ rowNumber++;
+ }
+
+ reader.close();
+ Assert.assertEquals(10000, rowNumber);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/dictionary/src/test/java/org/apache/kylin/dict/ITSnapshotManagerTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/ITSnapshotManagerTest.java b/dictionary/src/test/java/org/apache/kylin/dict/ITSnapshotManagerTest.java
new file mode 100644
index 0000000..e59fcda
--- /dev/null
+++ b/dictionary/src/test/java/org/apache/kylin/dict/ITSnapshotManagerTest.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.dict;
+
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.dict.lookup.HiveTable;
+import org.apache.kylin.dict.lookup.SnapshotManager;
+import org.apache.kylin.dict.lookup.SnapshotTable;
+import org.apache.kylin.dict.lookup.TableReader;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author yangli9
+ *
+ */
+public class ITSnapshotManagerTest extends HBaseMetadataTestCase {
+
+ SnapshotManager snapshotMgr;
+
+ @Before
+ public void setup() throws Exception {
+ createTestMetadata();
+ snapshotMgr = SnapshotManager.getInstance(getTestConfig());
+ }
+
+ @After
+ public void after() throws Exception {
+ cleanupTestMetadata();
+ }
+
+ @Test
+ public void basicTest() throws Exception {
+ String tableName = "EDW.TEST_SITES";
+ HiveTable hiveTable = new HiveTable(MetadataManager.getInstance(getTestConfig()), tableName);
+ TableDesc tableDesc = MetadataManager.getInstance(getTestConfig()).getTableDesc(tableName);
+ String snapshotPath = snapshotMgr.buildSnapshot(hiveTable, tableDesc).getResourcePath();
+
+ snapshotMgr.wipeoutCache();
+
+ SnapshotTable snapshot = snapshotMgr.getSnapshotTable(snapshotPath);
+
+ // compare hive & snapshot
+ TableReader hiveReader = hiveTable.getReader();
+ TableReader snapshotReader = snapshot.getReader();
+
+ while (true) {
+ boolean hiveNext = hiveReader.next();
+ boolean snapshotNext = snapshotReader.next();
+ assertEquals(hiveNext, snapshotNext);
+
+ if (hiveNext == false)
+ break;
+
+ String[] hiveRow = hiveReader.getRow();
+ String[] snapshotRow = snapshotReader.getRow();
+ assertArrayEquals(hiveRow, snapshotRow);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java b/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java
deleted file mode 100644
index 0ae8e95..0000000
--- a/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.dict;
-
-import static org.junit.Assert.*;
-
-import org.apache.kylin.dict.lookup.HiveTable;
-import org.apache.kylin.dict.lookup.SnapshotManager;
-import org.apache.kylin.dict.lookup.SnapshotTable;
-import org.apache.kylin.dict.lookup.TableReader;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.TableDesc;
-
-/**
- * @author yangli9
- *
- */
-public class SnapshotManagerTest extends HBaseMetadataTestCase {
-
- SnapshotManager snapshotMgr;
-
- @Before
- public void setup() throws Exception {
- createTestMetadata();
- snapshotMgr = SnapshotManager.getInstance(getTestConfig());
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- }
-
- @Test
- public void basicTest() throws Exception {
- String tableName = "EDW.TEST_SITES";
- HiveTable hiveTable = new HiveTable(MetadataManager.getInstance(getTestConfig()), tableName);
- TableDesc tableDesc = MetadataManager.getInstance(getTestConfig()).getTableDesc(tableName);
- String snapshotPath = snapshotMgr.buildSnapshot(hiveTable, tableDesc).getResourcePath();
-
- snapshotMgr.wipeoutCache();
-
- SnapshotTable snapshot = snapshotMgr.getSnapshotTable(snapshotPath);
-
- // compare hive & snapshot
- TableReader hiveReader = hiveTable.getReader();
- TableReader snapshotReader = snapshot.getReader();
-
- while (true) {
- boolean hiveNext = hiveReader.next();
- boolean snapshotNext = snapshotReader.next();
- assertEquals(hiveNext, snapshotNext);
-
- if (hiveNext == false)
- break;
-
- String[] hiveRow = hiveReader.getRow();
- String[] snapshotRow = snapshotReader.getRow();
- assertArrayEquals(hiveRow, snapshotRow);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/docs/Installation/Off Hadoop CLI Installation.md
----------------------------------------------------------------------
diff --git a/docs/Installation/Off Hadoop CLI Installation.md b/docs/Installation/Off Hadoop CLI Installation.md
index f83ed5d..a1877e3 100644
--- a/docs/Installation/Off Hadoop CLI Installation.md
+++ b/docs/Installation/Off Hadoop CLI Installation.md
@@ -53,7 +53,7 @@ An alternative to the host replacement is updating your `hosts` file to resolve
Run a end-to-end cube building test
- mvn test -Dtest=org.apache.kylin.job.BuildCubeWithEngineTest -DfailIfNoTests=false
+ mvn test -Dtest=org.apache.kylin.job.ITBuildCubeWithEngineTest -DfailIfNoTests=false
Run other tests, the end-to-end cube building test is exclueded
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/docs/MISC/FAQ on Kylin Installation and Usage.md
----------------------------------------------------------------------
diff --git a/docs/MISC/FAQ on Kylin Installation and Usage.md b/docs/MISC/FAQ on Kylin Installation and Usage.md
index a568614..6f24512 100644
--- a/docs/MISC/FAQ on Kylin Installation and Usage.md
+++ b/docs/MISC/FAQ on Kylin Installation and Usage.md
@@ -27,7 +27,7 @@ My Cluster is running on Cloudera Distribution CDH 5.2.0.
#### Unable to load a big cube as HTable, with java.lang.OutOfMemoryError: unable to create new native thread
HBase (as of writing) allocates one thread per region when bulk loading a HTable. Try reduce the number of regions of your cube by setting its "capacity" to "MEDIUM" or "LARGE". Also tweaks OS & JVM can allow more threads, for example see [this article](http://blog.egilh.com/2006/06/2811aspx.html).
-#### Failed to run BuildCubeWithEngineTest, saying failed to connect to hbase while hbase is active
+#### Failed to run ITBuildCubeWithEngineTest, saying failed to connect to hbase while hbase is active
User may get this error when first time run hbase client, please check the error trace to see whether there is an error saying couldn't access a folder like "/hadoop/hbase/local/jars"; If that folder doesn't exist, create it.
#### SUM(field) returns a negtive result while all the numbers in this field are > 0
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/docs/Test/Run Kylin test case with HBase Mini Cluster.md
----------------------------------------------------------------------
diff --git a/docs/Test/Run Kylin test case with HBase Mini Cluster.md b/docs/Test/Run Kylin test case with HBase Mini Cluster.md
index 6088eb9..107da81 100644
--- a/docs/Test/Run Kylin test case with HBase Mini Cluster.md
+++ b/docs/Test/Run Kylin test case with HBase Mini Cluster.md
@@ -8,7 +8,7 @@ mvn test -P sandbox
* Startup a HBase minicluster and update KylinConfig with the dynamic HBase configurations;
* Create Kylin metadata tables and import six example cube tables;
-* Import the hbase data from a tar ball from local: examples/test_case_data/minicluster/hbase-export.tar.gz (the hbase-export.tar.gz will be updated on complete of running BuildCubeWithEngineTest）
+* Import the hbase data from a tar ball from local: examples/test_case_data/minicluster/hbase-export.tar.gz (the hbase-export.tar.gz will be updated on complete of running ITBuildCubeWithEngineTest）
* After all test cases be completed, shutdown minicluster and cleanup KylinConfig cache;
### To ensure Mini cluster can run successfully, you need:
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/jdbc/kylin_jdbc.log.2014-12-22
----------------------------------------------------------------------
diff --git a/jdbc/kylin_jdbc.log.2014-12-22 b/jdbc/kylin_jdbc.log.2014-12-22
new file mode 100644
index 0000000..7c5e001
--- /dev/null
+++ b/jdbc/kylin_jdbc.log.2014-12-22
@@ -0,0 +1,18 @@
+[main]:[2014-12-22 11:47:05,477][DEBUG][com.kylinolap.jdbc.KylinConnectionImpl.<init>(KylinConnectionImpl.java:68)] - Kylin base url test_url, project name test_db
+[main]:[2014-12-22 11:47:05,486][DEBUG][com.kylinolap.jdbc.Driver$1.onConnectionInit(Driver.java:111)] - Connection inited.
+[main]:[2014-12-22 11:47:05,487][DEBUG][com.kylinolap.jdbc.KylinMetaImpl.getTables(KylinMetaImpl.java:108)] - Get tables with conn KylinConnectionImpl [baseUrl=test_url, project=test_db, metaProject=com.kylinolap.jdbc.KylinMetaImpl$MetaProject@21300853]
+[main]:[2014-12-22 11:47:05,487][DEBUG][com.kylinolap.jdbc.KylinMetaImpl$MetaProject.getMetaTables(KylinMetaImpl.java:489)] - getMetaTables with catalog:null, schema:null, table:null
+[main]:[2014-12-22 11:47:05,534][DEBUG][com.kylinolap.jdbc.KylinConnectionImpl.<init>(KylinConnectionImpl.java:68)] - Kylin base url test_url, project name test_db
+[main]:[2014-12-22 11:47:05,534][DEBUG][com.kylinolap.jdbc.Driver$1.onConnectionInit(Driver.java:111)] - Connection inited.
+[main]:[2014-12-22 15:57:49,077][DEBUG][com.kylinolap.jdbc.KylinConnectionImpl.<init>(KylinConnectionImpl.java:68)] - Kylin base url test_url, project name test_db
+[main]:[2014-12-22 15:57:49,086][DEBUG][com.kylinolap.jdbc.Driver$1.onConnectionInit(Driver.java:111)] - Connection inited.
+[main]:[2014-12-22 15:57:49,087][DEBUG][com.kylinolap.jdbc.KylinMetaImpl.getTables(KylinMetaImpl.java:108)] - Get tables with conn KylinConnectionImpl [baseUrl=test_url, project=test_db, metaProject=com.kylinolap.jdbc.KylinMetaImpl$MetaProject@3add750e]
+[main]:[2014-12-22 15:57:49,087][DEBUG][com.kylinolap.jdbc.KylinMetaImpl$MetaProject.getMetaTables(KylinMetaImpl.java:489)] - getMetaTables with catalog:null, schema:null, table:null
+[main]:[2014-12-22 15:57:49,134][DEBUG][com.kylinolap.jdbc.KylinConnectionImpl.<init>(KylinConnectionImpl.java:68)] - Kylin base url test_url, project name test_db
+[main]:[2014-12-22 15:57:49,135][DEBUG][com.kylinolap.jdbc.Driver$1.onConnectionInit(Driver.java:111)] - Connection inited.
+[main]:[2014-12-22 17:13:45,081][DEBUG][com.kylinolap.jdbc.KylinConnectionImpl.<init>(KylinConnectionImpl.java:68)] - Kylin base url test_url, project name test_db
+[main]:[2014-12-22 17:13:45,090][DEBUG][com.kylinolap.jdbc.Driver$1.onConnectionInit(Driver.java:111)] - Connection inited.
+[main]:[2014-12-22 17:13:45,091][DEBUG][com.kylinolap.jdbc.KylinMetaImpl.getTables(KylinMetaImpl.java:108)] - Get tables with conn KylinConnectionImpl [baseUrl=test_url, project=test_db, metaProject=com.kylinolap.jdbc.KylinMetaImpl$MetaProject@21300853]
+[main]:[2014-12-22 17:13:45,091][DEBUG][com.kylinolap.jdbc.KylinMetaImpl$MetaProject.getMetaTables(KylinMetaImpl.java:489)] - getMetaTables with catalog:null, schema:null, table:null
+[main]:[2014-12-22 17:13:45,141][DEBUG][com.kylinolap.jdbc.KylinConnectionImpl.<init>(KylinConnectionImpl.java:68)] - Kylin base url test_url, project name test_db
+[main]:[2014-12-22 17:13:45,141][DEBUG][com.kylinolap.jdbc.Driver$1.onConnectionInit(Driver.java:111)] - Connection inited.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
deleted file mode 100644
index 547a6e7..0000000
--- a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ /dev/null
@@ -1,279 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.job.cube.CubingJob;
-import org.apache.kylin.job.cube.CubingJobBuilder;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
-import org.apache.kylin.job.manager.ExecutableManager;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.File;
-import java.lang.reflect.Method;
-import java.text.SimpleDateFormat;
-import java.util.List;
-import java.util.TimeZone;
-import java.util.concurrent.*;
-
-import static org.junit.Assert.assertEquals;
-
-public class BuildCubeWithEngineTest {
-
- private JobEngineConfig jobEngineConfig;
-
- private CubeManager cubeManager;
-
- private DefaultScheduler scheduler;
-
- protected ExecutableManager jobService;
-
- private static final Log logger = LogFactory.getLog(BuildCubeWithEngineTest.class);
-
- protected void waitForJob(String jobId) {
- while (true) {
- AbstractExecutable job = jobService.getJob(jobId);
- if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) {
- break;
- } else {
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- }
- }
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
- }
-
- @Before
- public void before() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
-
- DeployUtil.initCliWorkDir();
- DeployUtil.deployMetadata();
- DeployUtil.overrideJobJarLocations();
-
-
- final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- jobService = ExecutableManager.getInstance(kylinConfig);
- scheduler = DefaultScheduler.getInstance();
- scheduler.init(new JobEngineConfig(kylinConfig));
- if (!scheduler.hasStarted()) {
- throw new RuntimeException("scheduler has not been started");
- }
- cubeManager = CubeManager.getInstance(kylinConfig);
- jobEngineConfig = new JobEngineConfig(kylinConfig);
- for (String jobId : jobService.getAllJobIds()) {
- if(jobService.getJob(jobId) instanceof CubingJob){
- jobService.deleteJob(jobId);
- }
- }
-
- }
-
- @After
- public void after() {
- HBaseMetadataTestCase.staticCleanupTestMetadata();
- }
-
- @Test
- public void test() throws Exception {
- DeployUtil.prepareTestData("left", "test_kylin_cube_with_slr_left_join_empty");
- testInner();
- testLeft();
- }
-
- private void testInner() throws Exception {
- String[] testCase = new String[]{
- "testInnerJoinCube",
- "testInnerJoinCube2",
- };
- runTestAndAssertSucceed(testCase);
- }
-
- private void testLeft() throws Exception {
- String[] testCase = new String[]{
- "testLeftJoinCube",
- "testLeftJoinCube2",
- };
- runTestAndAssertSucceed(testCase);
- }
-
- private void runTestAndAssertSucceed(String[] testCase) throws Exception {
- ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
- final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
- List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
- for (int i = 0; i < testCase.length; i++) {
- tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch)));
- }
- countDownLatch.await();
- try {
- for (int i = 0; i < tasks.size(); ++i) {
- Future<List<String>> task = tasks.get(i);
- final List<String> jobIds = task.get();
- for (String jobId : jobIds) {
- assertJobSucceed(jobId);
- }
- }
- } catch (Exception ex) {
- logger.error(ex);
- throw ex;
- }
- }
-
- private void assertJobSucceed(String jobId) {
- assertEquals("The job '" + jobId + "' is failed.", ExecutableState.SUCCEED, jobService.getOutput(jobId).getState());
- }
-
- private class TestCallable implements Callable<List<String>> {
-
- private final String methodName;
- private final CountDownLatch countDownLatch;
-
- public TestCallable(String methodName, CountDownLatch countDownLatch) {
- this.methodName = methodName;
- this.countDownLatch = countDownLatch;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public List<String> call() throws Exception {
- try {
- final Method method = BuildCubeWithEngineTest.class.getDeclaredMethod(methodName);
- method.setAccessible(true);
- return (List<String>) method.invoke(BuildCubeWithEngineTest.this);
- } finally {
- countDownLatch.countDown();
- }
- }
- }
-
- @SuppressWarnings("unused") // called by reflection
- private List<String> testInnerJoinCube2() throws Exception {
- clearSegment("test_kylin_cube_with_slr_empty");
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- long date1 = 0;
- long date2 = f.parse("2013-01-01").getTime();
- long date3 = f.parse("2022-01-01").getTime();
- List<String> result = Lists.newArrayList();
- result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date2));
- result.add(buildSegment("test_kylin_cube_with_slr_empty", date2, date3));
- return result;
- }
-
- @SuppressWarnings("unused") // called by reflection
- private List<String> testInnerJoinCube() throws Exception {
- clearSegment("test_kylin_cube_without_slr_empty");
-
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- // this cube's start date is 0, end date is 20501112000000
- long date1 = 0;
- long date2 = f.parse("2013-01-01").getTime();
-
-
- // this cube doesn't support incremental build, always do full build
-
- List<String> result = Lists.newArrayList();
- result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date2));
- return result;
- }
-
- @SuppressWarnings("unused") // called by reflection
- private List<String> testLeftJoinCube2() throws Exception {
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- List<String> result = Lists.newArrayList();
- final String cubeName = "test_kylin_cube_without_slr_left_join_empty";
- // this cube's start date is 0, end date is 20120601000000
- long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
- long dateEnd = f.parse("2012-06-01").getTime();
-
- clearSegment(cubeName);
- result.add(buildSegment(cubeName, dateStart, dateEnd));
-
- // then submit an append job, start date is 20120601000000, end
- // date is 20220101000000
- dateStart = f.parse("2012-06-01").getTime();
- dateEnd = f.parse("2022-01-01").getTime();
- result.add(buildSegment(cubeName, dateStart, dateEnd));
- return result;
-
- }
-
- @SuppressWarnings("unused") // called by reflection
- private List<String> testLeftJoinCube() throws Exception {
- String cubeName = "test_kylin_cube_with_slr_left_join_empty";
- clearSegment(cubeName);
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
- long dateEnd = f.parse("2050-11-12").getTime();
-
- // this cube's start date is 0, end date is 20501112000000
- List<String> result = Lists.newArrayList();
- result.add(buildSegment(cubeName, dateStart, dateEnd));
- return result;
-
- }
-
- private void clearSegment(String cubeName) throws Exception {
- CubeInstance cube = cubeManager.getCube(cubeName);
- cube.getSegments().clear();
- cubeManager.updateCube(cube,true);
- }
-
-
- private String buildSegment(String cubeName, long startDate, long endDate) throws Exception {
- CubeSegment segment = cubeManager.appendSegments(cubeManager.getCube(cubeName), endDate);
- CubingJobBuilder cubingJobBuilder = new CubingJobBuilder(jobEngineConfig);
- CubingJob job = cubingJobBuilder.buildJob(segment);
- jobService.addJob(job);
- waitForJob(job.getId());
- return job.getId();
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
deleted file mode 100644
index dc42dc5..0000000
--- a/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- *
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *
- * contributor license agreements. See the NOTICE file distributed with
- *
- * this work for additional information regarding copyright ownership.
- *
- * The ASF licenses this file to You under the Apache License, Version 2.0
- *
- * (the "License"); you may not use this file except in compliance with
- *
- * the License. You may obtain a copy of the License at
- *
- *
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- *
- *
- * Unless required by applicable law or agreed to in writing, software
- *
- * distributed under the License is distributed on an "AS IS" BASIS,
- *
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *
- * See the License for the specific language governing permissions and
- *
- * limitations under the License.
- *
- * /
- */
-
-package org.apache.kylin.job;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Maps;
-import com.google.common.collect.SetMultimap;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hive.hcatalog.data.schema.HCatSchema;
-import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.dict.DictionaryGenerator;
-import org.apache.kylin.dict.lookup.HiveTableReader;
-import org.apache.kylin.job.hadoop.cubev2.IGTRecordWriter;
-import org.apache.kylin.job.hadoop.cubev2.InMemCubeBuilder;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.gridtable.GTRecord;
-import org.junit.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.Nullable;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-/**
- *
- * This class is going to be deleted
- */
-@Ignore("For dev testing")
-public class BuildCubeWithStreamTest {
-
- private static final Logger logger = LoggerFactory.getLogger(BuildCubeWithStreamTest.class);
-
- private KylinConfig kylinConfig;
- private CubeManager cubeManager;
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
- }
-
- @Before
- public void before() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
- DeployUtil.overrideJobJarLocations();
-
- kylinConfig = KylinConfig.getInstanceFromEnv();
- cubeManager = CubeManager.getInstance(kylinConfig);
-
- }
-
- @After
- public void after() {
- HBaseMetadataTestCase.staticCleanupTestMetadata();
- }
-
- @Test
- public void test() throws Exception {
- CubeInstance cube = cubeManager.getCube("test_kylin_cube_without_slr_left_join_empty");
- final CubeDesc desc = cube.getDescriptor();
- // cube.getSegments().clear();
- // cubeManager.updateCube(cube);
-
- CubeSegment cubeSegment = cube.getSegment("19700101000000_20150401000000", SegmentStatusEnum.NEW);
- Map<TblColRef, Dictionary<?>> dictionaryMap = Maps.newHashMap();
-
-//
- for (DimensionDesc dim : desc.getDimensions()) {
- // dictionary
- for (TblColRef col : dim.getColumnRefs()) {
- if (desc.getRowkey().isUseDictionary(col)) {
- Dictionary dict = cubeSegment.getDictionary(col);
- if (dict == null) {
- throw new IllegalArgumentException("Dictionary for " + col + " was not found.");
- }
- logger.info("Dictionary for " + col + " was put into dictionary map.");
- dictionaryMap.put(col, cubeSegment.getDictionary(col));
- }
- }
- }
-
-// final String tableName = createIntermediateTable(desc, kylinConfig, null);
- String tableName = "kylin_intermediate_test_kylin_cube_without_slr_desc_19700101000000_20130112000000_a24dec89_efbd_425f_9a5f_8b78dd1412af"; // has 3089 records;
-// tableName = "kylin_intermediate_test_kylin_cube_without_slr_desc_19700101000000_20130112000000_a5e1eb5d_da6b_475d_9807_be0b61f03215"; // only 20 rows;
-// tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150302000000_0a183367_f245_43d1_8850_1c138c8514c3";
-// tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150301000000_ce061464_7962_4642_bd7d_7c3d8fbe9389";
- tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150401000000_fb7ae579_d987_4900_a3b7_c60c731cd269"; // 2 million records
- logger.info("intermediate table name:" + tableName);
-
-
- ArrayBlockingQueue queue = new ArrayBlockingQueue<List<String>>(10000);
-
- InMemCubeBuilder cubeBuilder = new InMemCubeBuilder(queue, cube, dictionaryMap, new ConsoleGTRecordWriter());
- ExecutorService executorService = Executors.newSingleThreadExecutor();
- Future<?> future = executorService.submit(cubeBuilder);
-
- final Configuration conf = new Configuration();
- HCatInputFormat.setInput(conf, "default", tableName);
- final HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
- logger.info(StringUtils.join(tableSchema.getFieldNames(), "\n"));
- HiveTableReader reader = new HiveTableReader("default", tableName);
- List<String> row;
- int counter = 0;
- while (reader.next()) {
- row = reader.getRowAsList();
- queue.put(row);
- counter++;
- if(counter == 200000)
- break;
- }
- queue.put(new ArrayList<String>(0));
- reader.close();
-
- try {
- future.get();
- } catch (Exception e) {
- logger.error("stream build failed", e);
- throw new IOException("Failed to build cube ", e);
- }
-
- logger.info("stream build finished");
- }
-
-
- private void buildDictionary(List<List<String>> table, CubeDesc desc, Map<TblColRef, Dictionary<?>> dictionaryMap) {
- SetMultimap<TblColRef, String> valueMap = HashMultimap.create();
-
- List<TblColRef> dimColumns = desc.listDimensionColumnsExcludingDerived();
- for (List<String> row : table) {
- for (int i = 0; i < dimColumns.size(); i++) {
- String cell = row.get(i);
- valueMap.put(dimColumns.get(i), cell);
- }
- }
-
- for (DimensionDesc dim : desc.getDimensions()) {
- // dictionary
- for (TblColRef col : dim.getColumnRefs()) {
- if (desc.getRowkey().isUseDictionary(col)) {
- Dictionary dict = DictionaryGenerator.buildDictionaryFromValueList(col.getType(), Collections2.transform(valueMap.get(col), new Function<String, byte[]>() {
- @Nullable
- @Override
- public byte[] apply(String input) {
- if (input == null)
- return null;
- return input.getBytes();
- }
- }));
-
- logger.info("Building dictionary for " + col);
- dictionaryMap.put(col, dict);
- }
- }
- }
-
- }
-
-
- class ConsoleGTRecordWriter implements IGTRecordWriter {
-
- boolean verbose = false;
-
- @Override
- public void write(Long cuboidId, GTRecord record) throws IOException {
- if (verbose)
- System.out.println(record.toString());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
deleted file mode 100644
index 9c0219a..0000000
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
+++ /dev/null
@@ -1,254 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
-import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
-import org.apache.kylin.job.invertedindex.IIJob;
-import org.apache.kylin.job.invertedindex.IIJobBuilder;
-import org.apache.kylin.job.manager.ExecutableManager;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.junit.*;
-
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.Method;
-import java.text.SimpleDateFormat;
-import java.util.List;
-import java.util.TimeZone;
-import java.util.concurrent.*;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * @author shaoshi
- */
-public class BuildIIWithEngineTest {
-
- private JobEngineConfig jobEngineConfig;
- private IIManager iiManager;
-
- private DefaultScheduler scheduler;
- protected ExecutableManager jobService;
-
- protected static final String[] TEST_II_INSTANCES = new String[] { "test_kylin_ii_inner_join", "test_kylin_ii_left_join" };
-
- private static final Log logger = LogFactory.getLog(BuildIIWithEngineTest.class);
-
- protected void waitForJob(String jobId) {
- while (true) {
- AbstractExecutable job = jobService.getJob(jobId);
- if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) {
- break;
- } else {
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- }
- }
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
- }
-
- @Before
- public void before() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
-
- //DeployUtil.initCliWorkDir();
- // DeployUtil.deployMetadata();
- DeployUtil.overrideJobJarLocations();
-
- final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- jobService = ExecutableManager.getInstance(kylinConfig);
- scheduler = DefaultScheduler.getInstance();
- scheduler.init(new JobEngineConfig(kylinConfig));
- if (!scheduler.hasStarted()) {
- throw new RuntimeException("scheduler has not been started");
- }
- jobEngineConfig = new JobEngineConfig(kylinConfig);
- for (String jobId : jobService.getAllJobIds()) {
- if (jobService.getJob(jobId) instanceof IIJob) {
- jobService.deleteJob(jobId);
- }
- }
-
- iiManager = IIManager.getInstance(kylinConfig);
- for (String iiInstance : TEST_II_INSTANCES) {
-
- IIInstance ii = iiManager.getII(iiInstance);
- if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
- ii.setStatus(RealizationStatusEnum.DISABLED);
- iiManager.updateII(ii, true);
- }
- }
- }
-
- @After
- public void after() throws Exception {
-
- for (String iiInstance : TEST_II_INSTANCES) {
- IIInstance ii = iiManager.getII(iiInstance);
- if (ii.getStatus() != RealizationStatusEnum.READY) {
- ii.setStatus(RealizationStatusEnum.READY);
- iiManager.updateII(ii, true);
- }
- }
- backup();
- }
-
- @Test
- @Ignore
- public void testBuildII() throws Exception {
-
- String[] testCase = new String[] { "buildIIInnerJoin", "buildIILeftJoin" };
- ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
- final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
- List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
- for (int i = 0; i < testCase.length; i++) {
- tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch)));
- }
- countDownLatch.await();
- for (int i = 0; i < tasks.size(); ++i) {
- Future<List<String>> task = tasks.get(i);
- final List<String> jobIds = task.get();
- for (String jobId : jobIds) {
- assertJobSucceed(jobId);
- }
- }
-
- }
-
- private void assertJobSucceed(String jobId) {
- assertEquals(ExecutableState.SUCCEED, jobService.getOutput(jobId).getState());
- }
-
- private class TestCallable implements Callable<List<String>> {
-
- private final String methodName;
- private final CountDownLatch countDownLatch;
-
- public TestCallable(String methodName, CountDownLatch countDownLatch) {
- this.methodName = methodName;
- this.countDownLatch = countDownLatch;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public List<String> call() throws Exception {
- try {
- final Method method = BuildIIWithEngineTest.class.getDeclaredMethod(methodName);
- method.setAccessible(true);
- return (List<String>) method.invoke(BuildIIWithEngineTest.this);
- } finally {
- countDownLatch.countDown();
- }
- }
- }
-
- protected List<String> buildIIInnerJoin() throws Exception {
- return buildII(TEST_II_INSTANCES[0]);
- }
-
- protected List<String> buildIILeftJoin() throws Exception {
- return buildII(TEST_II_INSTANCES[1]);
- }
-
- protected List<String> buildII(String iiName) throws Exception {
- clearSegment(iiName);
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- long date1 = 0;
- long date2 = f.parse("2015-01-01").getTime();
-
- List<String> result = Lists.newArrayList();
- result.add(buildSegment(iiName, date1, date2));
- return result;
- }
-
- private void clearSegment(String iiName) throws Exception {
- IIInstance ii = iiManager.getII(iiName);
- ii.getSegments().clear();
- iiManager.updateII(ii,true);
- }
-
- private String buildSegment(String iiName, long startDate, long endDate) throws Exception {
- IIInstance iiInstance = iiManager.getII(iiName);
- IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
- iiInstance.getSegments().add(segment);
- iiManager.updateII(iiInstance, true);
- IIJobBuilder iiJobBuilder = new IIJobBuilder(jobEngineConfig);
- IIJob job = iiJobBuilder.buildJob(segment);
- jobService.addJob(job);
- waitForJob(job.getId());
- return job.getId();
- }
-
- private int cleanupOldStorage() throws Exception {
- String[] args = { "--delete", "true" };
-
- int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
- return exitCode;
- }
-
- private void backup() throws Exception {
- int exitCode = cleanupOldStorage();
- if (exitCode == 0) {
- exportHBaseData();
- }
- }
-
- private void exportHBaseData() throws IOException {
- ExportHBaseData export = new ExportHBaseData();
- export.exportTables();
- }
-
- public static void main(String[] args) throws Exception {
- BuildIIWithEngineTest instance = new BuildIIWithEngineTest();
-
- BuildIIWithEngineTest.beforeClass();
- instance.before();
- instance.testBuildII();
- instance.after();
-
- }
-
-}
[03/11] incubator-kylin git commit: KYLIN-697 non-integration tests
all passed
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/query/src/test/java/org/apache/kylin/query/test/ITIIQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/ITIIQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/ITIIQueryTest.java
new file mode 100644
index 0000000..492b9bf
--- /dev/null
+++ b/query/src/test/java/org/apache/kylin/query/test/ITIIQueryTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.query.test;
+
+import com.google.common.collect.Maps;
+import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.query.routing.RoutingRules.RealizationPriorityRule;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ */
+@RunWith(Parameterized.class)
+public class ITIIQueryTest extends ITKylinQueryTest {
+ @BeforeClass
+ public static void setUp() throws Exception {
+
+ // give II higher priority than other realizations
+ Map<RealizationType, Integer> priorities = Maps.newHashMap();
+ priorities.put(RealizationType.INVERTED_INDEX, 0);
+ priorities.put(RealizationType.CUBE, 1);
+ priorities.put(RealizationType.HYBRID, 1);
+ RealizationPriorityRule.setPriorities(priorities);
+
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ ITKylinQueryTest.tearDown();//invoke super class
+
+ Map<RealizationType, Integer> priorities = Maps.newHashMap();
+ priorities.put(RealizationType.INVERTED_INDEX, 1);
+ priorities.put(RealizationType.CUBE, 0);
+ priorities.put(RealizationType.HYBRID, 0);
+ RealizationPriorityRule.setPriorities(priorities);
+ }
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> configs() {
+ return Arrays.asList(new Object[][] { { "inner" }, { "left" } });
+ }
+
+ public ITIIQueryTest(String joinType) throws Exception {
+
+ ITKylinQueryTest.clean();
+
+ ITKylinQueryTest.joinType = joinType;
+ ITKylinQueryTest.setupAll();
+
+ }
+
+ @Test
+ public void testSingleRunQuery() throws Exception {
+ super.testSingleRunQuery();
+ }
+
+ @Test
+ public void testDetailedQuery() throws Exception {
+ execAndCompQuery("src/test/resources/query/sql_ii", null, true);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java
new file mode 100644
index 0000000..5e14471
--- /dev/null
+++ b/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.query.test;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.query.enumerator.OLAPQuery;
+import org.apache.kylin.query.relnode.OLAPContext;
+import org.apache.kylin.query.schema.OLAPSchemaFactory;
+import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverEnabler;
+import org.dbunit.database.DatabaseConnection;
+import org.dbunit.database.IDatabaseConnection;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.File;
+import java.sql.DriverManager;
+import java.util.List;
+import java.util.Properties;
+
+import static org.junit.Assert.assertTrue;
+
+@Ignore("KylinQueryTest is contained by ITCombinationTest")
+public class ITKylinQueryTest extends KylinTestBase {
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ printInfo("setUp in KylinQueryTest");
+ joinType = "left";
+
+ setupAll();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ printInfo("tearDown");
+ printInfo("Closing connection...");
+ clean();
+ }
+
+ protected static void setupAll() throws Exception {
+ //setup env
+ HBaseMetadataTestCase.staticCreateTestMetadata();
+ config = KylinConfig.getInstanceFromEnv();
+
+ //setup cube conn
+ File olapTmp = OLAPSchemaFactory.createTempOLAPJson(ProjectInstance.DEFAULT_PROJECT_NAME, config);
+ Properties props = new Properties();
+ props.setProperty(OLAPQuery.PROP_SCAN_THRESHOLD, "10000");
+ cubeConnection = DriverManager.getConnection("jdbc:calcite:model=" + olapTmp.getAbsolutePath(), props);
+
+ //setup h2
+ h2Connection = DriverManager.getConnection("jdbc:h2:mem:db" + (h2InstanceCount++), "sa", "");
+ // Load H2 Tables (inner join)
+ H2Database h2DB = new H2Database(h2Connection, config);
+ h2DB.loadAllTables(joinType);
+ }
+
+ protected static void clean() {
+ if (cubeConnection != null)
+ closeConnection(cubeConnection);
+ if (h2Connection != null)
+ closeConnection(h2Connection);
+
+ ObserverEnabler.forceCoprocessorUnset();
+ HBaseMetadataTestCase.staticCleanupTestMetadata();
+ }
+
+ @Ignore("this is only for debug")
+ @Test
+ public void testTempQuery() throws Exception {
+ execAndCompQuery("src/test/resources/query/temp", null, true);
+ }
+
+ @Test
+ public void testSingleRunQuery() throws Exception {
+
+ String queryFileName = "src/test/resources/query/sql/query00.sql";
+
+ File sqlFile = new File(queryFileName);
+ if (sqlFile.exists()) {
+ runSQL(sqlFile, true, true);
+ runSQL(sqlFile, true, false);
+ }
+ }
+
+ @Test
+ public void testSingleExecuteQuery() throws Exception {
+
+ String queryFileName = "src/test/resources/query/sql/query53.sql";
+
+ File sqlFile = new File(queryFileName);
+ String sql = getTextFromFile(sqlFile);
+ IDatabaseConnection kylinConn = new DatabaseConnection(cubeConnection);
+
+ executeQuery(kylinConn, queryFileName, sql, true);
+ }
+
+ @Ignore
+ @Test
+ public void testTableauProbing() throws Exception {
+ batchExecuteQuery("src/test/resources/query/tableau_probing");
+ }
+
+ @Test
+ public void testCommonQuery() throws Exception {
+ execAndCompQuery("src/test/resources/query/sql", null, true);
+ }
+
+ @Ignore
+ @Test
+ public void testSimpleQuery() throws Exception {
+ verifyResultRowCount("src/test/resources/query/sql_verifyCount");
+ }
+
+ @Test
+ public void testOrderByQuery() throws Exception {
+ execAndCompQuery("src/test/resources/query/sql_orderby", null, true);
+ // FIXME
+ // as of optiq 0.8, we lost metadata type with "order by" clause, e.g. sql_orderby/query01.sql
+ // thus, temporarily the "order by" clause was cross out, and the needSort is set to true
+ // execAndCompQuery("src/test/resources/query/sql_orderby", null, false);
+ }
+
+ @Test
+ public void testLookupQuery() throws Exception {
+ execAndCompQuery("src/test/resources/query/sql_lookup", null, true);
+ }
+
+ @Test
+ public void testCachedQuery() throws Exception {
+ execAndCompQuery("src/test/resources/query/sql_cache", null, true);
+ }
+
+ @Test
+ public void testDerivedColumnQuery() throws Exception {
+ execAndCompQuery("src/test/resources/query/sql_derived", null, true);
+ }
+
+ @Test
+ public void testDistinctCountQuery() throws Exception {
+ batchExecuteQuery("src/test/resources/query/sql_distinct");
+ }
+
+ @Test
+ public void testTableauQuery() throws Exception {
+ batchExecuteQuery("src/test/resources/query/sql_tableau");
+ }
+
+ @Test
+ public void testSubQuery() throws Exception {
+ execAndCompQuery("src/test/resources/query/sql_subquery", null, true);
+ }
+
+ @Test
+ public void testCaseWhen() throws Exception {
+ execAndCompQuery("src/test/resources/query/sql_casewhen", null, true);
+ }
+
+ @Ignore
+ @Test
+ public void testHiveQuery() throws Exception {
+ execAndCompQuery("src/test/resources/query/sql_hive", null, true);
+ }
+
+ @Test
+ public void testH2Query() throws Exception {
+ this.execQueryUsingH2("src/test/resources/query/sql_orderby", false);
+ }
+
+ @Test
+ public void testInvalidQuery() throws Exception {
+
+ printInfo("-------------------- Test Invalid Query --------------------");
+ String queryFolder = "src/test/resources/query/sql_invalid";
+ List<File> sqlFiles = getFilesFromFolder(new File(queryFolder), ".sql");
+ for (File sqlFile : sqlFiles) {
+ String queryName = StringUtils.split(sqlFile.getName(), '.')[0];
+ printInfo("Testing Query " + queryName);
+ String sql = getTextFromFile(sqlFile);
+ IDatabaseConnection cubeConn = new DatabaseConnection(cubeConnection);
+ try {
+ cubeConn.createQueryTable(queryName, sql);
+ } catch (Throwable t) {
+ continue;
+ } finally {
+ cubeConn.close();
+ }
+ throw new IllegalStateException(queryName + " should be error!");
+ }
+ }
+
+ @Test
+ public void testDynamicQuery() throws Exception {
+ execAndCompDynamicQuery("src/test/resources/query/sql_dynamic", null, true);
+ }
+
+ @Ignore("simple query will be supported by ii")
+ @Test
+ public void testLimitEnabled() throws Exception {
+ runSqlFile("src/test/resources/query/sql_optimize/enable-limit01.sql");
+ assertLimitWasEnabled();
+ }
+
+ private void assertLimitWasEnabled() {
+ OLAPContext context = getFirstOLAPContext();
+ assertTrue(context.storageContext.isLimitEnabled());
+ }
+
+ private OLAPContext getFirstOLAPContext() {
+ return OLAPContext.getThreadLocalContexts().iterator().next();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
deleted file mode 100644
index cea8ee2..0000000
--- a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.query.test;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.query.enumerator.OLAPQuery;
-import org.apache.kylin.query.relnode.OLAPContext;
-import org.apache.kylin.query.schema.OLAPSchemaFactory;
-import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverEnabler;
-import org.dbunit.database.DatabaseConnection;
-import org.dbunit.database.IDatabaseConnection;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.io.File;
-import java.sql.DriverManager;
-import java.util.List;
-import java.util.Properties;
-
-import static org.junit.Assert.assertTrue;
-
-@Ignore("KylinQueryTest is contained by CombinationTest")
-public class KylinQueryTest extends KylinTestBase {
-
- @BeforeClass
- public static void setUp() throws Exception {
- printInfo("setUp in KylinQueryTest");
- joinType = "left";
-
- setupAll();
- }
-
- @AfterClass
- public static void tearDown() throws Exception {
- printInfo("tearDown");
- printInfo("Closing connection...");
- clean();
- }
-
- protected static void setupAll() throws Exception {
- //setup env
- HBaseMetadataTestCase.staticCreateTestMetadata();
- config = KylinConfig.getInstanceFromEnv();
-
- //setup cube conn
- File olapTmp = OLAPSchemaFactory.createTempOLAPJson(ProjectInstance.DEFAULT_PROJECT_NAME, config);
- Properties props = new Properties();
- props.setProperty(OLAPQuery.PROP_SCAN_THRESHOLD, "10000");
- cubeConnection = DriverManager.getConnection("jdbc:calcite:model=" + olapTmp.getAbsolutePath(), props);
-
- //setup h2
- h2Connection = DriverManager.getConnection("jdbc:h2:mem:db" + (h2InstanceCount++), "sa", "");
- // Load H2 Tables (inner join)
- H2Database h2DB = new H2Database(h2Connection, config);
- h2DB.loadAllTables(joinType);
- }
-
- protected static void clean() {
- if (cubeConnection != null)
- closeConnection(cubeConnection);
- if (h2Connection != null)
- closeConnection(h2Connection);
-
- ObserverEnabler.forceCoprocessorUnset();
- HBaseMetadataTestCase.staticCleanupTestMetadata();
- }
-
- @Ignore("this is only for debug")
- @Test
- public void testTempQuery() throws Exception {
- execAndCompQuery("src/test/resources/query/temp", null, true);
- }
-
- @Test
- public void testSingleRunQuery() throws Exception {
-
- String queryFileName = "src/test/resources/query/sql/query00.sql";
-
- File sqlFile = new File(queryFileName);
- if (sqlFile.exists()) {
- runSQL(sqlFile, true, true);
- runSQL(sqlFile, true, false);
- }
- }
-
- @Test
- public void testSingleExecuteQuery() throws Exception {
-
- String queryFileName = "src/test/resources/query/sql/query53.sql";
-
- File sqlFile = new File(queryFileName);
- String sql = getTextFromFile(sqlFile);
- IDatabaseConnection kylinConn = new DatabaseConnection(cubeConnection);
-
- executeQuery(kylinConn, queryFileName, sql, true);
- }
-
- @Ignore
- @Test
- public void testTableauProbing() throws Exception {
- batchExecuteQuery("src/test/resources/query/tableau_probing");
- }
-
- @Test
- public void testCommonQuery() throws Exception {
- execAndCompQuery("src/test/resources/query/sql", null, true);
- }
-
- @Ignore
- @Test
- public void testSimpleQuery() throws Exception {
- verifyResultRowCount("src/test/resources/query/sql_verifyCount");
- }
-
- @Test
- public void testOrderByQuery() throws Exception {
- execAndCompQuery("src/test/resources/query/sql_orderby", null, true);
- // FIXME
- // as of optiq 0.8, we lost metadata type with "order by" clause, e.g. sql_orderby/query01.sql
- // thus, temporarily the "order by" clause was cross out, and the needSort is set to true
- // execAndCompQuery("src/test/resources/query/sql_orderby", null, false);
- }
-
- @Test
- public void testLookupQuery() throws Exception {
- execAndCompQuery("src/test/resources/query/sql_lookup", null, true);
- }
-
- @Test
- public void testCachedQuery() throws Exception {
- execAndCompQuery("src/test/resources/query/sql_cache", null, true);
- }
-
- @Test
- public void testDerivedColumnQuery() throws Exception {
- execAndCompQuery("src/test/resources/query/sql_derived", null, true);
- }
-
- @Test
- public void testDistinctCountQuery() throws Exception {
- batchExecuteQuery("src/test/resources/query/sql_distinct");
- }
-
- @Test
- public void testTableauQuery() throws Exception {
- batchExecuteQuery("src/test/resources/query/sql_tableau");
- }
-
- @Test
- public void testSubQuery() throws Exception {
- execAndCompQuery("src/test/resources/query/sql_subquery", null, true);
- }
-
- @Test
- public void testCaseWhen() throws Exception {
- execAndCompQuery("src/test/resources/query/sql_casewhen", null, true);
- }
-
- @Ignore
- @Test
- public void testHiveQuery() throws Exception {
- execAndCompQuery("src/test/resources/query/sql_hive", null, true);
- }
-
- @Test
- public void testH2Query() throws Exception {
- this.execQueryUsingH2("src/test/resources/query/sql_orderby", false);
- }
-
- @Test
- public void testInvalidQuery() throws Exception {
-
- printInfo("-------------------- Test Invalid Query --------------------");
- String queryFolder = "src/test/resources/query/sql_invalid";
- List<File> sqlFiles = getFilesFromFolder(new File(queryFolder), ".sql");
- for (File sqlFile : sqlFiles) {
- String queryName = StringUtils.split(sqlFile.getName(), '.')[0];
- printInfo("Testing Query " + queryName);
- String sql = getTextFromFile(sqlFile);
- IDatabaseConnection cubeConn = new DatabaseConnection(cubeConnection);
- try {
- cubeConn.createQueryTable(queryName, sql);
- } catch (Throwable t) {
- continue;
- } finally {
- cubeConn.close();
- }
- throw new IllegalStateException(queryName + " should be error!");
- }
- }
-
- @Test
- public void testDynamicQuery() throws Exception {
- execAndCompDynamicQuery("src/test/resources/query/sql_dynamic", null, true);
- }
-
- @Ignore("simple query will be supported by ii")
- @Test
- public void testLimitEnabled() throws Exception {
- runSqlFile("src/test/resources/query/sql_optimize/enable-limit01.sql");
- assertLimitWasEnabled();
- }
-
- private void assertLimitWasEnabled() {
- OLAPContext context = getFirstOLAPContext();
- assertTrue(context.storageContext.isLimitEnabled());
- }
-
- private OLAPContext getFirstOLAPContext() {
- return OLAPContext.getThreadLocalContexts().iterator().next();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/server/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java b/server/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
new file mode 100644
index 0000000..9bdb510
--- /dev/null
+++ b/server/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
@@ -0,0 +1,245 @@
+package org.apache.kylin.jdbc;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.io.FileUtils;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.webapp.WebAppContext;
+import org.junit.*;
+
+import java.io.File;
+import java.sql.*;
+import java.util.List;
+import java.util.Properties;
+
+/**
+ */
+public class ITJDBCDriverTest extends HBaseMetadataTestCase {
+
+ private static Server server = null;
+
+ private static String previousSpringProfile = null;
+
+ private static String SPRING_PROFILE_PROPERTY = "spring.profiles.active";
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ previousSpringProfile = System.getProperty(SPRING_PROFILE_PROPERTY);
+ System.setProperty(SPRING_PROFILE_PROPERTY, "testing");
+ staticCreateTestMetadata();
+ startJetty();
+ }
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+ stopJetty();
+ staticCleanupTestMetadata();
+ if (previousSpringProfile == null) {
+ System.clearProperty(SPRING_PROFILE_PROPERTY);
+ } else {
+ System.setProperty(SPRING_PROFILE_PROPERTY, previousSpringProfile);
+ }
+ }
+
+ protected static void stopJetty() throws Exception {
+ if (server != null)
+ server.stop();
+
+ File workFolder = new File("work");
+ if(workFolder.isDirectory() && workFolder.exists()) {
+ FileUtils.deleteDirectory(workFolder);
+ }
+ }
+
+ protected static void startJetty() throws Exception {
+
+ server = new Server(7070);
+
+ WebAppContext context = new WebAppContext();
+ context.setDescriptor("./src/main/webapp/WEB-INF/web.xml");
+ context.setResourceBase("./src/main/webapp");
+ context.setContextPath("/kylin");
+ context.setParentLoaderPriority(true);
+
+ server.setHandler(context);
+
+ server.start();
+
+ }
+
+ protected Connection getConnection() throws Exception {
+
+ Driver driver = (Driver) Class.forName("org.apache.kylin.jdbc.Driver").newInstance();
+ Properties info = new Properties();
+ info.put("user", "ADMIN");
+ info.put("password", "KYLIN");
+ Connection conn = driver.connect("jdbc:kylin://localhost:7070/default", info);
+
+ return conn;
+ }
+
+ @Test
+ public void testMetadata1() throws Exception {
+
+ // check the JDBC API here: http://docs.oracle.com/javase/7/docs/api/java/sql/DatabaseMetaData.html
+ Connection conn = getConnection();
+
+ // test getSchemas();
+ List<String> schemaList = Lists.newArrayList();
+ DatabaseMetaData dbMetadata = conn.getMetaData();
+ ResultSet resultSet = dbMetadata.getSchemas();
+ while (resultSet.next()) {
+ String schema = resultSet.getString("TABLE_SCHEM");
+ String catalog = resultSet.getString("TABLE_CATALOG");
+
+ System.out.println("Get schema: schema=" + schema + ", catalog=" + catalog);
+ schemaList.add(schema);
+
+ }
+
+ resultSet.close();
+ Assert.assertTrue(schemaList.contains("DEFAULT"));
+ Assert.assertTrue(schemaList.contains("EDW"));
+
+ // test getCatalogs();
+ resultSet = dbMetadata.getCatalogs();
+
+ List<String> catalogList = Lists.newArrayList();
+ while (resultSet.next()) {
+ String catalog = resultSet.getString("TABLE_CAT");
+
+ System.out.println("Get catalog: catalog=" + catalog);
+ catalogList.add(catalog);
+
+ }
+ Assert.assertTrue(catalogList.size() > 0 && catalogList.contains("defaultCatalog"));
+
+ /** //Disable the test on getTableTypes() as it is not ready
+ resultSet = dbMetadata.getTableTypes();
+
+ List<String> tableTypes = Lists.newArrayList();
+ while (resultSet.next()) {
+ String type = resultSet.getString("TABLE_TYPE");
+
+ System.out.println("Get table type: type=" + type);
+ tableTypes.add(type);
+
+ }
+
+ Assert.assertTrue(tableTypes.size() > 0 && tableTypes.contains("TABLE"));
+ resultSet.close();
+
+ **/
+ conn.close();
+ }
+
+ @Test
+ public void testMetadata2() throws Exception {
+ Connection conn = getConnection();
+
+ List<String> tableList = Lists.newArrayList();
+ DatabaseMetaData dbMetadata = conn.getMetaData();
+ ResultSet resultSet = dbMetadata.getTables(null, "%", "%", new String[]{"TABLE"});
+ while (resultSet.next()) {
+ String schema = resultSet.getString("TABLE_SCHEM");
+ String name = resultSet.getString("TABLE_NAME");
+
+ System.out.println("Get table: schema=" + schema + ", name=" + name);
+ tableList.add(schema + "." + name);
+
+ }
+
+ resultSet.close();
+ Assert.assertTrue(tableList.contains("DEFAULT.TEST_KYLIN_FACT"));
+
+ resultSet = dbMetadata.getColumns(null, "%", "TEST_KYLIN_FACT", "%");
+
+ List<String> columns = Lists.newArrayList();
+ while (resultSet.next()) {
+ String name = resultSet.getString("COLUMN_NAME");
+ String type = resultSet.getString("TYPE_NAME");
+
+ System.out.println("Get column: name=" + name + ", data_type=" + type);
+ columns.add(name);
+
+ }
+
+ Assert.assertTrue(columns.size() > 0 && columns.contains("CAL_DT"));
+ resultSet.close();
+ conn.close();
+ }
+
+ @Test
+ public void testSimpleStatement() throws Exception {
+ Connection conn = getConnection();
+ Statement statement = conn.createStatement();
+
+ statement.execute("select count(*) from test_kylin_fact");
+
+ ResultSet rs = statement.getResultSet();
+
+ Assert.assertTrue(rs.next());
+ int result = rs.getInt(1);
+
+ Assert.assertTrue(result > 0);
+
+ rs.close();
+ statement.close();
+ conn.close();
+
+ }
+
+
+ @Test
+ public void testPreparedStatement() throws Exception {
+ Connection conn = getConnection();
+
+ PreparedStatement statement = conn.prepareStatement("select LSTG_FORMAT_NAME, sum(price) as GMV, count(1) as TRANS_CNT from test_kylin_fact " +
+ "where LSTG_FORMAT_NAME = ? group by LSTG_FORMAT_NAME");
+
+ statement.setString(1, "FP-GTC");
+
+ ResultSet rs = statement.executeQuery();
+
+ Assert.assertTrue(rs.next());
+
+ String format_name = rs.getString(1);
+
+ Assert.assertTrue("FP-GTC".equals(format_name));
+
+ rs.close();
+ statement.close();
+ conn.close();
+
+ }
+
+ @Test
+ public void testResultSet() throws Exception {
+ String sql = "select LSTG_FORMAT_NAME, sum(price) as GMV, count(1) as TRANS_CNT from test_kylin_fact \n" +
+ " group by LSTG_FORMAT_NAME ";
+
+ Connection conn = getConnection();
+ Statement statement = conn.createStatement();
+
+ statement.execute(sql);
+
+ ResultSet rs = statement.getResultSet();
+
+ int count = 0;
+ while (rs.next()) {
+ count++;
+ String lstg = rs.getString(1);
+ double gmv = rs.getDouble(2);
+ int trans_count = rs.getInt(3);
+
+ System.out.println("Get a line: LSTG_FORMAT_NAME=" + lstg + ", GMV=" + gmv + ", TRANS_CNT=" + trans_count);
+ }
+
+ Assert.assertTrue(count > 0);
+ statement.close();
+ rs.close();
+ conn.close();
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/server/src/test/java/org/apache/kylin/jdbc/JDBCDriverTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/jdbc/JDBCDriverTest.java b/server/src/test/java/org/apache/kylin/jdbc/JDBCDriverTest.java
deleted file mode 100644
index 2eff08f..0000000
--- a/server/src/test/java/org/apache/kylin/jdbc/JDBCDriverTest.java
+++ /dev/null
@@ -1,245 +0,0 @@
-package org.apache.kylin.jdbc;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.io.FileUtils;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.webapp.WebAppContext;
-import org.junit.*;
-
-import java.io.File;
-import java.sql.*;
-import java.util.List;
-import java.util.Properties;
-
-/**
- */
-public class JDBCDriverTest extends HBaseMetadataTestCase {
-
- private static Server server = null;
-
- private static String previousSpringProfile = null;
-
- private static String SPRING_PROFILE_PROPERTY = "spring.profiles.active";
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- previousSpringProfile = System.getProperty(SPRING_PROFILE_PROPERTY);
- System.setProperty(SPRING_PROFILE_PROPERTY, "testing");
- staticCreateTestMetadata();
- startJetty();
- }
-
- @AfterClass
- public static void afterClass() throws Exception {
- stopJetty();
- staticCleanupTestMetadata();
- if (previousSpringProfile == null) {
- System.clearProperty(SPRING_PROFILE_PROPERTY);
- } else {
- System.setProperty(SPRING_PROFILE_PROPERTY, previousSpringProfile);
- }
- }
-
- protected static void stopJetty() throws Exception {
- if (server != null)
- server.stop();
-
- File workFolder = new File("work");
- if(workFolder.isDirectory() && workFolder.exists()) {
- FileUtils.deleteDirectory(workFolder);
- }
- }
-
- protected static void startJetty() throws Exception {
-
- server = new Server(7070);
-
- WebAppContext context = new WebAppContext();
- context.setDescriptor("./src/main/webapp/WEB-INF/web.xml");
- context.setResourceBase("./src/main/webapp");
- context.setContextPath("/kylin");
- context.setParentLoaderPriority(true);
-
- server.setHandler(context);
-
- server.start();
-
- }
-
- protected Connection getConnection() throws Exception {
-
- Driver driver = (Driver) Class.forName("org.apache.kylin.jdbc.Driver").newInstance();
- Properties info = new Properties();
- info.put("user", "ADMIN");
- info.put("password", "KYLIN");
- Connection conn = driver.connect("jdbc:kylin://localhost:7070/default", info);
-
- return conn;
- }
-
- @Test
- public void testMetadata1() throws Exception {
-
- // check the JDBC API here: http://docs.oracle.com/javase/7/docs/api/java/sql/DatabaseMetaData.html
- Connection conn = getConnection();
-
- // test getSchemas();
- List<String> schemaList = Lists.newArrayList();
- DatabaseMetaData dbMetadata = conn.getMetaData();
- ResultSet resultSet = dbMetadata.getSchemas();
- while (resultSet.next()) {
- String schema = resultSet.getString("TABLE_SCHEM");
- String catalog = resultSet.getString("TABLE_CATALOG");
-
- System.out.println("Get schema: schema=" + schema + ", catalog=" + catalog);
- schemaList.add(schema);
-
- }
-
- resultSet.close();
- Assert.assertTrue(schemaList.contains("DEFAULT"));
- Assert.assertTrue(schemaList.contains("EDW"));
-
- // test getCatalogs();
- resultSet = dbMetadata.getCatalogs();
-
- List<String> catalogList = Lists.newArrayList();
- while (resultSet.next()) {
- String catalog = resultSet.getString("TABLE_CAT");
-
- System.out.println("Get catalog: catalog=" + catalog);
- catalogList.add(catalog);
-
- }
- Assert.assertTrue(catalogList.size() > 0 && catalogList.contains("defaultCatalog"));
-
- /** //Disable the test on getTableTypes() as it is not ready
- resultSet = dbMetadata.getTableTypes();
-
- List<String> tableTypes = Lists.newArrayList();
- while (resultSet.next()) {
- String type = resultSet.getString("TABLE_TYPE");
-
- System.out.println("Get table type: type=" + type);
- tableTypes.add(type);
-
- }
-
- Assert.assertTrue(tableTypes.size() > 0 && tableTypes.contains("TABLE"));
- resultSet.close();
-
- **/
- conn.close();
- }
-
- @Test
- public void testMetadata2() throws Exception {
- Connection conn = getConnection();
-
- List<String> tableList = Lists.newArrayList();
- DatabaseMetaData dbMetadata = conn.getMetaData();
- ResultSet resultSet = dbMetadata.getTables(null, "%", "%", new String[]{"TABLE"});
- while (resultSet.next()) {
- String schema = resultSet.getString("TABLE_SCHEM");
- String name = resultSet.getString("TABLE_NAME");
-
- System.out.println("Get table: schema=" + schema + ", name=" + name);
- tableList.add(schema + "." + name);
-
- }
-
- resultSet.close();
- Assert.assertTrue(tableList.contains("DEFAULT.TEST_KYLIN_FACT"));
-
- resultSet = dbMetadata.getColumns(null, "%", "TEST_KYLIN_FACT", "%");
-
- List<String> columns = Lists.newArrayList();
- while (resultSet.next()) {
- String name = resultSet.getString("COLUMN_NAME");
- String type = resultSet.getString("TYPE_NAME");
-
- System.out.println("Get column: name=" + name + ", data_type=" + type);
- columns.add(name);
-
- }
-
- Assert.assertTrue(columns.size() > 0 && columns.contains("CAL_DT"));
- resultSet.close();
- conn.close();
- }
-
- @Test
- public void testSimpleStatement() throws Exception {
- Connection conn = getConnection();
- Statement statement = conn.createStatement();
-
- statement.execute("select count(*) from test_kylin_fact");
-
- ResultSet rs = statement.getResultSet();
-
- Assert.assertTrue(rs.next());
- int result = rs.getInt(1);
-
- Assert.assertTrue(result > 0);
-
- rs.close();
- statement.close();
- conn.close();
-
- }
-
-
- @Test
- public void testPreparedStatement() throws Exception {
- Connection conn = getConnection();
-
- PreparedStatement statement = conn.prepareStatement("select LSTG_FORMAT_NAME, sum(price) as GMV, count(1) as TRANS_CNT from test_kylin_fact " +
- "where LSTG_FORMAT_NAME = ? group by LSTG_FORMAT_NAME");
-
- statement.setString(1, "FP-GTC");
-
- ResultSet rs = statement.executeQuery();
-
- Assert.assertTrue(rs.next());
-
- String format_name = rs.getString(1);
-
- Assert.assertTrue("FP-GTC".equals(format_name));
-
- rs.close();
- statement.close();
- conn.close();
-
- }
-
- @Test
- public void testResultSet() throws Exception {
- String sql = "select LSTG_FORMAT_NAME, sum(price) as GMV, count(1) as TRANS_CNT from test_kylin_fact \n" +
- " group by LSTG_FORMAT_NAME ";
-
- Connection conn = getConnection();
- Statement statement = conn.createStatement();
-
- statement.execute(sql);
-
- ResultSet rs = statement.getResultSet();
-
- int count = 0;
- while (rs.next()) {
- count++;
- String lstg = rs.getString(1);
- double gmv = rs.getDouble(2);
- int trans_count = rs.getInt(3);
-
- System.out.println("Get a line: LSTG_FORMAT_NAME=" + lstg + ", GMV=" + gmv + ", TRANS_CNT=" + trans_count);
- }
-
- Assert.assertTrue(count > 0);
- statement.close();
- rs.close();
- conn.close();
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/server/src/test/java/org/apache/kylin/rest/controller/ITTableControllerTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/ITTableControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/ITTableControllerTest.java
new file mode 100644
index 0000000..75defbf
--- /dev/null
+++ b/server/src/test/java/org/apache/kylin/rest/controller/ITTableControllerTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.controller;
+
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.rest.service.CubeService;
+import org.apache.kylin.rest.service.ServiceTestBase;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author shaoshi
+ */
+public class ITTableControllerTest extends ServiceTestBase {
+
+ private TableController tableController;
+ private CubeDescController cubeDescController;
+
+ @Autowired
+ CubeService cubeService;
+
+ @Before
+ public void setup() throws Exception {
+ super.setup();
+
+ tableController = new TableController();
+ tableController.setCubeService(cubeService);
+ }
+
+ @Test
+ public void testBasics() throws IOException {
+ List<TableDesc> tables = tableController.getHiveTables(true, "default");
+
+ Assert.assertTrue(tables != null && tables.size() > 0);
+
+ TableDesc factTable = null;
+ for (TableDesc t : tables) {
+ if (t.getName().equalsIgnoreCase("test_kylin_fact")) {
+ factTable = t;
+ break;
+ }
+ }
+
+ Assert.assertNotNull(factTable);
+
+
+ Map<String, String[]> loadResult = tableController.loadHiveTable("test_kylin_fact,TEST_CATEGORY_GROUPINGS", "default");
+ Assert.assertNotNull(loadResult);
+
+ Assert.assertTrue(loadResult.get("result.loaded").length ==2);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/server/src/test/java/org/apache/kylin/rest/controller/TableControllerTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/TableControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/TableControllerTest.java
deleted file mode 100644
index 4437abe..0000000
--- a/server/src/test/java/org/apache/kylin/rest/controller/TableControllerTest.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.rest.controller;
-
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.rest.service.CubeService;
-import org.apache.kylin.rest.service.ServiceTestBase;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-/**
- * @author shaoshi
- */
-public class TableControllerTest extends ServiceTestBase {
-
- private TableController tableController;
- private CubeDescController cubeDescController;
-
- @Autowired
- CubeService cubeService;
-
- @Before
- public void setup() throws Exception {
- super.setup();
-
- tableController = new TableController();
- tableController.setCubeService(cubeService);
- }
-
- @Test
- public void testBasics() throws IOException {
- List<TableDesc> tables = tableController.getHiveTables(true, "default");
-
- Assert.assertTrue(tables != null && tables.size() > 0);
-
- TableDesc factTable = null;
- for (TableDesc t : tables) {
- if (t.getName().equalsIgnoreCase("test_kylin_fact")) {
- factTable = t;
- break;
- }
- }
-
- Assert.assertNotNull(factTable);
-
-
- Map<String, String[]> loadResult = tableController.loadHiveTable("test_kylin_fact,TEST_CATEGORY_GROUPINGS", "default");
- Assert.assertNotNull(loadResult);
-
- Assert.assertTrue(loadResult.get("result.loaded").length ==2);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
index 559ec89..9f0598d 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
@@ -129,6 +129,7 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
@After
public void after() throws Exception {
+ cleanupTestMetadata();
}
private void waitForCounterAndClear(long count) {
@@ -145,16 +146,6 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
}
}
- @BeforeClass
- public static void startServer() throws Exception {
-
- }
-
- @AfterClass
- public static void stopServer() throws Exception {
-
- }
-
private static CubeManager getCubeManager(KylinConfig config) throws Exception {
return CubeManager.getInstance(config);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/storage/src/test/java/org/apache/kylin/storage/hbase/ITInvertedIndexHBaseTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hbase/ITInvertedIndexHBaseTest.java b/storage/src/test/java/org/apache/kylin/storage/hbase/ITInvertedIndexHBaseTest.java
new file mode 100644
index 0000000..c92921b
--- /dev/null
+++ b/storage/src/test/java/org/apache/kylin/storage/hbase/ITInvertedIndexHBaseTest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.storage.hbase;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.invertedindex.IISegment;
+import org.apache.kylin.invertedindex.index.RawTableRecord;
+import org.apache.kylin.invertedindex.index.Slice;
+import org.apache.kylin.invertedindex.index.TableRecord;
+import org.apache.kylin.invertedindex.index.TableRecordInfo;
+import org.apache.kylin.invertedindex.model.IIDesc;
+import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.List;
+
+/**
+ * @author yangli9
+ */
+public class ITInvertedIndexHBaseTest extends HBaseMetadataTestCase {
+
+ IIInstance ii;
+ IISegment seg;
+ HConnection hconn;
+
+ TableRecordInfo info;
+
+ @Before
+ public void setup() throws Exception {
+ this.createTestMetadata();
+
+ this.ii = IIManager.getInstance(getTestConfig()).getII("test_kylin_ii_left_join");
+ this.seg = ii.getFirstSegment();
+
+ String hbaseUrl = KylinConfig.getInstanceFromEnv().getStorageUrl();
+ Configuration hconf = HadoopUtil.newHBaseConfiguration(hbaseUrl);
+ hconn = HConnectionManager.createConnection(hconf);
+
+ this.info = new TableRecordInfo(seg);
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testLoad() throws Exception {
+
+ String tableName = seg.getStorageLocationIdentifier();
+ IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest());
+
+ List<Slice> slices = Lists.newArrayList();
+ HBaseClientKVIterator kvIterator = new HBaseClientKVIterator(hconn, tableName, IIDesc.HBASE_FAMILY_BYTES);
+ try {
+ for (Slice slice : codec.decodeKeyValue(kvIterator)) {
+ slices.add(slice);
+ }
+ } finally {
+ kvIterator.close();
+ }
+
+ List<TableRecord> records = iterateRecords(slices);
+ //dump(records);
+ System.out.println(records.size() + " records");
+ }
+
+ private List<TableRecord> iterateRecords(List<Slice> slices) {
+ List<TableRecord> records = Lists.newArrayList();
+ for (Slice slice : slices) {
+ for (RawTableRecord rec : slice) {
+ records.add(new TableRecord((RawTableRecord) rec.clone(), info));
+ }
+ }
+ return records;
+ }
+
+ private void dump(Iterable<TableRecord> records) {
+ for (TableRecord rec : records) {
+ byte[] x = rec.getBytes();
+ String y = BytesUtil.toReadableText(x);
+ System.out.println(y);
+ System.out.println();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java b/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
deleted file mode 100644
index fd24487..0000000
--- a/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase;
-
-import com.google.common.collect.Lists;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.invertedindex.index.RawTableRecord;
-import org.apache.kylin.invertedindex.index.Slice;
-import org.apache.kylin.invertedindex.index.TableRecord;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.List;
-
-/**
- * @author yangli9
- */
-public class InvertedIndexHBaseTest extends HBaseMetadataTestCase {
-
- IIInstance ii;
- IISegment seg;
- HConnection hconn;
-
- TableRecordInfo info;
-
- @Before
- public void setup() throws Exception {
- this.createTestMetadata();
-
- this.ii = IIManager.getInstance(getTestConfig()).getII("test_kylin_ii_left_join");
- this.seg = ii.getFirstSegment();
-
- String hbaseUrl = KylinConfig.getInstanceFromEnv().getStorageUrl();
- Configuration hconf = HadoopUtil.newHBaseConfiguration(hbaseUrl);
- hconn = HConnectionManager.createConnection(hconf);
-
- this.info = new TableRecordInfo(seg);
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testLoad() throws Exception {
-
- String tableName = seg.getStorageLocationIdentifier();
- IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest());
-
- List<Slice> slices = Lists.newArrayList();
- HBaseClientKVIterator kvIterator = new HBaseClientKVIterator(hconn, tableName, IIDesc.HBASE_FAMILY_BYTES);
- try {
- for (Slice slice : codec.decodeKeyValue(kvIterator)) {
- slices.add(slice);
- }
- } finally {
- kvIterator.close();
- }
-
- List<TableRecord> records = iterateRecords(slices);
- //dump(records);
- System.out.println(records.size() + " records");
- }
-
- private List<TableRecord> iterateRecords(List<Slice> slices) {
- List<TableRecord> records = Lists.newArrayList();
- for (Slice slice : slices) {
- for (RawTableRecord rec : slice) {
- records.add(new TableRecord((RawTableRecord) rec.clone(), info));
- }
- }
- return records;
- }
-
- private void dump(Iterable<TableRecord> records) {
- for (TableRecord rec : records) {
- byte[] x = rec.getBytes();
- String y = BytesUtil.toReadableText(x);
- System.out.println(y);
- System.out.println();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/storage/src/test/java/org/apache/kylin/storage/test/ITStorageTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/test/ITStorageTest.java b/storage/src/test/java/org/apache/kylin/storage/test/ITStorageTest.java
new file mode 100644
index 0000000..1e32ef5
--- /dev/null
+++ b/storage/src/test/java/org/apache/kylin/storage/test/ITStorageTest.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.storage.test;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.tuple.ITuple;
+import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.storage.IStorageEngine;
+import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.StorageEngineFactory;
+import org.apache.kylin.storage.hbase.ScanOutOfLimitException;
+import org.junit.*;
+
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.Assert.assertTrue;
+
+public class ITStorageTest extends HBaseMetadataTestCase {
+
+ private IStorageEngine storageEngine;
+ private CubeInstance cube;
+ private StorageContext context;
+
+ @BeforeClass
+ public static void setupResource() throws Exception {
+ }
+
+ @AfterClass
+ public static void tearDownResource() {
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+
+ CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
+ cube = cubeMgr.getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_EMPTY");
+ Assert.assertNotNull(cube);
+ storageEngine = StorageEngineFactory.getStorageEngine(cube);
+ String url = KylinConfig.getInstanceFromEnv().getStorageUrl();
+ context = new StorageContext();
+ context.setConnUrl(url);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test(expected = ScanOutOfLimitException.class)
+ @Ignore
+ public void testScanOutOfLimit() {
+ context.setThreshold(1);
+ List<TblColRef> groups = StorageMockUtils.buildGroups();
+ List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations();
+
+ search(groups, aggregations, null, context);
+ }
+
+ @Test
+ public void test01() {
+ List<TblColRef> groups = StorageMockUtils.buildGroups();
+ List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations();
+ TupleFilter filter = StorageMockUtils.buildFilter1(groups.get(0));
+
+ int count = search(groups, aggregations, filter, context);
+ assertTrue(count > 0);
+ }
+
+ /*
+ @Test
+ public void test02() {
+ List<TblColRef> groups = buildGroups();
+ List<FunctionDesc> aggregations = buildAggregations();
+ TupleFilter filter = buildFilter2(groups.get(1));
+
+ int count = search(groups, aggregations, filter, context);
+ assertTrue(count > 0);
+ }
+
+ @Test
+ public void test03() {
+ List<TblColRef> groups = buildGroups();
+ List<FunctionDesc> aggregations = buildAggregations();
+ TupleFilter filter = buildAndFilter(groups);
+
+ int count = search(groups, aggregations, filter, context);
+ assertTrue(count > 0);
+ }
+
+ @Test
+ public void test04() {
+ List<TblColRef> groups = buildGroups();
+ List<FunctionDesc> aggregations = buildAggregations();
+ TupleFilter filter = buildOrFilter(groups);
+
+ int count = search(groups, aggregations, filter, context);
+ assertTrue(count > 0);
+ }
+
+ @Test
+ public void test05() {
+ List<TblColRef> groups = buildGroups();
+ List<FunctionDesc> aggregations = buildAggregations();
+
+ int count = search(groups, aggregations, null, context);
+ assertTrue(count > 0);
+ }
+ */
+ private int search(List<TblColRef> groups, List<FunctionDesc> aggregations, TupleFilter filter, StorageContext context) {
+ int count = 0;
+ ITupleIterator iterator = null;
+ try {
+ SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations);
+ iterator = storageEngine.search(context, sqlDigest, StorageMockUtils.newTupleInfo(groups, aggregations));
+ while (iterator.hasNext()) {
+ ITuple tuple = iterator.next();
+ System.out.println("Tuple = " + tuple);
+ count++;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ if (iterator != null)
+ iterator.close();
+ }
+ return count;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java b/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
deleted file mode 100644
index f9a6212..0000000
--- a/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.test;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.SQLDigest;
-import org.apache.kylin.metadata.tuple.ITuple;
-import org.apache.kylin.metadata.tuple.ITupleIterator;
-import org.apache.kylin.storage.IStorageEngine;
-import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.StorageEngineFactory;
-import org.apache.kylin.storage.hbase.ScanOutOfLimitException;
-import org.junit.*;
-
-import java.util.Collections;
-import java.util.List;
-
-import static org.junit.Assert.assertTrue;
-
-public class StorageTest extends HBaseMetadataTestCase {
-
- private IStorageEngine storageEngine;
- private CubeInstance cube;
- private StorageContext context;
-
- @BeforeClass
- public static void setupResource() throws Exception {
- }
-
- @AfterClass
- public static void tearDownResource() {
- }
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
-
- CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
- cube = cubeMgr.getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_EMPTY");
- Assert.assertNotNull(cube);
- storageEngine = StorageEngineFactory.getStorageEngine(cube);
- String url = KylinConfig.getInstanceFromEnv().getStorageUrl();
- context = new StorageContext();
- context.setConnUrl(url);
- }
-
- @After
- public void tearDown() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test(expected = ScanOutOfLimitException.class)
- @Ignore
- public void testScanOutOfLimit() {
- context.setThreshold(1);
- List<TblColRef> groups = StorageMockUtils.buildGroups();
- List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations();
-
- search(groups, aggregations, null, context);
- }
-
- @Test
- public void test01() {
- List<TblColRef> groups = StorageMockUtils.buildGroups();
- List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations();
- TupleFilter filter = StorageMockUtils.buildFilter1(groups.get(0));
-
- int count = search(groups, aggregations, filter, context);
- assertTrue(count > 0);
- }
-
- /*
- @Test
- public void test02() {
- List<TblColRef> groups = buildGroups();
- List<FunctionDesc> aggregations = buildAggregations();
- TupleFilter filter = buildFilter2(groups.get(1));
-
- int count = search(groups, aggregations, filter, context);
- assertTrue(count > 0);
- }
-
- @Test
- public void test03() {
- List<TblColRef> groups = buildGroups();
- List<FunctionDesc> aggregations = buildAggregations();
- TupleFilter filter = buildAndFilter(groups);
-
- int count = search(groups, aggregations, filter, context);
- assertTrue(count > 0);
- }
-
- @Test
- public void test04() {
- List<TblColRef> groups = buildGroups();
- List<FunctionDesc> aggregations = buildAggregations();
- TupleFilter filter = buildOrFilter(groups);
-
- int count = search(groups, aggregations, filter, context);
- assertTrue(count > 0);
- }
-
- @Test
- public void test05() {
- List<TblColRef> groups = buildGroups();
- List<FunctionDesc> aggregations = buildAggregations();
-
- int count = search(groups, aggregations, null, context);
- assertTrue(count > 0);
- }
- */
- private int search(List<TblColRef> groups, List<FunctionDesc> aggregations, TupleFilter filter, StorageContext context) {
- int count = 0;
- ITupleIterator iterator = null;
- try {
- SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations);
- iterator = storageEngine.search(context, sqlDigest, StorageMockUtils.newTupleInfo(groups, aggregations));
- while (iterator.hasNext()) {
- ITuple tuple = iterator.next();
- System.out.println("Tuple = " + tuple);
- count++;
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- if (iterator != null)
- iterator.close();
- }
- return count;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/streaming/src/test/java/org/apache/kylin/streaming/ITKafkaConsumerTest.java
----------------------------------------------------------------------
diff --git a/streaming/src/test/java/org/apache/kylin/streaming/ITKafkaConsumerTest.java b/streaming/src/test/java/org/apache/kylin/streaming/ITKafkaConsumerTest.java
new file mode 100644
index 0000000..3008314
--- /dev/null
+++ b/streaming/src/test/java/org/apache/kylin/streaming/ITKafkaConsumerTest.java
@@ -0,0 +1,108 @@
+/*
+ *
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ * contributor license agreements. See the NOTICE file distributed with
+ *
+ * this work for additional information regarding copyright ownership.
+ *
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ * (the "License"); you may not use this file except in compliance with
+ *
+ * the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ *
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ *
+ * limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.streaming;
+
+import com.google.common.collect.Lists;
+import org.apache.kylin.common.KylinConfig;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ */
+public class ITKafkaConsumerTest extends KafkaBaseTest {
+
+ private OneOffStreamProducer producer;
+
+ private static final int TOTAL_SEND_COUNT = 100;
+
+ private KafkaConfig kafkaConfig;
+
+ @Before
+ public void before() throws IOException {
+ producer = new OneOffStreamProducer(TOTAL_SEND_COUNT);
+ producer.start();
+ kafkaConfig = StreamingManager.getInstance(KylinConfig.getInstanceFromEnv()).getKafkaConfig("kafka_test");
+ }
+
+ @After
+ public void after() {
+ producer.stop();
+ }
+
+ private void waitForProducerToStop(OneOffStreamProducer producer) {
+ while (!producer.isStopped()) {
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ @Test
+ @Ignore("since ci does not have the topic")
+ public void test() throws InterruptedException {
+ final TopicMeta kafkaTopicMeta = KafkaRequester.getKafkaTopicMeta(kafkaConfig);
+ final ExecutorService executorService = Executors.newFixedThreadPool(kafkaTopicMeta.getPartitionIds().size());
+ List<BlockingQueue<StreamMessage>> queues = Lists.newArrayList();
+ for (Integer partitionId : kafkaTopicMeta.getPartitionIds()) {
+ KafkaConsumer consumer = new KafkaConsumer(kafkaTopicMeta.getName(), partitionId, 0, kafkaConfig.getBrokers(), kafkaConfig);
+ queues.add(consumer.getStreamQueue(0));
+ executorService.execute(consumer);
+ }
+ waitForProducerToStop(producer);
+
+ //wait some time to ensure consumer has fetched all data
+ Thread.sleep(5000);
+ int count = 0;
+ for (BlockingQueue<StreamMessage> queue : queues) {
+ count += queue.size();
+ }
+
+ logger.info("count of messages are " + count);
+ //since there will be historical data
+ assertTrue(count >= TOTAL_SEND_COUNT && (count % TOTAL_SEND_COUNT == 0));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/streaming/src/test/java/org/apache/kylin/streaming/ITKafkaRequesterTest.java
----------------------------------------------------------------------
diff --git a/streaming/src/test/java/org/apache/kylin/streaming/ITKafkaRequesterTest.java b/streaming/src/test/java/org/apache/kylin/streaming/ITKafkaRequesterTest.java
new file mode 100644
index 0000000..d8853f6
--- /dev/null
+++ b/streaming/src/test/java/org/apache/kylin/streaming/ITKafkaRequesterTest.java
@@ -0,0 +1,76 @@
+/*
+ *
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ * contributor license agreements. See the NOTICE file distributed with
+ *
+ * this work for additional information regarding copyright ownership.
+ *
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ * (the "License"); you may not use this file except in compliance with
+ *
+ * the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ *
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ *
+ * limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.streaming;
+
+import org.apache.kylin.common.KylinConfig;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ */
+public class ITKafkaRequesterTest extends KafkaBaseTest {
+
+ private static final String NON_EXISTED_TOPIC = "non_existent_topic";
+ private KafkaConfig kafkaConfig;
+
+
+ @Before
+ public void before() {
+ kafkaConfig = StreamingManager.getInstance(KylinConfig.getInstanceFromEnv()).getKafkaConfig("kafka_test");
+ }
+
+ @AfterClass
+ public static void afterClass() {
+ }
+
+ @Test
+ @Ignore("since ci does not enable kafka")
+ public void testTopicMeta() throws Exception {
+ TopicMeta kafkaTopicMeta = KafkaRequester.getKafkaTopicMeta(kafkaConfig);
+ assertNotNull(kafkaTopicMeta);
+ assertEquals(2, kafkaTopicMeta.getPartitionIds().size());
+ assertEquals(kafkaConfig.getTopic(), kafkaTopicMeta.getName());
+
+ KafkaConfig anotherTopicConfig = kafkaConfig.clone();
+ anotherTopicConfig.setTopic(NON_EXISTED_TOPIC);
+
+ kafkaTopicMeta = KafkaRequester.getKafkaTopicMeta(anotherTopicConfig);
+ assertTrue(kafkaTopicMeta == null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java
----------------------------------------------------------------------
diff --git a/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java b/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java
deleted file mode 100644
index c782dae..0000000
--- a/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- *
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *
- * contributor license agreements. See the NOTICE file distributed with
- *
- * this work for additional information regarding copyright ownership.
- *
- * The ASF licenses this file to You under the Apache License, Version 2.0
- *
- * (the "License"); you may not use this file except in compliance with
- *
- * the License. You may obtain a copy of the License at
- *
- *
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- *
- *
- * Unless required by applicable law or agreed to in writing, software
- *
- * distributed under the License is distributed on an "AS IS" BASIS,
- *
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *
- * See the License for the specific language governing permissions and
- *
- * limitations under the License.
- *
- * /
- */
-
-package org.apache.kylin.streaming;
-
-import com.google.common.collect.Lists;
-import org.apache.kylin.common.KylinConfig;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import static org.junit.Assert.assertTrue;
-
-/**
- */
-public class KafkaConsumerTest extends KafkaBaseTest {
-
- private OneOffStreamProducer producer;
-
- private static final int TOTAL_SEND_COUNT = 100;
-
- private KafkaConfig kafkaConfig;
-
- @Before
- public void before() throws IOException {
- producer = new OneOffStreamProducer(TOTAL_SEND_COUNT);
- producer.start();
- kafkaConfig = StreamingManager.getInstance(KylinConfig.getInstanceFromEnv()).getKafkaConfig("kafka_test");
- }
-
- @After
- public void after() {
- producer.stop();
- }
-
- private void waitForProducerToStop(OneOffStreamProducer producer) {
- while (!producer.isStopped()) {
- try {
- Thread.sleep(100);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- }
-
- @Test
- @Ignore("since ci does not have the topic")
- public void test() throws InterruptedException {
- final TopicMeta kafkaTopicMeta = KafkaRequester.getKafkaTopicMeta(kafkaConfig);
- final ExecutorService executorService = Executors.newFixedThreadPool(kafkaTopicMeta.getPartitionIds().size());
- List<BlockingQueue<StreamMessage>> queues = Lists.newArrayList();
- for (Integer partitionId : kafkaTopicMeta.getPartitionIds()) {
- KafkaConsumer consumer = new KafkaConsumer(kafkaTopicMeta.getName(), partitionId, 0, kafkaConfig.getBrokers(), kafkaConfig);
- queues.add(consumer.getStreamQueue(0));
- executorService.execute(consumer);
- }
- waitForProducerToStop(producer);
-
- //wait some time to ensure consumer has fetched all data
- Thread.sleep(5000);
- int count = 0;
- for (BlockingQueue<StreamMessage> queue : queues) {
- count += queue.size();
- }
-
- logger.info("count of messages are " + count);
- //since there will be historical data
- assertTrue(count >= TOTAL_SEND_COUNT && (count % TOTAL_SEND_COUNT == 0));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/streaming/src/test/java/org/apache/kylin/streaming/KafkaRequesterTest.java
----------------------------------------------------------------------
diff --git a/streaming/src/test/java/org/apache/kylin/streaming/KafkaRequesterTest.java b/streaming/src/test/java/org/apache/kylin/streaming/KafkaRequesterTest.java
deleted file mode 100644
index 9d99825..0000000
--- a/streaming/src/test/java/org/apache/kylin/streaming/KafkaRequesterTest.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- *
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *
- * contributor license agreements. See the NOTICE file distributed with
- *
- * this work for additional information regarding copyright ownership.
- *
- * The ASF licenses this file to You under the Apache License, Version 2.0
- *
- * (the "License"); you may not use this file except in compliance with
- *
- * the License. You may obtain a copy of the License at
- *
- *
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- *
- *
- * Unless required by applicable law or agreed to in writing, software
- *
- * distributed under the License is distributed on an "AS IS" BASIS,
- *
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *
- * See the License for the specific language governing permissions and
- *
- * limitations under the License.
- *
- * /
- */
-
-package org.apache.kylin.streaming;
-
-import org.apache.kylin.common.KylinConfig;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import static org.junit.Assert.*;
-
-/**
- */
-public class KafkaRequesterTest extends KafkaBaseTest {
-
- private static final String NON_EXISTED_TOPIC = "non_existent_topic";
- private KafkaConfig kafkaConfig;
-
-
- @Before
- public void before() {
- kafkaConfig = StreamingManager.getInstance(KylinConfig.getInstanceFromEnv()).getKafkaConfig("kafka_test");
- }
-
- @AfterClass
- public static void afterClass() {
- }
-
- @Test
- @Ignore("since ci does not enable kafka")
- public void testTopicMeta() throws Exception {
- TopicMeta kafkaTopicMeta = KafkaRequester.getKafkaTopicMeta(kafkaConfig);
- assertNotNull(kafkaTopicMeta);
- assertEquals(2, kafkaTopicMeta.getPartitionIds().size());
- assertEquals(kafkaConfig.getTopic(), kafkaTopicMeta.getName());
-
- KafkaConfig anotherTopicConfig = kafkaConfig.clone();
- anotherTopicConfig.setTopic(NON_EXISTED_TOPIC);
-
- kafkaTopicMeta = KafkaRequester.getKafkaTopicMeta(anotherTopicConfig);
- assertTrue(kafkaTopicMeta == null);
- }
-}
[08/11] incubator-kylin git commit: KYLIN-697 fix mock htable bug
Posted by ma...@apache.org.
KYLIN-697 fix mock htable bug
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/4d323f8b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/4d323f8b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/4d323f8b
Branch: refs/heads/new697
Commit: 4d323f8b9763871f9acffd13137ae8df4e912c68
Parents: 6e74d6d
Author: honma <ho...@ebay.com>
Authored: Mon May 25 10:32:03 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon May 25 11:24:16 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/MockHTable.java | 7 +-
.../kylin/job/tools/HBaseRowDigestTest.java | 84 --------------------
2 files changed, 5 insertions(+), 86 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4d323f8b/common/src/main/java/org/apache/kylin/common/util/MockHTable.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/MockHTable.java b/common/src/main/java/org/apache/kylin/common/util/MockHTable.java
index d54fe4b..c60a7a5 100644
--- a/common/src/main/java/org/apache/kylin/common/util/MockHTable.java
+++ b/common/src/main/java/org/apache/kylin/common/util/MockHTable.java
@@ -516,7 +516,11 @@ public class MockHTable implements HTableInterface {
continue;
}
for (KeyValue kv : delete.getFamilyMap().get(family)) {
- data.get(row).get(kv.getFamily()).remove(kv.getQualifier());
+ if (kv.isDeleteFamily()) {
+ data.get(row).get(kv.getFamily()).clear();
+ } else {
+ data.get(row).get(kv.getFamily()).remove(kv.getQualifier());
+ }
}
if (data.get(row).get(family).isEmpty()) {
data.get(row).remove(family);
@@ -676,7 +680,6 @@ public class MockHTable implements HTableInterface {
public <R extends Message> void batchCoprocessorService(Descriptors.MethodDescriptor methodDescriptor, Message request, byte[] startKey, byte[] endKey, R responsePrototype, Batch.Callback<R> callback) throws ServiceException, Throwable {
throw new NotImplementedException();
-
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4d323f8b/job/src/test/java/org/apache/kylin/job/tools/HBaseRowDigestTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/tools/HBaseRowDigestTest.java b/job/src/test/java/org/apache/kylin/job/tools/HBaseRowDigestTest.java
deleted file mode 100644
index 95e4b7f..0000000
--- a/job/src/test/java/org/apache/kylin/job/tools/HBaseRowDigestTest.java
+++ /dev/null
@@ -1,84 +0,0 @@
-package org.apache.kylin.job.tools;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.persistence.HBaseConnection;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- */
-@Ignore
-public class HBaseRowDigestTest extends HBaseMetadataTestCase {
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- }
-
- @After
- public void tearDown() throws Exception {
- this.cleanupTestMetadata();
- }
-
- private static final byte[] CF = "f".getBytes();
- private static final byte[] QN = "c".getBytes();
- static ImmutableBytesWritable k = new ImmutableBytesWritable();
- static ImmutableBytesWritable v = new ImmutableBytesWritable();
-
- @Test
- public static void test() throws IOException {
- String hbaseUrl = "hbase"; // use hbase-site.xml on classpath
- HConnection conn = null;
- HTableInterface table = null;
- try {
- conn = HBaseConnection.get(hbaseUrl);
- table = conn.getTable("KYLIN_II_YTYWP3CQGJ");
- ResultScanner scanner = table.getScanner(CF, QN);
- StringBuffer sb = new StringBuffer();
- while (true) {
- Result r = scanner.next();
- if (r == null)
- break;
-
- Cell[] cells = r.rawCells();
- Cell c = cells[0];
-
- k.set(c.getRowArray(), c.getRowOffset(), c.getRowLength());
- v.set(c.getValueArray(), c.getValueOffset(), c.getValueLength());
-
- byte[] row = k.copyBytes();
- byte[] value = v.copyBytes();
- // byte[] row = r.getRow();
- // byte[] value = r.getValue(CF, QN);
- //
- sb.append("row length: " + row.length + "\r\n");
- sb.append(BytesUtil.toReadableText(row) + "\r\n");
- sb.append("value length: " + value.length + "\r\n");
- sb.append(BytesUtil.toReadableText(value) + "\r\n");
- }
- System.out.println(sb.toString());
- FileUtils.writeStringToFile(new File("/Users/honma/Desktop/a3"), sb.toString());
- } catch (IOException e) {
- e.printStackTrace();
- } finally {
- if (table != null)
- table.close();
- if (conn != null)
- conn.close();
- }
-
- }
-}
[11/11] incubator-kylin git commit: KYLIN-693 character all build
cube/ii test cases
Posted by ma...@apache.org.
KYLIN-693 character all build cube/ii test cases
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ada6b61e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ada6b61e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ada6b61e
Branch: refs/heads/new697
Commit: ada6b61e2217e7d94c168bacc5770b68fa54828c
Parents: 148576e
Author: honma <ho...@ebay.com>
Authored: Mon May 25 14:15:39 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon May 25 14:15:39 2015 +0800
----------------------------------------------------------------------
.../kylin/job/BuildCubeWithEngineTest.java | 279 +++++++++++++++++++
.../kylin/job/BuildCubeWithStreamTest.java | 224 +++++++++++++++
.../apache/kylin/job/BuildIIWithEngineTest.java | 254 +++++++++++++++++
.../apache/kylin/job/BuildIIWithStreamTest.java | 270 ++++++++++++++++++
.../kylin/job/ITBuildCubeWithEngineTest.java | 279 -------------------
.../kylin/job/ITBuildCubeWithStreamTest.java | 224 ---------------
.../kylin/job/ITBuildIIWithEngineTest.java | 254 -----------------
.../kylin/job/ITBuildIIWithStreamTest.java | 270 ------------------
pom.xml | 162 ++++++-----
9 files changed, 1122 insertions(+), 1094 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ada6b61e/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
new file mode 100644
index 0000000..547a6e7
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -0,0 +1,279 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.job.cube.CubingJob;
+import org.apache.kylin.job.cube.CubingJobBuilder;
+import org.apache.kylin.job.engine.JobEngineConfig;
+import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.execution.ExecutableState;
+import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
+import org.apache.kylin.job.manager.ExecutableManager;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.lang.reflect.Method;
+import java.text.SimpleDateFormat;
+import java.util.List;
+import java.util.TimeZone;
+import java.util.concurrent.*;
+
+import static org.junit.Assert.assertEquals;
+
+public class BuildCubeWithEngineTest {
+
+ private JobEngineConfig jobEngineConfig;
+
+ private CubeManager cubeManager;
+
+ private DefaultScheduler scheduler;
+
+ protected ExecutableManager jobService;
+
+ private static final Log logger = LogFactory.getLog(BuildCubeWithEngineTest.class);
+
+ protected void waitForJob(String jobId) {
+ while (true) {
+ AbstractExecutable job = jobService.getJob(jobId);
+ if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) {
+ break;
+ } else {
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
+ System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ }
+
+ @Before
+ public void before() throws Exception {
+ HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+
+ DeployUtil.initCliWorkDir();
+ DeployUtil.deployMetadata();
+ DeployUtil.overrideJobJarLocations();
+
+
+ final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ jobService = ExecutableManager.getInstance(kylinConfig);
+ scheduler = DefaultScheduler.getInstance();
+ scheduler.init(new JobEngineConfig(kylinConfig));
+ if (!scheduler.hasStarted()) {
+ throw new RuntimeException("scheduler has not been started");
+ }
+ cubeManager = CubeManager.getInstance(kylinConfig);
+ jobEngineConfig = new JobEngineConfig(kylinConfig);
+ for (String jobId : jobService.getAllJobIds()) {
+ if(jobService.getJob(jobId) instanceof CubingJob){
+ jobService.deleteJob(jobId);
+ }
+ }
+
+ }
+
+ @After
+ public void after() {
+ HBaseMetadataTestCase.staticCleanupTestMetadata();
+ }
+
+ @Test
+ public void test() throws Exception {
+ DeployUtil.prepareTestData("left", "test_kylin_cube_with_slr_left_join_empty");
+ testInner();
+ testLeft();
+ }
+
+ private void testInner() throws Exception {
+ String[] testCase = new String[]{
+ "testInnerJoinCube",
+ "testInnerJoinCube2",
+ };
+ runTestAndAssertSucceed(testCase);
+ }
+
+ private void testLeft() throws Exception {
+ String[] testCase = new String[]{
+ "testLeftJoinCube",
+ "testLeftJoinCube2",
+ };
+ runTestAndAssertSucceed(testCase);
+ }
+
+ private void runTestAndAssertSucceed(String[] testCase) throws Exception {
+ ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
+ final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
+ List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
+ for (int i = 0; i < testCase.length; i++) {
+ tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch)));
+ }
+ countDownLatch.await();
+ try {
+ for (int i = 0; i < tasks.size(); ++i) {
+ Future<List<String>> task = tasks.get(i);
+ final List<String> jobIds = task.get();
+ for (String jobId : jobIds) {
+ assertJobSucceed(jobId);
+ }
+ }
+ } catch (Exception ex) {
+ logger.error(ex);
+ throw ex;
+ }
+ }
+
+ private void assertJobSucceed(String jobId) {
+ assertEquals("The job '" + jobId + "' is failed.", ExecutableState.SUCCEED, jobService.getOutput(jobId).getState());
+ }
+
+ private class TestCallable implements Callable<List<String>> {
+
+ private final String methodName;
+ private final CountDownLatch countDownLatch;
+
+ public TestCallable(String methodName, CountDownLatch countDownLatch) {
+ this.methodName = methodName;
+ this.countDownLatch = countDownLatch;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public List<String> call() throws Exception {
+ try {
+ final Method method = BuildCubeWithEngineTest.class.getDeclaredMethod(methodName);
+ method.setAccessible(true);
+ return (List<String>) method.invoke(BuildCubeWithEngineTest.this);
+ } finally {
+ countDownLatch.countDown();
+ }
+ }
+ }
+
+ @SuppressWarnings("unused") // called by reflection
+ private List<String> testInnerJoinCube2() throws Exception {
+ clearSegment("test_kylin_cube_with_slr_empty");
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+ long date1 = 0;
+ long date2 = f.parse("2013-01-01").getTime();
+ long date3 = f.parse("2022-01-01").getTime();
+ List<String> result = Lists.newArrayList();
+ result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date2));
+ result.add(buildSegment("test_kylin_cube_with_slr_empty", date2, date3));
+ return result;
+ }
+
+ @SuppressWarnings("unused") // called by reflection
+ private List<String> testInnerJoinCube() throws Exception {
+ clearSegment("test_kylin_cube_without_slr_empty");
+
+
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+
+ // this cube's start date is 0, end date is 20501112000000
+ long date1 = 0;
+ long date2 = f.parse("2013-01-01").getTime();
+
+
+ // this cube doesn't support incremental build, always do full build
+
+ List<String> result = Lists.newArrayList();
+ result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date2));
+ return result;
+ }
+
+ @SuppressWarnings("unused") // called by reflection
+ private List<String> testLeftJoinCube2() throws Exception {
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+ List<String> result = Lists.newArrayList();
+ final String cubeName = "test_kylin_cube_without_slr_left_join_empty";
+ // this cube's start date is 0, end date is 20120601000000
+ long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
+ long dateEnd = f.parse("2012-06-01").getTime();
+
+ clearSegment(cubeName);
+ result.add(buildSegment(cubeName, dateStart, dateEnd));
+
+ // then submit an append job, start date is 20120601000000, end
+ // date is 20220101000000
+ dateStart = f.parse("2012-06-01").getTime();
+ dateEnd = f.parse("2022-01-01").getTime();
+ result.add(buildSegment(cubeName, dateStart, dateEnd));
+ return result;
+
+ }
+
+ @SuppressWarnings("unused") // called by reflection
+ private List<String> testLeftJoinCube() throws Exception {
+ String cubeName = "test_kylin_cube_with_slr_left_join_empty";
+ clearSegment(cubeName);
+
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+ long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
+ long dateEnd = f.parse("2050-11-12").getTime();
+
+ // this cube's start date is 0, end date is 20501112000000
+ List<String> result = Lists.newArrayList();
+ result.add(buildSegment(cubeName, dateStart, dateEnd));
+ return result;
+
+ }
+
+ private void clearSegment(String cubeName) throws Exception {
+ CubeInstance cube = cubeManager.getCube(cubeName);
+ cube.getSegments().clear();
+ cubeManager.updateCube(cube,true);
+ }
+
+
+ private String buildSegment(String cubeName, long startDate, long endDate) throws Exception {
+ CubeSegment segment = cubeManager.appendSegments(cubeManager.getCube(cubeName), endDate);
+ CubingJobBuilder cubingJobBuilder = new CubingJobBuilder(jobEngineConfig);
+ CubingJob job = cubingJobBuilder.buildJob(segment);
+ jobService.addJob(job);
+ waitForJob(job.getId());
+ return job.getId();
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ada6b61e/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
new file mode 100644
index 0000000..dc42dc5
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
@@ -0,0 +1,224 @@
+/*
+ *
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ * contributor license agreements. See the NOTICE file distributed with
+ *
+ * this work for additional information regarding copyright ownership.
+ *
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ * (the "License"); you may not use this file except in compliance with
+ *
+ * the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ *
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ *
+ * limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.job;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Maps;
+import com.google.common.collect.SetMultimap;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hive.hcatalog.data.schema.HCatSchema;
+import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.DimensionDesc;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.dict.DictionaryGenerator;
+import org.apache.kylin.dict.lookup.HiveTableReader;
+import org.apache.kylin.job.hadoop.cubev2.IGTRecordWriter;
+import org.apache.kylin.job.hadoop.cubev2.InMemCubeBuilder;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.gridtable.GTRecord;
+import org.junit.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+/**
+ *
+ * This class is going to be deleted
+ */
+@Ignore("For dev testing")
+public class BuildCubeWithStreamTest {
+
+ private static final Logger logger = LoggerFactory.getLogger(BuildCubeWithStreamTest.class);
+
+ private KylinConfig kylinConfig;
+ private CubeManager cubeManager;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ }
+
+ @Before
+ public void before() throws Exception {
+ HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+ DeployUtil.overrideJobJarLocations();
+
+ kylinConfig = KylinConfig.getInstanceFromEnv();
+ cubeManager = CubeManager.getInstance(kylinConfig);
+
+ }
+
+ @After
+ public void after() {
+ HBaseMetadataTestCase.staticCleanupTestMetadata();
+ }
+
+ @Test
+ public void test() throws Exception {
+ CubeInstance cube = cubeManager.getCube("test_kylin_cube_without_slr_left_join_empty");
+ final CubeDesc desc = cube.getDescriptor();
+ // cube.getSegments().clear();
+ // cubeManager.updateCube(cube);
+
+ CubeSegment cubeSegment = cube.getSegment("19700101000000_20150401000000", SegmentStatusEnum.NEW);
+ Map<TblColRef, Dictionary<?>> dictionaryMap = Maps.newHashMap();
+
+//
+ for (DimensionDesc dim : desc.getDimensions()) {
+ // dictionary
+ for (TblColRef col : dim.getColumnRefs()) {
+ if (desc.getRowkey().isUseDictionary(col)) {
+ Dictionary dict = cubeSegment.getDictionary(col);
+ if (dict == null) {
+ throw new IllegalArgumentException("Dictionary for " + col + " was not found.");
+ }
+ logger.info("Dictionary for " + col + " was put into dictionary map.");
+ dictionaryMap.put(col, cubeSegment.getDictionary(col));
+ }
+ }
+ }
+
+// final String tableName = createIntermediateTable(desc, kylinConfig, null);
+ String tableName = "kylin_intermediate_test_kylin_cube_without_slr_desc_19700101000000_20130112000000_a24dec89_efbd_425f_9a5f_8b78dd1412af"; // has 3089 records;
+// tableName = "kylin_intermediate_test_kylin_cube_without_slr_desc_19700101000000_20130112000000_a5e1eb5d_da6b_475d_9807_be0b61f03215"; // only 20 rows;
+// tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150302000000_0a183367_f245_43d1_8850_1c138c8514c3";
+// tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150301000000_ce061464_7962_4642_bd7d_7c3d8fbe9389";
+ tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150401000000_fb7ae579_d987_4900_a3b7_c60c731cd269"; // 2 million records
+ logger.info("intermediate table name:" + tableName);
+
+
+ ArrayBlockingQueue queue = new ArrayBlockingQueue<List<String>>(10000);
+
+ InMemCubeBuilder cubeBuilder = new InMemCubeBuilder(queue, cube, dictionaryMap, new ConsoleGTRecordWriter());
+ ExecutorService executorService = Executors.newSingleThreadExecutor();
+ Future<?> future = executorService.submit(cubeBuilder);
+
+ final Configuration conf = new Configuration();
+ HCatInputFormat.setInput(conf, "default", tableName);
+ final HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
+ logger.info(StringUtils.join(tableSchema.getFieldNames(), "\n"));
+ HiveTableReader reader = new HiveTableReader("default", tableName);
+ List<String> row;
+ int counter = 0;
+ while (reader.next()) {
+ row = reader.getRowAsList();
+ queue.put(row);
+ counter++;
+ if(counter == 200000)
+ break;
+ }
+ queue.put(new ArrayList<String>(0));
+ reader.close();
+
+ try {
+ future.get();
+ } catch (Exception e) {
+ logger.error("stream build failed", e);
+ throw new IOException("Failed to build cube ", e);
+ }
+
+ logger.info("stream build finished");
+ }
+
+
+ private void buildDictionary(List<List<String>> table, CubeDesc desc, Map<TblColRef, Dictionary<?>> dictionaryMap) {
+ SetMultimap<TblColRef, String> valueMap = HashMultimap.create();
+
+ List<TblColRef> dimColumns = desc.listDimensionColumnsExcludingDerived();
+ for (List<String> row : table) {
+ for (int i = 0; i < dimColumns.size(); i++) {
+ String cell = row.get(i);
+ valueMap.put(dimColumns.get(i), cell);
+ }
+ }
+
+ for (DimensionDesc dim : desc.getDimensions()) {
+ // dictionary
+ for (TblColRef col : dim.getColumnRefs()) {
+ if (desc.getRowkey().isUseDictionary(col)) {
+ Dictionary dict = DictionaryGenerator.buildDictionaryFromValueList(col.getType(), Collections2.transform(valueMap.get(col), new Function<String, byte[]>() {
+ @Nullable
+ @Override
+ public byte[] apply(String input) {
+ if (input == null)
+ return null;
+ return input.getBytes();
+ }
+ }));
+
+ logger.info("Building dictionary for " + col);
+ dictionaryMap.put(col, dict);
+ }
+ }
+ }
+
+ }
+
+
+ class ConsoleGTRecordWriter implements IGTRecordWriter {
+
+ boolean verbose = false;
+
+ @Override
+ public void write(Long cuboidId, GTRecord record) throws IOException {
+ if (verbose)
+ System.out.println(record.toString());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ada6b61e/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
new file mode 100644
index 0000000..9c0219a
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.invertedindex.IISegment;
+import org.apache.kylin.job.engine.JobEngineConfig;
+import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.execution.ExecutableState;
+import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
+import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
+import org.apache.kylin.job.invertedindex.IIJob;
+import org.apache.kylin.job.invertedindex.IIJobBuilder;
+import org.apache.kylin.job.manager.ExecutableManager;
+import org.apache.kylin.metadata.realization.RealizationStatusEnum;
+import org.junit.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.text.SimpleDateFormat;
+import java.util.List;
+import java.util.TimeZone;
+import java.util.concurrent.*;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author shaoshi
+ */
+public class BuildIIWithEngineTest {
+
+ private JobEngineConfig jobEngineConfig;
+ private IIManager iiManager;
+
+ private DefaultScheduler scheduler;
+ protected ExecutableManager jobService;
+
+ protected static final String[] TEST_II_INSTANCES = new String[] { "test_kylin_ii_inner_join", "test_kylin_ii_left_join" };
+
+ private static final Log logger = LogFactory.getLog(BuildIIWithEngineTest.class);
+
+ protected void waitForJob(String jobId) {
+ while (true) {
+ AbstractExecutable job = jobService.getJob(jobId);
+ if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) {
+ break;
+ } else {
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ }
+
+ @Before
+ public void before() throws Exception {
+ HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+
+ //DeployUtil.initCliWorkDir();
+ // DeployUtil.deployMetadata();
+ DeployUtil.overrideJobJarLocations();
+
+ final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ jobService = ExecutableManager.getInstance(kylinConfig);
+ scheduler = DefaultScheduler.getInstance();
+ scheduler.init(new JobEngineConfig(kylinConfig));
+ if (!scheduler.hasStarted()) {
+ throw new RuntimeException("scheduler has not been started");
+ }
+ jobEngineConfig = new JobEngineConfig(kylinConfig);
+ for (String jobId : jobService.getAllJobIds()) {
+ if (jobService.getJob(jobId) instanceof IIJob) {
+ jobService.deleteJob(jobId);
+ }
+ }
+
+ iiManager = IIManager.getInstance(kylinConfig);
+ for (String iiInstance : TEST_II_INSTANCES) {
+
+ IIInstance ii = iiManager.getII(iiInstance);
+ if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
+ ii.setStatus(RealizationStatusEnum.DISABLED);
+ iiManager.updateII(ii, true);
+ }
+ }
+ }
+
+ @After
+ public void after() throws Exception {
+
+ for (String iiInstance : TEST_II_INSTANCES) {
+ IIInstance ii = iiManager.getII(iiInstance);
+ if (ii.getStatus() != RealizationStatusEnum.READY) {
+ ii.setStatus(RealizationStatusEnum.READY);
+ iiManager.updateII(ii, true);
+ }
+ }
+ backup();
+ }
+
+ @Test
+ @Ignore
+ public void testBuildII() throws Exception {
+
+ String[] testCase = new String[] { "buildIIInnerJoin", "buildIILeftJoin" };
+ ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
+ final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
+ List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
+ for (int i = 0; i < testCase.length; i++) {
+ tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch)));
+ }
+ countDownLatch.await();
+ for (int i = 0; i < tasks.size(); ++i) {
+ Future<List<String>> task = tasks.get(i);
+ final List<String> jobIds = task.get();
+ for (String jobId : jobIds) {
+ assertJobSucceed(jobId);
+ }
+ }
+
+ }
+
+ private void assertJobSucceed(String jobId) {
+ assertEquals(ExecutableState.SUCCEED, jobService.getOutput(jobId).getState());
+ }
+
+ private class TestCallable implements Callable<List<String>> {
+
+ private final String methodName;
+ private final CountDownLatch countDownLatch;
+
+ public TestCallable(String methodName, CountDownLatch countDownLatch) {
+ this.methodName = methodName;
+ this.countDownLatch = countDownLatch;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public List<String> call() throws Exception {
+ try {
+ final Method method = BuildIIWithEngineTest.class.getDeclaredMethod(methodName);
+ method.setAccessible(true);
+ return (List<String>) method.invoke(BuildIIWithEngineTest.this);
+ } finally {
+ countDownLatch.countDown();
+ }
+ }
+ }
+
+ protected List<String> buildIIInnerJoin() throws Exception {
+ return buildII(TEST_II_INSTANCES[0]);
+ }
+
+ protected List<String> buildIILeftJoin() throws Exception {
+ return buildII(TEST_II_INSTANCES[1]);
+ }
+
+ protected List<String> buildII(String iiName) throws Exception {
+ clearSegment(iiName);
+
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+
+ long date1 = 0;
+ long date2 = f.parse("2015-01-01").getTime();
+
+ List<String> result = Lists.newArrayList();
+ result.add(buildSegment(iiName, date1, date2));
+ return result;
+ }
+
+ private void clearSegment(String iiName) throws Exception {
+ IIInstance ii = iiManager.getII(iiName);
+ ii.getSegments().clear();
+ iiManager.updateII(ii,true);
+ }
+
+ private String buildSegment(String iiName, long startDate, long endDate) throws Exception {
+ IIInstance iiInstance = iiManager.getII(iiName);
+ IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
+ iiInstance.getSegments().add(segment);
+ iiManager.updateII(iiInstance, true);
+ IIJobBuilder iiJobBuilder = new IIJobBuilder(jobEngineConfig);
+ IIJob job = iiJobBuilder.buildJob(segment);
+ jobService.addJob(job);
+ waitForJob(job.getId());
+ return job.getId();
+ }
+
+ private int cleanupOldStorage() throws Exception {
+ String[] args = { "--delete", "true" };
+
+ int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
+ return exitCode;
+ }
+
+ private void backup() throws Exception {
+ int exitCode = cleanupOldStorage();
+ if (exitCode == 0) {
+ exportHBaseData();
+ }
+ }
+
+ private void exportHBaseData() throws IOException {
+ ExportHBaseData export = new ExportHBaseData();
+ export.exportTables();
+ }
+
+ public static void main(String[] args) throws Exception {
+ BuildIIWithEngineTest instance = new BuildIIWithEngineTest();
+
+ BuildIIWithEngineTest.beforeClass();
+ instance.before();
+ instance.testBuildII();
+ instance.after();
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ada6b61e/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
new file mode 100644
index 0000000..6bceab7
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -0,0 +1,270 @@
+/*
+ *
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ * contributor license agreements. See the NOTICE file distributed with
+ *
+ * this work for additional information regarding copyright ownership.
+ *
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ * (the "License"); you may not use this file except in compliance with
+ *
+ * the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ *
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ *
+ * limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.job;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hive.hcatalog.data.schema.HCatSchema;
+import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.dict.lookup.HiveTableReader;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.invertedindex.IISegment;
+import org.apache.kylin.invertedindex.model.IIDesc;
+import org.apache.kylin.invertedindex.model.IIJoinedFlatTableDesc;
+import org.apache.kylin.job.common.ShellExecutable;
+import org.apache.kylin.job.constant.ExecutableConstants;
+import org.apache.kylin.job.engine.JobEngineConfig;
+import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
+import org.apache.kylin.job.hadoop.invertedindex.IICreateHTableJob;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.RealizationStatusEnum;
+import org.apache.kylin.streaming.StreamMessage;
+import org.apache.kylin.streaming.invertedindex.IIStreamBuilder;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingDeque;
+
+import static org.junit.Assert.fail;
+
+/**
+ */
+public class BuildIIWithStreamTest {
+
+ private static final Logger logger = LoggerFactory.getLogger(BuildIIWithStreamTest.class);
+
+ private static final String[] II_NAME = new String[] { "test_kylin_ii_left_join", "test_kylin_ii_inner_join" };
+ private IIManager iiManager;
+ private KylinConfig kylinConfig;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ }
+
+ @Before
+ public void before() throws Exception {
+ HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+ DeployUtil.overrideJobJarLocations();
+
+ kylinConfig = KylinConfig.getInstanceFromEnv();
+ iiManager = IIManager.getInstance(kylinConfig);
+ iiManager = IIManager.getInstance(kylinConfig);
+ for (String iiInstance : II_NAME) {
+
+ IIInstance ii = iiManager.getII(iiInstance);
+ if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
+ ii.setStatus(RealizationStatusEnum.DISABLED);
+ iiManager.updateII(ii,true);
+ }
+ }
+ }
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+ backup();
+ }
+
+ private static int cleanupOldStorage() throws Exception {
+ String[] args = { "--delete", "true" };
+ int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
+ return exitCode;
+ }
+
+ private static void backup() throws Exception {
+ int exitCode = cleanupOldStorage();
+ if (exitCode == 0) {
+ exportHBaseData();
+ }
+ }
+
+ private static void exportHBaseData() throws IOException {
+ ExportHBaseData export = new ExportHBaseData();
+ export.exportTables();
+ }
+
+ private String createIntermediateTable(IIDesc desc, KylinConfig kylinConfig) throws IOException {
+ IIJoinedFlatTableDesc intermediateTableDesc = new IIJoinedFlatTableDesc(desc);
+ JobEngineConfig jobEngineConfig = new JobEngineConfig(kylinConfig);
+ final String uuid = UUID.randomUUID().toString();
+ final String dropTableHql = JoinedFlatTable.generateDropTableStatement(intermediateTableDesc, uuid);
+ final String createTableHql = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, jobEngineConfig.getHdfsWorkingDirectory() + "/kylin-" + uuid, uuid);
+ String insertDataHqls;
+ try {
+ insertDataHqls = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, uuid, jobEngineConfig);
+ } catch (IOException e1) {
+ e1.printStackTrace();
+ throw new RuntimeException("Failed to generate insert data SQL for intermediate table.");
+ }
+
+ ShellExecutable step = new ShellExecutable();
+ StringBuffer buf = new StringBuffer();
+ buf.append("hive -e \"");
+ buf.append(dropTableHql + "\n");
+ buf.append(createTableHql + "\n");
+ buf.append(insertDataHqls + "\n");
+ buf.append("\"");
+
+ step.setCmd(buf.toString());
+ logger.info(step.getCmd());
+ step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);
+ kylinConfig.getCliCommandExecutor().execute(step.getCmd(), null);
+ return intermediateTableDesc.getTableName(uuid);
+ }
+
+ private void clearSegment(String iiName) throws Exception {
+ IIInstance ii = iiManager.getII(iiName);
+ ii.getSegments().clear();
+ iiManager.updateII(ii,true);
+ }
+
+ private IISegment createSegment(String iiName) throws Exception {
+ clearSegment(iiName);
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+
+ long date1 = 0;
+ long date2 = f.parse("2015-01-01").getTime();
+ return buildSegment(iiName, date1, date2);
+ }
+
+ private IISegment buildSegment(String iiName, long startDate, long endDate) throws Exception {
+ IIInstance iiInstance = iiManager.getII(iiName);
+ IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
+ iiInstance.getSegments().add(segment);
+ iiManager.updateII(iiInstance,true);
+ return segment;
+ }
+
+ private void buildII(String iiName) throws Exception {
+ final IIDesc desc = iiManager.getII(iiName).getDescriptor();
+ final String tableName = createIntermediateTable(desc, kylinConfig);
+ logger.info("intermediate table name:" + tableName);
+ final Configuration conf = new Configuration();
+ HCatInputFormat.setInput(conf, "default", tableName);
+ final HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
+ logger.info(StringUtils.join(tableSchema.getFieldNames(), "\n"));
+ HiveTableReader reader = new HiveTableReader("default", tableName);
+ final List<TblColRef> tblColRefs = desc.listAllColumns();
+ for (TblColRef tblColRef : tblColRefs) {
+ if (desc.isMetricsCol(tblColRef)) {
+ logger.info("matrix:" + tblColRef.getName());
+ } else {
+ logger.info("measure:" + tblColRef.getName());
+ }
+ }
+ LinkedBlockingDeque<StreamMessage> queue = new LinkedBlockingDeque<StreamMessage>();
+ final IISegment segment = createSegment(iiName);
+ String[] args = new String[] { "-iiname", iiName, "-htablename", segment.getStorageLocationIdentifier() };
+ ToolRunner.run(new IICreateHTableJob(), args);
+
+ ExecutorService executorService = Executors.newSingleThreadExecutor();
+ final IIStreamBuilder streamBuilder = new IIStreamBuilder(queue, iiName, segment.getStorageLocationIdentifier(), segment.getIIDesc(), 0);
+
+ List<String[]> sorted = getSortedRows(reader, desc.getTimestampColumn());
+ int count = sorted.size();
+ for (String[] row : sorted) {
+ logger.info("another row: " + StringUtils.join(row, ","));
+ queue.put(parse(row));
+ }
+
+ reader.close();
+ logger.info("total record count:" + count + " htable:" + segment.getStorageLocationIdentifier());
+ queue.put(StreamMessage.EOF);
+ final Future<?> future = executorService.submit(streamBuilder);
+ try {
+ future.get();
+ } catch (Exception e) {
+ logger.error("stream build failed", e);
+ fail("stream build failed");
+ }
+
+ logger.info("stream build finished, htable name:" + segment.getStorageLocationIdentifier());
+ }
+
+ @Test
+ public void test() throws Exception {
+ for (String iiName : II_NAME) {
+ buildII(iiName);
+ IIInstance ii = iiManager.getII(iiName);
+ if (ii.getStatus() != RealizationStatusEnum.READY) {
+ ii.setStatus(RealizationStatusEnum.READY);
+ iiManager.updateII(ii,true);
+ }
+ }
+ }
+
+ private StreamMessage parse(String[] row) {
+ return new StreamMessage(System.currentTimeMillis(), StringUtils.join(row, ",").getBytes());
+ }
+
+ private List<String[]> getSortedRows(HiveTableReader reader, final int tsCol) throws IOException {
+ List<String[]> unsorted = Lists.newArrayList();
+ while (reader.next()) {
+ unsorted.add(reader.getRow());
+ }
+ Collections.sort(unsorted, new Comparator<String[]>() {
+ @Override
+ public int compare(String[] o1, String[] o2) {
+ long t1 = DateFormat.stringToMillis(o1[tsCol]);
+ long t2 = DateFormat.stringToMillis(o2[tsCol]);
+ return Long.compare(t1, t2);
+ }
+ });
+ return unsorted;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ada6b61e/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithEngineTest.java
deleted file mode 100644
index e99bd01..0000000
--- a/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithEngineTest.java
+++ /dev/null
@@ -1,279 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.job.cube.CubingJob;
-import org.apache.kylin.job.cube.CubingJobBuilder;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
-import org.apache.kylin.job.manager.ExecutableManager;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.File;
-import java.lang.reflect.Method;
-import java.text.SimpleDateFormat;
-import java.util.List;
-import java.util.TimeZone;
-import java.util.concurrent.*;
-
-import static org.junit.Assert.assertEquals;
-
-public class ITBuildCubeWithEngineTest {
-
- private JobEngineConfig jobEngineConfig;
-
- private CubeManager cubeManager;
-
- private DefaultScheduler scheduler;
-
- protected ExecutableManager jobService;
-
- private static final Log logger = LogFactory.getLog(ITBuildCubeWithEngineTest.class);
-
- protected void waitForJob(String jobId) {
- while (true) {
- AbstractExecutable job = jobService.getJob(jobId);
- if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) {
- break;
- } else {
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- }
- }
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
- }
-
- @Before
- public void before() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
-
- DeployUtil.initCliWorkDir();
- DeployUtil.deployMetadata();
- DeployUtil.overrideJobJarLocations();
-
-
- final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- jobService = ExecutableManager.getInstance(kylinConfig);
- scheduler = DefaultScheduler.getInstance();
- scheduler.init(new JobEngineConfig(kylinConfig));
- if (!scheduler.hasStarted()) {
- throw new RuntimeException("scheduler has not been started");
- }
- cubeManager = CubeManager.getInstance(kylinConfig);
- jobEngineConfig = new JobEngineConfig(kylinConfig);
- for (String jobId : jobService.getAllJobIds()) {
- if(jobService.getJob(jobId) instanceof CubingJob){
- jobService.deleteJob(jobId);
- }
- }
-
- }
-
- @After
- public void after() {
- HBaseMetadataTestCase.staticCleanupTestMetadata();
- }
-
- @Test
- public void test() throws Exception {
- DeployUtil.prepareTestData("left", "test_kylin_cube_with_slr_left_join_empty");
- testInner();
- testLeft();
- }
-
- private void testInner() throws Exception {
- String[] testCase = new String[]{
- "testInnerJoinCube",
- "testInnerJoinCube2",
- };
- runTestAndAssertSucceed(testCase);
- }
-
- private void testLeft() throws Exception {
- String[] testCase = new String[]{
- "testLeftJoinCube",
- "testLeftJoinCube2",
- };
- runTestAndAssertSucceed(testCase);
- }
-
- private void runTestAndAssertSucceed(String[] testCase) throws Exception {
- ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
- final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
- List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
- for (int i = 0; i < testCase.length; i++) {
- tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch)));
- }
- countDownLatch.await();
- try {
- for (int i = 0; i < tasks.size(); ++i) {
- Future<List<String>> task = tasks.get(i);
- final List<String> jobIds = task.get();
- for (String jobId : jobIds) {
- assertJobSucceed(jobId);
- }
- }
- } catch (Exception ex) {
- logger.error(ex);
- throw ex;
- }
- }
-
- private void assertJobSucceed(String jobId) {
- assertEquals("The job '" + jobId + "' is failed.", ExecutableState.SUCCEED, jobService.getOutput(jobId).getState());
- }
-
- private class TestCallable implements Callable<List<String>> {
-
- private final String methodName;
- private final CountDownLatch countDownLatch;
-
- public TestCallable(String methodName, CountDownLatch countDownLatch) {
- this.methodName = methodName;
- this.countDownLatch = countDownLatch;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public List<String> call() throws Exception {
- try {
- final Method method = ITBuildCubeWithEngineTest.class.getDeclaredMethod(methodName);
- method.setAccessible(true);
- return (List<String>) method.invoke(ITBuildCubeWithEngineTest.this);
- } finally {
- countDownLatch.countDown();
- }
- }
- }
-
- @SuppressWarnings("unused") // called by reflection
- private List<String> testInnerJoinCube2() throws Exception {
- clearSegment("test_kylin_cube_with_slr_empty");
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- long date1 = 0;
- long date2 = f.parse("2013-01-01").getTime();
- long date3 = f.parse("2022-01-01").getTime();
- List<String> result = Lists.newArrayList();
- result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date2));
- result.add(buildSegment("test_kylin_cube_with_slr_empty", date2, date3));
- return result;
- }
-
- @SuppressWarnings("unused") // called by reflection
- private List<String> testInnerJoinCube() throws Exception {
- clearSegment("test_kylin_cube_without_slr_empty");
-
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- // this cube's start date is 0, end date is 20501112000000
- long date1 = 0;
- long date2 = f.parse("2013-01-01").getTime();
-
-
- // this cube doesn't support incremental build, always do full build
-
- List<String> result = Lists.newArrayList();
- result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date2));
- return result;
- }
-
- @SuppressWarnings("unused") // called by reflection
- private List<String> testLeftJoinCube2() throws Exception {
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- List<String> result = Lists.newArrayList();
- final String cubeName = "test_kylin_cube_without_slr_left_join_empty";
- // this cube's start date is 0, end date is 20120601000000
- long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
- long dateEnd = f.parse("2012-06-01").getTime();
-
- clearSegment(cubeName);
- result.add(buildSegment(cubeName, dateStart, dateEnd));
-
- // then submit an append job, start date is 20120601000000, end
- // date is 20220101000000
- dateStart = f.parse("2012-06-01").getTime();
- dateEnd = f.parse("2022-01-01").getTime();
- result.add(buildSegment(cubeName, dateStart, dateEnd));
- return result;
-
- }
-
- @SuppressWarnings("unused") // called by reflection
- private List<String> testLeftJoinCube() throws Exception {
- String cubeName = "test_kylin_cube_with_slr_left_join_empty";
- clearSegment(cubeName);
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
- long dateEnd = f.parse("2050-11-12").getTime();
-
- // this cube's start date is 0, end date is 20501112000000
- List<String> result = Lists.newArrayList();
- result.add(buildSegment(cubeName, dateStart, dateEnd));
- return result;
-
- }
-
- private void clearSegment(String cubeName) throws Exception {
- CubeInstance cube = cubeManager.getCube(cubeName);
- cube.getSegments().clear();
- cubeManager.updateCube(cube,true);
- }
-
-
- private String buildSegment(String cubeName, long startDate, long endDate) throws Exception {
- CubeSegment segment = cubeManager.appendSegments(cubeManager.getCube(cubeName), endDate);
- CubingJobBuilder cubingJobBuilder = new CubingJobBuilder(jobEngineConfig);
- CubingJob job = cubingJobBuilder.buildJob(segment);
- jobService.addJob(job);
- waitForJob(job.getId());
- return job.getId();
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ada6b61e/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithStreamTest.java
deleted file mode 100644
index 8a26397..0000000
--- a/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithStreamTest.java
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- *
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *
- * contributor license agreements. See the NOTICE file distributed with
- *
- * this work for additional information regarding copyright ownership.
- *
- * The ASF licenses this file to You under the Apache License, Version 2.0
- *
- * (the "License"); you may not use this file except in compliance with
- *
- * the License. You may obtain a copy of the License at
- *
- *
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- *
- *
- * Unless required by applicable law or agreed to in writing, software
- *
- * distributed under the License is distributed on an "AS IS" BASIS,
- *
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *
- * See the License for the specific language governing permissions and
- *
- * limitations under the License.
- *
- * /
- */
-
-package org.apache.kylin.job;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Maps;
-import com.google.common.collect.SetMultimap;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hive.hcatalog.data.schema.HCatSchema;
-import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.dict.DictionaryGenerator;
-import org.apache.kylin.dict.lookup.HiveTableReader;
-import org.apache.kylin.job.hadoop.cubev2.IGTRecordWriter;
-import org.apache.kylin.job.hadoop.cubev2.InMemCubeBuilder;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.gridtable.GTRecord;
-import org.junit.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.Nullable;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-/**
- *
- * This class is going to be deleted
- */
-@Ignore("For dev testing")
-public class ITBuildCubeWithStreamTest {
-
- private static final Logger logger = LoggerFactory.getLogger(ITBuildCubeWithStreamTest.class);
-
- private KylinConfig kylinConfig;
- private CubeManager cubeManager;
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
- }
-
- @Before
- public void before() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
- DeployUtil.overrideJobJarLocations();
-
- kylinConfig = KylinConfig.getInstanceFromEnv();
- cubeManager = CubeManager.getInstance(kylinConfig);
-
- }
-
- @After
- public void after() {
- HBaseMetadataTestCase.staticCleanupTestMetadata();
- }
-
- @Test
- public void test() throws Exception {
- CubeInstance cube = cubeManager.getCube("test_kylin_cube_without_slr_left_join_empty");
- final CubeDesc desc = cube.getDescriptor();
- // cube.getSegments().clear();
- // cubeManager.updateCube(cube);
-
- CubeSegment cubeSegment = cube.getSegment("19700101000000_20150401000000", SegmentStatusEnum.NEW);
- Map<TblColRef, Dictionary<?>> dictionaryMap = Maps.newHashMap();
-
-//
- for (DimensionDesc dim : desc.getDimensions()) {
- // dictionary
- for (TblColRef col : dim.getColumnRefs()) {
- if (desc.getRowkey().isUseDictionary(col)) {
- Dictionary dict = cubeSegment.getDictionary(col);
- if (dict == null) {
- throw new IllegalArgumentException("Dictionary for " + col + " was not found.");
- }
- logger.info("Dictionary for " + col + " was put into dictionary map.");
- dictionaryMap.put(col, cubeSegment.getDictionary(col));
- }
- }
- }
-
-// final String tableName = createIntermediateTable(desc, kylinConfig, null);
- String tableName = "kylin_intermediate_test_kylin_cube_without_slr_desc_19700101000000_20130112000000_a24dec89_efbd_425f_9a5f_8b78dd1412af"; // has 3089 records;
-// tableName = "kylin_intermediate_test_kylin_cube_without_slr_desc_19700101000000_20130112000000_a5e1eb5d_da6b_475d_9807_be0b61f03215"; // only 20 rows;
-// tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150302000000_0a183367_f245_43d1_8850_1c138c8514c3";
-// tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150301000000_ce061464_7962_4642_bd7d_7c3d8fbe9389";
- tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150401000000_fb7ae579_d987_4900_a3b7_c60c731cd269"; // 2 million records
- logger.info("intermediate table name:" + tableName);
-
-
- ArrayBlockingQueue queue = new ArrayBlockingQueue<List<String>>(10000);
-
- InMemCubeBuilder cubeBuilder = new InMemCubeBuilder(queue, cube, dictionaryMap, new ConsoleGTRecordWriter());
- ExecutorService executorService = Executors.newSingleThreadExecutor();
- Future<?> future = executorService.submit(cubeBuilder);
-
- final Configuration conf = new Configuration();
- HCatInputFormat.setInput(conf, "default", tableName);
- final HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
- logger.info(StringUtils.join(tableSchema.getFieldNames(), "\n"));
- HiveTableReader reader = new HiveTableReader("default", tableName);
- List<String> row;
- int counter = 0;
- while (reader.next()) {
- row = reader.getRowAsList();
- queue.put(row);
- counter++;
- if(counter == 200000)
- break;
- }
- queue.put(new ArrayList<String>(0));
- reader.close();
-
- try {
- future.get();
- } catch (Exception e) {
- logger.error("stream build failed", e);
- throw new IOException("Failed to build cube ", e);
- }
-
- logger.info("stream build finished");
- }
-
-
- private void buildDictionary(List<List<String>> table, CubeDesc desc, Map<TblColRef, Dictionary<?>> dictionaryMap) {
- SetMultimap<TblColRef, String> valueMap = HashMultimap.create();
-
- List<TblColRef> dimColumns = desc.listDimensionColumnsExcludingDerived();
- for (List<String> row : table) {
- for (int i = 0; i < dimColumns.size(); i++) {
- String cell = row.get(i);
- valueMap.put(dimColumns.get(i), cell);
- }
- }
-
- for (DimensionDesc dim : desc.getDimensions()) {
- // dictionary
- for (TblColRef col : dim.getColumnRefs()) {
- if (desc.getRowkey().isUseDictionary(col)) {
- Dictionary dict = DictionaryGenerator.buildDictionaryFromValueList(col.getType(), Collections2.transform(valueMap.get(col), new Function<String, byte[]>() {
- @Nullable
- @Override
- public byte[] apply(String input) {
- if (input == null)
- return null;
- return input.getBytes();
- }
- }));
-
- logger.info("Building dictionary for " + col);
- dictionaryMap.put(col, dict);
- }
- }
- }
-
- }
-
-
- class ConsoleGTRecordWriter implements IGTRecordWriter {
-
- boolean verbose = false;
-
- @Override
- public void write(Long cuboidId, GTRecord record) throws IOException {
- if (verbose)
- System.out.println(record.toString());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ada6b61e/job/src/test/java/org/apache/kylin/job/ITBuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITBuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/ITBuildIIWithEngineTest.java
deleted file mode 100644
index 23f97d0..0000000
--- a/job/src/test/java/org/apache/kylin/job/ITBuildIIWithEngineTest.java
+++ /dev/null
@@ -1,254 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
-import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
-import org.apache.kylin.job.invertedindex.IIJob;
-import org.apache.kylin.job.invertedindex.IIJobBuilder;
-import org.apache.kylin.job.manager.ExecutableManager;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.junit.*;
-
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.Method;
-import java.text.SimpleDateFormat;
-import java.util.List;
-import java.util.TimeZone;
-import java.util.concurrent.*;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * @author shaoshi
- */
-public class ITBuildIIWithEngineTest {
-
- private JobEngineConfig jobEngineConfig;
- private IIManager iiManager;
-
- private DefaultScheduler scheduler;
- protected ExecutableManager jobService;
-
- protected static final String[] TEST_II_INSTANCES = new String[] { "test_kylin_ii_inner_join", "test_kylin_ii_left_join" };
-
- private static final Log logger = LogFactory.getLog(ITBuildIIWithEngineTest.class);
-
- protected void waitForJob(String jobId) {
- while (true) {
- AbstractExecutable job = jobService.getJob(jobId);
- if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) {
- break;
- } else {
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- }
- }
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
- }
-
- @Before
- public void before() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
-
- //DeployUtil.initCliWorkDir();
- // DeployUtil.deployMetadata();
- DeployUtil.overrideJobJarLocations();
-
- final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- jobService = ExecutableManager.getInstance(kylinConfig);
- scheduler = DefaultScheduler.getInstance();
- scheduler.init(new JobEngineConfig(kylinConfig));
- if (!scheduler.hasStarted()) {
- throw new RuntimeException("scheduler has not been started");
- }
- jobEngineConfig = new JobEngineConfig(kylinConfig);
- for (String jobId : jobService.getAllJobIds()) {
- if (jobService.getJob(jobId) instanceof IIJob) {
- jobService.deleteJob(jobId);
- }
- }
-
- iiManager = IIManager.getInstance(kylinConfig);
- for (String iiInstance : TEST_II_INSTANCES) {
-
- IIInstance ii = iiManager.getII(iiInstance);
- if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
- ii.setStatus(RealizationStatusEnum.DISABLED);
- iiManager.updateII(ii, true);
- }
- }
- }
-
- @After
- public void after() throws Exception {
-
- for (String iiInstance : TEST_II_INSTANCES) {
- IIInstance ii = iiManager.getII(iiInstance);
- if (ii.getStatus() != RealizationStatusEnum.READY) {
- ii.setStatus(RealizationStatusEnum.READY);
- iiManager.updateII(ii, true);
- }
- }
- backup();
- }
-
- @Test
- @Ignore
- public void testBuildII() throws Exception {
-
- String[] testCase = new String[] { "buildIIInnerJoin", "buildIILeftJoin" };
- ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
- final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
- List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
- for (int i = 0; i < testCase.length; i++) {
- tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch)));
- }
- countDownLatch.await();
- for (int i = 0; i < tasks.size(); ++i) {
- Future<List<String>> task = tasks.get(i);
- final List<String> jobIds = task.get();
- for (String jobId : jobIds) {
- assertJobSucceed(jobId);
- }
- }
-
- }
-
- private void assertJobSucceed(String jobId) {
- assertEquals(ExecutableState.SUCCEED, jobService.getOutput(jobId).getState());
- }
-
- private class TestCallable implements Callable<List<String>> {
-
- private final String methodName;
- private final CountDownLatch countDownLatch;
-
- public TestCallable(String methodName, CountDownLatch countDownLatch) {
- this.methodName = methodName;
- this.countDownLatch = countDownLatch;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public List<String> call() throws Exception {
- try {
- final Method method = ITBuildIIWithEngineTest.class.getDeclaredMethod(methodName);
- method.setAccessible(true);
- return (List<String>) method.invoke(ITBuildIIWithEngineTest.this);
- } finally {
- countDownLatch.countDown();
- }
- }
- }
-
- protected List<String> buildIIInnerJoin() throws Exception {
- return buildII(TEST_II_INSTANCES[0]);
- }
-
- protected List<String> buildIILeftJoin() throws Exception {
- return buildII(TEST_II_INSTANCES[1]);
- }
-
- protected List<String> buildII(String iiName) throws Exception {
- clearSegment(iiName);
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- long date1 = 0;
- long date2 = f.parse("2015-01-01").getTime();
-
- List<String> result = Lists.newArrayList();
- result.add(buildSegment(iiName, date1, date2));
- return result;
- }
-
- private void clearSegment(String iiName) throws Exception {
- IIInstance ii = iiManager.getII(iiName);
- ii.getSegments().clear();
- iiManager.updateII(ii,true);
- }
-
- private String buildSegment(String iiName, long startDate, long endDate) throws Exception {
- IIInstance iiInstance = iiManager.getII(iiName);
- IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
- iiInstance.getSegments().add(segment);
- iiManager.updateII(iiInstance, true);
- IIJobBuilder iiJobBuilder = new IIJobBuilder(jobEngineConfig);
- IIJob job = iiJobBuilder.buildJob(segment);
- jobService.addJob(job);
- waitForJob(job.getId());
- return job.getId();
- }
-
- private int cleanupOldStorage() throws Exception {
- String[] args = { "--delete", "true" };
-
- int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
- return exitCode;
- }
-
- private void backup() throws Exception {
- int exitCode = cleanupOldStorage();
- if (exitCode == 0) {
- exportHBaseData();
- }
- }
-
- private void exportHBaseData() throws IOException {
- ExportHBaseData export = new ExportHBaseData();
- export.exportTables();
- }
-
- public static void main(String[] args) throws Exception {
- ITBuildIIWithEngineTest instance = new ITBuildIIWithEngineTest();
-
- ITBuildIIWithEngineTest.beforeClass();
- instance.before();
- instance.testBuildII();
- instance.after();
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ada6b61e/job/src/test/java/org/apache/kylin/job/ITBuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITBuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/ITBuildIIWithStreamTest.java
deleted file mode 100644
index cfb7bdb..0000000
--- a/job/src/test/java/org/apache/kylin/job/ITBuildIIWithStreamTest.java
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- *
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *
- * contributor license agreements. See the NOTICE file distributed with
- *
- * this work for additional information regarding copyright ownership.
- *
- * The ASF licenses this file to You under the Apache License, Version 2.0
- *
- * (the "License"); you may not use this file except in compliance with
- *
- * the License. You may obtain a copy of the License at
- *
- *
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- *
- *
- * Unless required by applicable law or agreed to in writing, software
- *
- * distributed under the License is distributed on an "AS IS" BASIS,
- *
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *
- * See the License for the specific language governing permissions and
- *
- * limitations under the License.
- *
- * /
- */
-
-package org.apache.kylin.job;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hive.hcatalog.data.schema.HCatSchema;
-import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.dict.lookup.HiveTableReader;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.invertedindex.model.IIJoinedFlatTableDesc;
-import org.apache.kylin.job.common.ShellExecutable;
-import org.apache.kylin.job.constant.ExecutableConstants;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
-import org.apache.kylin.job.hadoop.invertedindex.IICreateHTableJob;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.apache.kylin.streaming.StreamMessage;
-import org.apache.kylin.streaming.invertedindex.IIStreamBuilder;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingDeque;
-
-import static org.junit.Assert.fail;
-
-/**
- */
-public class ITBuildIIWithStreamTest {
-
- private static final Logger logger = LoggerFactory.getLogger(ITBuildIIWithStreamTest.class);
-
- private static final String[] II_NAME = new String[] { "test_kylin_ii_left_join", "test_kylin_ii_inner_join" };
- private IIManager iiManager;
- private KylinConfig kylinConfig;
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
- }
-
- @Before
- public void before() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
- DeployUtil.overrideJobJarLocations();
-
- kylinConfig = KylinConfig.getInstanceFromEnv();
- iiManager = IIManager.getInstance(kylinConfig);
- iiManager = IIManager.getInstance(kylinConfig);
- for (String iiInstance : II_NAME) {
-
- IIInstance ii = iiManager.getII(iiInstance);
- if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
- ii.setStatus(RealizationStatusEnum.DISABLED);
- iiManager.updateII(ii,true);
- }
- }
- }
-
- @AfterClass
- public static void afterClass() throws Exception {
- backup();
- }
-
- private static int cleanupOldStorage() throws Exception {
- String[] args = { "--delete", "true" };
- int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
- return exitCode;
- }
-
- private static void backup() throws Exception {
- int exitCode = cleanupOldStorage();
- if (exitCode == 0) {
- exportHBaseData();
- }
- }
-
- private static void exportHBaseData() throws IOException {
- ExportHBaseData export = new ExportHBaseData();
- export.exportTables();
- }
-
- private String createIntermediateTable(IIDesc desc, KylinConfig kylinConfig) throws IOException {
- IIJoinedFlatTableDesc intermediateTableDesc = new IIJoinedFlatTableDesc(desc);
- JobEngineConfig jobEngineConfig = new JobEngineConfig(kylinConfig);
- final String uuid = UUID.randomUUID().toString();
- final String dropTableHql = JoinedFlatTable.generateDropTableStatement(intermediateTableDesc, uuid);
- final String createTableHql = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, jobEngineConfig.getHdfsWorkingDirectory() + "/kylin-" + uuid, uuid);
- String insertDataHqls;
- try {
- insertDataHqls = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, uuid, jobEngineConfig);
- } catch (IOException e1) {
- e1.printStackTrace();
- throw new RuntimeException("Failed to generate insert data SQL for intermediate table.");
- }
-
- ShellExecutable step = new ShellExecutable();
- StringBuffer buf = new StringBuffer();
- buf.append("hive -e \"");
- buf.append(dropTableHql + "\n");
- buf.append(createTableHql + "\n");
- buf.append(insertDataHqls + "\n");
- buf.append("\"");
-
- step.setCmd(buf.toString());
- logger.info(step.getCmd());
- step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);
- kylinConfig.getCliCommandExecutor().execute(step.getCmd(), null);
- return intermediateTableDesc.getTableName(uuid);
- }
-
- private void clearSegment(String iiName) throws Exception {
- IIInstance ii = iiManager.getII(iiName);
- ii.getSegments().clear();
- iiManager.updateII(ii,true);
- }
-
- private IISegment createSegment(String iiName) throws Exception {
- clearSegment(iiName);
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- long date1 = 0;
- long date2 = f.parse("2015-01-01").getTime();
- return buildSegment(iiName, date1, date2);
- }
-
- private IISegment buildSegment(String iiName, long startDate, long endDate) throws Exception {
- IIInstance iiInstance = iiManager.getII(iiName);
- IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
- iiInstance.getSegments().add(segment);
- iiManager.updateII(iiInstance,true);
- return segment;
- }
-
- private void buildII(String iiName) throws Exception {
- final IIDesc desc = iiManager.getII(iiName).getDescriptor();
- final String tableName = createIntermediateTable(desc, kylinConfig);
- logger.info("intermediate table name:" + tableName);
- final Configuration conf = new Configuration();
- HCatInputFormat.setInput(conf, "default", tableName);
- final HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
- logger.info(StringUtils.join(tableSchema.getFieldNames(), "\n"));
- HiveTableReader reader = new HiveTableReader("default", tableName);
- final List<TblColRef> tblColRefs = desc.listAllColumns();
- for (TblColRef tblColRef : tblColRefs) {
- if (desc.isMetricsCol(tblColRef)) {
- logger.info("matrix:" + tblColRef.getName());
- } else {
- logger.info("measure:" + tblColRef.getName());
- }
- }
- LinkedBlockingDeque<StreamMessage> queue = new LinkedBlockingDeque<StreamMessage>();
- final IISegment segment = createSegment(iiName);
- String[] args = new String[] { "-iiname", iiName, "-htablename", segment.getStorageLocationIdentifier() };
- ToolRunner.run(new IICreateHTableJob(), args);
-
- ExecutorService executorService = Executors.newSingleThreadExecutor();
- final IIStreamBuilder streamBuilder = new IIStreamBuilder(queue, iiName, segment.getStorageLocationIdentifier(), segment.getIIDesc(), 0);
-
- List<String[]> sorted = getSortedRows(reader, desc.getTimestampColumn());
- int count = sorted.size();
- for (String[] row : sorted) {
- logger.info("another row: " + StringUtils.join(row, ","));
- queue.put(parse(row));
- }
-
- reader.close();
- logger.info("total record count:" + count + " htable:" + segment.getStorageLocationIdentifier());
- queue.put(StreamMessage.EOF);
- final Future<?> future = executorService.submit(streamBuilder);
- try {
- future.get();
- } catch (Exception e) {
- logger.error("stream build failed", e);
- fail("stream build failed");
- }
-
- logger.info("stream build finished, htable name:" + segment.getStorageLocationIdentifier());
- }
-
- @Test
- public void test() throws Exception {
- for (String iiName : II_NAME) {
- buildII(iiName);
- IIInstance ii = iiManager.getII(iiName);
- if (ii.getStatus() != RealizationStatusEnum.READY) {
- ii.setStatus(RealizationStatusEnum.READY);
- iiManager.updateII(ii,true);
- }
- }
- }
-
- private StreamMessage parse(String[] row) {
- return new StreamMessage(System.currentTimeMillis(), StringUtils.join(row, ",").getBytes());
- }
-
- private List<String[]> getSortedRows(HiveTableReader reader, final int tsCol) throws IOException {
- List<String[]> unsorted = Lists.newArrayList();
- while (reader.next()) {
- unsorted.add(reader.getRow());
- }
- Collections.sort(unsorted, new Comparator<String[]>() {
- @Override
- public int compare(String[] o1, String[] o2) {
- long t1 = DateFormat.stringToMillis(o1[tsCol]);
- long t2 = DateFormat.stringToMillis(o2[tsCol]);
- return Long.compare(t1, t2);
- }
- });
- return unsorted;
- }
-
-}
[09/11] incubator-kylin git commit: KYLIN-697 non-integration tests
passed after merging codes
Posted by ma...@apache.org.
KYLIN-697 non-integration tests passed after merging codes
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/148576ea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/148576ea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/148576ea
Branch: refs/heads/new697
Commit: 148576eaaa415e6d0b75d06127c65823595630f5
Parents: 42f124f
Author: honma <ho...@ebay.com>
Authored: Mon May 25 13:19:51 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon May 25 13:19:51 2015 +0800
----------------------------------------------------------------------
.../kylin/common/restclient/AbstractRestCache.java | 1 +
.../kylin/common/restclient/SingleValueCache.java | 13 ++++++++-----
.../java/org/apache/kylin/cube/CubeManagerTest.java | 6 +++---
.../apache/kylin/rest/service/CacheServiceTest.java | 1 +
4 files changed, 13 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/148576ea/common/src/main/java/org/apache/kylin/common/restclient/AbstractRestCache.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/restclient/AbstractRestCache.java b/common/src/main/java/org/apache/kylin/common/restclient/AbstractRestCache.java
index d9e98fd..c728ff2 100644
--- a/common/src/main/java/org/apache/kylin/common/restclient/AbstractRestCache.java
+++ b/common/src/main/java/org/apache/kylin/common/restclient/AbstractRestCache.java
@@ -24,6 +24,7 @@ package org.apache.kylin.common.restclient;
*/
public abstract class AbstractRestCache<K, V> {
+
protected final Broadcaster.TYPE syncType;
protected AbstractRestCache(Broadcaster.TYPE syncType) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/148576ea/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java b/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
index fb44206..00f8a3e 100644
--- a/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
+++ b/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
@@ -43,10 +43,12 @@ public abstract class SingleValueCache<K, V> extends AbstractRestCache<K, V> {
}
public void put(K key, V value) {
- //enforce all cache changes coming from REST
- //final V result = innerCache.put(key, value);
+ boolean exists = innerCache.containsKey(key);
+ //The put operation will be duplicated when REST request is received.
+ //It is intended so because many test cases does not have REST env
+ innerCache.put(key, value);
- if (!innerCache.containsKey(key)) {
+ if (!exists) {
syncRemote(key, Broadcaster.EVENT.CREATE);
} else {
syncRemote(key, Broadcaster.EVENT.UPDATE);
@@ -59,8 +61,9 @@ public abstract class SingleValueCache<K, V> extends AbstractRestCache<K, V> {
public void remove(K key) {
if (innerCache.containsKey(key)) {
- //enforce all cache changes coming from REST
- //innerCache.remove(key);
+ //The remove operation will be duplicated when REST request is received.
+ //It is intended so because many test cases does not have REST env
+ innerCache.remove(key);
syncRemote(key, Broadcaster.EVENT.DROP);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/148576ea/cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java b/cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
index 8970295..3b0bd3f 100644
--- a/cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
+++ b/cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
@@ -18,8 +18,6 @@
package org.apache.kylin.cube;
-import static org.junit.Assert.*;
-
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.JsonUtil;
@@ -31,6 +29,8 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import static org.junit.Assert.*;
+
/**
* @author yangli9
*/
@@ -75,7 +75,7 @@ public class CubeManagerTest extends LocalFileMetadataTestCase {
assertTrue(prjMgr.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME).contains(createdCube));
- CubeInstance droppedCube = CubeManager.getInstance(getTestConfig()).dropCube("a_whole_new_cube", true);
+ CubeInstance droppedCube = CubeManager.getInstance(getTestConfig()).dropCube("a_whole_new_cube", false);
assertTrue(createdCube == droppedCube);
assertTrue(!prjMgr.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME).contains(droppedCube));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/148576ea/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
index 9f0598d..795afaf 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
@@ -125,6 +125,7 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
@Before
public void setUp() throws Exception {
counter.set(0L);
+ createTestMetadata();
}
@After
[07/11] incubator-kylin git commit: fix ci: new model API changed
Posted by ma...@apache.org.
fix ci: new model API changed
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/08eedde3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/08eedde3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/08eedde3
Branch: refs/heads/new697
Commit: 08eedde320fb54b0785dd436e4a41d3a7068995a
Parents: 4d323f8
Author: honma <ho...@ebay.com>
Authored: Mon May 25 10:33:23 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon May 25 11:24:16 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/rest/controller/CubeControllerTest.java | 8 ++------
1 file changed, 2 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/08eedde3/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
index 1ec27d5..d2a948b 100644
--- a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
@@ -22,7 +22,6 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.rest.request.CubeRequest;
import org.apache.kylin.rest.service.CubeService;
import org.apache.kylin.rest.service.JobService;
@@ -66,7 +65,7 @@ public class CubeControllerTest extends ServiceTestBase {
CubeDesc[] cubes = (CubeDesc[]) cubeDescController.getCube("test_kylin_cube_with_slr_ready");
Assert.assertNotNull(cubes);
Assert.assertNotNull(cubeController.getSql("test_kylin_cube_with_slr_ready", "20130331080000_20131212080000"));
- Assert.assertNotNull(cubeController.getCubes(null, null, null,0, 5));
+ Assert.assertNotNull(cubeController.getCubes(null, null, null, 0, 5));
CubeDesc cube = cubes[0];
CubeDesc newCube = new CubeDesc();
@@ -87,8 +86,6 @@ public class CubeControllerTest extends ServiceTestBase {
newCube.setConfig(cube.getConfig());
newCube.setRowkey(cube.getRowkey());
- String newModelName = newCubeName + "_model_desc";
- newCube.getModel().setName(newModelName);//generate a random model
newCube.getModel().setLastModified(0);
ObjectMapper cubeDescMapper = new ObjectMapper();
@@ -103,13 +100,12 @@ public class CubeControllerTest extends ServiceTestBase {
cubeRequest.setCubeDescData(cubeDescWriter.toString());
cubeRequest = cubeController.saveCubeDesc(cubeRequest);
-
List<String> notifyList = Lists.newArrayList();
notifyList.add("john@example.com");
cubeController.updateNotifyList(newCubeName, notifyList);
cubeController.updateCubeCost(newCubeName, 80);
- List<CubeInstance> cubeInstances = cubeController.getCubes(newCubeName, "default",null, 1, 0);
+ List<CubeInstance> cubeInstances = cubeController.getCubes(newCubeName, cube.getModelName(),"default", 1, 0);
CubeInstance cubeInstance = cubeInstances.get(0);
Assert.assertTrue(cubeInstance.getDescriptor().getNotifyList().contains("john@example.com"));