You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/06/09 08:28:21 UTC
[1/4] incubator-kylin git commit: Metadata check change for 0.6
migration
Repository: incubator-kylin
Updated Branches:
refs/heads/0.7-staging b3b0f2f95 -> b32f9b301
Metadata check change for 0.6 migration
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/3647e6bc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/3647e6bc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/3647e6bc
Branch: refs/heads/0.7-staging
Commit: 3647e6bc8363693e98b0a6d15d633ada7c6c7b0a
Parents: f5dc6b9
Author: shaofengshi <sh...@apache.org>
Authored: Fri Jun 5 14:33:03 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Jun 5 14:33:03 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/model/CubeDesc.java | 46 ++++++++++--
.../kylin/metadata/MetadataUpgradeTest.java | 17 +++--
.../apache/kylin/job/CubeMetadataUpgrade.java | 77 +++++++++++---------
3 files changed, 95 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3647e6bc/cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index f236c4e..4959685 100644
--- a/cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -456,7 +456,7 @@ public class CubeDesc extends RootPersistentEntity {
// check all dimension columns are presented on rowkey
List<TblColRef> dimCols = listDimensionColumnsExcludingDerived();
if (rowkey.getRowKeyColumns().length != dimCols.size()) {
- addError("RowKey columns count (" + rowkey.getRowKeyColumns().length + ") does not match dimension columns count (" + dimCols.size() + "). ");
+ addError("RowKey columns count (" + rowkey.getRowKeyColumns().length + ") does not match dimension columns count (" + dimCols.size() + "); rowkeys are " + Arrays.toString(rowkey.getRowKeyColumns()) + ", dimension columns are " + dimCols.toString());
}
}
@@ -467,7 +467,7 @@ public class CubeDesc extends RootPersistentEntity {
// init dimension columns
ArrayList<TblColRef> dimCols = Lists.newArrayList();
String[] colStrs = dim.getColumn();
-
+
// when column is omitted, special case
if (colStrs == null && dim.isDerived() || ArrayUtils.contains(colStrs, "{FK}")) {
for (TblColRef col : join.getForeignKeyColumns()) {
@@ -478,18 +478,18 @@ public class CubeDesc extends RootPersistentEntity {
else {
if (colStrs == null || colStrs.length == 0)
throw new IllegalStateException("Dimension column must not be blank " + dim);
-
+
for (String colStr : colStrs) {
dimCols.add(initDimensionColRef(dim, colStr));
}
-
+
// fill back column ref in hierarchy
if (dim.isHierarchy()) {
for (int i = 0; i < dimCols.size(); i++)
dim.getHierarchy()[i].setColumnRef(dimCols.get(i));
}
}
-
+
TblColRef[] dimColArray = (TblColRef[]) dimCols.toArray(new TblColRef[dimCols.size()]);
dim.setColumnRefs(dimColArray);
@@ -518,6 +518,8 @@ public class CubeDesc extends RootPersistentEntity {
initDerivedMap(hostCols[find], DeriveType.PK_FK, dim, derivedCol);
}
}
+
+ /** disable this code as we don't need fk be derived from pk
for (int i = 0; i < pk.length; i++) {
int find = ArrayUtils.indexOf(hostCols, pk[i]);
if (find >= 0) {
@@ -525,6 +527,40 @@ public class CubeDesc extends RootPersistentEntity {
initDerivedMap(hostCols[find], DeriveType.PK_FK, dim, derivedCol);
}
}
+ */
+ }
+ }
+
+
+ {
+ // drop those columns (like lookup table's PK but used as a dimension) that can be derived
+ List<TblColRef> derivedCols = Lists.newArrayList();
+ for (TblColRef col : dimensionColumns) {
+ if (isDerived(col) == true)
+ derivedCols.add(col);
+ }
+
+ if (derivedCols.size() > 0) {
+ dimensionColumns.removeAll(derivedCols);
+ }
+ }
+
+ {
+ // for hierarchy columns, if a column can be derived, use the host col to replace
+ for (DimensionDesc dim : dimensions) {
+ if (dim.isHierarchy()) {
+ for (int i = 0; i < dim.getColumnRefs().length; i++) {
+ TblColRef colRef = dim.getColumnRefs()[i];
+ if (isDerived(colRef)) {
+ TblColRef[] hostCols = derivedToHostMap.get(colRef).columns;
+ if (hostCols.length == 1) {
+ dim.getHierarchy()[i].setColumnRef(hostCols[0]);
+ } else {
+ throw new IllegalStateException();
+ }
+ }
+ }
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3647e6bc/cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java b/cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java
index 4b1459c..e8feb87 100644
--- a/cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java
+++ b/cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java
@@ -22,6 +22,7 @@ import java.util.List;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.project.ProjectManager;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -47,7 +48,7 @@ public class MetadataUpgradeTest extends LocalFileMetadataTestCase {
@Before
public void setUp() throws Exception {
- createTestMetadata(LOCALMETA_TEST_DATA_V1);
+ createTestMetadata("../meta_upgrade/");
}
@After
@@ -58,14 +59,16 @@ public class MetadataUpgradeTest extends LocalFileMetadataTestCase {
@Test
public void testCubeDescUpgrade() throws Exception {
- String[] sampleCubeDescs = new String[] { "test_kylin_cube_with_slr_desc", "test_kylin_cube_with_slr_left_join_desc", "test_kylin_cube_without_slr_desc", "test_kylin_cube_without_slr_left_join_desc" };
-
- for (String name : sampleCubeDescs)
- checkCubeDesc(name);
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ MetadataManager.getInstance(config).reload();
+ CubeDescManager.clearCache();
+ CubeDescManager.getInstance(config);
+// CubeManager cubeManager = CubeManager.getInstance(config);
+// ProjectManager.getInstance(config);
}
-
- @Test
+
+ //@Test
public void testTableDescUpgrade() throws Exception {
MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3647e6bc/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java b/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
index ae03e26..e817dd8 100644
--- a/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
+++ b/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
@@ -38,10 +38,6 @@ import org.apache.kylin.cube.model.v1.CubeInstance;
import org.apache.kylin.cube.model.v1.CubeSegment;
import org.apache.kylin.cube.model.v1.CubeSegmentStatusEnum;
import org.apache.kylin.cube.model.v1.CubeStatusEnum;
-import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.dict.lookup.SnapshotManager;
-import org.apache.kylin.dict.lookup.SnapshotTable;
-import org.apache.kylin.dict.lookup.TableReader;
import org.apache.kylin.job.common.HadoopShellExecutable;
import org.apache.kylin.job.common.MapReduceExecutable;
import org.apache.kylin.job.common.ShellExecutable;
@@ -69,7 +65,10 @@ import org.apache.kylin.metadata.realization.RealizationType;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
-import java.util.*;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
/**
@@ -104,13 +103,13 @@ public class CubeMetadataUpgrade {
upgradeCubeDesc();
upgradeProjectInstance();
upgradeCubeInstance();
- // upgradeJobInstance();
+ upgradeJobInstance();
verify();
}
- private void verify() {
+ public void verify() {
MetadataManager.getInstance(config).reload();
CubeDescManager.clearCache();
CubeDescManager.getInstance(config);
@@ -629,49 +628,61 @@ public class CubeMetadataUpgrade {
public static void main(String[] args) {
- if (!(args != null && args.length == 1)) {
- System.out.println("Usage: java CubeMetadataUpgrade <metadata_export_folder>; e.g, /export/kylin/meta");
+ if (!(args != null && (args.length == 1 || args.length == 2))) {
+ System.out.println("Usage: java CubeMetadataUpgrade <metadata_export_folder> <verify>; e.g, /export/kylin/meta ");
return;
}
String exportFolder = args[0];
-
- File oldMetaFolder = new File(exportFolder);
- if (!oldMetaFolder.exists()) {
- System.out.println("Provided folder doesn't exist: '" + exportFolder + "'");
- return;
+ boolean verify = false;
+ if (args.length == 2 && "verify".equals(args[1])) {
+ System.out.println("Only verify the metadata in folder " + exportFolder);
+ verify = true;
}
- if (!oldMetaFolder.isDirectory()) {
- System.out.println("Provided folder is not a directory: '" + exportFolder + "'");
- return;
- }
+ CubeMetadataUpgrade instance = null;
+ if (verify) {
+ instance = new CubeMetadataUpgrade(exportFolder);
+ instance.verify();
+ } else {
+ File oldMetaFolder = new File(exportFolder);
+ if (!oldMetaFolder.exists()) {
+ System.out.println("Provided folder doesn't exist: '" + exportFolder + "'");
+ return;
+ }
+ if (!oldMetaFolder.isDirectory()) {
+ System.out.println("Provided folder is not a directory: '" + exportFolder + "'");
+ return;
+ }
- String newMetadataUrl = oldMetaFolder.getAbsolutePath() + "_v2";
- try {
- FileUtils.deleteDirectory(new File(newMetadataUrl));
- FileUtils.copyDirectory(oldMetaFolder, new File(newMetadataUrl));
- } catch (IOException e) {
- e.printStackTrace();
- }
+ String newMetadataUrl = oldMetaFolder.getAbsolutePath() + "_v2";
+ try {
+ FileUtils.deleteDirectory(new File(newMetadataUrl));
+ FileUtils.copyDirectory(oldMetaFolder, new File(newMetadataUrl));
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
- CubeMetadataUpgrade instance = new CubeMetadataUpgrade(newMetadataUrl);
+ instance = new CubeMetadataUpgrade(newMetadataUrl);
+ instance.upgrade();
+ logger.info("=================================================================");
+ logger.info("Run CubeMetadataUpgrade completed; The following resources have been successfully updated : ");
+ for (String s : instance.updatedResources) {
+ logger.info(s);
+ }
- instance.upgrade();
- logger.info("=================================================================");
- logger.info("Run CubeMetadataUpgrade completed; The following resources have been successfully updated in : " + newMetadataUrl);
- for (String s : instance.updatedResources) {
- logger.info(s);
}
logger.info("=================================================================");
- if (instance.errorMsgs.size() > 0) {
+ if (instance.errorMsgs.size() > 0)
+ {
logger.info("Here are the error/warning messages, you may need check:");
for (String s : instance.errorMsgs) {
logger.warn(s);
}
- } else {
+ } else
+ {
logger.info("No error or warning messages; The migration is success.");
}
}
[3/4] incubator-kylin git commit: Misc minor changes,
mainly for 0.6 metadata upgrade
Posted by sh...@apache.org.
Misc minor changes, mainly for 0.6 metadata upgrade
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/f7823911
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/f7823911
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/f7823911
Branch: refs/heads/0.7-staging
Commit: f78239118f3d6871679f465607d03d825bb22e1b
Parents: a6b17c4
Author: shaofengshi <sh...@apache.org>
Authored: Tue Jun 9 14:19:36 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Jun 9 14:19:36 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfig.java | 3 +-
.../apache/kylin/job/CubeMetadataUpgrade.java | 117 ++++++++++++-------
.../org/apache/kylin/job/cube/CubingJob.java | 2 +-
.../kylin/metadata/model/PartitionDesc.java | 27 ++++-
.../kylin/rest/controller/CubeController.java | 2 -
5 files changed, 106 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f7823911/common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 08a4d66..cfddbe7 100644
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -19,7 +19,6 @@
package org.apache.kylin.common;
import com.google.common.collect.Sets;
-import jodd.util.StringUtil;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.io.IOUtils;
@@ -473,7 +472,7 @@ public class KylinConfig {
private String[] getOptionalStringArray(String prop) {
final String property = System.getProperty(prop);
- if (!StringUtil.isBlank(property))
+ if (!StringUtils.isBlank(property))
return property.split("\\s*,\\s*");
return kylinConfig.getStringArray(prop);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f7823911/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java b/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
index e817dd8..0b06323 100644
--- a/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
+++ b/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
@@ -65,10 +65,7 @@ import org.apache.kylin.metadata.realization.RealizationType;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
/**
@@ -103,52 +100,90 @@ public class CubeMetadataUpgrade {
upgradeCubeDesc();
upgradeProjectInstance();
upgradeCubeInstance();
- upgradeJobInstance();
-
+ //upgradeJobInstance();
verify();
}
- public void verify() {
+ public void cleanup() {
MetadataManager.getInstance(config).reload();
- CubeDescManager.clearCache();
CubeDescManager.getInstance(config);
CubeManager cubeManager = CubeManager.getInstance(config);
- ProjectManager.getInstance(config);
- /*
- DictionaryManager dictManager = DictionaryManager.getInstance(config);
- SnapshotManager snapshotManager = SnapshotManager.getInstance(config);
- List<org.apache.kylin.cube.CubeInstance> allCubes = cubeManager.listAllCubes();
- for (org.apache.kylin.cube.CubeInstance cube : allCubes) {
- for (org.apache.kylin.cube.CubeSegment cubeSegment : cube.getSegments()) {
- Collection<String> snapshots = cubeSegment.getSnapshots().values();
- for (String s : snapshots) {
- try {
- SnapshotTable t = snapshotManager.getSnapshotTable(s);
- TableReader reader = t.getReader();
- while (reader.next()) {
- System.out.println(Arrays.toString(reader.getRow()));
+ List<String> activeResourceList = Lists.newArrayList();
+ for (org.apache.kylin.cube.CubeInstance cube : cubeManager.listAllCubes()) {
+ for (org.apache.kylin.cube.CubeSegment segment : cube.getSegments()) {
+ activeResourceList.addAll(segment.getSnapshotPaths());
+ activeResourceList.addAll(segment.getDictionaryPaths());
+ }
+ }
+
+ List<String> toDeleteResource = Lists.newArrayList();
+ List<String> activeResource = Lists.newArrayList();
+ try {
+ ArrayList<String> snapshotTables = getStore().listResources(ResourceStore.SNAPSHOT_RESOURCE_ROOT);
+
+ for (String snapshotTable : snapshotTables) {
+ ArrayList<String> snapshotNames = getStore().listResources(snapshotTable);
+ if (snapshotNames != null)
+ for (String snapshot : snapshotNames) {
+ if (!activeResourceList.contains(snapshot)) {
+ toDeleteResource.add(snapshot);
+
+ } else {
+ activeResource.add(snapshot);
}
- } catch (IOException e) {
- e.printStackTrace();
}
- }
-
- Collection<String> dicts = cubeSegment.getDictionaries().values();
- for (String s : dicts) {
- try {
- org.apache.kylin.dict.Dictionary<?> dict = dictManager.getDictionary(s);
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
- System.out.println(dict.getMaxId());
- } catch (IOException e) {
- e.printStackTrace();
+ try {
+ ArrayList<String> dictTables = getStore().listResources(ResourceStore.DICT_RESOURCE_ROOT);
+
+ for (String table : dictTables) {
+ ArrayList<String> tableColNames = getStore().listResources(table);
+ if (tableColNames != null)
+ for (String tableCol : tableColNames) {
+ ArrayList<String> dictionaries = getStore().listResources(tableCol);
+ if (dictionaries != null)
+ for (String dict : dictionaries)
+ if (!activeResourceList.contains(dict)) {
+ toDeleteResource.add(dict);
+ } else {
+ activeResource.add(dict);
+ }
}
- }
}
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+
+ if (toDeleteResource.size() > 0) {
+ logger.info("The following resources is never needed, will be dropped, number :" + toDeleteResource.size());
+ for (String s : toDeleteResource) {
+ logger.info(s);
+ try {
+ getStore().deleteResource(s);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
}
- */
+
+ }
+
+ public void verify() {
+ MetadataManager.getInstance(config).reload();
+ CubeDescManager.clearCache();
+ CubeDescManager.getInstance(config);
+ CubeManager cubeManager = CubeManager.getInstance(config);
+ ProjectManager.getInstance(config);
+ //cleanup();
+
}
private List<String> listResourceStore(String pathRoot) {
@@ -478,9 +513,15 @@ public class CubeMetadataUpgrade {
private void upgradeJobInstance(String path) throws IOException {
JobInstance job = getStore().getResource(path, JobInstance.class, new JsonSerializer<JobInstance>(JobInstance.class));
+ long lastModified = job.getLastModified();
+ if (System.currentTimeMillis() - lastModified > 2592000000l) {
+ // old than 30 days, skip;
+ return;
+ }
CubingJob cubingJob = new CubingJob();
cubingJob.setId(job.getId());
cubingJob.setName(job.getName());
+ cubingJob.setCubeName(job.getRelatedCube());
cubingJob.setSubmitter(job.getSubmitter());
for (JobInstance.JobStep step : job.getSteps()) {
final AbstractExecutable executable = parseToExecutable(step);
@@ -675,14 +716,12 @@ public class CubeMetadataUpgrade {
}
logger.info("=================================================================");
- if (instance.errorMsgs.size() > 0)
- {
+ if (instance.errorMsgs.size() > 0) {
logger.info("Here are the error/warning messages, you may need check:");
for (String s : instance.errorMsgs) {
logger.warn(s);
}
- } else
- {
+ } else {
logger.info("No error or warning messages; The migration is success.");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f7823911/job/src/main/java/org/apache/kylin/job/cube/CubingJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/CubingJob.java b/job/src/main/java/org/apache/kylin/job/cube/CubingJob.java
index bf6ae11..483b44d 100644
--- a/job/src/main/java/org/apache/kylin/job/cube/CubingJob.java
+++ b/job/src/main/java/org/apache/kylin/job/cube/CubingJob.java
@@ -47,7 +47,7 @@ public class CubingJob extends DefaultChainedExecutable {
public static final String MAP_REDUCE_WAIT_TIME = "mapReduceWaitTime";
- void setCubeName(String name) {
+ public void setCubeName(String name) {
setParam(CUBE_INSTANCE_NAME, name);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f7823911/metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index 73db41f..b943622 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -73,7 +73,7 @@ public class PartitionDesc {
throw new IllegalStateException("The 'partition_date_column' format is invalid: " + partitionDateColumn + ", it should be {db}.{table}.{column}.");
}
- partitionConditionBuilder = (IPartitionConditionBuilder) ClassUtil.newInstance(partitionConditionBuilderClz);
+ partitionConditionBuilder = (IPartitionConditionBuilder) ClassUtil.newInstance(partitionConditionBuilderClz);
}
public boolean isPartitioned() {
@@ -144,4 +144,29 @@ public class PartitionDesc {
}
}
+
+ public static class YearMonthDayPartitionConditionBuilder implements PartitionDesc.IPartitionConditionBuilder {
+
+ @Override
+ public String buildDateRangeCondition(PartitionDesc partDesc, long l, long l1, Map<String, String> map) {
+
+ String partitionColumnName = partDesc.getPartitionDateColumn();
+ String partitionTableName;
+
+ // convert to use table alias
+ int indexOfDot = partitionColumnName.lastIndexOf(".");
+ if (indexOfDot > 0) {
+ partitionTableName = partitionColumnName.substring(0, indexOfDot);
+ } else {
+ throw new IllegalStateException("The partitionColumnName is invalid: " + partitionColumnName);
+ }
+
+ if (map.containsKey(partitionTableName)) {
+ partitionTableName = map.get(partitionTableName);
+ }
+
+ String sql = String.format("CONCAT(%s.YEAR,'-',%s.MONTH,'-',%s.DAY)", partitionTableName, partitionTableName, partitionTableName);
+ return sql;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f7823911/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 51224b8..d56fb4a 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -52,7 +52,6 @@ import org.apache.kylin.rest.response.HBaseResponse;
import org.apache.kylin.rest.service.CubeService;
import org.apache.kylin.rest.service.JobService;
import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverEnabler;
-import org.quartz.SchedulerException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -190,7 +189,6 @@ public class CubeController extends BasicController {
*
* @param cubeName Cube ID
* @return
- * @throws SchedulerException
* @throws IOException
*/
@RequestMapping(value = "/{cubeName}/rebuild", method = {RequestMethod.PUT})
[2/4] incubator-kylin git commit: Merge branch '0.7-staging' of
https://git-wip-us.apache.org/repos/asf/incubator-kylin into 0.7.1-staging
Posted by sh...@apache.org.
Merge branch '0.7-staging' of https://git-wip-us.apache.org/repos/asf/incubator-kylin into 0.7.1-staging
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a6b17c4c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a6b17c4c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a6b17c4c
Branch: refs/heads/0.7-staging
Commit: a6b17c4c8ef0bbee5b72ee7d54a46f444e2cf6f8
Parents: 3647e6b 1dc1a37
Author: shaofengshi <sh...@apache.org>
Authored: Mon Jun 8 14:56:47 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jun 8 14:56:47 2015 +0800
----------------------------------------------------------------------
bin/find-hive-dependency.sh | 8 +-
.../org/apache/kylin/common/KylinConfig.java | 20 +++-
.../org/apache/kylin/common/util/ClassUtil.java | 7 ++
.../apache/kylin/common/util/DateFormat.java | 101 +++++++++++++++++++
.../apache/kylin/dict/DateStrDictionary.java | 66 +-----------
docs/Installation/Hadoop Environment.md | 7 +-
.../kylin/invertedindex/index/TableRecord.java | 9 +-
.../org/apache/kylin/job/JoinedFlatTable.java | 50 +++------
.../kylin/job/common/ShellExecutable.java | 6 +-
.../kylin/job/hadoop/hbase/BulkLoadJob.java | 7 +-
.../hadoop/hive/CubeJoinedFlatTableDesc.java | 12 ++-
.../kylin/job/tools/HadoopStatusGetter.java | 2 +
.../kylin/metadata/model/PartitionDesc.java | 79 +++++++++++----
.../java/org/apache/kylin/rest/DebugTomcat.java | 87 +++++++++++-----
.../kylin/storage/hbase/HBaseKeyRange.java | 14 +--
.../org/apache/kylin/storage/tuple/Tuple.java | 6 +-
16 files changed, 309 insertions(+), 172 deletions(-)
----------------------------------------------------------------------
[4/4] incubator-kylin git commit: Merge branch '0.7-staging' of
https://git-wip-us.apache.org/repos/asf/incubator-kylin into 0.7.1-staging
Posted by sh...@apache.org.
Merge branch '0.7-staging' of https://git-wip-us.apache.org/repos/asf/incubator-kylin into 0.7.1-staging
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b32f9b30
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b32f9b30
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b32f9b30
Branch: refs/heads/0.7-staging
Commit: b32f9b3012d58fb9db27f840ee459b968493054e
Parents: f782391 b3b0f2f
Author: shaofengshi <sh...@apache.org>
Authored: Tue Jun 9 14:20:35 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Jun 9 14:20:35 2015 +0800
----------------------------------------------------------------------
atopcalcite/pom.xml | 2 +-
bin/find-hbase-dependency.sh | 18 +
common/pom.xml | 2 +-
cube/pom.xml | 2 +-
dictionary/pom.xml | 2 +-
.../apache/kylin/dict/TrieDictionaryTest.java | 2 +-
.../english-words.80 (scowl-2015.05.18).dic | 139304 ++++++++++++++++
.../english-words.80 (scowl-2015.05.18).txt | 139304 ----------------
invertedindex/pom.xml | 2 +-
jdbc/pom.xml | 2 +-
.../org/apache/kylin/jdbc/util/SQLTypeMap.java | 366 +-
job/pom.xml | 2 +-
metadata/pom.xml | 2 +-
pom.xml | 9 +-
query/pom.xml | 2 +-
server/pom.xml | 2 +-
.../kylin/rest/controller/CubeController.java | 2 +
.../kylin/rest/response/HBaseResponse.java | 26 +
src/main/config/assemblies/source-assembly.xml | 3 +
storage/pom.xml | 2 +-
webapp/app/index.html | 1 -
webapp/app/js/controllers/cubeSchema.js | 2 +-
webapp/app/js/controllers/index.js | 44 -
webapp/app/less/app.less | 2 +-
webapp/app/less/layout.less | 5 +
webapp/app/partials/cubes/cube_detail.html | 2 +
webapp/app/partials/header.html | 2 +-
webapp/app/partials/home.html | 65 -
webapp/app/partials/query/query_detail.html | 5 +-
webapp/app/routes.json | 9 +-
webapp/bower.json | 2 +-
31 files changed, 139568 insertions(+), 139627 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b32f9b30/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------