You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2017/01/03 06:42:42 UTC
[06/15] kylin git commit: KYLIN-2323 refactor Table Ext Info,
display table type
KYLIN-2323 refactor Table Ext Info, display table type
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b052d6f2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b052d6f2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b052d6f2
Branch: refs/heads/sparkcubing-rebase
Commit: b052d6f22930c45bd34f1d7a2fd493cbf4330791
Parents: e7e29f1
Author: Billy Liu <bi...@apache.org>
Authored: Tue Jan 3 11:15:06 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Tue Jan 3 11:15:06 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/metadata/MetadataManager.java | 4 +-
.../kylin/metadata/model/TableExtDesc.java | 52 +-------------------
.../kylin/rest/response/TableDescResponse.java | 1 +
.../apache/kylin/rest/service/TableService.java | 21 ++++----
.../source/hive/HiveSourceTableLoader.java | 16 ++----
webapp/app/js/services/tables.js | 2 -
webapp/app/partials/tables/table_detail.html | 16 +++---
7 files changed, 28 insertions(+), 84 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b052d6f2/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 3b6eb61..49ec96e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -504,7 +504,7 @@ public class MetadataManager {
List<String> models = new ArrayList<>();
for (DataModelDesc modelDesc : getModels(projectName)) {
for (TableRef tableRef : modelDesc.getAllTables()) {
- if (tableRef.getTableName().equalsIgnoreCase(tableName)) {
+ if (tableRef.getTableIdentity().equalsIgnoreCase(tableName)) {
models.add(modelDesc.getName());
}
}
@@ -515,7 +515,7 @@ public class MetadataManager {
public boolean isTableInAnyModel(String tableName) {
for (DataModelDesc modelDesc : getModels()) {
for (TableRef tableRef : modelDesc.getAllTables()) {
- if (tableRef.getTableName().equalsIgnoreCase(tableName)) {
+ if (tableRef.getTableIdentity().equalsIgnoreCase(tableName)) {
return true;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b052d6f2/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
index 6393dbf..97dd2c1 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
@@ -47,18 +47,8 @@ public class TableExtDesc extends RootPersistentEntity {
@JsonProperty("sample_rows")
private List<String[]> sampleRows = new ArrayList<>();
- @JsonProperty("storage_location")
- private String storageLocation;
- @JsonProperty("owner")
- private String owner;
- @JsonProperty("last_access_time")
- private String lastAccessTime;
@JsonProperty("last_modified_time")
private String lastModifiedTime;
- @JsonProperty("partition_column")
- private String partitionColumn;
- @JsonProperty("total_file_size")
- private String totalFileSize;
@JsonProperty("total_rows")
private String totalRows;
@JsonProperty("data_source_properties")
@@ -161,22 +151,6 @@ public class TableExtDesc extends RootPersistentEntity {
this.tableName = this.tableName.toUpperCase();
}
- public void setStorageLocation(String storageLocation) {
- this.storageLocation = storageLocation;
- }
-
- public String getStorageLocation() {
- return this.storageLocation;
- }
-
- public void setOwner(String owner) {
- this.owner = owner;
- }
-
- public String getOwner() {
- return this.owner;
- }
-
public void setLastModifiedTime(String lastModifiedTime) {
this.lastModifiedTime = lastModifiedTime;
}
@@ -185,32 +159,8 @@ public class TableExtDesc extends RootPersistentEntity {
return this.lastModifiedTime;
}
- public void setLastAccessTime(String lastAccessTime) {
- this.lastAccessTime = lastAccessTime;
- }
-
- public String getLastAccessTime() {
- return this.lastAccessTime;
- }
-
- public void setPartitionColumn(String partitionColumn) {
- this.partitionColumn = partitionColumn;
- }
-
- public String getPartitionColumn() {
- return this.partitionColumn;
- }
-
public boolean isPartitioned() {
- return this.partitionColumn == null ? false : !this.partitionColumn.isEmpty();
- }
-
- public void setTotalFileSize(String totalFileSize) {
- this.totalFileSize = totalFileSize;
- }
-
- public String getTotalFileSize() {
- return this.totalFileSize;
+ return this.dataSourceProps.get("partition_column") == null ? false : !this.dataSourceProps.get("partition_column").isEmpty();
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/b052d6f2/server-base/src/main/java/org/apache/kylin/rest/response/TableDescResponse.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/response/TableDescResponse.java b/server-base/src/main/java/org/apache/kylin/rest/response/TableDescResponse.java
index f53b951..9bc5360 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/response/TableDescResponse.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/response/TableDescResponse.java
@@ -76,6 +76,7 @@ public class TableDescResponse extends TableDesc {
this.setName(table.getName());
this.setSourceType(table.getSourceType());
this.setUuid(table.getUuid());
+ this.setTableType(table.getTableType());
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/b052d6f2/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
index 3325907..e3df60a 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
@@ -29,6 +29,7 @@ import java.util.Set;
import java.util.UUID;
import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.common.HadoopShellExecutable;
import org.apache.kylin.engine.mr.common.MapReduceExecutable;
@@ -91,7 +92,7 @@ public class TableService extends BasicService {
return table;
}
- @PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER + " or " + Constant.ACCESS_HAS_ROLE_ADMIN)
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
public String[] loadHiveTablesToProject(String[] tables, String project) throws IOException {
Set<String> loaded = HiveSourceTableLoader.loadHiveTables(tables, getConfig());
String[] result = (String[]) loaded.toArray(new String[loaded.size()]);
@@ -99,18 +100,17 @@ public class TableService extends BasicService {
return result;
}
- @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
private void unLoadHiveTable(String tableName) throws IOException {
tableName = normalizeHiveTableName(tableName);
- HiveSourceTableLoader.unLoadHiveTable(tableName);
+ MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ metaMgr.removeSourceTable(tableName);
+ metaMgr.removeTableExt(tableName);
}
- @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
private void syncTableToProject(String[] tables, String project) throws IOException {
getProjectManager().addTableDescToProject(tables, project);
}
- @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
private void removeTableFromProject(String tableName, String projectName) throws IOException {
tableName = normalizeHiveTableName(tableName);
getProjectManager().removeTableDescFromProject(tableName, projectName);
@@ -123,6 +123,7 @@ public class TableService extends BasicService {
* @param project
* @return
*/
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
public boolean unLoadHiveTable(String tableName, String project) {
boolean rtn = false;
int tableType = 0;
@@ -178,6 +179,7 @@ public class TableService extends BasicService {
* @param project
* @throws IOException
*/
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
public void addStreamingTable(TableDesc desc, String project) throws IOException {
desc.setUuid(UUID.randomUUID().toString());
getMetadataManager().saveSourceTable(desc);
@@ -230,11 +232,6 @@ public class TableService extends BasicService {
rtableDesc.setCardinality(cardinality);
}
dataSourceProp.putAll(tableExtDesc.getDataSourceProp());
- dataSourceProp.put("location", tableExtDesc.getStorageLocation());
- dataSourceProp.put("owner", tableExtDesc.getOwner());
- dataSourceProp.put("last_access_time", tableExtDesc.getLastAccessTime());
- dataSourceProp.put("partition_column", tableExtDesc.getPartitionColumn());
- dataSourceProp.put("total_file_size", tableExtDesc.getTotalFileSize());
rtableDesc.setDescExd(dataSourceProp);
return rtableDesc;
}
@@ -277,8 +274,8 @@ public class TableService extends BasicService {
TableDesc table = getMetadataManager().getTableDesc(tableName);
final TableExtDesc tableExt = getMetadataManager().getTableExt(tableName);
if (table == null) {
- IllegalArgumentException e = new IllegalArgumentException("Cannot find table descirptor " + tableName);
- logger.error("Cannot find table descirptor " + tableName, e);
+ IllegalArgumentException e = new IllegalArgumentException("Cannot find table descriptor " + tableName);
+ logger.error("Cannot find table descriptor " + tableName, e);
throw e;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b052d6f2/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
index b56009a..286ffac 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
@@ -76,12 +76,6 @@ public class HiveSourceTableLoader {
return loadedTables;
}
- public static void unLoadHiveTable(String hiveTable) throws IOException {
- MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
- metaMgr.removeSourceTable(hiveTable);
- metaMgr.removeTableExt(hiveTable);
- }
-
private static List<String> extractHiveTables(String database, Set<String> tables, IHiveClient hiveClient) throws IOException {
List<String> loadedTables = Lists.newArrayList();
@@ -132,11 +126,11 @@ public class HiveSourceTableLoader {
}
TableExtDesc tableExtDesc = metaMgr.getTableExt(tableDesc.getIdentity());
- tableExtDesc.setStorageLocation(hiveTableMeta.sdLocation);
- tableExtDesc.setOwner(hiveTableMeta.owner);
- tableExtDesc.setLastAccessTime(String.valueOf(hiveTableMeta.lastAccessTime));
- tableExtDesc.setPartitionColumn(partitionColumnString.toString());
- tableExtDesc.setTotalFileSize(String.valueOf(hiveTableMeta.fileSize));
+ tableExtDesc.addDataSourceProp("location", hiveTableMeta.sdLocation);
+ tableExtDesc.addDataSourceProp("owner", hiveTableMeta.owner);
+ tableExtDesc.addDataSourceProp("last_access_time", String.valueOf(hiveTableMeta.lastAccessTime));
+ tableExtDesc.addDataSourceProp("partition_column", partitionColumnString.toString());
+ tableExtDesc.addDataSourceProp("total_file_size", String.valueOf(hiveTableMeta.fileSize));
tableExtDesc.addDataSourceProp("total_file_number", String.valueOf(hiveTableMeta.fileNum));
tableExtDesc.addDataSourceProp("hive_inputFormat", hiveTableMeta.sdInputFormat);
tableExtDesc.addDataSourceProp("hive_outputFormat", hiveTableMeta.sdOutputFormat);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b052d6f2/webapp/app/js/services/tables.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/services/tables.js b/webapp/app/js/services/tables.js
index 4e7a7c4..38bf8a0 100755
--- a/webapp/app/js/services/tables.js
+++ b/webapp/app/js/services/tables.js
@@ -20,8 +20,6 @@ KylinApp.factory('TableService', ['$resource', function ($resource, config) {
return $resource(Config.service.url + 'tables/:tableName/:action/:database', {}, {
list: {method: 'GET', params: {}, cache: true, isArray: true},
get: {method: 'GET', params: {}, isArray: false},
- getExd: {method: 'GET', params: {action: 'exd-map'}, isArray: false},
- reload: {method: 'PUT', params: {action: 'reload'}, isArray: false},
loadHiveTable: {method: 'POST', params: {}, isArray: false},
unLoadHiveTable: {method: 'DELETE', params: {}, isArray: false},
genCardinality: {method: 'PUT', params: {action: 'cardinality'}, isArray: false},
http://git-wip-us.apache.org/repos/asf/kylin/blob/b052d6f2/webapp/app/partials/tables/table_detail.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/tables/table_detail.html b/webapp/app/partials/tables/table_detail.html
index 6399422..f227826 100644
--- a/webapp/app/partials/tables/table_detail.html
+++ b/webapp/app/partials/tables/table_detail.html
@@ -39,7 +39,7 @@
<table class="table">
<tbody>
<tr>
- <th style="width:20%">NAME</th>
+ <th style="width:20%">TABLE NAME</th>
<td>{{ tableModel.selectedSrcTable.name}}</td>
</tr>
<tr>
@@ -47,6 +47,10 @@
<td>{{tableModel.selectedSrcTable.database}}</td>
</tr>
<tr>
+ <th>TABLE TYPE</th>
+ <td>{{tableModel.selectedSrcTable.table_type}}</td>
+ </tr>
+ <tr>
<th>SNAPSHOT TIME</th>
<td>{{tableModel.selectedSrcTable.exd.lastUpdateTime | utcToConfigTimeZone}}</td>
</tr>
@@ -56,11 +60,11 @@
</tr>
<tr>
<th>INPUT FORMAT</th>
- <td>{{tableModel.selectedSrcTable.exd.inputformat}}</td>
+ <td>{{tableModel.selectedSrcTable.exd.hive_inputFormat}}</td>
</tr>
<tr>
<th>OUTPUT FORMAT</th>
- <td>{{tableModel.selectedSrcTable.exd.outputformat}}</td>
+ <td>{{tableModel.selectedSrcTable.exd.hive_outputFormat}}</td>
</tr>
<tr>
<th>OWNER</th>
@@ -70,11 +74,11 @@
</tr>
<tr>
<th>TOTAL FILE NUMBER</th>
- <td>{{tableModel.selectedSrcTable.exd.totalNumberFiles}}</td>
+ <td>{{tableModel.selectedSrcTable.exd.total_file_number}}</td>
</tr>
<tr>
<th>TOTAL FILE SIZE</th>
- <td>{{tableModel.selectedSrcTable.exd.totalFileSize}}</td>
+ <td>{{tableModel.selectedSrcTable.exd.total_file_size}}</td>
</tr>
<tr>
<th>PARTITIONED</th>
@@ -82,7 +86,7 @@
</tr>
<tr>
<th>PARTITION COLUMNS</th>
- <td>{{tableModel.selectedSrcTable.exd.partitionColumns}}</td>
+ <td>{{tableModel.selectedSrcTable.exd.partition_column}}</td>
</tr>
</tbody>
</table>