You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by bi...@apache.org on 2016/12/27 02:23:00 UTC
kylin git commit: refactor
Repository: kylin
Updated Branches:
refs/heads/hive-load-refactor 0b66ef1d6 -> 877b538a5
refactor
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/877b538a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/877b538a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/877b538a
Branch: refs/heads/hive-load-refactor
Commit: 877b538a508f60ab72d97eaf7af6e39d55a4b4bd
Parents: 0b66ef1
Author: Billy Liu <bi...@apache.org>
Authored: Tue Dec 27 10:22:50 2016 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Tue Dec 27 10:22:50 2016 +0800
----------------------------------------------------------------------
.../kylin/rest/controller/TableController.java | 76 ++++++++++----------
.../apache/kylin/rest/service/TableService.java | 72 +++++++++++--------
webapp/app/js/controllers/sourceMeta.js | 16 ++---
webapp/app/js/services/tables.js | 1 -
4 files changed, 86 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/877b538a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
index 0debad7..964cd4c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
@@ -24,12 +24,11 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.rest.exception.InternalErrorException;
+import org.apache.kylin.rest.exception.NotFoundException;
import org.apache.kylin.rest.request.CardinalityRequest;
import org.apache.kylin.rest.request.HiveTableRequest;
-import org.apache.kylin.rest.request.StreamingRequest;
import org.apache.kylin.rest.service.TableService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,14 +65,12 @@ public class TableController extends BasicController {
@RequestMapping(value = "", method = { RequestMethod.GET })
@ResponseBody
public List<TableDesc> getTableDesc(@RequestParam(value = "ext", required = false) boolean withExt, @RequestParam(value = "project", required = true) String project) throws IOException {
- List<TableDesc> tables = null;
try {
- tables = tableService.getTableDescByProject(project, withExt);
+ return tableService.getTableDescByProject(project, withExt);
} catch (IOException e) {
logger.error("Failed to get Hive Tables", e);
throw new InternalErrorException(e.getLocalizedMessage());
}
- return tables;
}
/**
@@ -85,21 +82,28 @@ public class TableController extends BasicController {
@RequestMapping(value = "/{tableName:.+}", method = { RequestMethod.GET })
@ResponseBody
public TableDesc getTableDesc(@PathVariable String tableName) {
- return tableService.getTableDescByName(tableName);
+ TableDesc table = tableService.getTableDescByName(tableName, false);
+ if (table == null)
+ throw new NotFoundException("Could not find Hive table: " + tableName);
+ return table;
}
@RequestMapping(value = "/{tables}/{project}", method = { RequestMethod.POST })
@ResponseBody
public Map<String, String[]> loadHiveTables(@PathVariable String tables, @PathVariable String project, @RequestBody HiveTableRequest request) throws IOException {
String submitter = SecurityContextHolder.getContext().getAuthentication().getName();
+ Map<String, String[]> result = new HashMap<String, String[]>();
String[] tableNames = tables.split(",");
- String[] loaded = tableService.loadHiveTablesToProject(tableNames, project);
- if (request.isCalculate()) {
- tableService.calculateCardinalityIfNotPresent(loaded, submitter);
+ try {
+ String[] loaded = tableService.loadHiveTablesToProject(tableNames, project);
+ result.put("result.loaded", loaded);
+ if (request.isCalculate()) {
+ tableService.calculateCardinalityIfNotPresent(loaded, submitter);
+ }
+ } catch (Exception e) {
+ logger.error("Failed to load Hive Table", e);
+ throw new InternalErrorException(e.getLocalizedMessage());
}
- Map<String, String[]> result = new HashMap<String, String[]>();
- result.put("result.loaded", loaded);
- result.put("result.unloaded", new String[] {});
return result;
}
@@ -109,29 +113,23 @@ public class TableController extends BasicController {
Set<String> unLoadSuccess = Sets.newHashSet();
Set<String> unLoadFail = Sets.newHashSet();
Map<String, String[]> result = new HashMap<String, String[]>();
- for (String tableName : tables.split(",")) {
- if (tableService.unLoadHiveTable(tableName, project)) {
- unLoadSuccess.add(tableName);
- } else {
- unLoadFail.add(tableName);
+ try {
+ for (String tableName : tables.split(",")) {
+ if (tableService.unLoadHiveTable(tableName, project)) {
+ unLoadSuccess.add(tableName);
+ } else {
+ unLoadFail.add(tableName);
+ }
}
+ } catch (Exception e) {
+ logger.error("Failed to unload Hive Table", e);
+ throw new InternalErrorException(e.getLocalizedMessage());
}
result.put("result.unload.success", (String[]) unLoadSuccess.toArray(new String[unLoadSuccess.size()]));
result.put("result.unload.fail", (String[]) unLoadFail.toArray(new String[unLoadFail.size()]));
return result;
}
- @RequestMapping(value = "/addStreamingSrc", method = { RequestMethod.POST })
- @ResponseBody
- public Map<String, String> addStreamingTable(@RequestBody StreamingRequest request) throws IOException {
- Map<String, String> result = new HashMap<String, String>();
- String project = request.getProject();
- TableDesc desc = JsonUtil.readValue(request.getTableData(), TableDesc.class);
- tableService.addStreamingTable(desc, project);
- result.put("success", "true");
- return result;
- }
-
/**
* Regenerate table cardinality
*
@@ -143,8 +141,13 @@ public class TableController extends BasicController {
public CardinalityRequest generateCardinality(@PathVariable String tableNames, @RequestBody CardinalityRequest request) throws IOException {
String submitter = SecurityContextHolder.getContext().getAuthentication().getName();
String[] tables = tableNames.split(",");
- for (String table : tables) {
- tableService.calculateCardinality(table.trim().toUpperCase(), submitter);
+ try {
+ for (String table : tables) {
+ tableService.calculateCardinality(table.trim().toUpperCase(), submitter);
+ }
+ } catch (IOException e) {
+ logger.error("Failed to calculate cardinality", e);
+ throw new InternalErrorException(e.getLocalizedMessage());
}
return request;
}
@@ -158,13 +161,11 @@ public class TableController extends BasicController {
@RequestMapping(value = "/hive", method = { RequestMethod.GET })
@ResponseBody
private List<String> showHiveDatabases() throws IOException {
- List<String> results = null;
try {
- results = tableService.getHiveDbNames();
+ return tableService.getHiveDbNames();
} catch (Exception e) {
- throw new IOException(e);
+ throw new InternalErrorException(e.getLocalizedMessage());
}
- return results;
}
/**
@@ -176,14 +177,11 @@ public class TableController extends BasicController {
@RequestMapping(value = "/hive/{database}", method = { RequestMethod.GET })
@ResponseBody
private List<String> showHiveTables(@PathVariable String database) throws IOException {
- List<String> results = null;
-
try {
- results = tableService.getHiveTableNames(database);
+ return tableService.getHiveTableNames(database);
} catch (Exception e) {
- throw new IOException(e);
+ throw new InternalErrorException(e.getLocalizedMessage());
}
- return results;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/877b538a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
index 6accfdc..8868bff 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
@@ -83,8 +83,12 @@ public class TableService extends BasicService {
return tables;
}
- public TableDesc getTableDescByName(String tableName) {
- return getMetadataManager().getTableDesc(tableName);
+ public TableDesc getTableDescByName(String tableName, boolean withExt) {
+ TableDesc table = getMetadataManager().getTableDesc(tableName);
+ if(withExt){
+ table = cloneTableDesc(table);
+ }
+ return table;
}
@PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER + " or " + Constant.ACCESS_HAS_ROLE_ADMIN)
@@ -128,7 +132,7 @@ public class TableService extends BasicService {
//remove streaming info
String[] dbTableName = HadoopUtil.parseHiveTableName(tableName);
tableName = dbTableName[0] + "." + dbTableName[1];
- TableDesc desc = getTableDescByName(tableName);
+ TableDesc desc = getMetadataManager().getTableDesc(tableName);
if (desc == null)
return false;
tableType = desc.getSourceType();
@@ -206,39 +210,45 @@ public class TableService extends BasicService {
return results;
}
+ private TableDescResponse cloneTableDesc(TableDesc table) {
+ TableExtDesc tableExtDesc = getMetadataManager().getTableExt(table.getIdentity());
+
+ // Clone TableDesc
+ TableDescResponse rtableDesc = new TableDescResponse(table);
+ Map<String, Long> cardinality = new HashMap<String, Long>();
+ Map<String, String> dataSourceProp = new HashMap<>();
+ String scard = tableExtDesc.getCardinality();
+ if (!StringUtils.isEmpty(scard)) {
+ String[] cards = StringUtils.split(scard, ",");
+ ColumnDesc[] cdescs = rtableDesc.getColumns();
+ for (int i = 0; i < cdescs.length; i++) {
+ ColumnDesc columnDesc = cdescs[i];
+ if (cards.length > i) {
+ cardinality.put(columnDesc.getName(), Long.parseLong(cards[i]));
+ } else {
+ logger.error("The result cardinality is not identical with hive table metadata, cardinality : " + scard + " column array length: " + cdescs.length);
+ break;
+ }
+ }
+ rtableDesc.setCardinality(cardinality);
+ }
+ dataSourceProp.putAll(tableExtDesc.getDataSourceProp());
+ dataSourceProp.put("location", tableExtDesc.getStorageLocation());
+ dataSourceProp.put("owner", tableExtDesc.getOwner());
+ dataSourceProp.put("last_access_time", tableExtDesc.getLastAccessTime());
+ dataSourceProp.put("partition_column", tableExtDesc.getPartitionColumn());
+ dataSourceProp.put("total_file_size", tableExtDesc.getTotalFileSize());
+ rtableDesc.setDescExd(dataSourceProp);
+ return rtableDesc;
+ }
+
+
private List<TableDesc> cloneTableDesc(List<TableDesc> tables) throws IOException {
List<TableDesc> descs = new ArrayList<TableDesc>();
Iterator<TableDesc> it = tables.iterator();
while (it.hasNext()) {
TableDesc table = it.next();
- TableExtDesc tableExtDesc = getMetadataManager().getTableExt(table.getIdentity());
-
- // Clone TableDesc
- TableDescResponse rtableDesc = new TableDescResponse(table);
- Map<String, Long> cardinality = new HashMap<String, Long>();
- Map<String, String> dataSourceProp = new HashMap<>();
- String scard = tableExtDesc.getCardinality();
- if (!StringUtils.isEmpty(scard)) {
- String[] cards = StringUtils.split(scard, ",");
- ColumnDesc[] cdescs = rtableDesc.getColumns();
- for (int i = 0; i < cdescs.length; i++) {
- ColumnDesc columnDesc = cdescs[i];
- if (cards.length > i) {
- cardinality.put(columnDesc.getName(), Long.parseLong(cards[i]));
- } else {
- logger.error("The result cardinality is not identical with hive table metadata, cardinality : " + scard + " column array length: " + cdescs.length);
- break;
- }
- }
- rtableDesc.setCardinality(cardinality);
- }
- dataSourceProp.putAll(tableExtDesc.getDataSourceProp());
- dataSourceProp.put("location", tableExtDesc.getStorageLocation());
- dataSourceProp.put("owner", tableExtDesc.getOwner());
- dataSourceProp.put("last_access_time", tableExtDesc.getLastAccessTime());
- dataSourceProp.put("partition_column", tableExtDesc.getPartitionColumn());
- dataSourceProp.put("total_file_size", tableExtDesc.getTotalFileSize());
- rtableDesc.setDescExd(dataSourceProp);
+ TableDescResponse rtableDesc = cloneTableDesc(table);
descs.add(rtableDesc);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/877b538a/webapp/app/js/controllers/sourceMeta.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/sourceMeta.js b/webapp/app/js/controllers/sourceMeta.js
index bbb9915..a53a35f 100755
--- a/webapp/app/js/controllers/sourceMeta.js
+++ b/webapp/app/js/controllers/sourceMeta.js
@@ -330,7 +330,7 @@ KylinApp
}
if ($scope.tableNames.trim() === "") {
- SweetAlert.swal('', 'Please input table(s) you want to synchronize.', 'info');
+ SweetAlert.swal('', 'Please input table(s) you want to load.', 'info');
return;
}
@@ -352,13 +352,13 @@ KylinApp
})
if (result['result.unloaded'].length != 0 && result['result.loaded'].length == 0) {
- SweetAlert.swal('Failed!', 'Failed to synchronize following table(s): ' + unloadedTableInfo, 'error');
+ SweetAlert.swal('Failed!', 'Failed to load following table(s): ' + unloadedTableInfo, 'error');
}
if (result['result.loaded'].length != 0 && result['result.unloaded'].length == 0) {
- SweetAlert.swal('Success!', 'The following table(s) have been successfully synchronized: ' + loadTableInfo, 'success');
+ SweetAlert.swal('Success!', 'The following table(s) have been successfully loaded: ' + loadTableInfo, 'success');
}
if (result['result.loaded'].length != 0 && result['result.unloaded'].length != 0) {
- SweetAlert.swal('Partial loaded!', 'The following table(s) have been successfully synchronized: ' + loadTableInfo + "\n\n Failed to synchronize following table(s):" + unloadedTableInfo, 'warning');
+ SweetAlert.swal('Partial loaded!', 'The following table(s) have been successfully loaded: ' + loadTableInfo + "\n\n Failed to load following table(s):" + unloadedTableInfo, 'warning');
}
loadingRequest.hide();
scope.aceSrcTbLoaded(true);
@@ -378,7 +378,7 @@ KylinApp
$scope.remove = function () {
if ($scope.tableNames.trim() === "") {
- SweetAlert.swal('', 'Please input table(s) you want to synchronize.', 'info');
+ SweetAlert.swal('', 'Please input table(s) you want to unload.', 'info');
return;
}
@@ -400,13 +400,13 @@ KylinApp
})
if (result['result.unload.fail'].length != 0 && result['result.unload.success'].length == 0) {
- SweetAlert.swal('Failed!', 'Failed to synchronize following table(s): ' + unRemovedTableInfo, 'error');
+ SweetAlert.swal('Failed!', 'Failed to unload following table(s): ' + unRemovedTableInfo, 'error');
}
if (result['result.unload.success'].length != 0 && result['result.unload.fail'].length == 0) {
- SweetAlert.swal('Success!', 'The following table(s) have been successfully synchronized: ' + removedTableInfo, 'success');
+ SweetAlert.swal('Success!', 'The following table(s) have been successfully unloaded: ' + removedTableInfo, 'success');
}
if (result['result.unload.success'].length != 0 && result['result.unload.fail'].length != 0) {
- SweetAlert.swal('Partial unloaded!', 'The following table(s) have been successfully synchronized: ' + removedTableInfo + "\n\n Failed to synchronize following table(s):" + unRemovedTableInfo, 'warning');
+ SweetAlert.swal('Partial unloaded!', 'The following table(s) have been successfully unloaded: ' + removedTableInfo + "\n\n Failed to unload following table(s):" + unRemovedTableInfo, 'warning');
}
loadingRequest.hide();
scope.aceSrcTbLoaded(true);
http://git-wip-us.apache.org/repos/asf/kylin/blob/877b538a/webapp/app/js/services/tables.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/services/tables.js b/webapp/app/js/services/tables.js
index 4199d6c..4e7a7c4 100755
--- a/webapp/app/js/services/tables.js
+++ b/webapp/app/js/services/tables.js
@@ -24,7 +24,6 @@ KylinApp.factory('TableService', ['$resource', function ($resource, config) {
reload: {method: 'PUT', params: {action: 'reload'}, isArray: false},
loadHiveTable: {method: 'POST', params: {}, isArray: false},
unLoadHiveTable: {method: 'DELETE', params: {}, isArray: false},
- addStreamingSrc: {method: 'POST', params: {action:'addStreamingSrc'}, isArray: false},
genCardinality: {method: 'PUT', params: {action: 'cardinality'}, isArray: false},
showHiveDatabases: {method: 'GET', params: {action:'hive'}, cache: true, isArray: true},
showHiveTables: {method: 'GET', params: {action:'hive'}, cache: true, isArray: true}