You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@griffin.apache.org by gu...@apache.org on 2018/10/26 09:07:28 UTC
incubator-griffin git commit: [GRIFFIN-194] single call to fetch all
tables of all dbs
Repository: incubator-griffin
Updated Branches:
refs/heads/master 2637ca885 -> 4370fd86e
[GRIFFIN-194] single call to fetch all tables of all dbs
Significantly improves opening speed for measure dialogs.
Previously taking 10+ seconds on huge schemas, with this call
it takes fraction of second to load.
Author: Nikolay Sokolov <ch...@gmail.com>
Closes #447 from chemikadze/GRIFFIN-194.
Project: http://git-wip-us.apache.org/repos/asf/incubator-griffin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-griffin/commit/4370fd86
Tree: http://git-wip-us.apache.org/repos/asf/incubator-griffin/tree/4370fd86
Diff: http://git-wip-us.apache.org/repos/asf/incubator-griffin/diff/4370fd86
Branch: refs/heads/master
Commit: 4370fd86eb7c7433f6f1d3e9c6ed5913d774ea5a
Parents: 2637ca8
Author: Nikolay Sokolov <ch...@gmail.com>
Authored: Fri Oct 26 02:07:08 2018 -0700
Committer: Yuepeng <yu...@griffin-2454820.phx02.dev.ebayc3.com>
Committed: Fri Oct 26 02:07:08 2018 -0700
----------------------------------------------------------------------
.../metastore/hive/HiveMetaStoreController.java | 5 ++
.../metastore/hive/HiveMetaStoreService.java | 2 +
.../hive/HiveMetaStoreServiceImpl.java | 14 +++++-
.../measure/create-measure/ac/ac.component.ts | 51 ++++++++-----------
.../create-measure/pr/step1/step1.component.ts | 52 ++++++++------------
ui/angular/src/app/service/service.service.ts | 1 +
6 files changed, 60 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/4370fd86/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreController.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreController.java b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreController.java
index 2371044..ba425ec 100644
--- a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreController.java
+++ b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreController.java
@@ -57,6 +57,11 @@ public class HiveMetaStoreController {
return hiveMetaStoreService.getAllTable();
}
+ @RequestMapping(value = "/dbs/tables/names", method = RequestMethod.GET)
+ public Map<String, List<String>> getAllTableNames() {
+ return hiveMetaStoreService.getAllTableNames();
+ }
+
@RequestMapping(value = "/table", method = RequestMethod.GET)
public Table getTable(@RequestParam("db") String dbName,
@RequestParam("table") String tableName) {
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/4370fd86/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreService.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreService.java b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreService.java
index 4ef7690..609a34e 100644
--- a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreService.java
+++ b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreService.java
@@ -30,6 +30,8 @@ public interface HiveMetaStoreService {
Iterable<String> getAllTableNames(String dbName);
+ Map<String, List<String>> getAllTableNames();
+
List<Table> getAllTable(String db);
Map<String, List<Table>> getAllTable();
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/4370fd86/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java
index 72d8c56..48a78a4 100644
--- a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java
+++ b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java
@@ -24,6 +24,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import com.google.common.collect.Lists;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.Table;
import org.slf4j.Logger;
@@ -99,8 +100,17 @@ public class HiveMetaStoreServiceImpl implements HiveMetaStoreService {
public List<Table> getAllTable(String db) {
return getTables(db);
}
-
-
+
+ @Override
+ @Cacheable(unless = "#result==null || #result.isEmpty()")
+ public Map<String, List<String>> getAllTableNames() {
+ Map<String, List<String>> result = new HashMap<>();
+ for (String dbName: getAllDatabases()) {
+ result.put(dbName, Lists.newArrayList(getAllTableNames(dbName)));
+ }
+ return result;
+ }
+
@Override
@Cacheable(unless = "#result==null")
public Map<String, List<Table>> getAllTable() {
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/4370fd86/ui/angular/src/app/measure/create-measure/ac/ac.component.ts
----------------------------------------------------------------------
diff --git a/ui/angular/src/app/measure/create-measure/ac/ac.component.ts b/ui/angular/src/app/measure/create-measure/ac/ac.component.ts
index 3007f34..72d0138 100644
--- a/ui/angular/src/app/measure/create-measure/ac/ac.component.ts
+++ b/ui/angular/src/app/measure/create-measure/ac/ac.component.ts
@@ -644,48 +644,37 @@ export class AcComponent implements OnInit, AfterViewChecked {
}
ngOnInit() {
- let allDatabases = this.serviceService.config.uri.dblist;
- let getTableNames = this.serviceService.config.uri.tablenames;
+ let getTableNames = this.serviceService.config.uri.dbtablenames;
- this.http.get(allDatabases).subscribe((databases: Array<string>) => {
+ this.http.get(getTableNames).subscribe((databases) => {
this.nodeList = new Array();
this.nodeListTarget = this.nodeList; // share same model instead of copying(?)
let i = 1;
- let pending = databases.length;
- if (databases.length > 10) {
- this.options.animateExpand = false;
- this.updateTrees();
- }
- for (let dbName of databases) {
+ for (let dbName in databases) {
+ if (!databases.hasOwnProperty(dbName)) {
+ continue;
+ }
let dbNode = new node();
dbNode.name = dbName;
dbNode.id = i++;
dbNode.isExpanded = false;
dbNode.children = new Array();
- let params = new HttpParams({fromString: "db="+dbName});
- this.http.get(getTableNames, {params: params}).subscribe((tables: Array<string>) => {
- for (let tableName of tables) {
- let tableNode = new node();
- tableNode.name = tableName;
- dbNode.children.push(tableNode);
- tableNode.isExpanded = true;
- tableNode.location = null;
- tableNode.parent = dbName;
- tableNode.cols = null;
- }
- pending -= 1;
- if (pending == 0) {
- this.updateTrees();
- }
- },
- () => {
- pending -= 1;
- if (pending == 0) {
- this.updateTrees();
- }
- });
+ for (let tableName of databases[dbName]) {
+ let tableNode = new node();
+ tableNode.name = tableName;
+ dbNode.children.push(tableNode);
+ tableNode.isExpanded = true;
+ tableNode.location = null;
+ tableNode.parent = dbName;
+ tableNode.cols = null;
+ }
this.nodeList.push(dbNode);
}
+ if (i >= 10) {
+ this.options.animateExpand = false;
+ this.targetOptions.animateExpand = false;
+ }
+ this.updateTrees();
});
this.src_size = "1day";
this.tgt_size = "1day";
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/4370fd86/ui/angular/src/app/measure/create-measure/pr/step1/step1.component.ts
----------------------------------------------------------------------
diff --git a/ui/angular/src/app/measure/create-measure/pr/step1/step1.component.ts b/ui/angular/src/app/measure/create-measure/pr/step1/step1.component.ts
index a5b052e..2f4c3cd 100644
--- a/ui/angular/src/app/measure/create-measure/pr/step1/step1.component.ts
+++ b/ui/angular/src/app/measure/create-measure/pr/step1/step1.component.ts
@@ -188,47 +188,35 @@ export class PrStep1Component implements AfterViewChecked, OnInit {
ngOnInit() {
if (this.step1.nodeList.length !== 0) return;
- let allDatabases = this.serviceService.config.uri.dblist;
- let getTableNames = this.serviceService.config.uri.tablenames;
-
- this.http.get(allDatabases).subscribe((databases: Array<string>) => {
+ let getTableNames = this.serviceService.config.uri.dbtablenames;
+
+ this.http.get(getTableNames).subscribe((databases) => {
this.step1.nodeList = new Array();
let i = 1;
- let pending = databases.length;
- if (databases.length > 10) {
- this.options.animateExpand = false;
- this.tree.treeModel.update();
- }
- for (let dbName of databases) {
+ for (let dbName in databases) {
+ if (!databases.hasOwnProperty(dbName)) {
+ continue;
+ }
let dbNode = new node();
dbNode.name = dbName;
dbNode.id = i++;
dbNode.isExpanded = false;
dbNode.children = new Array();
- let params = new HttpParams({fromString: "db="+dbName});
- this.http.get(getTableNames, {params: params}).subscribe((tables: Array<string>) => {
- for (let tableName of tables) {
- let tableNode = new node();
- tableNode.name = tableName;
- dbNode.children.push(tableNode);
- tableNode.isExpanded = true;
- tableNode.location = null;
- tableNode.parent = dbName;
- tableNode.cols = null;
- }
- pending -= 1;
- if (pending == 0) {
- this.tree.treeModel.update();
- }
- },
- () => {
- pending -= 1;
- if (pending == 0) {
- this.tree.treeModel.update();
- }
- });
+ for (let tableName of databases[dbName]) {
+ let tableNode = new node();
+ tableNode.name = tableName;
+ dbNode.children.push(tableNode);
+ tableNode.isExpanded = true;
+ tableNode.location = null;
+ tableNode.parent = dbName;
+ tableNode.cols = null;
+ }
this.step1.nodeList.push(dbNode);
}
+ if (i >= 10) {
+ this.options.animateExpand = false;
+ }
+ this.tree.treeModel.update();
});
}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/4370fd86/ui/angular/src/app/service/service.service.ts
----------------------------------------------------------------------
diff --git a/ui/angular/src/app/service/service.service.ts b/ui/angular/src/app/service/service.service.ts
index 6161af0..7d50b4b 100644
--- a/ui/angular/src/app/service/service.service.ts
+++ b/ui/angular/src/app/service/service.service.ts
@@ -38,6 +38,7 @@ export class ServiceService {
dataassetlist: this.BACKEND_SERVER + this.API_ROOT_PATH + "/metadata/hive/dbs/tables",
dblist: this.BACKEND_SERVER + this.API_ROOT_PATH + "/metadata/hive/dbs",
tablenames: this.BACKEND_SERVER + this.API_ROOT_PATH + "/metadata/hive/tables/names", // ?db=...
+ dbtablenames: this.BACKEND_SERVER + this.API_ROOT_PATH + "/metadata/hive/dbs/tables/names",
dbtable: this.BACKEND_SERVER + this.API_ROOT_PATH + "/metadata/hive/table", // ?db=...&table=...
getdataasset: this.BACKEND_SERVER + this.API_ROOT_PATH + "/dataassets",