You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2020/09/01 09:53:03 UTC
[kylin] branch kylin-on-parquet-v2 updated (e573ed3 -> 5eb9b37)
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a change to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from e573ed3 KYLIN-4705 add rest api for sparder-context url monitor
new f48a032 KYLIN-4733 The cube size is inconsistent with the size of all segments
new 5eb9b37 KYLIN-4732 The cube size is wrong after disabling/enabling the cube
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../kylin/engine/spark/utils/BuildUtils.scala | 3 +-
.../kylin/rest/controller/CubeController.java | 45 ++++++++++++----------
.../{HBaseResponse.java => StorageResponse.java} | 4 +-
.../org/apache/kylin/rest/service/CubeService.java | 33 ++++++++--------
.../apache/kylin/rest/service/HBaseInfoUtil.java | 6 +--
webapp/app/js/controllers/cube.js | 4 +-
webapp/app/js/controllers/cubes.js | 2 +
webapp/app/js/filters/filter.js | 4 +-
webapp/app/js/services/cubes.js | 2 +-
webapp/app/partials/cubes/cube_detail.html | 2 +-
10 files changed, 56 insertions(+), 49 deletions(-)
rename server-base/src/main/java/org/apache/kylin/rest/response/{HBaseResponse.java => StorageResponse.java} (97%)
[kylin] 02/02: KYLIN-4732 The cube size is wrong after
disabling/enabling the cube
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 5eb9b37cf4164f4cf94076aa84fd47f346ae4b17
Author: Zhichao Zhang <44...@qq.com>
AuthorDate: Tue Sep 1 16:10:12 2020 +0800
KYLIN-4732 The cube size is wrong after disabling/enabling the cube
---
webapp/app/js/controllers/cubes.js | 2 ++
1 file changed, 2 insertions(+)
diff --git a/webapp/app/js/controllers/cubes.js b/webapp/app/js/controllers/cubes.js
index 7000581..21d86e1 100644
--- a/webapp/app/js/controllers/cubes.js
+++ b/webapp/app/js/controllers/cubes.js
@@ -48,10 +48,12 @@ KylinApp.controller('CubesCtrl', function ($scope, $q, $routeParams, $location,
CubeService.getCube(queryParam, function(newCube){
var segmentsLen = newCube.segments && newCube.segments.length || 0
newCube.input_records_count = 0;
+ newCube.size_kb = 0;
for(var i = segmentsLen - 1;i >= 0;i--){
var curSeg = newCube.segments[i]
if(curSeg.status === "READY"){
newCube.input_records_count += curSeg.input_records
+ newCube.size_kb += curSeg.size_kb
if(newCube.last_build_time === undefined || newCube.last_build_time < curSeg.last_build_time) {
newCube.last_build_time = curSeg.last_build_time;
}
[kylin] 01/02: KYLIN-4733 The cube size is inconsistent with the
size of all segments
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit f48a0321b314e63516ac1a0774d68b3836ba5978
Author: Zhichao Zhang <44...@qq.com>
AuthorDate: Tue Sep 1 16:05:09 2020 +0800
KYLIN-4733 The cube size is inconsistent with the size of all segments
---
.../kylin/engine/spark/utils/BuildUtils.scala | 3 +-
.../kylin/rest/controller/CubeController.java | 45 ++++++++++++----------
.../{HBaseResponse.java => StorageResponse.java} | 4 +-
.../org/apache/kylin/rest/service/CubeService.java | 33 ++++++++--------
.../apache/kylin/rest/service/HBaseInfoUtil.java | 6 +--
webapp/app/js/controllers/cube.js | 4 +-
webapp/app/js/filters/filter.js | 4 +-
webapp/app/js/services/cubes.js | 2 +-
webapp/app/partials/cubes/cube_detail.html | 2 +-
9 files changed, 54 insertions(+), 49 deletions(-)
diff --git a/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/kylin/engine/spark/utils/BuildUtils.scala b/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/kylin/engine/spark/utils/BuildUtils.scala
index 980c4b3..4d0db98 100644
--- a/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/kylin/engine/spark/utils/BuildUtils.scala
+++ b/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/kylin/engine/spark/utils/BuildUtils.scala
@@ -81,7 +81,8 @@ object BuildUtils extends Logging {
@throws[IOException]
def fillCuboidInfo(cuboid: LayoutEntity, strPath: String): Unit = {
val fs = HadoopUtil.getWorkingFileSystem
- if (fs.exists(new Path(strPath))) {
+ // when cuboid.getRows == 0, it means there is no data written, so set byte size to 0
+ if (fs.exists(new Path(strPath)) && (cuboid.getRows > 0)) {
val cs = HadoopUtil.getContentSummary(fs, new Path(strPath))
cuboid.setFileCount(cs.getFileCount)
cuboid.setByteSize(cs.getLength)
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index aac0b47..a8f59f3 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -72,7 +72,7 @@ import org.apache.kylin.rest.response.CubeInstanceResponse;
import org.apache.kylin.rest.response.CuboidTreeResponse;
import org.apache.kylin.rest.response.EnvelopeResponse;
import org.apache.kylin.rest.response.GeneralResponse;
-import org.apache.kylin.rest.response.HBaseResponse;
+import org.apache.kylin.rest.response.StorageResponse;
import org.apache.kylin.rest.response.ResponseCode;
import org.apache.kylin.rest.service.CubeService;
import org.apache.kylin.rest.service.JobService;
@@ -713,15 +713,16 @@ public class CubeController extends BasicController {
}
/**
- * get Hbase Info
+ * get storage Info
*
* @return true
* @throws IOException
*/
- @RequestMapping(value = "/{cubeName}/hbase", method = { RequestMethod.GET }, produces = { "application/json" })
+ @RequestMapping(value = "/{cubeName}/storage", method = { RequestMethod.GET }, produces = {
+ "application/json" })
@ResponseBody
- public List<HBaseResponse> getHBaseInfo(@PathVariable String cubeName) {
- List<HBaseResponse> hbase = new ArrayList<HBaseResponse>();
+ public List<StorageResponse> getStorageInfo(@PathVariable String cubeName) {
+ List<StorageResponse> storage = new ArrayList<StorageResponse>();
CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
if (null == cube) {
@@ -733,38 +734,40 @@ public class CubeController extends BasicController {
for (CubeSegment segment : segments) {
Map<String, String> addInfo = segment.getAdditionalInfo();
String tableName = segment.getStorageLocationIdentifier();
- HBaseResponse hr = null;
+ StorageResponse sr = null;
if (("" + IStorageAware.ID_PARQUET).equals(addInfo.get("storageType"))) {
- hr = new HBaseResponse();
- hr.setStorageType("parquet");
+ sr = new StorageResponse();
+ sr.setStorageType("parquet");
+ // unit: Byte
+ sr.setTableSize(segment.getSizeKB() * 1024);
} else {
// Get info of given table.
try {
- hr = cubeService.getHTableInfo(cubeName, tableName);
+ sr = cubeService.getHTableInfo(cubeName, tableName);
} catch (IOException e) {
logger.error("Failed to calcuate size of HTable \"" + tableName + "\".", e);
}
- if (null == hr) {
+ if (null == sr) {
logger.info("Failed to calcuate size of HTable \"" + tableName + "\".");
- hr = new HBaseResponse();
+ sr = new StorageResponse();
}
}
- hr.setTableName(tableName);
- hr.setDateRangeStart(segment.getTSRange().start.v);
- hr.setDateRangeEnd(segment.getTSRange().end.v);
- hr.setSegmentName(segment.getName());
- hr.setSegmentStatus(segment.getStatus().toString());
- hr.setSourceCount(segment.getInputRecords());
+ sr.setTableName(tableName);
+ sr.setDateRangeStart(segment.getTSRange().start.v);
+ sr.setDateRangeEnd(segment.getTSRange().end.v);
+ sr.setSegmentName(segment.getName());
+ sr.setSegmentStatus(segment.getStatus().toString());
+ sr.setSourceCount(segment.getInputRecords());
if (segment.isOffsetCube()) {
- hr.setSourceOffsetStart((Long) segment.getSegRange().start.v);
- hr.setSourceOffsetEnd((Long) segment.getSegRange().end.v);
+ sr.setSourceOffsetStart((Long) segment.getSegRange().start.v);
+ sr.setSourceOffsetEnd((Long) segment.getSegRange().end.v);
}
- hbase.add(hr);
+ storage.add(sr);
}
- return hbase;
+ return storage;
}
/**
diff --git a/server-base/src/main/java/org/apache/kylin/rest/response/HBaseResponse.java b/server-base/src/main/java/org/apache/kylin/rest/response/StorageResponse.java
similarity index 97%
rename from server-base/src/main/java/org/apache/kylin/rest/response/HBaseResponse.java
rename to server-base/src/main/java/org/apache/kylin/rest/response/StorageResponse.java
index cb2a108..30784b5 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/response/HBaseResponse.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/response/StorageResponse.java
@@ -20,7 +20,7 @@ package org.apache.kylin.rest.response;
import java.io.Serializable;
-public class HBaseResponse implements Serializable {
+public class StorageResponse implements Serializable {
private static final long serialVersionUID = 7263557115683263492L;
private String storageType;
private String segmentName;
@@ -35,7 +35,7 @@ public class HBaseResponse implements Serializable {
private long sourceOffsetEnd;
private long sourceCount;
- public HBaseResponse() {
+ public StorageResponse() {
}
public String getStorageType() {
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index b50bdc0..d286cfd 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -82,7 +82,7 @@ import org.apache.kylin.rest.request.PrepareSqlRequest;
import org.apache.kylin.rest.response.CubeInstanceResponse;
import org.apache.kylin.rest.response.CuboidTreeResponse;
import org.apache.kylin.rest.response.CuboidTreeResponse.NodeInfo;
-import org.apache.kylin.rest.response.HBaseResponse;
+import org.apache.kylin.rest.response.StorageResponse;
import org.apache.kylin.rest.response.MetricsResponse;
import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.rest.util.ValidateUtil;
@@ -113,7 +113,7 @@ public class CubeService extends BasicService implements InitializingBean {
private static final Logger logger = LoggerFactory.getLogger(CubeService.class);
- protected Cache<String, HBaseResponse> htableInfoCache = CacheBuilder.newBuilder().build();
+ protected Cache<String, StorageResponse> storageInfoCache = CacheBuilder.newBuilder().build();
@Autowired
@Qualifier("projectService")
@@ -530,18 +530,18 @@ public class CubeService extends BasicService implements InitializingBean {
* table.
*
* @param tableName The table name.
- * @return The HBaseResponse object contains table size, region count. null
+ * @return The StorageResponse object contains table size, region count. null
* if error happens
* @throws IOException Exception when HTable resource is not closed correctly.
*/
- public HBaseResponse getHTableInfo(String cubeName, String tableName) throws IOException {
+ public StorageResponse getHTableInfo(String cubeName, String tableName) throws IOException {
String key = cubeName + "/" + tableName;
- HBaseResponse hr = htableInfoCache.getIfPresent(key);
- if (null != hr) {
- return hr;
+ StorageResponse sr = storageInfoCache.getIfPresent(key);
+ if (null != sr) {
+ return sr;
}
- hr = new HBaseResponse();
+ sr = new StorageResponse();
CubeInstance cube = CubeManager.getInstance(getConfig()).getCube(cubeName);
if (cube.getStorageType() == IStorageAware.ID_HBASE || cube.getStorageType() == IStorageAware.ID_SHARDED_HBASE
|| cube.getStorageType() == IStorageAware.ID_REALTIME_AND_HBASE) {
@@ -549,17 +549,18 @@ public class CubeService extends BasicService implements InitializingBean {
logger.debug("Loading HTable info " + cubeName + ", " + tableName);
// use reflection to isolate NoClassDef errors when HBase is not available
- hr = (HBaseResponse) Class.forName("org.apache.kylin.rest.service.HBaseInfoUtil")//
- .getMethod("getHBaseInfo", new Class[] { String.class, KylinConfig.class })//
+ sr = (StorageResponse)
+ Class.forName("org.apache.kylin.rest.service.HBaseInfoUtil")
+ .getMethod("getHBaseInfo", new Class[] { String.class, KylinConfig.class })
.invoke(null, tableName, this.getConfig());
- hr.setStorageType("hbase");
+ sr.setStorageType("hbase");
} catch (Throwable e) {
throw new IOException(e);
}
}
- htableInfoCache.put(key, hr);
- return hr;
+ storageInfoCache.put(key, sr);
+ return sr;
}
public void updateCubeNotifyList(CubeInstance cube, List<String> notifyList) throws IOException {
@@ -1133,7 +1134,7 @@ public class CubeService extends BasicService implements InitializingBean {
private class HTableInfoSyncListener extends Broadcaster.Listener {
@Override
public void onClearAll(Broadcaster broadcaster) throws IOException {
- htableInfoCache.invalidateAll();
+ storageInfoCache.invalidateAll();
}
@Override
@@ -1141,9 +1142,9 @@ public class CubeService extends BasicService implements InitializingBean {
throws IOException {
String cubeName = cacheKey;
String keyPrefix = cubeName + "/";
- for (String k : htableInfoCache.asMap().keySet()) {
+ for (String k : storageInfoCache.asMap().keySet()) {
if (k.startsWith(keyPrefix))
- htableInfoCache.invalidate(k);
+ storageInfoCache.invalidate(k);
}
}
}
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/HBaseInfoUtil.java b/server-base/src/main/java/org/apache/kylin/rest/service/HBaseInfoUtil.java
index e984295..76ddbc0 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/HBaseInfoUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/HBaseInfoUtil.java
@@ -21,12 +21,12 @@ package org.apache.kylin.rest.service;
public class HBaseInfoUtil {
// @SuppressWarnings("unused") // used by reflection
-// public static HBaseResponse getHBaseInfo(String tableName, KylinConfig config) throws IOException {
+// public static StorageResponse getHBaseInfo(String tableName, KylinConfig config) throws IOException {
// if (!config.getStorageUrl().getScheme().equals("hbase"))
// return null;
//
// Connection conn = HBaseUnionUtil.getConnection(config, tableName);
-// HBaseResponse hr = null;
+// StorageResponse hr = null;
// long tableSize = 0;
// int regionCount = 0;
//
@@ -40,7 +40,7 @@ public class HBaseInfoUtil {
// regionCount = sizeMap.size();
//
// // Set response.
-// hr = new HBaseResponse();
+// hr = new StorageResponse();
// hr.setTableSize(tableSize);
// hr.setRegionCount(regionCount);
// return hr;
diff --git a/webapp/app/js/controllers/cube.js b/webapp/app/js/controllers/cube.js
index 0d48dbc..60df772 100755
--- a/webapp/app/js/controllers/cube.js
+++ b/webapp/app/js/controllers/cube.js
@@ -88,9 +88,9 @@ KylinApp.controller('CubeCtrl', function ($scope, $rootScope, AccessService, Mes
});
};
- $scope.getHbaseInfo = function (cube) {
+ $scope.getStorageInfo = function (cube) {
if (!cube.hbase) {
- CubeService.getHbaseInfo({cubeId: cube.name, propValue: null, action: null}, function (hbase) {
+ CubeService.getStorageInfo({cubeId: cube.name, propValue: null, action: null}, function (hbase) {
cube.hbase = hbase;
TableService.get({pro:cube.model.project, tableName:cube.model.fact_table},function(table) {
if (table && table.source_type == 1) {
diff --git a/webapp/app/js/filters/filter.js b/webapp/app/js/filters/filter.js
index 0f8382d..a239c17 100755
--- a/webapp/app/js/filters/filter.js
+++ b/webapp/app/js/filters/filter.js
@@ -87,7 +87,7 @@ KylinApp
.filter('bytes', function () {
return function (bytes, precision) {
if (bytes === 0) {
- return 'less than 1 MB';
+ return '0 KB';
}
if (isNaN(parseFloat(bytes)) || !isFinite(bytes)) {
return '-';
@@ -97,7 +97,7 @@ KylinApp
precision = 3;
}
- var units = ['bytes', 'kB', 'MB', 'GB', 'TB', 'PB'],
+ var units = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB'],
number = Math.floor(Math.log(bytes) / Math.log(1024));
switch(number){
case 0:
diff --git a/webapp/app/js/services/cubes.js b/webapp/app/js/services/cubes.js
index 52cb8fe..53f490d 100644
--- a/webapp/app/js/services/cubes.js
+++ b/webapp/app/js/services/cubes.js
@@ -54,7 +54,7 @@ KylinApp.factory('CubeService', ['$resource', function ($resource, config) {
drop: {method: 'DELETE', params: {}, isArray: false},
save: {method: 'POST', params: {}, isArray: false},
update: {method: 'PUT', params: {}, isArray: false},
- getHbaseInfo: {method: 'GET', params: {propName: 'hbase'}, isArray: true},
+ getStorageInfo: {method: 'GET', params: {propName: 'storage'}, isArray: true},
getCurrentCuboids: {
method: 'GET',
params: {
diff --git a/webapp/app/partials/cubes/cube_detail.html b/webapp/app/partials/cubes/cube_detail.html
index 7cb0351..0076069 100755
--- a/webapp/app/partials/cubes/cube_detail.html
+++ b/webapp/app/partials/cubes/cube_detail.html
@@ -39,7 +39,7 @@
</li>
<li class="{{cube.visiblePage=='hbase'? 'active':''}}"
ng-if="userService.hasRole('ROLE_ADMIN') || hasPermission('cube' ,cube, permissions.ADMINISTRATION.mask) && !newAccess">
- <a href="" ng-click="cube.visiblePage='hbase';getHbaseInfo(cube)">Storage</a>
+ <a href="" ng-click="cube.visiblePage='hbase';getStorageInfo(cube)">Storage</a>
</li>
<li class="{{cube.visiblePage=='planner'? 'active':''}}" ng-if="(userService.hasRole('ROLE_ADMIN') || hasPermission('cube', cube, permissions.ADMINISTRATION.mask) && !newAccess) && isShowCubeplanner && cube.detail.engine_type != 6">
<a href="" ng-click="cube.visiblePage='planner';getCubePlanner(cube);">Planner</a>