You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2020/01/12 09:59:46 UTC
[kylin] branch master updated: KYLIN-4331 Fix noncompliant method
names in child class
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new 2a9aa87 KYLIN-4331 Fix noncompliant method names in child class
2a9aa87 is described below
commit 2a9aa87bb3833f80c29d462e5a1cd8c7a238b21e
Author: etherge <et...@163.com>
AuthorDate: Wed Jan 8 18:11:00 2020 -0500
KYLIN-4331 Fix noncompliant method names in child class
---
.../gridtable/CuboidToGridTableMappingExt.java | 4 +-
.../framework/FixedCachedRowSetImpl.java | 106 ++++++++++-----------
.../apache/kylin/engine/spark/SparkExecutable.java | 34 ++++---
.../kylin/engine/spark/SparkExecutableLivy.java | 28 +++---
4 files changed, 90 insertions(+), 82 deletions(-)
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMappingExt.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMappingExt.java
index 32c4ca0..23ec8cf 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMappingExt.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMappingExt.java
@@ -51,10 +51,10 @@ public class CuboidToGridTableMappingExt extends CuboidToGridTableMapping {
super(cuboid);
this.dynDims = dynDims;
this.dynFuncs = dynFuncs;
- init();
+ initialize();
}
- private void init() {
+ private void initialize() {
dynGtDataTypes = Lists.newArrayList();
dynGtColBlocks = Lists.newArrayList();
dynDim2gt = Maps.newHashMap();
diff --git a/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/FixedCachedRowSetImpl.java b/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/FixedCachedRowSetImpl.java
index 100dd27..79fcbf5 100644
--- a/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/FixedCachedRowSetImpl.java
+++ b/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/FixedCachedRowSetImpl.java
@@ -47,7 +47,7 @@ public class FixedCachedRowSetImpl extends CachedRowSetImpl {
super(env);
}
- private int getColIdxByName(String name) throws SQLException {
+ private int getColumnIdxByName(String name) throws SQLException {
rowSetMD = (RowSetMetaDataImpl) this.getMetaData();
int cols = rowSetMD.getColumnCount();
@@ -64,262 +64,262 @@ public class FixedCachedRowSetImpl extends CachedRowSetImpl {
@Override
public Collection<?> toCollection(String column) throws SQLException {
- return toCollection(getColIdxByName(column));
+ return toCollection(getColumnIdxByName(column));
}
@Override
public String getString(String columnName) throws SQLException {
- return getString(getColIdxByName(columnName));
+ return getString(getColumnIdxByName(columnName));
}
@Override
public boolean getBoolean(String columnName) throws SQLException {
- return getBoolean(getColIdxByName(columnName));
+ return getBoolean(getColumnIdxByName(columnName));
}
@Override
public byte getByte(String columnName) throws SQLException {
- return getByte(getColIdxByName(columnName));
+ return getByte(getColumnIdxByName(columnName));
}
@Override
public short getShort(String columnName) throws SQLException {
- return getShort(getColIdxByName(columnName));
+ return getShort(getColumnIdxByName(columnName));
}
@Override
public int getInt(String columnName) throws SQLException {
- return getInt(getColIdxByName(columnName));
+ return getInt(getColumnIdxByName(columnName));
}
@Override
public long getLong(String columnName) throws SQLException {
- return getLong(getColIdxByName(columnName));
+ return getLong(getColumnIdxByName(columnName));
}
@Override
public float getFloat(String columnName) throws SQLException {
- return getFloat(getColIdxByName(columnName));
+ return getFloat(getColumnIdxByName(columnName));
}
@Override
public double getDouble(String columnName) throws SQLException {
- return getDouble(getColIdxByName(columnName));
+ return getDouble(getColumnIdxByName(columnName));
}
@Override
public byte[] getBytes(String columnName) throws SQLException {
- return getBytes(getColIdxByName(columnName));
+ return getBytes(getColumnIdxByName(columnName));
}
@Override
public java.sql.Date getDate(String columnName) throws SQLException {
- return getDate(getColIdxByName(columnName));
+ return getDate(getColumnIdxByName(columnName));
}
@Override
public java.sql.Time getTime(String columnName) throws SQLException {
- return getTime(getColIdxByName(columnName));
+ return getTime(getColumnIdxByName(columnName));
}
@Override
public java.sql.Timestamp getTimestamp(String columnName) throws SQLException {
- return getTimestamp(getColIdxByName(columnName));
+ return getTimestamp(getColumnIdxByName(columnName));
}
@Override
public java.io.InputStream getAsciiStream(String columnName) throws SQLException {
- return getAsciiStream(getColIdxByName(columnName));
+ return getAsciiStream(getColumnIdxByName(columnName));
}
@Override
public java.io.InputStream getBinaryStream(String columnName) throws SQLException {
- return getBinaryStream(getColIdxByName(columnName));
+ return getBinaryStream(getColumnIdxByName(columnName));
}
@Override
public Object getObject(String columnName) throws SQLException {
- return getObject(getColIdxByName(columnName));
+ return getObject(getColumnIdxByName(columnName));
}
@Override
public int findColumn(String columnName) throws SQLException {
- return getColIdxByName(columnName);
+ return getColumnIdxByName(columnName);
}
@Override
public java.io.Reader getCharacterStream(String columnName) throws SQLException {
- return getCharacterStream(getColIdxByName(columnName));
+ return getCharacterStream(getColumnIdxByName(columnName));
}
@Override
public BigDecimal getBigDecimal(String columnName) throws SQLException {
- return getBigDecimal(getColIdxByName(columnName));
+ return getBigDecimal(getColumnIdxByName(columnName));
}
@Override
public boolean columnUpdated(String columnName) throws SQLException {
- return columnUpdated(getColIdxByName(columnName));
+ return columnUpdated(getColumnIdxByName(columnName));
}
@Override
public void updateNull(String columnName) throws SQLException {
- updateNull(getColIdxByName(columnName));
+ updateNull(getColumnIdxByName(columnName));
}
@Override
public void updateBoolean(String columnName, boolean x) throws SQLException {
- updateBoolean(getColIdxByName(columnName), x);
+ updateBoolean(getColumnIdxByName(columnName), x);
}
@Override
public void updateByte(String columnName, byte x) throws SQLException {
- updateByte(getColIdxByName(columnName), x);
+ updateByte(getColumnIdxByName(columnName), x);
}
@Override
public void updateShort(String columnName, short x) throws SQLException {
- updateShort(getColIdxByName(columnName), x);
+ updateShort(getColumnIdxByName(columnName), x);
}
@Override
public void updateInt(String columnName, int x) throws SQLException {
- updateInt(getColIdxByName(columnName), x);
+ updateInt(getColumnIdxByName(columnName), x);
}
@Override
public void updateLong(String columnName, long x) throws SQLException {
- updateLong(getColIdxByName(columnName), x);
+ updateLong(getColumnIdxByName(columnName), x);
}
@Override
public void updateFloat(String columnName, float x) throws SQLException {
- updateFloat(getColIdxByName(columnName), x);
+ updateFloat(getColumnIdxByName(columnName), x);
}
@Override
public void updateDouble(String columnName, double x) throws SQLException {
- updateDouble(getColIdxByName(columnName), x);
+ updateDouble(getColumnIdxByName(columnName), x);
}
@Override
public void updateBigDecimal(String columnName, BigDecimal x) throws SQLException {
- updateBigDecimal(getColIdxByName(columnName), x);
+ updateBigDecimal(getColumnIdxByName(columnName), x);
}
@Override
public void updateString(String columnName, String x) throws SQLException {
- updateString(getColIdxByName(columnName), x);
+ updateString(getColumnIdxByName(columnName), x);
}
@Override
public void updateBytes(String columnName, byte[] x) throws SQLException {
- updateBytes(getColIdxByName(columnName), x);
+ updateBytes(getColumnIdxByName(columnName), x);
}
@Override
public void updateDate(String columnName, java.sql.Date x) throws SQLException {
- updateDate(getColIdxByName(columnName), x);
+ updateDate(getColumnIdxByName(columnName), x);
}
@Override
public void updateTime(String columnName, java.sql.Time x) throws SQLException {
- updateTime(getColIdxByName(columnName), x);
+ updateTime(getColumnIdxByName(columnName), x);
}
@Override
public void updateTimestamp(String columnName, java.sql.Timestamp x) throws SQLException {
- updateTimestamp(getColIdxByName(columnName), x);
+ updateTimestamp(getColumnIdxByName(columnName), x);
}
@Override
public void updateAsciiStream(String columnName, java.io.InputStream x, int length) throws SQLException {
- updateAsciiStream(getColIdxByName(columnName), x, length);
+ updateAsciiStream(getColumnIdxByName(columnName), x, length);
}
@Override
public void updateBinaryStream(String columnName, java.io.InputStream x, int length) throws SQLException {
- updateBinaryStream(getColIdxByName(columnName), x, length);
+ updateBinaryStream(getColumnIdxByName(columnName), x, length);
}
@Override
public void updateCharacterStream(String columnName, java.io.Reader reader, int length) throws SQLException {
- updateCharacterStream(getColIdxByName(columnName), reader, length);
+ updateCharacterStream(getColumnIdxByName(columnName), reader, length);
}
@Override
public void updateObject(String columnName, Object x, int scale) throws SQLException {
- updateObject(getColIdxByName(columnName), x, scale);
+ updateObject(getColumnIdxByName(columnName), x, scale);
}
@Override
public void updateObject(String columnName, Object x) throws SQLException {
- updateObject(getColIdxByName(columnName), x);
+ updateObject(getColumnIdxByName(columnName), x);
}
@Override
public Object getObject(String columnName, java.util.Map<String, Class<?>> map) throws SQLException {
- return getObject(getColIdxByName(columnName), map);
+ return getObject(getColumnIdxByName(columnName), map);
}
@Override
public Ref getRef(String colName) throws SQLException {
- return getRef(getColIdxByName(colName));
+ return getRef(getColumnIdxByName(colName));
}
@Override
public Blob getBlob(String colName) throws SQLException {
- return getBlob(getColIdxByName(colName));
+ return getBlob(getColumnIdxByName(colName));
}
@Override
public Clob getClob(String colName) throws SQLException {
- return getClob(getColIdxByName(colName));
+ return getClob(getColumnIdxByName(colName));
}
@Override
public Array getArray(String colName) throws SQLException {
- return getArray(getColIdxByName(colName));
+ return getArray(getColumnIdxByName(colName));
}
@Override
public java.sql.Date getDate(String columnName, Calendar cal) throws SQLException {
- return getDate(getColIdxByName(columnName), cal);
+ return getDate(getColumnIdxByName(columnName), cal);
}
@Override
public java.sql.Time getTime(String columnName, Calendar cal) throws SQLException {
- return getTime(getColIdxByName(columnName), cal);
+ return getTime(getColumnIdxByName(columnName), cal);
}
@Override
public java.sql.Timestamp getTimestamp(String columnName, Calendar cal) throws SQLException {
- return getTimestamp(getColIdxByName(columnName), cal);
+ return getTimestamp(getColumnIdxByName(columnName), cal);
}
@Override
public void updateRef(String columnName, java.sql.Ref ref) throws SQLException {
- updateRef(getColIdxByName(columnName), ref);
+ updateRef(getColumnIdxByName(columnName), ref);
}
@Override
public void updateClob(String columnName, Clob c) throws SQLException {
- updateClob(getColIdxByName(columnName), c);
+ updateClob(getColumnIdxByName(columnName), c);
}
@Override
public void updateBlob(String columnName, Blob b) throws SQLException {
- updateBlob(getColIdxByName(columnName), b);
+ updateBlob(getColumnIdxByName(columnName), b);
}
@Override
public void updateArray(String columnName, Array a) throws SQLException {
- updateArray(getColIdxByName(columnName), a);
+ updateArray(getColumnIdxByName(columnName), a);
}
@Override
public java.net.URL getURL(String columnName) throws SQLException {
- return getURL(getColIdxByName(columnName));
+ return getURL(getColumnIdxByName(columnName));
}
}
\ No newline at end of file
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
index 1eeab04..a95d3d6 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
@@ -79,7 +79,6 @@ public class SparkExecutable extends AbstractExecutable {
this.setParam(CLASS_NAME, className);
}
-
public void setJobId(String jobId) {
this.setParam(JOB_ID, jobId);
}
@@ -160,7 +159,7 @@ public class SparkExecutable extends AbstractExecutable {
}
}
- private ExecuteResult onResumed(String appId, ExecutableManager mgr) throws ExecuteException {
+ protected ExecuteResult onResumed(String appId, ExecutableManager mgr) throws ExecuteException {
Map<String, String> info = new HashMap<>();
try {
logger.info("spark_job_id:" + appId + " resumed");
@@ -210,7 +209,7 @@ public class SparkExecutable extends AbstractExecutable {
CubeInstance cube;
if (cubeName != null) {
cube = CubeManager.getInstance(context.getConfig()).getCube(cubeName);
- } else { // Cube name can't be got when loading hive table
+ } else { // Cube name can't be got when loading hive table
cube = null;
}
final KylinConfig config;
@@ -280,7 +279,8 @@ public class SparkExecutable extends AbstractExecutable {
if (StringUtils.isNotEmpty(config.getHBaseClusterFs())) {
String fileSystems = sparkConfs.get("spark.yarn.access.hadoopFileSystems");
if (StringUtils.isNotEmpty(fileSystems)) {
- sparkConfs.put("spark.yarn.access.hadoopFileSystems", fileSystems + "," + config.getHBaseClusterFs());
+ sparkConfs.put("spark.yarn.access.hadoopFileSystems",
+ fileSystems + "," + config.getHBaseClusterFs());
} else {
sparkConfs.put("spark.yarn.access.hadoopFileSystems", config.getHBaseClusterFs());
}
@@ -372,10 +372,10 @@ public class SparkExecutable extends AbstractExecutable {
readCounters(joblogInfo);
getManager().addJobInfo(getId(), joblogInfo);
if (joblogInfo.containsKey(ExecutableConstants.SPARK_DIMENSION_DIC_SEGMENT_ID)) {
- updateSparkDimensionDicMetadata(config, cube, joblogInfo.get(ExecutableConstants.SPARK_DIMENSION_DIC_SEGMENT_ID));
+ updateSparkDimensionDicMetadata(config, cube,
+ joblogInfo.get(ExecutableConstants.SPARK_DIMENSION_DIC_SEGMENT_ID));
logger.info("Finished update dictionaries and snapshot info from {} to {}.",
- this.getParam(SparkBuildDictionary.OPTION_META_URL.getOpt()),
- config.getMetadataUrl());
+ this.getParam(SparkBuildDictionary.OPTION_META_URL.getOpt()), config.getMetadataUrl());
}
return new ExecuteResult(ExecuteResult.State.SUCCEED, patternedLogger.getBufferedLog());
}
@@ -414,14 +414,16 @@ public class SparkExecutable extends AbstractExecutable {
}
//to update metadata from hdfs due to the step build dimension dic using spark dump metadata to hdfs
- private void updateSparkDimensionDicMetadata(KylinConfig config, CubeInstance cube, String segmentId) throws IOException{
- KylinConfig hdfsConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(this.getParam(SparkBuildDictionary.OPTION_META_URL.getOpt()));
+ private void updateSparkDimensionDicMetadata(KylinConfig config, CubeInstance cube, String segmentId)
+ throws IOException {
+ KylinConfig hdfsConfig = AbstractHadoopJob
+ .loadKylinConfigFromHdfs(this.getParam(SparkBuildDictionary.OPTION_META_URL.getOpt()));
CubeInstance cubeInstance = CubeManager.getInstance(hdfsConfig).reloadCube(cube.getName());
CubeSegment segment = cubeInstance.getSegmentById(segmentId);
CubeSegment oldSeg = cube.getSegmentById(segmentId);
oldSeg.setDictionaries((ConcurrentHashMap<String, String>) segment.getDictionaries());
- oldSeg.setSnapshots((ConcurrentHashMap)segment.getSnapshots());
+ oldSeg.setSnapshots((ConcurrentHashMap) segment.getSnapshots());
oldSeg.getRowkeyStats().addAll(segment.getRowkeyStats());
CubeInstance cubeCopy = cube.latestCopyForWrite();
CubeUpdate update = new CubeUpdate(cubeCopy);
@@ -432,7 +434,8 @@ public class SparkExecutable extends AbstractExecutable {
dumpList.addAll(segment.getDictionaryPaths());
dumpList.addAll(segment.getSnapshotPaths());
- JobRelatedMetaUtil.dumpAndUploadKylinPropsAndMetadata(dumpList, (KylinConfigExt) segment.getConfig(), config.getMetadataUrl().toString());
+ JobRelatedMetaUtil.dumpAndUploadKylinPropsAndMetadata(dumpList, (KylinConfigExt) segment.getConfig(),
+ config.getMetadataUrl().toString());
}
// Spark Cubing can only work in layer algorithm
@@ -442,7 +445,7 @@ public class SparkExecutable extends AbstractExecutable {
cubingJob.setAlgorithm(CubingJob.AlgorithmEnum.LAYER);
}
- private String getAppState(String appId) throws IOException {
+ protected String getAppState(String appId) throws IOException {
if (StringUtils.isEmpty(appId)) {
throw new IOException("The app is is null or empty");
}
@@ -454,13 +457,13 @@ public class SparkExecutable extends AbstractExecutable {
return info.get(ExecutableConstants.YARN_APP_STATE);
}
- private void killApp(String appId) throws IOException, InterruptedException {
+ protected void killApp(String appId) throws IOException, InterruptedException {
CliCommandExecutor executor = KylinConfig.getInstanceFromEnv().getCliCommandExecutor();
String killCmd = String.format(Locale.ROOT, "yarn application -kill %s", appId);
executor.execute(killCmd);
}
- private int killAppRetry(String appId) throws IOException, InterruptedException {
+ protected int killAppRetry(String appId) throws IOException, InterruptedException {
if (StringUtils.isEmpty(appId)) {
logger.warn("The app is is null or empty");
return 0;
@@ -508,7 +511,8 @@ public class SparkExecutable extends AbstractExecutable {
}
private void attachSegmentsMetadataWithDict(List<CubeSegment> segments) throws IOException {
- Set<String> dumpList = new LinkedHashSet<>(JobRelatedMetaUtil.collectCubeMetadata(segments.get(0).getCubeInstance()));
+ Set<String> dumpList = new LinkedHashSet<>(
+ JobRelatedMetaUtil.collectCubeMetadata(segments.get(0).getCubeInstance()));
ResourceStore rs = ResourceStore.getStore(segments.get(0).getConfig());
for (CubeSegment segment : segments) {
dumpList.addAll(segment.getDictionaryPaths());
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutableLivy.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutableLivy.java
index 709c1b7..37938b0 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutableLivy.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutableLivy.java
@@ -17,6 +17,11 @@
*/
package org.apache.kylin.engine.spark;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.KylinConfig;
@@ -41,11 +46,6 @@ import org.apache.kylin.metadata.model.Segments;
import org.apache.parquet.Strings;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
/**
*
*/
@@ -107,7 +107,8 @@ public class SparkExecutableLivy extends SparkExecutable {
}
}
- private ExecuteResult onResumed(String appId, ExecutableManager mgr) throws ExecuteException {
+ @Override
+ protected ExecuteResult onResumed(String appId, ExecutableManager mgr) throws ExecuteException {
Map<String, String> info = new HashMap<>();
try {
logger.info("livy spark_job_id:" + appId + " resumed");
@@ -176,7 +177,8 @@ public class SparkExecutableLivy extends SparkExecutable {
}
for (Map.Entry<String, String> entry : sparkConfs.entrySet()) {
- if (entry.getKey().equals("spark.submit.deployMode") || entry.getKey().equals("spark.master") || entry.getKey().equals("spark.yarn.archive")) {
+ if (entry.getKey().equals("spark.submit.deployMode") || entry.getKey().equals("spark.master")
+ || entry.getKey().equals("spark.yarn.archive")) {
continue;
} else {
livyRestBuilder.addConf(entry.getKey(), entry.getValue());
@@ -187,8 +189,7 @@ public class SparkExecutableLivy extends SparkExecutable {
final LivyRestExecutor executor = new LivyRestExecutor();
final PatternedLogger patternedLogger = new PatternedLogger(logger, (infoKey, info) -> {
// only care three properties here
- if (ExecutableConstants.SPARK_JOB_ID.equals(infoKey)
- || ExecutableConstants.YARN_APP_ID.equals(infoKey)
+ if (ExecutableConstants.SPARK_JOB_ID.equals(infoKey) || ExecutableConstants.YARN_APP_ID.equals(infoKey)
|| ExecutableConstants.YARN_APP_URL.equals(infoKey)) {
getManager().addJobInfo(getId(), info);
}
@@ -230,17 +231,20 @@ public class SparkExecutableLivy extends SparkExecutable {
}
}
- private String getAppState(String appId) throws IOException {
+ @Override
+ protected String getAppState(String appId) throws IOException {
LivyRestExecutor executor = new LivyRestExecutor();
return executor.state(appId);
}
- private void killApp(String appId) throws IOException, InterruptedException {
+ @Override
+ protected void killApp(String appId) throws IOException, InterruptedException {
LivyRestExecutor executor = new LivyRestExecutor();
executor.kill(appId);
}
- private int killAppRetry(String appId) throws IOException, InterruptedException {
+ @Override
+ protected int killAppRetry(String appId) throws IOException, InterruptedException {
String status = getAppState(appId);
if (Strings.isNullOrEmpty(status) || LivyStateEnum.dead.name().equalsIgnoreCase(status)
|| LivyStateEnum.error.name().equalsIgnoreCase(status)