You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/12/31 13:19:41 UTC
[24/50] [abbrv] kylin git commit: KYLIN-3092 fix cyclic lock
KYLIN-3092 fix cyclic lock
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f37fbce1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f37fbce1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f37fbce1
Branch: refs/heads/master
Commit: f37fbce17f6e25775877290ba1f0de076c62f32f
Parents: 9f603d3
Author: Li Yang <li...@apache.org>
Authored: Wed Dec 20 21:50:54 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Thu Dec 21 11:06:34 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/common/util/HiveCmdBuilder.java | 4 ++--
.../main/java/org/apache/kylin/cube/CubeDescManager.java | 9 ---------
.../kylin/job/impl/threadpool/DefaultScheduler.java | 5 +++--
.../org/apache/kylin/metadata/TableMetadataManager.java | 10 +++++++---
4 files changed, 12 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f37fbce1/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
index 10d6798..77dc579 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
@@ -188,8 +188,8 @@ public class HiveCmdBuilder {
hiveConfFile = new File(path + File.separator + "conf", hiveConfFileName);
}
- if (hiveConfFile == null || !hiveConfFile.exists()) {
- throw new RuntimeException("Failed to read " + HIVE_CONF_FILENAME + ".xml");
+ if (!hiveConfFile.exists()) {
+ throw new RuntimeException("Missing config file: " + hiveConfFile.getAbsolutePath());
}
String fileUrl = OptionsHelper.convertToFileURL(hiveConfFile.getAbsolutePath());
http://git-wip-us.apache.org/repos/asf/kylin/blob/f37fbce1/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index a58ba40..a416499 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -46,7 +46,6 @@ import org.apache.kylin.metadata.model.ParameterDesc;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -166,14 +165,6 @@ public class CubeDescManager {
if (ndesc.isBroken())
throw new IllegalStateException("CubeDesc " + name + " is broken");
- // if related cube is in DESCBROKEN state before, change it back to DISABLED
- CubeManager cubeManager = CubeManager.getInstance(config);
- for (CubeInstance cube : cubeManager.getCubesByDesc(name)) {
- if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
- cube.init(config);
- }
- }
-
return ndesc;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f37fbce1/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
index 78f10bf..c8ab7ea 100644
--- a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
+++ b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
@@ -163,8 +163,6 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
try (SetThreadName ignored = new SetThreadName("Scheduler %s Job %s",
System.identityHashCode(DefaultScheduler.this), executable.getId())) {
executable.execute(context);
- // trigger the next step asap
- fetcherPool.schedule(fetcher, 0, TimeUnit.SECONDS);
} catch (ExecuteException e) {
logger.error("ExecuteException job:" + executable.getId(), e);
} catch (Exception e) {
@@ -172,6 +170,9 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
} finally {
context.removeRunningJob(executable);
}
+
+ // trigger the next step asap
+ fetcherPool.schedule(fetcher, 0, TimeUnit.SECONDS);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f37fbce1/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java
index 844159d..efbc33e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java
@@ -156,6 +156,9 @@ public class TableMetadataManager {
}
public Map<String, TableDesc> getAllTablesMap(String prj) {
+ // avoid cyclic locks
+ ProjectInstance project = (prj == null) ? null : ProjectManager.getInstance(config).getProject(prj);
+
try (AutoLock lock = srcTableMapLock.lockForWrite()) {
//TODO prj == null case is now only used by test case and CubeMetaIngester
//should refactor these test case and tool ASAP and stop supporting null case
@@ -168,7 +171,6 @@ public class TableMetadataManager {
return globalTables;
}
- ProjectInstance project = ProjectManager.getInstance(config).getProject(prj);
Set<String> prjTableNames = project.getTables();
Map<String, TableDesc> ret = new LinkedHashMap<>();
@@ -248,8 +250,10 @@ public class TableMetadataManager {
* again
*/
public void resetProjectSpecificTableDesc(String prj) throws IOException {
+ // avoid cyclic locks
+ ProjectInstance project = ProjectManager.getInstance(config).getProject(prj);
+
try (AutoLock lock = srcTableMapLock.lockForWrite()) {
- ProjectInstance project = ProjectManager.getInstance(config).getProject(prj);
for (String tableName : project.getTables()) {
String tableIdentity = getTableIdentity(tableName);
String key = mapKey(tableIdentity, prj);
@@ -338,7 +342,7 @@ public class TableMetadataManager {
result.setUuid(UUID.randomUUID().toString());
result.setLastModified(0);
result.init(t.getProject());
- srcExtMap.put(mapKey(t.getIdentity(), t.getProject()), result);
+ srcExtMap.putLocal(mapKey(t.getIdentity(), t.getProject()), result);
}
return result;
}