You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/07/25 09:45:04 UTC
[17/24] kylin git commit: KYLIN-2748 Calcite code generation can not
gc cause OOM
KYLIN-2748 Calcite code generation can not gc cause OOM
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8898213f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8898213f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8898213f
Branch: refs/heads/2.1.x
Commit: 8898213f9d8229382b805574784e566fb841c7eb
Parents: 057bca4
Author: yiming.xu <10...@qq.com>
Authored: Thu Jul 20 18:12:44 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Mon Jul 24 17:56:04 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/query/QueryConnection.java | 42 ++++++++++++++++++++
.../apache/kylin/rest/service/CacheService.java | 1 +
.../apache/kylin/rest/service/QueryService.java | 10 ++---
.../kylin/rest/service/CubeServiceTest.java | 6 ++-
.../kylin/rest/service/JobServiceTest.java | 6 ++-
.../kylin/rest/service/QueryServiceTest.java | 3 +-
6 files changed, 57 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8898213f/query/src/main/java/org/apache/kylin/query/QueryConnection.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/QueryConnection.java b/query/src/main/java/org/apache/kylin/query/QueryConnection.java
new file mode 100644
index 0000000..bf8d970
--- /dev/null
+++ b/query/src/main/java/org/apache/kylin/query/QueryConnection.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.query;
+
+import java.io.File;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.Properties;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.query.schema.OLAPSchemaFactory;
+import org.apache.log4j.Logger;
+
+public class QueryConnection {
+ private static final Logger logger = Logger.getLogger(QueryConnection.class);
+
+ public static Connection getConnection(String project) throws SQLException {
+ File olapTmp = OLAPSchemaFactory.createTempOLAPJson(ProjectInstance.getNormalizedProjectName(project),
+ KylinConfig.getInstanceFromEnv());
+ Properties info = new Properties();
+ info.put("model", olapTmp.getAbsolutePath());
+ return DriverManager.getConnection("jdbc:calcite:", info);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/8898213f/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
index 742efcf..7758987 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -147,6 +147,7 @@ public class CacheService extends BasicService {
queryDataSource.clearCache();
}
+ @Deprecated
public DataSource getOLAPDataSource(String project) {
project = ProjectInstance.getNormalizedProjectName(project);
http://git-wip-us.apache.org/repos/asf/kylin/blob/8898213f/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 4061c82..75c16bd 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -44,7 +44,6 @@ import java.util.Map;
import java.util.Set;
import javax.annotation.PostConstruct;
-import javax.sql.DataSource;
import org.apache.calcite.avatica.ColumnMetaData.Rep;
import org.apache.calcite.config.CalciteConnectionConfig;
@@ -81,6 +80,7 @@ import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.apache.kylin.metadata.querymeta.TableMeta;
import org.apache.kylin.metadata.querymeta.TableMetaWithType;
import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.query.QueryConnection;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.rest.constant.Constant;
@@ -517,8 +517,7 @@ public class QueryService extends BasicService {
}
ResultSet JDBCTableMeta = null;
try {
- DataSource dataSource = cacheService.getOLAPDataSource(project);
- conn = dataSource.getConnection();
+ conn = QueryConnection.getConnection(project);
DatabaseMetaData metaData = conn.getMetaData();
JDBCTableMeta = metaData.getTables(null, null, null, null);
@@ -593,8 +592,7 @@ public class QueryService extends BasicService {
}
ResultSet JDBCTableMeta = null;
try {
- DataSource dataSource = cacheService.getOLAPDataSource(project);
- conn = dataSource.getConnection();
+ conn = QueryConnection.getConnection(project);
DatabaseMetaData metaData = conn.getMetaData();
JDBCTableMeta = metaData.getTables(null, null, null, null);
@@ -757,7 +755,7 @@ public class QueryService extends BasicService {
List<SelectedColumnMeta> columnMetas = Lists.newArrayList();
try {
- conn = cacheService.getOLAPDataSource(sqlRequest.getProject()).getConnection();
+ conn = QueryConnection.getConnection(sqlRequest.getProject());
// special case for prepare query.
if (BackdoorToggles.getPrepareOnly()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/8898213f/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
index 96146ef..769d9d1 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
@@ -19,11 +19,13 @@
package org.apache.kylin.rest.service;
import java.net.UnknownHostException;
+import java.sql.SQLException;
import java.util.List;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.job.exception.JobException;
import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.query.QueryConnection;
import org.junit.Assert;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
@@ -45,11 +47,11 @@ public class CubeServiceTest extends ServiceTestBase {
private CacheService cacheService;
@Test
- public void testBasics() throws JsonProcessingException, JobException, UnknownHostException {
+ public void testBasics() throws JsonProcessingException, JobException, UnknownHostException, SQLException {
Assert.assertNotNull(cubeService.getConfig());
Assert.assertNotNull(cubeService.getConfig());
Assert.assertNotNull(cubeService.getMetadataManager());
- Assert.assertNotNull(cacheService.getOLAPDataSource(ProjectInstance.DEFAULT_PROJECT_NAME));
+ Assert.assertNotNull(QueryConnection.getConnection(ProjectInstance.DEFAULT_PROJECT_NAME));
List<CubeInstance> cubes = cubeService.listAllCubes(null, null, null, true);
Assert.assertNotNull(cubes);
http://git-wip-us.apache.org/repos/asf/kylin/blob/8898213f/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
index 0932748..b47f05f 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
@@ -19,10 +19,12 @@
package org.apache.kylin.rest.service;
import java.io.IOException;
+import java.sql.SQLException;
import org.apache.kylin.job.constant.JobTimeFilterEnum;
import org.apache.kylin.job.exception.JobException;
import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.query.QueryConnection;
import org.junit.Assert;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
@@ -42,11 +44,11 @@ public class JobServiceTest extends ServiceTestBase {
private CacheService cacheService;
@Test
- public void testBasics() throws JobException, IOException {
+ public void testBasics() throws JobException, IOException, SQLException {
Assert.assertNotNull(jobService.getConfig());
Assert.assertNotNull(jobService.getConfig());
Assert.assertNotNull(jobService.getMetadataManager());
- Assert.assertNotNull(cacheService.getOLAPDataSource(ProjectInstance.DEFAULT_PROJECT_NAME));
+ Assert.assertNotNull(QueryConnection.getConnection(ProjectInstance.DEFAULT_PROJECT_NAME));
Assert.assertNull(jobService.getJobInstance("job_not_exist"));
Assert.assertNotNull(jobService.searchJobs(null, null, null, 0, 0, JobTimeFilterEnum.ALL));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8898213f/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
index 1e5a7a5..c980959 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
@@ -23,6 +23,7 @@ import java.sql.SQLException;
import org.apache.kylin.job.exception.JobException;
import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.query.QueryConnection;
import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.response.SQLResponse;
import org.junit.Assert;
@@ -48,7 +49,7 @@ public class QueryServiceTest extends ServiceTestBase {
Assert.assertNotNull(queryService.getConfig());
Assert.assertNotNull(queryService.getConfig());
Assert.assertNotNull(queryService.getMetadataManager());
- Assert.assertNotNull(cacheService.getOLAPDataSource(ProjectInstance.DEFAULT_PROJECT_NAME));
+ Assert.assertNotNull(QueryConnection.getConnection(ProjectInstance.DEFAULT_PROJECT_NAME));
// Assert.assertTrue(queryService.getQueries("ADMIN").size() == 0);
//