You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/04/26 06:27:13 UTC
[31/50] incubator-kylin git commit: KYLIN-688 fix memory leak for
segmentIterator
KYLIN-688 fix memory leak for segmentIterator
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/48a87797
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/48a87797
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/48a87797
Branch: refs/heads/streaming-localdict
Commit: 48a8779708347eea9f783313206ac43603a9c5d6
Parents: dea04bf
Author: honma <ho...@ebay.com>
Authored: Wed Apr 15 13:00:49 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Apr 22 16:45:38 2015 +0800
----------------------------------------------------------------------
.../metadata/tuple/CompoundTupleIterator.java | 7 ++--
.../kylin/metadata/tuple/ITupleIterator.java | 8 ++--
.../kylin/query/enumerator/OLAPEnumerator.java | 3 +-
.../storage/hbase/CubeSegmentTupleIterator.java | 35 ++++++-----------
.../hbase/SerializedHBaseTupleIterator.java | 37 +++++++++---------
.../endpoint/EndpointTupleIterator.java | 4 +-
.../apache/kylin/storage/test/StorageTest.java | 40 ++++++--------------
7 files changed, 51 insertions(+), 83 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/48a87797/metadata/src/main/java/org/apache/kylin/metadata/tuple/CompoundTupleIterator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/tuple/CompoundTupleIterator.java b/metadata/src/main/java/org/apache/kylin/metadata/tuple/CompoundTupleIterator.java
index 08562ca..3b35ac2 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/tuple/CompoundTupleIterator.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/tuple/CompoundTupleIterator.java
@@ -1,11 +1,10 @@
package org.apache.kylin.metadata.tuple;
-import com.google.common.collect.Iterators;
-
-import java.io.IOException;
import java.util.Iterator;
import java.util.List;
+import com.google.common.collect.Iterators;
+
/**
* Created by Hongbin Ma(Binmahone) on 4/14/15.
*/
@@ -19,7 +18,7 @@ public class CompoundTupleIterator implements ITupleIterator {
}
@Override
- public void close() throws IOException {
+ public void close() {
for (ITupleIterator i : backends) {
i.close();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/48a87797/metadata/src/main/java/org/apache/kylin/metadata/tuple/ITupleIterator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/tuple/ITupleIterator.java b/metadata/src/main/java/org/apache/kylin/metadata/tuple/ITupleIterator.java
index 5ba8d9e..4a00780 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/tuple/ITupleIterator.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/tuple/ITupleIterator.java
@@ -18,14 +18,13 @@
package org.apache.kylin.metadata.tuple;
-import java.io.Closeable;
-import java.io.IOException;
import java.util.Iterator;
/**
* @author xjiang
+ *
*/
-public interface ITupleIterator extends Iterator<ITuple>, Closeable {
+public interface ITupleIterator extends Iterator<ITuple> {
public static final ITupleIterator EMPTY_TUPLE_ITERATOR = new ITupleIterator() {
@Override
public boolean hasNext() {
@@ -47,6 +46,5 @@ public interface ITupleIterator extends Iterator<ITuple>, Closeable {
}
};
- @Override
- public void close() throws IOException;
+ public void close();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/48a87797/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index 0300d70..f71409d 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -90,7 +90,8 @@ public class OLAPEnumerator implements Enumerator<Object[]> {
@Override
public void close() {
- IOUtils.closeQuietly(cursor);
+ if (cursor != null)
+ cursor.close();
}
private Object[] convertCurrentRow(ITuple tuple) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/48a87797/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
index e19db9e..f8e4efc 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
@@ -20,22 +20,10 @@ package org.apache.kylin.storage.hbase;
import java.io.IOException;
import java.text.MessageFormat;
-import java.util.BitSet;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import java.util.Map.Entry;
-import org.apache.commons.lang.NotImplementedException;
-import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverEnabler;
-
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
@@ -43,11 +31,6 @@ import org.apache.hadoop.hbase.filter.FuzzyRowFilter;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
-import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.tuple.TupleInfo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.kylin.common.persistence.StorageException;
import org.apache.kylin.common.util.Array;
import org.apache.kylin.cube.CubeInstance;
@@ -56,18 +39,23 @@ import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.kv.RowKeyDecoder;
import org.apache.kylin.cube.kv.RowValueDecoder;
+import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.cube.model.HBaseColumnDesc;
+import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverEnabler;
import org.apache.kylin.storage.tuple.Tuple;
import org.apache.kylin.storage.tuple.Tuple.IDerivedColumnFiller;
+import org.apache.kylin.storage.tuple.TupleInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* @author xjiang
- *
+ *
*/
public class CubeSegmentTupleIterator implements ITupleIterator {
@@ -128,7 +116,8 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
byte[] metricsBytes = scan.getAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA);
if (metricsBytes != null) {
ScanMetrics scanMetrics = ProtobufUtil.toScanMetrics(metricsBytes);
- logger.debug("HBase Metrics: " + "count={}, ms={}, bytes={}, remote_bytes={}, regions={}, not_serving_region={}, rpc={}, rpc_retries={}, remote_rpc={}, remote_rpc_retries={}", new Object[] { scanCount, scanMetrics.sumOfMillisSecBetweenNexts, scanMetrics.countOfBytesInResults, scanMetrics.countOfBytesInRemoteResults, scanMetrics.countOfRegions, scanMetrics.countOfNSRE, scanMetrics.countOfRPCcalls, scanMetrics.countOfRPCRetries, scanMetrics.countOfRemoteRPCcalls, scanMetrics.countOfRemoteRPCRetries });
+ logger.debug("HBase Metrics: " + "count={}, ms={}, bytes={}, remote_bytes={}, regions={}, not_serving_region={}, rpc={}, rpc_retries={}, remote_rpc={}, remote_rpc_retries={}", //
+ new Object[] { scanCount, scanMetrics.sumOfMillisSecBetweenNexts, scanMetrics.countOfBytesInResults, scanMetrics.countOfBytesInRemoteResults, scanMetrics.countOfRegions, scanMetrics.countOfNSRE, scanMetrics.countOfRPCcalls, scanMetrics.countOfRPCRetries, scanMetrics.countOfRemoteRPCcalls, scanMetrics.countOfRemoteRPCRetries });
}
}
try {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/48a87797/storage/src/main/java/org/apache/kylin/storage/hbase/SerializedHBaseTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/SerializedHBaseTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/SerializedHBaseTupleIterator.java
index 7e8892d..8c37347 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/SerializedHBaseTupleIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/SerializedHBaseTupleIterator.java
@@ -18,25 +18,20 @@
package org.apache.kylin.storage.hbase;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import org.apache.hadoop.hbase.client.HConnection;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.kv.RowValueDecoder;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.storage.StorageContext;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
/**
* @author xjiang
@@ -107,9 +102,7 @@ public class SerializedHBaseTupleIterator implements ITupleIterator {
}
@Override
- public ITuple next() {
- //TODO: last not closed
-
+ public ITuple next() {
ITuple t = null;
while (hasNext()) {
if (segmentIterator.hasNext()) {
@@ -117,14 +110,11 @@ public class SerializedHBaseTupleIterator implements ITupleIterator {
scanCount++;
break;
} else {
- try {
- segmentIterator.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
+ segmentIterator.close();
segmentIterator = segmentIteratorIterator.next();
}
}
+
return t;
}
@@ -134,8 +124,15 @@ public class SerializedHBaseTupleIterator implements ITupleIterator {
}
@Override
- public void close() throws IOException {
+ public void close() {
context.setTotalScanCount(scanCount);
+
+ //hasNext() is complex, the loop might exited because of limit, threshold, etc.
+ //close all the remaining segmentIterator
segmentIterator.close();
+ while (segmentIteratorIterator.hasNext()) {
+ segmentIterator = segmentIteratorIterator.next();
+ segmentIterator.close();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/48a87797/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
index 5461de3..df1b90f 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
@@ -205,7 +205,7 @@ public class EndpointTupleIterator implements ITupleIterator {
}
@Override
- public void close() throws IOException {
+ public void close() {
IOUtils.closeQuietly(table);
logger.info("Closed after " + rowsInAllMetric + " rows are fetched");
}
@@ -288,6 +288,7 @@ public class EndpointTupleIterator implements ITupleIterator {
/**
* Internal class to handle iterators for a single region's returned rows
+ *
*/
class SingleRegionTupleIterator implements ITupleIterator {
private List<IIProtos.IIResponse.IIRow> rows;
@@ -335,7 +336,6 @@ public class EndpointTupleIterator implements ITupleIterator {
@Override
public void remove() {
throw new UnsupportedOperationException();
-
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/48a87797/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java b/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
index 5212d2e..99b5540 100644
--- a/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
@@ -18,44 +18,27 @@
package org.apache.kylin.storage.test;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertTrue;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import org.apache.commons.io.IOUtils;
-import org.apache.kylin.metadata.realization.SQLDigest;
-
-import org.apache.kylin.storage.IStorageEngine;
-import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.StorageEngineFactory;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.ParameterDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.filter.*;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
-import org.apache.kylin.storage.hbase.ScanOutOfLimitException;
+import org.apache.kylin.metadata.model.*;
+import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.storage.IStorageEngine;
+import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.StorageEngineFactory;
+import org.apache.kylin.storage.hbase.ScanOutOfLimitException;
+import org.junit.*;
public class StorageTest extends HBaseMetadataTestCase {
@@ -78,7 +61,7 @@ public class StorageTest extends HBaseMetadataTestCase {
CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
cube = cubeMgr.getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_EMPTY");
Assert.assertNotNull(cube);
- storageEngine = StorageEngineFactory.getStorageEngine(cube,true);
+ storageEngine = StorageEngineFactory.getStorageEngine(cube, true);
String url = KylinConfig.getInstanceFromEnv().getStorageUrl();
context = new StorageContext();
context.setConnUrl(url);
@@ -163,7 +146,8 @@ public class StorageTest extends HBaseMetadataTestCase {
} catch (Exception e) {
e.printStackTrace();
} finally {
- IOUtils.closeQuietly(iterator);
+ if (iterator != null)
+ iterator.close();
}
return count;
}