You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ni...@apache.org on 2020/01/27 14:16:46 UTC

[kylin] branch master updated (7019774 -> c5192c4)

This is an automated email from the ASF dual-hosted git repository.

nic pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git.


    from 7019774  Set repo of spring-snapshots with url https
     new 8bdbdf2  minor, sonar issues for ignoring initial values of method parameters
     new 17e6c56  minor, sonar issues for unchecked stream read
     new d532e5e  minor, sonar issues for throwing exceptions in finnally block
     new 5f6e93c  minor, sonar issues, volatile variables should not be used with compound operators
     new c5192c4  minor, should use BigDecimal.valueOf instead of constructor to avoid imprecision

The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../org/apache/kylin/common/util/DecimalUtil.java  |  7 ++-
 .../apache/kylin/gridtable/GTAggregateScanner.java | 15 ++---
 .../org/apache/kylin/dict/NumberDictionary.java    | 10 ++-
 .../org/apache/kylin/dict/NumberDictionary2.java   |  4 +-
 .../org/apache/kylin/metadata/tuple/Tuple.java     |  2 +-
 .../datasource/framework/conv/ConvSqlWriter.java   |  8 +--
 .../flink/util/PercentileCounterSerializer.java    | 18 ++++--
 .../java/org/apache/kylin/engine/mr/CubingJob.java |  1 -
 .../spark/util/PercentileCounterSerializer.java    | 11 +++-
 .../kylin/rest/controller/StreamingController.java | 21 ++++---
 .../rest/controller/StreamingV2Controller.java     | 31 ++++++----
 .../v2/coprocessor/endpoint/CubeVisitService.java  | 16 +++--
 .../core/consumer/StreamingConsumerChannel.java    | 33 +++++-----
 .../core/storage/columnar/FragmentFilesMerger.java | 72 ++++++++++++++--------
 14 files changed, 155 insertions(+), 94 deletions(-)


[kylin] 05/05: minor, should use BigDecimal.valueOf instead of constructor to avoid imprecision

Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git

commit c5192c4b1b5012d5e6eabeda303ae98c9b68bcc0
Author: etherge <et...@163.com>
AuthorDate: Fri Jan 24 10:22:00 2020 -0500

    minor, should use BigDecimal.valueOf instead of constructor to avoid imprecision
---
 .../src/main/java/org/apache/kylin/common/util/DecimalUtil.java    | 7 ++++---
 .../src/main/java/org/apache/kylin/metadata/tuple/Tuple.java       | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/core-common/src/main/java/org/apache/kylin/common/util/DecimalUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/DecimalUtil.java
index 0891960..e92cfff 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/DecimalUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/DecimalUtil.java
@@ -32,9 +32,10 @@ public class DecimalUtil {
         // There are some values of "long" that cannot be represented as "double".
         // Not so "int". If it isn't a long, go straight to double.
         return number instanceof BigDecimal ? (BigDecimal) number
-                : number instanceof BigInteger ? new BigDecimal((BigInteger) number)
-                        : number instanceof Long ? new BigDecimal(number.longValue())
-                                : new BigDecimal(number.doubleValue());
+                : number instanceof Integer ? new BigDecimal((Integer) number)
+                        : number instanceof BigInteger ? new BigDecimal((BigInteger) number)
+                                : number instanceof Long ? new BigDecimal(number.longValue())
+                                        : BigDecimal.valueOf(number.doubleValue());
     }
 
     public static BigDecimal toBigDecimal(Object o) {
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
index 86db79b..6a0cda9 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
@@ -119,7 +119,7 @@ public class Tuple implements ITuple {
             if (fieldValue instanceof BigDecimal) {
                 fieldValue = normalizeDecimal((BigDecimal) fieldValue);
             } else if (fieldValue instanceof Number) {
-                fieldValue = new BigDecimal(((Number) fieldValue).doubleValue());
+                fieldValue = BigDecimal.valueOf(((Number) fieldValue).doubleValue());
             }
         } else if ("float".equals(dataType) && fieldValue instanceof BigDecimal) {
             fieldValue = ((BigDecimal) fieldValue).floatValue();


[kylin] 02/05: minor, sonar issues for unchecked stream read

Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git

commit 17e6c565529f2c04bd883bb12fea67e4069dfc05
Author: etherge <et...@163.com>
AuthorDate: Thu Jan 23 17:51:50 2020 -0500

    minor, sonar issues for unchecked stream read
---
 .../apache/kylin/gridtable/GTAggregateScanner.java | 15 ++---
 .../flink/util/PercentileCounterSerializer.java    | 18 ++++--
 .../spark/util/PercentileCounterSerializer.java    | 11 +++-
 .../core/storage/columnar/FragmentFilesMerger.java | 72 ++++++++++++++--------
 4 files changed, 78 insertions(+), 38 deletions(-)

diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index 75e79d0..78e413a 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -434,8 +434,8 @@ public class GTAggregateScanner implements IGTScanner, IGTBypassChecker {
                 sumSpilledSize += dump.size();
                 // when spilled data is too much, we can modify it by other strategy.
                 // this means, all spilled data is bigger than half of original spillThreshold.
-                if(sumSpilledSize > spillThreshold) {
-                    for(Dump current : dumps) {
+                if (sumSpilledSize > spillThreshold) {
+                    for (Dump current : dumps) {
                         current.spill();
                     }
                     spillThreshold += sumSpilledSize;
@@ -678,7 +678,7 @@ public class GTAggregateScanner implements IGTScanner, IGTBypassChecker {
                                 + (dumpedFile == null ? "<null>" : dumpedFile.getAbsolutePath()));
                     }
 
-                    if(spillBuffer == null) {
+                    if (spillBuffer == null) {
                         dis = new DataInputStream(new FileInputStream(dumpedFile));
                     } else {
                         dis = new DataInputStream(new ByteArrayInputStream(spillBuffer));
@@ -698,10 +698,10 @@ public class GTAggregateScanner implements IGTScanner, IGTBypassChecker {
                                 cursorIdx++;
                                 int keyLen = dis.readInt();
                                 byte[] key = new byte[keyLen];
-                                dis.read(key);
+                                dis.readFully(key);
                                 int valueLen = dis.readInt();
                                 byte[] value = new byte[valueLen];
-                                dis.read(value);
+                                dis.readFully(value);
                                 return new Pair<>(key, value);
                             } catch (Exception e) {
                                 throw new RuntimeException(
@@ -720,7 +720,8 @@ public class GTAggregateScanner implements IGTScanner, IGTBypassChecker {
             }
 
             public void spill() throws IOException {
-                if(spillBuffer == null) return;
+                if (spillBuffer == null)
+                    return;
                 OutputStream ops = new FileOutputStream(dumpedFile);
                 InputStream ips = new ByteArrayInputStream(spillBuffer);
                 IOUtils.copy(ips, ops);
@@ -729,7 +730,7 @@ public class GTAggregateScanner implements IGTScanner, IGTBypassChecker {
                 IOUtils.closeQuietly(ops);
 
                 logger.info("Spill buffer to disk, location: {}, size = {}.", dumpedFile.getAbsolutePath(),
-                    dumpedFile.length());
+                        dumpedFile.length());
             }
 
             public int size() {
diff --git a/engine-flink/src/main/java/org/apache/kylin/engine/flink/util/PercentileCounterSerializer.java b/engine-flink/src/main/java/org/apache/kylin/engine/flink/util/PercentileCounterSerializer.java
index 684f85e..a0d8f1c 100644
--- a/engine-flink/src/main/java/org/apache/kylin/engine/flink/util/PercentileCounterSerializer.java
+++ b/engine-flink/src/main/java/org/apache/kylin/engine/flink/util/PercentileCounterSerializer.java
@@ -19,13 +19,14 @@
 
 package org.apache.kylin.engine.flink.util;
 
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.measure.percentile.PercentileCounter;
+
 import com.esotericsoftware.kryo.Kryo;
 import com.esotericsoftware.kryo.Serializer;
 import com.esotericsoftware.kryo.io.Input;
 import com.esotericsoftware.kryo.io.Output;
-import org.apache.kylin.measure.percentile.PercentileCounter;
-
-import java.nio.ByteBuffer;
 
 /**
  * A customized kryo serializer for {@link PercentileCounter}
@@ -49,7 +50,16 @@ public class PercentileCounterSerializer extends Serializer<PercentileCounter> {
         double quantileRatio = input.readDouble();
         int length = input.readInt();
         byte[] buffer = new byte[length];
-        input.read(buffer);
+
+        int offset = 0;
+        int bytesRead;
+        while ((bytesRead = input.read(buffer, offset, buffer.length - offset)) != -1) {
+            offset += bytesRead;
+            if (offset >= buffer.length) {
+                break;
+            }
+        }
+
         PercentileCounter counter = new PercentileCounter(compression, quantileRatio);
         counter.readRegisters(ByteBuffer.wrap(buffer));
         return counter;
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/PercentileCounterSerializer.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/PercentileCounterSerializer.java
index c9ba0f3..1e256f2 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/PercentileCounterSerializer.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/PercentileCounterSerializer.java
@@ -47,7 +47,16 @@ public class PercentileCounterSerializer extends Serializer<PercentileCounter> {
         double quantileRatio = input.readDouble();
         int length = input.readInt();
         byte[] buffer = new byte[length];
-        input.read(buffer);
+
+        int offset = 0;
+        int bytesRead;
+        while ((bytesRead = input.read(buffer, offset, buffer.length - offset)) != -1) {
+            offset += bytesRead;
+            if (offset >= buffer.length) {
+                break;
+            }
+        }
+
         PercentileCounter counter = new PercentileCounter(compression, quantileRatio);
         counter.readRegisters(ByteBuffer.wrap(buffer));
         return counter;
diff --git a/stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/FragmentFilesMerger.java b/stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/FragmentFilesMerger.java
index 4a9c398..627781b 100644
--- a/stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/FragmentFilesMerger.java
+++ b/stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/FragmentFilesMerger.java
@@ -93,8 +93,8 @@ public class FragmentFilesMerger {
         }
 
         Collections.sort(fragmentList);
-        FragmentId mergedFragmentId = new FragmentId(fragmentList.get(0).getFragmentId().getStartId(), fragmentList
-                .get(fragmentList.size() - 1).getFragmentId().getEndId());
+        FragmentId mergedFragmentId = new FragmentId(fragmentList.get(0).getFragmentId().getStartId(),
+                fragmentList.get(fragmentList.size() - 1).getFragmentId().getEndId());
         List<FragmentData> fragmentDataList = Lists.newArrayList();
         Map<TblColRef, List<Dictionary<String>>> dimDictListMap = Maps.newHashMap();
         Map<FragmentId, Map<TblColRef, Dictionary<String>>> fragmentDictionaryMaps = Maps.newHashMap();
@@ -146,8 +146,8 @@ public class FragmentFilesMerger {
         File mergedFragmentMetaFile = new File(mergeWorkingDirectory, mergedFragmentId + Constants.META_FILE_SUFFIX);
         try {
             FragmentMetaInfo mergedFragmentMeta = new FragmentMetaInfo();
-            CountingOutputStream fragmentDataOutput = new CountingOutputStream(new BufferedOutputStream(
-                    FileUtils.openOutputStream(mergedFragmentDataFile)));
+            CountingOutputStream fragmentDataOutput = new CountingOutputStream(
+                    new BufferedOutputStream(FileUtils.openOutputStream(mergedFragmentDataFile)));
             // merge dictionaries
             Map<TblColRef, Dictionary<String>> mergedDictMap = mergeAndPersistDictionaries(mergedFragmentMeta,
                     dimDictListMap, fragmentDataOutput);
@@ -202,8 +202,8 @@ public class FragmentFilesMerger {
         List<DimDictionaryMetaInfo> dimDictionaryMetaInfos = Lists.newArrayList();
         for (TblColRef dimension : parsedCubeInfo.dimensionsUseDictEncoding) {
             List<Dictionary<String>> dicts = dimDictListMap.get(dimension);
-            MultipleDictionaryValueEnumerator multipleDictionaryValueEnumerator = new MultipleDictionaryValueEnumerator(dimension.getType(),
-                    dicts);
+            MultipleDictionaryValueEnumerator multipleDictionaryValueEnumerator = new MultipleDictionaryValueEnumerator(
+                    dimension.getType(), dicts);
             Dictionary<String> mergedDict = DictionaryGenerator.buildDictionary(dimension.getType(),
                     multipleDictionaryValueEnumerator);
             mergedDictMap.put(dimension, mergedDict);
@@ -240,10 +240,10 @@ public class FragmentFilesMerger {
             } else {
                 cuboidMetaInfo = fragmentMetaInfo.getCuboidMetaInfo(cuboidId);
             }
-            Map<TblColRef, Dictionary<String>> dictMap = fragmentDictionaryMaps.get(FragmentId.parse(fragmentMetaInfo
-                    .getFragmentId()));
-            DimensionEncoding[] dimensionEncodings = ParsedStreamingCubeInfo.getDimensionEncodings(
-                    parsedCubeInfo.cubeDesc, dimensions, dictMap);
+            Map<TblColRef, Dictionary<String>> dictMap = fragmentDictionaryMaps
+                    .get(FragmentId.parse(fragmentMetaInfo.getFragmentId()));
+            DimensionEncoding[] dimensionEncodings = ParsedStreamingCubeInfo
+                    .getDimensionEncodings(parsedCubeInfo.cubeDesc, dimensions, dictMap);
             FragmentCuboidReader fragmentCuboidReader = new FragmentCuboidReader(parsedCubeInfo.cubeDesc, fragmentData,
                     cuboidMetaInfo, cuboidInfo.getDimensions(), parsedCubeInfo.measureDescs, dimensionEncodings);
             fragmentCuboidReaders.add(fragmentCuboidReader);
@@ -264,7 +264,8 @@ public class FragmentFilesMerger {
                 if (dict instanceof TrieDictionary) {
                     invertIndexColDescs[i] = new SeqIIColumnDescriptor(dim.getName(), dict.getMinId(), dict.getMaxId());
                 } else {
-                    invertIndexColDescs[i] = new FixLenIIColumnDescriptor(dim.getName(), encoding.getLengthOfEncoding());
+                    invertIndexColDescs[i] = new FixLenIIColumnDescriptor(dim.getName(),
+                            encoding.getLengthOfEncoding());
                 }
             } else {
                 invertIndexColDescs[i] = new FixLenIIColumnDescriptor(dim.getName(), encoding.getLengthOfEncoding());
@@ -282,8 +283,8 @@ public class FragmentFilesMerger {
         for (int i = 0; i < metricDataWriters.length; i++) {
             metricDataWriters[i] = new CuboidMetricDataWriter(cuboidId, parsedCubeInfo.measureDescs[i].getName(),
                     parsedCubeInfo.getMeasureTypeSerializer(i).maxLength());
-            metricsEncodings[i] = ColumnarMetricsEncodingFactory.create(parsedCubeInfo.measureDescs[i].getFunction()
-                    .getReturnDataType());
+            metricsEncodings[i] = ColumnarMetricsEncodingFactory
+                    .create(parsedCubeInfo.measureDescs[i].getFunction().getReturnDataType());
         }
 
         FragmentCuboidDataMerger fragmentCuboidDataMerger = new FragmentCuboidDataMerger(cuboidInfo,
@@ -324,19 +325,29 @@ public class FragmentFilesMerger {
         for (int i = 0; i < dimDataWriters.length; i++) {
             DimensionEncoding encoding = mergedDimEncodings[i];
             int dimFixLen = encoding.getLengthOfEncoding();
-            InputStream dimInput = new BufferedInputStream(FileUtils.openInputStream(dimDataWriters[i].getOutputFile()));
+            InputStream dimInput = new BufferedInputStream(
+                    FileUtils.openInputStream(dimDataWriters[i].getOutputFile()));
             try {
                 DimensionMetaInfo dimensionMeta = new DimensionMetaInfo();
                 dimensionMeta.setName(dimensions[i].getName());
                 int startOffset = (int) fragmentDataOutput.getCount();
                 dimensionMeta.setStartOffset(startOffset);
 
-                ColumnarStoreDimDesc cStoreDimDesc = ColumnarStoreDimDesc.getDefaultCStoreDimDesc(parsedCubeInfo.cubeDesc,
-                        dimensions[i].getName(), encoding);
+                ColumnarStoreDimDesc cStoreDimDesc = ColumnarStoreDimDesc
+                        .getDefaultCStoreDimDesc(parsedCubeInfo.cubeDesc, dimensions[i].getName(), encoding);
                 ColumnDataWriter columnDataWriter = cStoreDimDesc.getDimWriter(fragmentDataOutput, rowCnt);
                 for (int j = 0; j < rowCnt; j++) {
                     byte[] dimValue = new byte[dimFixLen];
-                    dimInput.read(dimValue);
+
+                    int offset = 0;
+                    int bytesRead;
+                    while ((bytesRead = dimInput.read(dimValue, offset, dimValue.length - offset)) != -1) {
+                        offset += bytesRead;
+                        if (offset >= dimValue.length) {
+                            break;
+                        }
+                    }
+
                     if (DimensionEncoding.isNull(dimValue, 0, dimValue.length)) {
                         dimensionMeta.setHasNull(true);
                     }
@@ -358,8 +369,8 @@ public class FragmentFilesMerger {
         }
 
         for (int i = 0; i < metricDataWriters.length; i++) {
-            DataInputStream metricInput = new DataInputStream(new BufferedInputStream(
-                    FileUtils.openInputStream(metricDataWriters[i].getOutputFile())));
+            DataInputStream metricInput = new DataInputStream(
+                    new BufferedInputStream(FileUtils.openInputStream(metricDataWriters[i].getOutputFile())));
             try {
                 ColumnarMetricsEncoding metricsEncoding = ColumnarMetricsEncodingFactory
                         .create(parsedCubeInfo.measureDescs[i].getFunction().getReturnDataType());
@@ -373,7 +384,16 @@ public class FragmentFilesMerger {
                 for (int j = 0; j < rowCnt; j++) {
                     int metricLen = metricInput.readInt();
                     byte[] metricValue = new byte[metricLen];
-                    metricInput.read(metricValue);
+
+                    int offset = 0;
+                    int bytesRead;
+                    while ((bytesRead = metricInput.read(metricValue, offset, metricValue.length - offset)) != -1) {
+                        offset += bytesRead;
+                        if (offset >= metricValue.length) {
+                            break;
+                        }
+                    }
+
                     columnDataWriter.write(metricValue);
                 }
                 columnDataWriter.flush();
@@ -404,7 +424,8 @@ public class FragmentFilesMerger {
             this.colName = colName;
 
             this.tmpColDataFile = new File(mergeWorkingDirectory, cuboidId + "-" + colName + ".data");
-            this.output = new CountingOutputStream(new BufferedOutputStream(FileUtils.openOutputStream(tmpColDataFile)));
+            this.output = new CountingOutputStream(
+                    new BufferedOutputStream(FileUtils.openOutputStream(tmpColDataFile)));
         }
 
         public void write(byte[] value) throws IOException {
@@ -437,8 +458,8 @@ public class FragmentFilesMerger {
             this.metricName = metricName;
             this.maxValLen = maxValLen;
             this.tmpMetricDataFile = new File(mergeWorkingDirectory, cuboidId + "-" + metricName + ".data");
-            this.countingOutput = new CountingOutputStream(new BufferedOutputStream(
-                    FileUtils.openOutputStream(tmpMetricDataFile)));
+            this.countingOutput = new CountingOutputStream(
+                    new BufferedOutputStream(FileUtils.openOutputStream(tmpMetricDataFile)));
             this.output = new DataOutputStream(countingOutput);
         }
 
@@ -534,9 +555,8 @@ public class FragmentFilesMerger {
             enqueueFromFragment(currRecordEntry.getSecond());
             boolean needAggregate = false;
             boolean first = true;
-            while ((!minHeap.isEmpty())
-                    && StringArrayComparator.INSTANCE.compare(currRecord.dimensions,
-                            minHeap.peek().getFirst().dimensions) == 0) {
+            while ((!minHeap.isEmpty()) && StringArrayComparator.INSTANCE.compare(currRecord.dimensions,
+                    minHeap.peek().getFirst().dimensions) == 0) {
                 if (first) {
                     doAggregate(currRecord);
                     first = false;


[kylin] 04/05: minor, sonar issues, volatile variables should not be used with compound operators

Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git

commit 5f6e93cc5c43e7d583156626094a8706eac75d1d
Author: etherge <et...@163.com>
AuthorDate: Thu Jan 23 19:02:00 2020 -0500

    minor, sonar issues, volatile variables should not be used with compound operators
---
 .../core/consumer/StreamingConsumerChannel.java    | 33 +++++++++++-----------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/stream-core/src/main/java/org/apache/kylin/stream/core/consumer/StreamingConsumerChannel.java b/stream-core/src/main/java/org/apache/kylin/stream/core/consumer/StreamingConsumerChannel.java
index f5d70e3..b27d830 100644
--- a/stream-core/src/main/java/org/apache/kylin/stream/core/consumer/StreamingConsumerChannel.java
+++ b/stream-core/src/main/java/org/apache/kylin/stream/core/consumer/StreamingConsumerChannel.java
@@ -22,6 +22,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.kylin.stream.core.exception.StreamingException;
 import org.apache.kylin.stream.core.metrics.StreamingMetrics;
@@ -52,10 +53,10 @@ public class StreamingConsumerChannel implements Runnable {
     private StreamingSegmentManager cubeSegmentManager;
     private volatile IStopConsumptionCondition stopCondition;
     private volatile long minAcceptEventTime;
-    private volatile long parseEventErrorCnt;
-    private volatile long addEventErrorCnt;
-    private volatile long incomingEventCnt;
-    private volatile long dropEventCnt;
+    private AtomicLong parseEventErrorCnt = new AtomicLong(0);
+    private AtomicLong addEventErrorCnt = new AtomicLong(0);
+    private AtomicLong incomingEventCnt = new AtomicLong(0);
+    private AtomicLong dropEventCnt = new AtomicLong(0);
     private Map<Integer, Meter> eventConsumeMeters;
 
     public StreamingConsumerChannel(String cubeName, IStreamingConnector connector,
@@ -96,7 +97,7 @@ public class StreamingConsumerChannel implements Runnable {
                         Thread.sleep(100);
                         continue;
                     }
-                    incomingEventCnt++;
+                    incomingEventCnt.incrementAndGet();
                     recordConsumeMetric(event.getSourcePosition().getPartition(), event.getParams());
                     if (!stopCondition.isSatisfied(event)) {
                         if (!isFilter(event)) {
@@ -107,16 +108,16 @@ public class StreamingConsumerChannel implements Runnable {
                         this.stopped = true;
                     }
                 } catch (MessageFormatException mfe) {
-                    parseEventErrorCnt++;
-                    if (parseEventErrorCnt % 1000 < 3) {
+                    long countValue = parseEventErrorCnt.incrementAndGet();
+                    if (countValue % 1000 < 3) {
                         logger.error(mfe.getMessage(), mfe);
                     }
                 } catch (InterruptedException ie) {
                     logger.warn("interrupted!");
                     stopped = true;
                 } catch (Exception e) {
-                    addEventErrorCnt++;
-                    if (addEventErrorCnt % 1000 < 3) {
+                    long countValue = addEventErrorCnt.incrementAndGet();
+                    if (countValue % 1000 < 3) {
                         logger.error("error happens when save event:" + event, e);
                     }
                 }
@@ -146,8 +147,7 @@ public class StreamingConsumerChannel implements Runnable {
     private void removeMetrics() {
         for (Map.Entry<Integer, Meter> meterEntry : eventConsumeMeters.entrySet()) {
             StreamingMetrics.getInstance().getMetricRegistry().remove(MetricRegistry
-                    .name(StreamingMetrics.CONSUME_RATE_PFX,
-                    cubeName, String.valueOf(meterEntry.getKey())));
+                    .name(StreamingMetrics.CONSUME_RATE_PFX, cubeName, String.valueOf(meterEntry.getKey())));
         }
     }
 
@@ -163,9 +163,9 @@ public class StreamingConsumerChannel implements Runnable {
 
     private boolean isFilter(StreamingMessage event) {
         if (minAcceptEventTime != 0 && event.getTimestamp() < minAcceptEventTime) {
-            dropEventCnt++;
+            long countValue = dropEventCnt.incrementAndGet();
             // drop events is less than the min accepted event time
-            if (dropEventCnt % 1000 <= 1) {
+            if (countValue % 1000 <= 1) {
                 logger.warn("event dropped, event time {}, min event accept time {}", event.getTimestamp(),
                         minAcceptEventTime);
             }
@@ -269,7 +269,8 @@ public class StreamingConsumerChannel implements Runnable {
     }
 
     public ConsumerStats getConsumerStats() {
-        Map<Integer, Long> consumeLagMap = connector.getSource().calConsumeLag(cubeName, cubeSegmentManager.getConsumePosition());
+        Map<Integer, Long> consumeLagMap = connector.getSource().calConsumeLag(cubeName,
+                cubeSegmentManager.getConsumePosition());
         long totalLag = 0L;
         Map<Integer, PartitionConsumeStats> partitionConsumeStatsMap = Maps.newHashMap();
         for (Map.Entry<Integer, Meter> meterEntry : eventConsumeMeters.entrySet()) {
@@ -295,8 +296,8 @@ public class StreamingConsumerChannel implements Runnable {
         ConsumerStats stats = new ConsumerStats();
         stats.setStopped(stopped);
         stats.setPaused(paused);
-        stats.setTotalIncomingEvents(incomingEventCnt);
-        stats.setTotalExceptionEvents(parseEventErrorCnt + addEventErrorCnt);
+        stats.setTotalIncomingEvents(incomingEventCnt.get());
+        stats.setTotalExceptionEvents(parseEventErrorCnt.get() + addEventErrorCnt.get());
         stats.setPartitionConsumeStatsMap(partitionConsumeStatsMap);
         stats.setConsumeOffsetInfo(getSourceConsumeInfo());
         stats.setConsumeLag(totalLag);


[kylin] 03/05: minor, sonar issues for throwing exceptions in finnally block

Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git

commit d532e5eaf93f2e1751f6e7ccc735da8352ef26c5
Author: etherge <et...@163.com>
AuthorDate: Thu Jan 23 18:33:02 2020 -0500

    minor, sonar issues for throwing exceptions in finnally block
---
 .../kylin/rest/controller/StreamingController.java | 21 ++++++++++-----
 .../rest/controller/StreamingV2Controller.java     | 31 ++++++++++++++--------
 .../v2/coprocessor/endpoint/CubeVisitService.java  | 16 +++++++----
 3 files changed, 45 insertions(+), 23 deletions(-)

diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
index 36a8a7c..6d98434 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
@@ -113,7 +113,6 @@ public class StreamingController extends BasicController {
     @RequestMapping(value = "", method = { RequestMethod.POST }, produces = { "application/json" })
     @ResponseBody
     public StreamingRequest saveStreamingConfig(@RequestBody StreamingRequest streamingRequest) {
-
         String project = streamingRequest.getProject();
         TableDesc tableDesc = deserializeTableDesc(streamingRequest);
         if (null == tableDesc) {
@@ -138,6 +137,8 @@ public class StreamingController extends BasicController {
 
         streamingConfig.setName(tableDesc.getIdentity());
         kafkaConfig.setName(tableDesc.getIdentity());
+
+        InternalErrorException exception = null;
         try {
             if (StringUtils.isEmpty(streamingConfig.getName())) {
                 logger.info("StreamingConfig should not be empty.");
@@ -160,23 +161,24 @@ public class StreamingController extends BasicController {
                     streamingService.dropStreamingConfig(streamingConfig, project);
                 } catch (IOException e1) {
                     throw new InternalErrorException(
-                            "StreamingConfig is created, but failed to create KafkaConfig: " + e.getLocalizedMessage(), e);
+                            "StreamingConfig is created, but failed to create KafkaConfig: " + e.getLocalizedMessage(),
+                            e);
                 }
                 logger.error("Failed to save KafkaConfig:" + e.getLocalizedMessage(), e);
                 throw new InternalErrorException("Failed to save KafkaConfig: " + e.getLocalizedMessage(), e);
             }
         } finally {
             if (saveKafkaSuccess == false || saveStreamingSuccess == false) {
-
                 if (saveStreamingSuccess == true) {
                     StreamingConfig sConfig = streamingService.getStreamingManager()
                             .getStreamingConfig(streamingConfig.getName());
                     try {
                         streamingService.dropStreamingConfig(sConfig, project);
                     } catch (IOException e) {
-                        throw new InternalErrorException(
+                        exception = new InternalErrorException(
                                 "Action failed and failed to rollback the created streaming config: "
-                                        + e.getLocalizedMessage(), e);
+                                        + e.getLocalizedMessage(),
+                                e);
                     }
                 }
                 if (saveKafkaSuccess == true) {
@@ -184,14 +186,19 @@ public class StreamingController extends BasicController {
                         KafkaConfig kConfig = kafkaConfigService.getKafkaConfig(kafkaConfig.getName(), project);
                         kafkaConfigService.dropKafkaConfig(kConfig, project);
                     } catch (IOException e) {
-                        throw new InternalErrorException(
+                        exception = new InternalErrorException(
                                 "Action failed and failed to rollback the created kafka config: "
-                                        + e.getLocalizedMessage(), e);
+                                        + e.getLocalizedMessage(),
+                                e);
                     }
                 }
             }
+        }
 
+        if (null != exception) {
+            throw exception;
         }
+
         streamingRequest.setSuccessful(true);
         return streamingRequest;
     }
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingV2Controller.java b/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingV2Controller.java
index 2843cac..935d273 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingV2Controller.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingV2Controller.java
@@ -52,8 +52,8 @@ import org.apache.kylin.rest.service.CubeService;
 import org.apache.kylin.rest.service.StreamingV2Service;
 import org.apache.kylin.rest.service.TableService;
 import org.apache.kylin.stream.core.model.CubeAssignment;
-import org.apache.kylin.stream.core.model.ReplicaSet;
 import org.apache.kylin.stream.core.model.Node;
+import org.apache.kylin.stream.core.model.ReplicaSet;
 import org.apache.kylin.stream.core.model.stats.ClusterState;
 import org.apache.kylin.stream.core.model.stats.CubeRealTimeState;
 import org.apache.kylin.stream.core.model.stats.ReceiverStats;
@@ -78,10 +78,10 @@ import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
-import com.google.common.base.Preconditions;
 
 /**
  * StreamingController is defined as Restful API entrance for UI.
@@ -136,6 +136,8 @@ public class StreamingV2Controller extends BasicController {
         boolean saveStreamingSuccess = false, saveTableSuccess = false;
         KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
         ProjectInstance projectInstance = ProjectManager.getInstance(kylinConfig).getProject(project);
+
+        InternalErrorException shouldThrow = null;
         try {
             try {
                 tableDesc.setUuid(UUID.randomUUID().toString());
@@ -160,22 +162,27 @@ public class StreamingV2Controller extends BasicController {
                     try {
                         tableService.unloadHiveTable(tableDesc.getIdentity(), project);
                     } catch (IOException e) {
-                        throw new InternalErrorException("Action failed and failed to rollback the create table "
-                                + e.getLocalizedMessage(), e);
+                        shouldThrow = new InternalErrorException(
+                                "Action failed and failed to rollback the create table " + e.getLocalizedMessage(), e);
                     }
                 }
                 if (saveStreamingSuccess) {
                     try {
                         streamingService.dropStreamingConfig(streamingSourceConfig);
                     } catch (IOException e) {
-                        throw new InternalErrorException(
+                        shouldThrow = new InternalErrorException(
                                 "Action failed and failed to rollback the created streaming config: "
-                                        + e.getLocalizedMessage(), e);
+                                        + e.getLocalizedMessage(),
+                                e);
                     }
                 }
             }
+        }
 
+        if (null != shouldThrow) {
+            throw shouldThrow;
         }
+
         streamingRequest.setSuccessful(true);
         return streamingRequest;
     }
@@ -197,8 +204,9 @@ public class StreamingV2Controller extends BasicController {
             } catch (NoSuchObjectException noObjectException) {
                 logger.info("table not exist in hive meta store for table:" + tableDesc.getIdentity(),
                         noObjectException);
-                throw new BadRequestException("table doesn't exist in hive meta store for table:"
-                        + tableDesc.getIdentity(), ResponseCode.CODE_UNDEFINED, noObjectException);
+                throw new BadRequestException(
+                        "table doesn't exist in hive meta store for table:" + tableDesc.getIdentity(),
+                        ResponseCode.CODE_UNDEFINED, noObjectException);
             } catch (Exception e) {
                 logger.error("error when get metadata from hive meta store for table:" + tableDesc.getIdentity(), e);
                 throw new BadRequestException("error when connect hive meta store", ResponseCode.CODE_UNDEFINED, e);
@@ -230,7 +238,8 @@ public class StreamingV2Controller extends BasicController {
             }
             if (!incompatibleMsgs.isEmpty()) {
                 logger.info("incompatible for hive and input table schema:{}", incompatibleMsgs);
-                throw new BadRequestException("incompatible for hive schema and input table schema:" + incompatibleMsgs);
+                throw new BadRequestException(
+                        "incompatible for hive schema and input table schema:" + incompatibleMsgs);
             }
         }
     }
@@ -299,8 +308,8 @@ public class StreamingV2Controller extends BasicController {
             streamingService.dropStreamingConfig(config);
         } catch (Exception e) {
             logger.error(e.getLocalizedMessage(), e);
-            throw new InternalErrorException("Failed to delete StreamingSourceConfig. " + " Caused by: "
-                    + e.getMessage(), e);
+            throw new InternalErrorException(
+                    "Failed to delete StreamingSourceConfig. " + " Caused by: " + e.getMessage(), e);
         }
     }
 
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index fd54e2b..f0a8eb9 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -150,8 +150,8 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
         private long rowCount;
         private long rowBytes;
 
-        ResourceTrackingCellListIterator(Iterator<List<Cell>> delegate,
-                                         long rowCountLimit, long bytesLimit, long deadline) {
+        ResourceTrackingCellListIterator(Iterator<List<Cell>> delegate, long rowCountLimit, long bytesLimit,
+                long deadline) {
             this.delegate = delegate;
             this.rowCountLimit = rowCountLimit;
             this.bytesLimit = bytesLimit;
@@ -242,12 +242,14 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
 
         // if user change kylin.properties on kylin server, need to manually redeploy coprocessor jar to update KylinConfig of Env.
         KylinConfig kylinConfig = KylinConfig.createKylinConfig(request.getKylinProperties());
-        
+
         String queryId = request.hasQueryId() ? request.getQueryId() : "UnknownId";
         logger.info("start query {} in thread {}", queryId, Thread.currentThread().getName());
+
+        RuntimeException shouldThrow = null;
         try (SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(kylinConfig);
                 SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
-            
+
             final long serviceStartTime = System.currentTimeMillis();
 
             region = (HRegion) env.getRegion();
@@ -428,10 +430,14 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
                 try {
                     region.closeRegionOperation();
                 } catch (IOException e) {
-                    throw new RuntimeException(e);
+                    shouldThrow = new RuntimeException(e);
                 }
             }
         }
+
+        if (null != shouldThrow) {
+            throw shouldThrow;
+        }
     }
 
     @Override


[kylin] 01/05: minor, sonar issues for ignoring initial values of method parameters

Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git

commit 8bdbdf287ef64afc726e46fba67f4a3ccd3f13e3
Author: etherge <et...@163.com>
AuthorDate: Thu Jan 23 17:50:36 2020 -0500

    minor, sonar issues for ignoring initial values of method parameters
---
 .../src/main/java/org/apache/kylin/dict/NumberDictionary.java  | 10 ++++------
 .../src/main/java/org/apache/kylin/dict/NumberDictionary2.java |  4 ++--
 .../kylin/sdk/datasource/framework/conv/ConvSqlWriter.java     |  8 ++++----
 .../src/main/java/org/apache/kylin/engine/mr/CubingJob.java    |  1 -
 4 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
index 8e59792..083f272 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
@@ -18,7 +18,6 @@
 
 package org.apache.kylin.dict;
 
-
 import org.apache.kylin.common.util.ClassUtil;
 
 /**
@@ -29,7 +28,6 @@ import org.apache.kylin.common.util.ClassUtil;
 @Deprecated
 public class NumberDictionary<T> extends TrieDictionary<T> {
 
-
     // ============================================================================
 
     public NumberDictionary() { // default constructor for Writable interface
@@ -48,10 +46,10 @@ public class NumberDictionary<T> extends TrieDictionary<T> {
 
     @Override
     protected void setConverterByName(String converterName) throws Exception {
-        converterName = "org.apache.kylin.dict.Number2BytesConverter";
-        this.bytesConvert = ClassUtil.forName(converterName, BytesConverter.class).getDeclaredConstructor().newInstance();
-        ((Number2BytesConverter)this.bytesConvert).setMaxDigitsBeforeDecimalPoint(Number2BytesConverter.MAX_DIGITS_BEFORE_DECIMAL_POINT_LEGACY);
+        this.bytesConvert = ClassUtil.forName("org.apache.kylin.dict.Number2BytesConverter", BytesConverter.class).getDeclaredConstructor()
+                .newInstance();
+        ((Number2BytesConverter) this.bytesConvert)
+                .setMaxDigitsBeforeDecimalPoint(Number2BytesConverter.MAX_DIGITS_BEFORE_DECIMAL_POINT_LEGACY);
     }
 
-
 }
\ No newline at end of file
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java
index 24700ca..16cd44f 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java
@@ -39,8 +39,8 @@ public class NumberDictionary2<T> extends NumberDictionary<T> {
 
     @Override
     protected void setConverterByName(String converterName) throws Exception {
-        converterName = "org.apache.kylin.dict.Number2BytesConverter";
-        this.bytesConvert = ClassUtil.forName(converterName, BytesConverter.class).getDeclaredConstructor().newInstance();
+        this.bytesConvert = ClassUtil.forName("org.apache.kylin.dict.Number2BytesConverter", BytesConverter.class).getDeclaredConstructor()
+                .newInstance();
     }
 
 }
diff --git a/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/conv/ConvSqlWriter.java b/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/conv/ConvSqlWriter.java
index eb0630c..4045b93 100644
--- a/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/conv/ConvSqlWriter.java
+++ b/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/framework/conv/ConvSqlWriter.java
@@ -217,11 +217,11 @@ public class ConvSqlWriter extends SqlPrettyWriter {
     public void writeWithItem(SqlCall call, SqlWithItem.SqlWithItemOperator sqlWithItemOperator, int leftPrec,
                               int rightPrec) {
         final SqlWithItem withItem = (SqlWithItem) call;
-        leftPrec = sqlWithItemOperator.getLeftPrec();
-        rightPrec = sqlWithItemOperator.getRightPrec();
-        withItem.name.unparse(this, leftPrec, rightPrec);
+        int leftP = sqlWithItemOperator.getLeftPrec();
+        int rightP = sqlWithItemOperator.getRightPrec();
+        withItem.name.unparse(this, leftP, rightP);
         if (withItem.columnList != null) {
-            withItem.columnList.unparse(this, leftPrec, rightPrec);
+            withItem.columnList.unparse(this, leftP, rightP);
         }
         this.keyword("AS");
         Frame frame = this.startList(FrameTypeEnum.WITH_ITEM, "(", ")");
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
index c0fc183..568392e 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
@@ -202,7 +202,6 @@ public class CubingJob extends DefaultChainedExecutable {
         CubeInstance cubeInstance = CubeManager.getInstance(context.getConfig())
                 .getCube(CubingExecutableUtil.getCubeName(this.getParams()));
         final Output output = getManager().getOutput(getId());
-        state = output.getState();
         if (state != ExecutableState.ERROR
                 && !cubeInstance.getDescriptor().getStatusNeedNotify().contains(state.toString())) {
             logger.info("state:" + state + " no need to notify users");