You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rocketmq.apache.org by du...@apache.org on 2021/12/22 12:21:50 UTC
[rocketmq-streams] branch main updated: fix code base
This is an automated email from the ASF dual-hosted git repository.
duhengforever pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/rocketmq-streams.git
The following commit(s) were added to refs/heads/main by this push:
new 190106b fix code base
190106b is described below
commit 190106b05bf8a7c3e797411b8476056b22d600a9
Author: duhenglucky <du...@apache.org>
AuthorDate: Wed Dec 22 20:11:50 2021 +0800
fix code base
---
.../common/cache/compress/impl/IntValueKV.java | 112 ---------------------
.../common/optimization/HyperscanEngine.java | 2 +-
.../common/optimization/OptimizationRegex.java | 10 +-
.../rocketmq/streams/common/utils/FileUtil.java | 7 +-
4 files changed, 3 insertions(+), 128 deletions(-)
diff --git a/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/cache/compress/impl/IntValueKV.java b/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/cache/compress/impl/IntValueKV.java
index ef365e2..2189b9f 100644
--- a/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/cache/compress/impl/IntValueKV.java
+++ b/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/cache/compress/impl/IntValueKV.java
@@ -16,23 +16,11 @@
*/
package org.apache.rocketmq.streams.common.cache.compress.impl;
-import java.io.BufferedReader;
-import java.io.FileInputStream;
-import java.io.InputStreamReader;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Random;
-import java.util.Set;
-import java.util.UUID;
import org.apache.rocketmq.streams.common.cache.compress.AdditionStore;
import org.apache.rocketmq.streams.common.cache.compress.ByteArray;
import org.apache.rocketmq.streams.common.cache.compress.CacheKV;
import org.apache.rocketmq.streams.common.cache.compress.MapAddress;
import org.apache.rocketmq.streams.common.utils.NumberUtils;
-import org.junit.Assert;
/**
* 支持key是string，value是int的场景，支持size不大于10000000.只支持int，long，boolean，string类型
@@ -91,105 +79,5 @@ public class IntValueKV extends CacheKV<Integer> {
super(capacity);
}
- public static void main(String[] args) throws Exception {
- IntValueKV cache = new IntValueKV(5);
- cache.put("A", 0);
- cache.put("B", 1);
- cache.put("C", 2);
- cache.put("D", 3);
- cache.put("E", 4);
- cache.put("F", 5);
- cache.put("G", 6);
-
- System.exit(0);
-
- int size = 10000000;
- int sampleSize = 1024;
- int dataSize = 3974534;
- IntValueKV compressByteMap = new IntValueKV(size);
- Map<String, Integer> dataMap = new HashMap<>(size);
- Set<Integer> whiteSet = new HashSet<>(1024);
- Map<String, Integer> sample1Map = new HashMap<>(1024);
- Map<String, Integer> sample2Map = new HashMap<>(1024);
- //init data
- Random random = new Random();
- while (true) {
- if (whiteSet.size() >= sampleSize) {
- break;
- }
- int seed = random.nextInt(dataSize);
- if (!whiteSet.contains(seed)) {
- whiteSet.add(seed);
- }
- }
-
- long originWriteCounter = 0;
- long compressWriteCounter = 0;
- String path = "/Users/arthur/Downloads/";
- String blackFile = "2020-11-11-14-08-32_EXPORT_CSV_16231630_392_0.csv";
- try (BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(path + blackFile)))) {
- reader.readLine();
- String line = null;
- int counter = 0;
- while ((line = reader.readLine()) != null) {
- line = line.replaceAll("\"", "");
- String[] parts = line.split(",", 2);
- long begin = System.nanoTime();
- dataMap.put(parts[1].trim(), Integer.parseInt(parts[0]));
- originWriteCounter += (System.nanoTime() - begin);
- if (whiteSet.contains(counter++)) {
- sample1Map.put(parts[1].trim(), Integer.parseInt(parts[0]));
- }
- }
- }
- for (int i = 0; i < sampleSize * 100; i++) {
- sample2Map.put(UUID.randomUUID().toString(), -1);
- }
- System.out.println("sample1 size:\t" + sample1Map.size());
- System.out.println("sample2 size:\t" + sample2Map.size());
- //System.out.println(
- // "origin map size(computed by third party):\t" + RamUsageEstimator.humanSizeOf(dataMap) + "\tline's\t"
- // + dataMap.size());
- //
- Iterator<Entry<String, Integer>> iterator = dataMap.entrySet().iterator();
- while (iterator.hasNext()) {
- Entry<String, Integer> entry = iterator.next();
- long begin = System.nanoTime();
- compressByteMap.put(entry.getKey(), entry.getValue());
- compressWriteCounter += (System.nanoTime() - begin);
- }
- //System.out.println(
- // "compressed map size(computed by third party):\t" + RamUsageEstimator.humanSizeOf(compressByteMap)
- // + "\tline's\t"
- // + compressByteMap.size);
- System.out.println("compressed map size(computed by it's self)\t" + compressByteMap.calMemory() + " MB");
- System.out.println(
- "origin write cost:\t" + originWriteCounter / 1000 + "\tcompress write cost:\t"
- + compressWriteCounter / 1000);
- //
- long originSearchCounter = 0;
- long compressCounter = 0;
- Iterator<Entry<String, Integer>> iterator1 = sample1Map.entrySet().iterator();
- Iterator<Entry<String, Integer>> iterator2 = sample2Map.entrySet().iterator();
- while (iterator1.hasNext() && iterator2.hasNext()) {
- Entry<String, Integer> entry1 = iterator1.next();
- String key1 = entry1.getKey();
- Integer value1 = entry1.getValue();
- Entry<String, Integer> entry2 = iterator2.next();
- String key2 = entry2.getKey();
- Integer value2 = entry2.getValue();
- long begin = System.nanoTime();
- Assert.assertEquals(value1, dataMap.get(key1));
- Assert.assertNotEquals(value2, dataMap.get(key2));
- originSearchCounter += (System.nanoTime() - begin);
- begin = System.nanoTime();
- Assert.assertEquals(value1, compressByteMap.get(key1));
- Assert.assertNotEquals(value2, compressByteMap.get(key2));
- compressCounter += (System.nanoTime() - begin);
- }
- System.out.println(
- "origin search cost:\t" + originSearchCounter / 1000 + "\tcompress search cost:\t"
- + compressCounter / 1000);
- }
}
diff --git a/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/optimization/HyperscanEngine.java b/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/optimization/HyperscanEngine.java
index 152affe..c22e3db 100644
--- a/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/optimization/HyperscanEngine.java
+++ b/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/optimization/HyperscanEngine.java
@@ -77,7 +77,7 @@ public class HyperscanEngine<T> implements IStreamRegex<T> {
this.scanner = scanner;
break;
} catch (CompileErrorException e) {
- if(scanner!=null){
+ if (scanner != null) {
scanner.close();
}
Expression expression = e.getFailedExpression();
diff --git a/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/optimization/OptimizationRegex.java b/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/optimization/OptimizationRegex.java
index bcaa111..db890d6 100644
--- a/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/optimization/OptimizationRegex.java
+++ b/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/optimization/OptimizationRegex.java
@@ -80,10 +80,6 @@ public class OptimizationRegex {
}
}
}
- //String regex2Optimization=regex+":"+this.toString();
- //List<String> allRegexs=new ArrayList<>();
- //allRegexs.add(regex2Optimization);
- //FileUtil.write("/Users/yuanxiaodong/Documents/dipper_engine/regex.txt",allRegexs,true);
}
@@ -113,15 +109,11 @@ public class OptimizationRegex {
String word = orWord.get(i);
if (contentLen > word.length() && content.indexOf(word) != -1) {
isMatch = true;
- //AtomicInteger count=keyword2Count.get(word);
- //count.incrementAndGet();
+
break;
}
}
if (!isMatch) {
- //String keyword=orIndex2Keyword.get(index);
- //AtomicInteger count=keyword2Count.get(keyword);
- //count.incrementAndGet();
return false;
}
//index++;
diff --git a/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/utils/FileUtil.java b/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/utils/FileUtil.java
index 1a64d2d..dad47e7 100644
--- a/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/utils/FileUtil.java
+++ b/rocketmq-streams-commons/src/main/java/org/apache/rocketmq/streams/common/utils/FileUtil.java
@@ -236,10 +236,7 @@ public class FileUtil {
System.out.println("创建目标目录成功");
}
File targetFile = new File(targetPath, targetFileName);
- // System.out.println(targetFile.getName());
- // if (!(targetFile.getParentFile().exists())) {
- // targetFile.getParentFile().mkdirs();
- // }
+
System.out.println("创建目标目录成功:" + targetFile.getParent());
if (!targetFile.exists()) {
targetFile.createNewFile();
@@ -270,8 +267,6 @@ public class FileUtil {
} catch (IOException e) {
e.printStackTrace();
}
- // jar:file:/Users/yuanxiaodong/alibaba/rule-engine-feature/5/rules-engine/engine/target/ruleengine
- // .jar!/com/aliyun/filter/function/expression
}
}