You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by jx...@apache.org on 2017/12/20 17:54:17 UTC
[14/50] [abbrv] helix git commit: Optimize ClusterDataCache's data
refresh strategy by: 1) cacheing CurrentStates locally and update only these
that have been changed from zk. 2) Controller listens on ResourceConfig
changes 3) Cache resource configs loca
http://git-wip-us.apache.org/repos/asf/helix/blob/d6f4943e/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestInstanceHistory.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestInstanceHistory.java b/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestInstanceHistory.java
index 0f21a13..f48fb0f 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestInstanceHistory.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/paticipant/TestInstanceHistory.java
@@ -43,7 +43,7 @@ public class TestInstanceHistory extends ZkStandAloneCMTestBase {
List<String> list = history.getRecord().getListField("HISTORY");
Assert.assertEquals(list.size(), 1);
- for (int i = 0; i <= 12; i++) {
+ for (int i = 0; i <= 22; i++) {
_participants[0].disconnect();
_participants[0].connect();
}
@@ -51,7 +51,9 @@ public class TestInstanceHistory extends ZkStandAloneCMTestBase {
history = manager.getHelixDataAccessor().getProperty(propertyKey);
Assert.assertNotNull(history);
list = history.getRecord().getListField("HISTORY");
- Assert.assertEquals(list.size(), 10);
+ Assert.assertEquals(list.size(), 20);
+ list = history.getRecord().getListField("OFFLINE");
+ Assert.assertEquals(list.size(), 20);
manager.disconnect();
}
}
http://git-wip-us.apache.org/repos/asf/helix/blob/d6f4943e/helix-core/src/test/java/org/apache/helix/integration/rebalancer/TestDelayedAutoRebalance.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/TestDelayedAutoRebalance.java b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/TestDelayedAutoRebalance.java
index d926e01..d698647 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/TestDelayedAutoRebalance.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/TestDelayedAutoRebalance.java
@@ -228,8 +228,7 @@ public class TestDelayedAutoRebalance extends ZkIntegrationTestBase {
Thread.sleep(1000);
for (String db : _testDBs) {
- IdealState is = _setupTool.getClusterManagementTool().getResourceIdealState(
- CLUSTER_NAME, db);
+ IdealState is = _setupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, db);
Map<String, List<String>> preferenceLists = is.getPreferenceLists();
for (List<String> instances : preferenceLists.values()) {
Assert.assertFalse(instances.contains(disabledInstanceName));
http://git-wip-us.apache.org/repos/asf/helix/blob/d6f4943e/helix-core/src/test/java/org/apache/helix/integration/task/TestTaskRebalancerStopResume.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/integration/task/TestTaskRebalancerStopResume.java b/helix-core/src/test/java/org/apache/helix/integration/task/TestTaskRebalancerStopResume.java
index 38ef373..54a7fe9 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/task/TestTaskRebalancerStopResume.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/task/TestTaskRebalancerStopResume.java
@@ -256,7 +256,7 @@ public class TestTaskRebalancerStopResume extends TaskTestBase {
LOG.info("Flusing job-queue: " + queueName);
_driver.flushQueue(queueName);
- TimeUnit.MILLISECONDS.sleep(200);
+ TimeUnit.MILLISECONDS.sleep(1000);
// verify the cleanup
for (int i = 0; i < currentJobNames.size(); i++) {
String namedSpaceJobName = String.format("%s_%s", queueName, currentJobNames.get(i));
@@ -522,8 +522,8 @@ public class TestTaskRebalancerStopResume extends TaskTestBase {
HelixDataAccessor accessor = _manager.getHelixDataAccessor();
PropertyKey.Builder keyBuilder = accessor.keyBuilder();
- Assert.assertNull(accessor.getProperty(keyBuilder.idealStates(jobName)));
- Assert.assertNull(accessor.getProperty(keyBuilder.resourceConfig(jobName)));
+ Assert.assertNull(accessor.getProperty(keyBuilder.idealStates(jobName)), jobName + "'s idealstate has not been deleted!");
+ Assert.assertNull(accessor.getProperty(keyBuilder.resourceConfig(jobName)), jobName + "'s resourceConfig has not been deleted!");
TaskTestUtil.pollForEmptyJobState(_driver, queueName, jobName);
}
http://git-wip-us.apache.org/repos/asf/helix/blob/d6f4943e/helix-core/src/test/java/org/apache/helix/mock/MockBaseDataAccessor.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/mock/MockBaseDataAccessor.java b/helix-core/src/test/java/org/apache/helix/mock/MockBaseDataAccessor.java
index 6468006..f84565b 100644
--- a/helix-core/src/test/java/org/apache/helix/mock/MockBaseDataAccessor.java
+++ b/helix-core/src/test/java/org/apache/helix/mock/MockBaseDataAccessor.java
@@ -33,14 +33,48 @@ import org.apache.helix.ZNRecord;
import org.apache.zookeeper.data.Stat;
public class MockBaseDataAccessor implements BaseDataAccessor<ZNRecord> {
- Map<String, ZNRecord> map = new HashMap<String, ZNRecord>();
- @Override public boolean create(String path, ZNRecord record, int options) {
+ class ZNode {
+ private ZNRecord _record;
+ private Stat _stat;
+
+ public ZNode (ZNRecord record) {
+ _record = record;
+ _stat = new Stat();
+ _stat.setCtime(System.currentTimeMillis());
+ }
+
+ public void set (ZNRecord record) {
+ _record = record;
+ _stat.setMtime(System.currentTimeMillis());
+ _stat.setVersion(_stat.getVersion() + 1);
+ }
+
+ public ZNRecord getRecord() {
+ return _record;
+ }
+
+ public Stat getStat() {
+ return _stat;
+ }
+ }
+
+ Map<String, ZNode> _recordMap = new HashMap<>();
+
+ @Override
+ public boolean create(String path, ZNRecord record, int options) {
return set(path, record, options);
}
- @Override public boolean set(String path, ZNRecord record, int options) {
- map.put(path, record);
+ @Override
+ public boolean set(String path, ZNRecord record, int options) {
+ ZNode zNode = _recordMap.get(path);
+ if (zNode == null) {
+ _recordMap.put(path, new ZNode(record));
+ } else {
+ zNode.set(record);
+ _recordMap.put(path, zNode);
+ }
try {
Thread.sleep(50);
} catch (InterruptedException e) {
@@ -49,22 +83,20 @@ public class MockBaseDataAccessor implements BaseDataAccessor<ZNRecord> {
return true;
}
- @Override public boolean update(String path, DataUpdater<ZNRecord> updater, int options) {
- ZNRecord current = map.get(path);
+ @Override
+ public boolean update(String path, DataUpdater<ZNRecord> updater, int options) {
+ ZNode zNode = _recordMap.get(path);
+ ZNRecord current = zNode != null ? zNode.getRecord() : null;
ZNRecord newRecord = updater.update(current);
if (newRecord != null) {
- map.put(path, newRecord);
+ return set(path, newRecord, options);
}
- try {
- Thread.sleep(50);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- return true;
+ return false;
}
- @Override public boolean remove(String path, int options) {
- map.remove(path);
+ @Override
+ public boolean remove(String path, int options) {
+ _recordMap.remove(path);
try {
Thread.sleep(50);
} catch (InterruptedException e) {
@@ -73,12 +105,14 @@ public class MockBaseDataAccessor implements BaseDataAccessor<ZNRecord> {
return true;
}
- @Override public boolean[] createChildren(List<String> paths, List<ZNRecord> records,
+ @Override
+ public boolean[] createChildren(List<String> paths, List<ZNRecord> records,
int options) {
return setChildren(paths, records, options);
}
- @Override public boolean[] setChildren(List<String> paths, List<ZNRecord> records, int options) {
+ @Override
+ public boolean[] setChildren(List<String> paths, List<ZNRecord> records, int options) {
boolean [] ret = new boolean[paths.size()];
for (int i = 0; i < paths.size(); i++) {
boolean success = create(paths.get(i), records.get(i), options);
@@ -87,7 +121,8 @@ public class MockBaseDataAccessor implements BaseDataAccessor<ZNRecord> {
return ret;
}
- @Override public boolean[] updateChildren(List<String> paths,
+ @Override
+ public boolean[] updateChildren(List<String> paths,
List<DataUpdater<ZNRecord>> updaters, int options) {
boolean [] ret = new boolean[paths.size()];
for (int i = 0; i < paths.size(); i++) {
@@ -97,7 +132,8 @@ public class MockBaseDataAccessor implements BaseDataAccessor<ZNRecord> {
return ret;
}
- @Override public boolean[] remove(List<String> paths, int options) {
+ @Override
+ public boolean[] remove(List<String> paths, int options) {
boolean [] ret = new boolean[paths.size()];
for (int i = 0; i < paths.size(); i++) {
boolean success = remove(paths.get(i), options);
@@ -106,11 +142,14 @@ public class MockBaseDataAccessor implements BaseDataAccessor<ZNRecord> {
return ret;
}
- @Override public ZNRecord get(String path, Stat stat, int options) {
- return map.get(path);
+ @Override
+ public ZNRecord get(String path, Stat stat, int options) {
+ ZNode zNode = _recordMap.get(path);
+ return zNode != null ? zNode.getRecord() : null;
}
- @Override public List<ZNRecord> get(List<String> paths, List<Stat> stats, int options) {
+ @Override
+ public List<ZNRecord> get(List<String> paths, List<Stat> stats, int options) {
List<ZNRecord> records = new ArrayList<ZNRecord>();
for (int i = 0; i < paths.size(); i++) {
ZNRecord record = get(paths.get(i), stats.get(i), options);
@@ -119,14 +158,16 @@ public class MockBaseDataAccessor implements BaseDataAccessor<ZNRecord> {
return records;
}
- @Override public List<ZNRecord> getChildren(String parentPath, List<Stat> stats, int options) {
+ @Override
+ public List<ZNRecord> getChildren(String parentPath, List<Stat> stats, int options) {
List<ZNRecord> children = new ArrayList<ZNRecord>();
- for (String key : map.keySet()) {
+ for (String key : _recordMap.keySet()) {
if (key.startsWith(parentPath)) {
String[] keySplit = key.split("\\/");
String[] pathSplit = parentPath.split("\\/");
if (keySplit.length - pathSplit.length == 1) {
- ZNRecord record = map.get(key);
+ ZNode zNode = _recordMap.get(key);
+ ZNRecord record = zNode != null ? zNode.getRecord() : null;
if (record != null) {
children.add(record);
}
@@ -139,9 +180,10 @@ public class MockBaseDataAccessor implements BaseDataAccessor<ZNRecord> {
return children;
}
- @Override public List<String> getChildNames(String parentPath, int options) {
+ @Override
+ public List<String> getChildNames(String parentPath, int options) {
List<String> child = new ArrayList<String>();
- for (String key : map.keySet()) {
+ for (String key : _recordMap.keySet()) {
if (key.startsWith(parentPath)) {
String[] keySplit = key.split("\\/");
String[] pathSplit = parentPath.split("\\/");
@@ -153,26 +195,31 @@ public class MockBaseDataAccessor implements BaseDataAccessor<ZNRecord> {
return child;
}
- @Override public boolean exists(String path, int options) {
- return map.containsKey(path);
+ @Override
+ public boolean exists(String path, int options) {
+ return _recordMap.containsKey(path);
}
- @Override public boolean[] exists(List<String> paths, int options) {
+ @Override
+ public boolean[] exists(List<String> paths, int options) {
boolean [] ret = new boolean[paths.size()];
for (int i = 0; i < paths.size(); i++) {
- ret[i] = map.containsKey(paths.get(i));
+ ret[i] = _recordMap.containsKey(paths.get(i));
}
return ret;
}
@Override public Stat[] getStats(List<String> paths, int options) {
- // TODO Auto-generated method stub
- return null;
+ Stat [] stats = new Stat[paths.size()];
+ for (int i = 0; i < paths.size(); i++) {
+ stats[i] = getStat(paths.get(i), options);
+ }
+ return stats;
}
@Override public Stat getStat(String path, int options) {
- // TODO Auto-generated method stub
- return null;
+ ZNode zNode = _recordMap.get(path);
+ return zNode != null ? zNode.getStat() : null;
}
@Override public void subscribeDataChanges(String path, IZkDataListener listener) {
@@ -196,7 +243,7 @@ public class MockBaseDataAccessor implements BaseDataAccessor<ZNRecord> {
}
@Override public void reset() {
- map.clear();
+ _recordMap.clear();
}
@Override public boolean set(String path, ZNRecord record, int options, int expectVersion) {
http://git-wip-us.apache.org/repos/asf/helix/blob/d6f4943e/helix-core/src/test/java/org/apache/helix/monitoring/TestClusterStatusMonitorLifecycle.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/monitoring/TestClusterStatusMonitorLifecycle.java b/helix-core/src/test/java/org/apache/helix/monitoring/TestClusterStatusMonitorLifecycle.java
index 17a04ec..29df0b7 100644
--- a/helix-core/src/test/java/org/apache/helix/monitoring/TestClusterStatusMonitorLifecycle.java
+++ b/helix-core/src/test/java/org/apache/helix/monitoring/TestClusterStatusMonitorLifecycle.java
@@ -216,7 +216,7 @@ public class TestClusterStatusMonitorLifecycle extends ZkIntegrationTestBase {
}
}
firstController.disconnect();
- Thread.sleep(1000);
+ Thread.sleep(2000);
// 1 cluster status monitor, 1 resource monitor, 5 instances
// Unregister 1+4+1 per-instance resource mbean
@@ -231,7 +231,7 @@ public class TestClusterStatusMonitorLifecycle extends ZkIntegrationTestBase {
// 1 participant comes back
// No change in instance/resource mbean
// Register 1 per-instance resource mbean
- Thread.sleep(1000);
+ Thread.sleep(2000);
Assert.assertEquals(nMbeansUnregistered, listener._nMbeansUnregistered - 32);
Assert.assertEquals(nMbeansRegistered, listener._nMbeansRegistered - 30);