You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by xy...@apache.org on 2023/10/05 21:47:30 UTC
[helix] branch metaclient updated (113b99237 -> 6bd97a98c)
This is an automated email from the ASF dual-hosted git repository.
xyuanlu pushed a change to branch metaclient
in repository https://gitbox.apache.org/repos/asf/helix.git
discard 113b99237 Add recursiveCreate functionality to metaclient (#2607)
omit c147eb4c0 Lattice cache - caching just data implementation (#2619)
omit 34ec3c6c9 Skip one time listener re-register for exists for ZkClient - MetaClient usage. (#2637)
omit 0555e9307 MetaClientCache Part 1 - API's, configs, and builders (#2612)
omit ddd688252 Leader election client - Proactively send leader gone event when disconnect from ZK (#2585)
omit 1f9cae890 Add stress test for Metaclient leader election (#2574)
omit a1fa50283 Lattice Puppy Stress Test Race Condition Fix and Code Cleanup (#2579)
omit 8fa86e9f9 Lattice LockClient LockInfoSerializer Implementation(#2580)
omit dd95ca441 Metaclient - Leader election - leader change event notification (#2560)
omit c22d4eb0e Metaclient - Leader election - Track participants and add participant info (#2562)
omit 785c61ad5 Multithreading stress test lattice - CRUD puppies and Listener Tests (#2548)
omit 294fae2c0 Metaclient leader election - basics (#2558)
omit abb867ab2 MultiThreading Stress Test Lattice - Puppy Logic and Skeleton (Part 1) (#2549)
omit def0bb6d6 ZkClient - only register one time watcher for read data when not using persist listener. (#2555)
omit e49908731 MetaClient LockClient and Semaphore Creation Edge Cases and Added Exception(#2550)
omit 14283032f Lattice MetaClient Distributed Semaphore Implementation (#2515)
omit 8af1b60ab Add API for getDataAndStat (#2535)
omit 14d83b064 MetaClient Distributed Semaphore API and Permit object (#2520)
omit 1e9b19ba6 [helix/metaclient] ZkMetaClient Stress Tests (Part 2) (#2523)
omit b35811011 set test iteration count to 500
omit 1dee8eee3 refactor, update with dataRecord
omit 6797d9151 deserialize test
omit e7de35e5c tests for get, create, set, update, and data change listeners
omit e70905698 Inline
omit f16a523e0 DataRecord Serializer
add ff4c5cadd DataRecord Serializer
add df1eb6ae9 Inline
add b93f89a06 tests for get, create, set, update, and data change listeners
add 1f80503b6 deserialize test
add b45361ba3 refactor, update with dataRecord
add 17f8ab3fd set test iteration count to 500
add 09649b34c [helix/metaclient] ZkMetaClient Stress Tests (Part 2) (#2523)
add 0e917599c MetaClient Distributed Semaphore API and Permit object (#2520)
add 53f9fa7c3 Add API for getDataAndStat (#2535)
add c7a0d28fc Lattice MetaClient Distributed Semaphore Implementation (#2515)
add f41a2c606 MetaClient LockClient and Semaphore Creation Edge Cases and Added Exception(#2550)
add 6aa7c2cce ZkClient - only register one time watcher for read data when not using persist listener. (#2555)
add 5389b1739 MultiThreading Stress Test Lattice - Puppy Logic and Skeleton (Part 1) (#2549)
add 36231faeb Metaclient leader election - basics (#2558)
add b1f59e6f8 Multithreading stress test lattice - CRUD puppies and Listener Tests (#2548)
add 52fe73818 Metaclient - Leader election - Track participants and add participant info (#2562)
add 4eacde331 Metaclient - Leader election - leader change event notification (#2560)
add 26f319608 Lattice LockClient LockInfoSerializer Implementation(#2580)
add c3afa7f27 Lattice Puppy Stress Test Race Condition Fix and Code Cleanup (#2579)
add 7a96d9ba8 Add stress test for Metaclient leader election (#2574)
add 2e53694f7 Leader election client - Proactively send leader gone event when disconnect from ZK (#2585)
add fbefa6490 Change TopStateHandoffReportStage to be an Async Stage as it is the slowest stage in the pipeline. It can be run async since it does not add to ClusterEvent and simply computes and reports metrics. No following stages depend on it. (#2610)
add d00a37c0d Changed Log level to DEBUG for the top-state metrics logs, to avoid logging for every partition.(#2611)
add 0e82bd3fd Fix flaky test TestP2PNoDuplicatedMessage (#2587)
add c9402ea1e Fix the 'BadVersion' exception handling in doSet() code path. (#2614)
add 279c3f39e Bump jackson-databind in /helix-view-aggregator (#2280)
add c607364c8 Fix TestClusterAggregateMetrics (#2602)
add 989c20c4b Add API for Instance evacuation (#2588)
add ba858b2a0 Implement the on-demand rebalance service (#2595)
add 059f1f60c Instance Evacuation operation sanity check (#2598)
add 7fe7d0a30 Add CI test config for ApplicationClusterManager branch (#2605)
add eb62d1511 Add support for node evacuation. (#2604)
add 7ab07d034 Fix NPR error in test for TestDelayedAutoRebalanceWithRackaware (#2606)
add a51c587f7 Add on-demand rebalance API (#2601)
add eff214d92 Change instance operation orthogonal to instance enable (#2615)
add 96cd0d0e3 Add supporting APIs a isEvacuateFinished and isReadyForPreparingJoiningCluster for evacuation. (#2618)
add 0922e1f2f Add support for providing default Instance Config to be used with auto-join and auto-reg (#2609)
add 045deb60f Support persist instance info collected by CloudInstanceInformationProcessor during auto-reg (#2622)
add 56f230e9c Exclude on-operation instance from computing min active replica in WAGED. (#2621)
add 79cefa3eb Refine TestInstanceOperation by using verifier (#2625)
add fd6a039c9 [maven-release-plugin] prepare release helix-1.3.1
add 38ac277c4 [maven-release-plugin] prepare for next development iteration
add 84d89efaf Fix CurrentStateComputationStage when mis configure(#2635)
add 0db2f3ae3 Release note 1.3.1
add 47c1718c9 Remove 1.2.0 docs
add 4a464b8b9 Update menu bar
add 6241bcbf5 Bump snapshot
new a6252856b MetaClientCache Part 1 - API's, configs, and builders (#2612)
new 2649e6e6a Skip one time listener re-register for exists for ZkClient - MetaClient usage. (#2637)
new 827ca822d Lattice cache - caching just data implementation (#2619)
new 6bd97a98c Add recursiveCreate functionality to metaclient (#2607)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (113b99237)
\
N -- N -- N refs/heads/metaclient (6bd97a98c)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/Helix-PR-CI.yml | 2 +-
...T.ivy => helix-admin-webapp-1.3.2-SNAPSHOT.ivy} | 4 +-
helix-admin-webapp/pom.xml | 2 +-
...SNAPSHOT.ivy => helix-agent-1.3.2-SNAPSHOT.ivy} | 12 +-
helix-agent/pom.xml | 2 +-
...NAPSHOT.ivy => helix-common-1.3.2-SNAPSHOT.ivy} | 2 +-
helix-common/pom.xml | 2 +-
.../apache/helix/constants/InstanceConstants.java | 6 +
...-SNAPSHOT.ivy => helix-core-1.3.2-SNAPSHOT.ivy} | 2 +-
helix-core/pom.xml | 2 +-
.../src/main/java/org/apache/helix/HelixAdmin.java | 26 +
.../org/apache/helix/HelixManagerProperty.java | 21 +-
...mation.java => CloudInstanceInformationV2.java} | 26 +-
.../helix/controller/pipeline/AsyncWorkerType.java | 1 +
.../rebalancer/DelayedAutoRebalancer.java | 36 +-
.../rebalancer/util/DelayedRebalanceUtil.java | 80 ++-
.../rebalancer/waged/WagedInstanceCapacity.java | 12 +-
.../rebalancer/waged/WagedRebalancer.java | 15 +-
.../stages/CurrentStateComputationStage.java | 2 +
.../stages/TopStateHandoffReportStage.java | 24 +-
.../helix/manager/zk/ParticipantManager.java | 41 +-
.../org/apache/helix/manager/zk/ZKHelixAdmin.java | 124 ++++-
.../helix/manager/zk/ZkBaseDataAccessor.java | 3 +
.../java/org/apache/helix/model/ClusterConfig.java | 27 +-
.../org/apache/helix/model/InstanceConfig.java | 250 ++++++++-
.../org/apache/helix/util/ConfigStringUtil.java | 2 +-
.../main/java/org/apache/helix/util/HelixUtil.java | 15 +-
.../java/org/apache/helix/TestConfigAccessor.java | 9 +-
.../org/apache/helix/TestHelixConfigAccessor.java | 21 +-
.../java/org/apache/helix/common/ZkTestBase.java | 8 +
.../helix/controller/stages/BaseStageTest.java | 11 +-
.../stages/TestCurrentStateComputationStage.java | 14 +-
.../manager/MockParticipantManager.java | 13 +-
.../messaging/TestP2PNoDuplicatedMessage.java | 33 +-
.../paticipant/CustomCloudInstanceInformation.java | 23 +-
.../CustomCloudInstanceInformationProcessor.java | 3 +-
.../paticipant/TestInstanceAutoJoin.java | 74 ++-
.../TestDelayedAutoRebalance.java | 144 ++++++
...stDelayedAutoRebalanceWithDisabledInstance.java | 29 +-
.../TestDelayedAutoRebalanceWithRackaware.java | 14 +
.../rebalancer/TestInstanceOperation.java | 559 +++++++++++++++++++++
.../WagedRebalancer/TestDelayedWagedRebalance.java | 15 +
...tDelayedWagedRebalanceWithDisabledInstance.java | 15 +
.../helix/manager/zk/TestZkBaseDataAccessor.java | 31 ++
.../java/org/apache/helix/mock/MockHelixAdmin.java | 23 +-
.../org/apache/helix/model/TestClusterConfig.java | 22 +
.../org/apache/helix/model/TestInstanceConfig.java | 32 +-
.../mbeans/TestClusterAggregateMetrics.java | 12 +-
helix-front/pom.xml | 2 +-
...-SNAPSHOT.ivy => helix-lock-1.3.2-SNAPSHOT.ivy} | 4 +-
helix-lock/pom.xml | 2 +-
...-SNAPSHOT.ivy => helix-rest-1.3.2-SNAPSHOT.ivy} | 4 +-
helix-rest/pom.xml | 4 +-
.../rest/server/resources/AbstractResource.java | 4 +-
.../server/resources/helix/ClusterAccessor.java | 9 +
.../server/resources/helix/InstancesAccessor.java | 1 +
.../resources/helix/PerInstanceAccessor.java | 4 +
.../helix/rest/server/TestClusterAccessor.java | 20 +
.../helix/rest/server/TestPerInstanceAccessor.java | 14 +
...vy => helix-view-aggregator-1.3.2-SNAPSHOT.ivy} | 4 +-
helix-view-aggregator/pom.xml | 4 +-
...SNAPSHOT.ivy => meta-client-1.3.2-SNAPSHOT.ivy} | 4 +-
meta-client/pom.xml | 2 +-
...data-store-directory-common-1.3.2-SNAPSHOT.ivy} | 4 +-
metadata-store-directory-common/pom.xml | 2 +-
...PSHOT.ivy => metrics-common-1.3.2-SNAPSHOT.ivy} | 2 +-
metrics-common/pom.xml | 2 +-
pom.xml | 6 +-
recipes/distributed-lock-manager/pom.xml | 2 +-
recipes/pom.xml | 2 +-
recipes/rabbitmq-consumer-group/pom.xml | 2 +-
recipes/rsync-replicated-file-system/pom.xml | 2 +-
recipes/service-discovery/pom.xml | 2 +-
recipes/task-execution/pom.xml | 2 +-
website/1.2.0/src/site/xdoc/download.xml.vm | 226 ---------
website/{1.2.0 => 1.3.1}/pom.xml | 4 +-
.../src/site/apt/privacy-policy.apt | 0
.../src/site/apt/releasenotes/release-1.3.1.apt} | 39 +-
.../{1.2.0 => 1.3.1}/src/site/markdown/Building.md | 4 +-
.../{1.2.0 => 1.3.1}/src/site/markdown/Features.md | 0
.../{1.2.0 => 1.3.1}/src/site/markdown/Metrics.md | 0
.../src/site/markdown/Quickstart.md | 6 +-
.../{1.2.0 => 1.3.1}/src/site/markdown/Tutorial.md | 0
.../src/site/markdown/auto_exit_maintenance.md | 0
.../src/site/markdown/design_crushed.md | 0
.../{1.2.0 => 1.3.1}/src/site/markdown/index.md | 6 +-
.../src/site/markdown/quota_scheduling.md | 0
.../src/site/markdown/recipes/lock_manager.md | 2 +-
.../markdown/recipes/rabbitmq_consumer_group.md | 2 +-
.../recipes/rsync_replicated_file_store.md | 2 +-
.../src/site/markdown/recipes/service_discovery.md | 0
.../site/markdown/recipes/task_dag_execution.md | 2 +-
.../src/site/markdown/tutorial_admin.md | 6 +-
.../src/site/markdown/tutorial_agent.md | 0
.../src/site/markdown/tutorial_cloud_support.md | 0
.../src/site/markdown/tutorial_controller.md | 0
.../src/site/markdown/tutorial_customized_view.md | 0
.../src/site/markdown/tutorial_distributed_lock.md | 0
.../src/site/markdown/tutorial_health.md | 0
.../src/site/markdown/tutorial_messaging.md | 2 +-
.../src/site/markdown/tutorial_participant.md | 0
.../src/site/markdown/tutorial_propstore.md | 2 +-
.../src/site/markdown/tutorial_rebalance.md | 0
.../src/site/markdown/tutorial_rest_service.md | 6 +-
.../src/site/markdown/tutorial_spectator.md | 0
.../src/site/markdown/tutorial_state.md | 0
.../src/site/markdown/tutorial_task_framework.md | 0
.../src/site/markdown/tutorial_task_throttling.md | 0
.../src/site/markdown/tutorial_throttling.md | 0
.../src/site/markdown/tutorial_ui.md | 2 +-
.../site/markdown/tutorial_user_content_store.md | 0
.../site/markdown/tutorial_user_def_rebalancer.md | 0
.../src/site/markdown/tutorial_yaml.md | 0
.../{1.2.0 => 1.3.1}/src/site/resources/.htaccess | 0
.../src/site/resources/JobExample.json | 0
.../src/site/resources/WorkflowExample.json | 0
.../resources/css/bootstrap-responsive.min.css | 0
.../src/site/resources/css/bootstrap.min.css | 0
.../src/site/resources/download.cgi | 0
.../images/CustomizedViewSystemArchitecture.jpeg | Bin
.../images/HelixPriorityLockWorkflow.jpeg | Bin
.../src/site/resources/images/PFS-Generic.png | Bin
.../images/ParticipantAutoRegistrationLogic.png | Bin
.../src/site/resources/images/RSYNC_BASED_PFS.png | Bin
.../site/resources/images/TaskFrameworkLayers.png | Bin
.../src/site/resources/images/UIScreenshot.png | Bin
.../src/site/resources/images/UIScreenshot2.png | Bin
.../resources/images/auto-exit-maintenance.jpg | Bin
.../images/design/crushed/after-using-crushed.png | Bin
.../images/design/crushed/before-using-crush.png | Bin
.../resources/images/design/crushed/classes.png | Bin
.../images/design/crushed/crushed-master-dist.png | Bin
.../design/crushed/crushed-partition-dist.png | Bin
.../images/design/crushed/cursh-master-dist.png | Bin
.../images/design/crushed/cursh-partition-dist.png | Bin
.../crushed/example-cluster-master-dist-after.png | Bin
.../design/crushed/example-cluster-master-dist.png | Bin
.../crushed/example-cluster-partition-dist.png | Bin
.../crushed/example-movement-on-expansion.png | Bin
.../design/crushed/node-down-master-move.png | Bin
.../design/crushed/node-down-partition-move.png | Bin
.../images/design/crushed/performance.png | Bin
.../images/quota_InstanceCapacityManager.jpeg | Bin
.../src/site/resources/images/quota_intro.png | Bin
.../src/site/resources/js/bootstrap.min.js | 0
.../src/site/resources/js/jquery.min.js | 0
website/{1.2.0 => 1.3.1}/src/site/site.xml | 8 +-
.../{1.3.0 => 1.3.1}/src/site/xdoc/download.xml.vm | 0
website/{1.2.0 => 1.3.1}/src/test/conf/testng.xml | 0
website/pom.xml | 6 +-
website/src/site/markdown/index.md | 10 +-
website/src/site/resources/.htaccess | 4 +-
website/src/site/site.xml | 14 +-
zookeeper-api/pom.xml | 2 +-
...APSHOT.ivy => zookeeper-api-1.3.2-SNAPSHOT.ivy} | 2 +-
155 files changed, 1790 insertions(+), 523 deletions(-)
rename helix-admin-webapp/{helix-admin-webapp-1.3.1-SNAPSHOT.ivy => helix-admin-webapp-1.3.2-SNAPSHOT.ivy} (97%)
rename helix-agent/{helix-agent-1.3.1-SNAPSHOT.ivy => helix-agent-1.3.2-SNAPSHOT.ivy} (96%)
rename helix-common/{helix-common-1.3.1-SNAPSHOT.ivy => helix-common-1.3.2-SNAPSHOT.ivy} (99%)
rename helix-core/{helix-core-1.3.1-SNAPSHOT.ivy => helix-core-1.3.2-SNAPSHOT.ivy} (99%)
copy helix-core/src/main/java/org/apache/helix/api/cloud/{CloudInstanceInformation.java => CloudInstanceInformationV2.java} (57%)
create mode 100644 helix-core/src/test/java/org/apache/helix/integration/rebalancer/TestInstanceOperation.java
rename helix-lock/{helix-lock-1.3.1-SNAPSHOT.ivy => helix-lock-1.3.2-SNAPSHOT.ivy} (97%)
rename helix-rest/{helix-rest-1.3.1-SNAPSHOT.ivy => helix-rest-1.3.2-SNAPSHOT.ivy} (97%)
rename helix-view-aggregator/{helix-view-aggregator-1.3.1-SNAPSHOT.ivy => helix-view-aggregator-1.3.2-SNAPSHOT.ivy} (97%)
rename meta-client/{meta-client-1.3.1-SNAPSHOT.ivy => meta-client-1.3.2-SNAPSHOT.ivy} (97%)
rename metadata-store-directory-common/{metadata-store-directory-common-1.3.1-SNAPSHOT.ivy => metadata-store-directory-common-1.3.2-SNAPSHOT.ivy} (97%)
rename metrics-common/{metrics-common-1.3.1-SNAPSHOT.ivy => metrics-common-1.3.2-SNAPSHOT.ivy} (99%)
delete mode 100644 website/1.2.0/src/site/xdoc/download.xml.vm
rename website/{1.2.0 => 1.3.1}/pom.xml (95%)
rename website/{1.2.0 => 1.3.1}/src/site/apt/privacy-policy.apt (100%)
rename website/{1.2.0/src/site/apt/releasenotes/release-1.2.0.apt => 1.3.1/src/site/apt/releasenotes/release-1.3.1.apt} (54%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/Building.md (95%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/Features.md (100%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/Metrics.md (100%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/Quickstart.md (99%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/Tutorial.md (100%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/auto_exit_maintenance.md (100%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/design_crushed.md (100%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/index.md (89%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/quota_scheduling.md (100%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/recipes/lock_manager.md (99%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/recipes/rabbitmq_consumer_group.md (99%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/recipes/rsync_replicated_file_store.md (99%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/recipes/service_discovery.md (100%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/recipes/task_dag_execution.md (99%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/tutorial_admin.md (99%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/tutorial_agent.md (100%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/tutorial_cloud_support.md (100%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/tutorial_controller.md (100%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/tutorial_customized_view.md (100%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/tutorial_distributed_lock.md (100%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/tutorial_health.md (100%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/tutorial_messaging.md (98%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/tutorial_participant.md (100%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/tutorial_propstore.md (95%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/tutorial_rebalance.md (100%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/tutorial_rest_service.md (99%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/tutorial_spectator.md (100%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/tutorial_state.md (100%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/tutorial_task_framework.md (100%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/tutorial_task_throttling.md (100%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/tutorial_throttling.md (100%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/tutorial_ui.md (99%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/tutorial_user_content_store.md (100%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/tutorial_user_def_rebalancer.md (100%)
rename website/{1.2.0 => 1.3.1}/src/site/markdown/tutorial_yaml.md (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/.htaccess (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/JobExample.json (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/WorkflowExample.json (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/css/bootstrap-responsive.min.css (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/css/bootstrap.min.css (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/download.cgi (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/images/CustomizedViewSystemArchitecture.jpeg (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/images/HelixPriorityLockWorkflow.jpeg (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/images/PFS-Generic.png (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/images/ParticipantAutoRegistrationLogic.png (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/images/RSYNC_BASED_PFS.png (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/images/TaskFrameworkLayers.png (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/images/UIScreenshot.png (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/images/UIScreenshot2.png (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/images/auto-exit-maintenance.jpg (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/images/design/crushed/after-using-crushed.png (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/images/design/crushed/before-using-crush.png (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/images/design/crushed/classes.png (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/images/design/crushed/crushed-master-dist.png (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/images/design/crushed/crushed-partition-dist.png (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/images/design/crushed/cursh-master-dist.png (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/images/design/crushed/cursh-partition-dist.png (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/images/design/crushed/example-cluster-master-dist-after.png (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/images/design/crushed/example-cluster-master-dist.png (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/images/design/crushed/example-cluster-partition-dist.png (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/images/design/crushed/example-movement-on-expansion.png (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/images/design/crushed/node-down-master-move.png (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/images/design/crushed/node-down-partition-move.png (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/images/design/crushed/performance.png (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/images/quota_InstanceCapacityManager.jpeg (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/images/quota_intro.png (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/js/bootstrap.min.js (100%)
rename website/{1.2.0 => 1.3.1}/src/site/resources/js/jquery.min.js (100%)
rename website/{1.2.0 => 1.3.1}/src/site/site.xml (96%)
copy website/{1.3.0 => 1.3.1}/src/site/xdoc/download.xml.vm (100%)
rename website/{1.2.0 => 1.3.1}/src/test/conf/testng.xml (100%)
rename zookeeper-api/{zookeeper-api-1.3.1-SNAPSHOT.ivy => zookeeper-api-1.3.2-SNAPSHOT.ivy} (99%)
[helix] 01/04: MetaClientCache Part 1 - API's, configs, and builders (#2612)
Posted by xy...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xyuanlu pushed a commit to branch metaclient
in repository https://gitbox.apache.org/repos/asf/helix.git
commit a6252856b14c8e7164cd4d70da2ebfd72218472b
Author: Marcos Rico Peng <55...@users.noreply.github.com>
AuthorDate: Tue Sep 12 22:24:22 2023 +0200
MetaClientCache Part 1 - API's, configs, and builders (#2612)
MetaClientCache Part 1 - API's, configs, and builders
---------
Co-authored-by: mapeng <ma...@linkedin.com>
---
.../metaclient/api/MetaClientCacheInterface.java | 61 ++++++++++++++
.../MetaClientCacheConfig.java} | 43 ++++++----
.../metaclient/factories/MetaClientFactory.java | 17 ++++
.../metaclient/impl/zk/ZkMetaClientCache.java | 94 ++++++++++++++++++++++
.../impl/zk/factory/ZkMetaClientFactory.java | 15 ++++
.../metaclient/impl/zk/TestZkMetaClientCache.java | 57 +++++++++++++
6 files changed, 272 insertions(+), 15 deletions(-)
diff --git a/meta-client/src/main/java/org/apache/helix/metaclient/api/MetaClientCacheInterface.java b/meta-client/src/main/java/org/apache/helix/metaclient/api/MetaClientCacheInterface.java
new file mode 100644
index 000000000..348bd0929
--- /dev/null
+++ b/meta-client/src/main/java/org/apache/helix/metaclient/api/MetaClientCacheInterface.java
@@ -0,0 +1,61 @@
+package org.apache.helix.metaclient.api;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.HashMap;
+import java.util.Map;
+
+public interface MetaClientCacheInterface<T> extends MetaClientInterface<T> {
+
+ /**
+ * TrieNode class to store the children of the entries to be cached.
+ */
+ class TrieNode {
+ // A mapping between trie key and children nodes.
+ private Map<String, TrieNode> _children;
+
+ // the complete path/prefix leading to the current node.
+ private final String _path;
+
+ private final String _nodeKey;
+
+ TrieNode(String path, String nodeKey) {
+ _path = path;
+ _nodeKey = nodeKey;
+ _children = new HashMap<>();
+ }
+
+ public Map<String, TrieNode> getChildren() {
+ return _children;
+ }
+
+ public String getPath() {
+ return _path;
+ }
+
+ public String getNodeKey() {
+ return _nodeKey;
+ }
+
+ public void addChild(String key, TrieNode node) {
+ _children.put(key, node);
+ }
+ }
+}
diff --git a/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/factory/ZkMetaClientFactory.java b/meta-client/src/main/java/org/apache/helix/metaclient/factories/MetaClientCacheConfig.java
similarity index 50%
copy from meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/factory/ZkMetaClientFactory.java
copy to meta-client/src/main/java/org/apache/helix/metaclient/factories/MetaClientCacheConfig.java
index 9eba28b91..9e0323601 100644
--- a/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/factory/ZkMetaClientFactory.java
+++ b/meta-client/src/main/java/org/apache/helix/metaclient/factories/MetaClientCacheConfig.java
@@ -1,4 +1,4 @@
-package org.apache.helix.metaclient.impl.zk.factory;
+package org.apache.helix.metaclient.factories;
/*
* Licensed to the Apache Software Foundation (ASF) under one
@@ -19,21 +19,34 @@ package org.apache.helix.metaclient.impl.zk.factory;
* under the License.
*/
-import org.apache.helix.metaclient.api.MetaClientInterface;
-import org.apache.helix.metaclient.factories.MetaClientConfig;
-import org.apache.helix.metaclient.factories.MetaClientFactory;
-import org.apache.helix.metaclient.impl.zk.ZkMetaClient;
-public class ZkMetaClientFactory extends MetaClientFactory {
- @Override
- public MetaClientInterface getMetaClient(MetaClientConfig config) {
- if (config == null) {
- throw new IllegalArgumentException("MetaClientConfig cannot be null.");
+
+public class MetaClientCacheConfig {
+ private final String _rootEntry;
+ private boolean _cacheData = false;
+ private boolean _cacheChildren = false;
+ private boolean _lazyCaching = true;
+
+ public MetaClientCacheConfig(String rootEntry, boolean cacheData, boolean cacheChildren, boolean lazyCaching) {
+ _rootEntry = rootEntry;
+ _cacheData = cacheData;
+ _cacheChildren = cacheChildren;
+ _lazyCaching = lazyCaching;
}
- if (MetaClientConfig.StoreType.ZOOKEEPER.equals(config.getStoreType())
- && config instanceof ZkMetaClientConfig) {
- return new ZkMetaClient((ZkMetaClientConfig) config);
+
+ public String getRootEntry() {
+ return _rootEntry;
+ }
+
+ public boolean getCacheData() {
+ return _cacheData;
+ }
+
+ public boolean getCacheChildren() {
+ return _cacheChildren;
+ }
+
+ public boolean getLazyCaching() {
+ return _lazyCaching;
}
- throw new IllegalArgumentException("Invalid MetaClientConfig type.");
- }
}
diff --git a/meta-client/src/main/java/org/apache/helix/metaclient/factories/MetaClientFactory.java b/meta-client/src/main/java/org/apache/helix/metaclient/factories/MetaClientFactory.java
index 045374332..ebb4549da 100644
--- a/meta-client/src/main/java/org/apache/helix/metaclient/factories/MetaClientFactory.java
+++ b/meta-client/src/main/java/org/apache/helix/metaclient/factories/MetaClientFactory.java
@@ -20,6 +20,7 @@ package org.apache.helix.metaclient.factories;
*/
+import org.apache.helix.metaclient.api.MetaClientCacheInterface;
import org.apache.helix.metaclient.api.MetaClientInterface;
import org.apache.helix.metaclient.impl.zk.factory.ZkMetaClientConfig;
import org.apache.helix.metaclient.impl.zk.factory.ZkMetaClientFactory;
@@ -49,4 +50,20 @@ public class MetaClientFactory {
}
return null;
}
+
+ public MetaClientCacheInterface getMetaClientCache(MetaClientConfig config, MetaClientCacheConfig cacheConfig) {
+ if (config == null) {
+ throw new IllegalArgumentException("MetaClientConfig cannot be null.");
+ }
+ if (MetaClientConfig.StoreType.ZOOKEEPER.equals(config.getStoreType())) {
+ ZkMetaClientConfig zkMetaClientConfig = new ZkMetaClientConfig.ZkMetaClientConfigBuilder().
+ setConnectionAddress(config.getConnectionAddress())
+ .setMetaClientReconnectPolicy(config.getMetaClientReconnectPolicy())
+ .setConnectionInitTimeoutInMillis(config.getConnectionInitTimeoutInMillis())
+ .setSessionTimeoutInMillis(config.getSessionTimeoutInMillis())
+ .build();
+ return new ZkMetaClientFactory().getMetaClientCache(zkMetaClientConfig, cacheConfig);
+ }
+ return null;
+ }
}
diff --git a/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/ZkMetaClientCache.java b/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/ZkMetaClientCache.java
new file mode 100644
index 000000000..af1c9d791
--- /dev/null
+++ b/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/ZkMetaClientCache.java
@@ -0,0 +1,94 @@
+package org.apache.helix.metaclient.impl.zk;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.helix.metaclient.api.ChildChangeListener;
+import org.apache.helix.metaclient.api.MetaClientCacheInterface;
+import org.apache.helix.metaclient.datamodel.DataRecord;
+import org.apache.helix.metaclient.exception.MetaClientException;
+import org.apache.helix.metaclient.factories.MetaClientCacheConfig;
+import org.apache.helix.metaclient.factories.MetaClientConfig;
+import org.apache.helix.metaclient.impl.zk.factory.ZkMetaClientConfig;
+import org.apache.helix.metaclient.impl.zk.factory.ZkMetaClientFactory;
+import org.apache.helix.metaclient.recipes.lock.LockInfoSerializer;
+import org.apache.helix.zookeeper.zkclient.ZkClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Map;
+
+public class ZkMetaClientCache<T> extends ZkMetaClient<T> implements MetaClientCacheInterface<T> {
+
+ private Map<String, DataRecord> _dataCacheMap;
+ private final String _rootEntry;
+ private TrieNode _childrenCacheTree;
+ private ChildChangeListener _eventListener;
+ private boolean _cacheData;
+ private boolean _cacheChildren;
+ private boolean _lazyCaching;
+ private static final Logger LOG = LoggerFactory.getLogger(ZkMetaClientCache.class);
+ private ZkClient _cacheClient;
+
+ /**
+ * Constructor for ZkMetaClientCache.
+ * @param config ZkMetaClientConfig
+ * @param cacheConfig MetaClientCacheConfig
+ */
+ public ZkMetaClientCache(ZkMetaClientConfig config, MetaClientCacheConfig cacheConfig) {
+ super(config);
+ _cacheClient = getZkClient();
+ _rootEntry = cacheConfig.getRootEntry();
+ _lazyCaching = cacheConfig.getLazyCaching();
+ _cacheData = cacheConfig.getCacheData();
+ _cacheChildren = cacheConfig.getCacheChildren();
+ }
+
+ @Override
+ public Stat exists(String key) {
+ throw new MetaClientException("Not implemented yet.");
+ }
+
+ @Override
+ public T get(final String key) {
+ throw new MetaClientException("Not implemented yet.");
+ }
+
+ @Override
+ public List<String> getDirectChildrenKeys(final String key) {
+ throw new MetaClientException("Not implemented yet.");
+ }
+
+ @Override
+ public int countDirectChildren(final String key) {
+ throw new MetaClientException("Not implemented yet.");
+ }
+
+ @Override
+ public List<T> get(List<String> keys) {
+ throw new MetaClientException("Not implemented yet.");
+ }
+
+ @Override
+ public List<Stat> exists(List<String> keys) {
+ throw new MetaClientException("Not implemented yet.");
+ }
+
+}
diff --git a/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/factory/ZkMetaClientFactory.java b/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/factory/ZkMetaClientFactory.java
index 9eba28b91..c4018eb2f 100644
--- a/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/factory/ZkMetaClientFactory.java
+++ b/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/factory/ZkMetaClientFactory.java
@@ -19,10 +19,13 @@ package org.apache.helix.metaclient.impl.zk.factory;
* under the License.
*/
+import org.apache.helix.metaclient.api.MetaClientCacheInterface;
import org.apache.helix.metaclient.api.MetaClientInterface;
+import org.apache.helix.metaclient.factories.MetaClientCacheConfig;
import org.apache.helix.metaclient.factories.MetaClientConfig;
import org.apache.helix.metaclient.factories.MetaClientFactory;
import org.apache.helix.metaclient.impl.zk.ZkMetaClient;
+import org.apache.helix.metaclient.impl.zk.ZkMetaClientCache;
public class ZkMetaClientFactory extends MetaClientFactory {
@Override
@@ -36,4 +39,16 @@ public class ZkMetaClientFactory extends MetaClientFactory {
}
throw new IllegalArgumentException("Invalid MetaClientConfig type.");
}
+
+ @Override
+ public MetaClientCacheInterface getMetaClientCache(MetaClientConfig config, MetaClientCacheConfig cacheConfig) {
+ if (config == null) {
+ throw new IllegalArgumentException("MetaClientConfig cannot be null.");
+ }
+ if (MetaClientConfig.StoreType.ZOOKEEPER.equals(config.getStoreType())
+ && config instanceof ZkMetaClientConfig) {
+ return new ZkMetaClientCache((ZkMetaClientConfig) config, cacheConfig);
+ }
+ throw new IllegalArgumentException("Invalid MetaClientConfig type.");
+ }
}
diff --git a/meta-client/src/test/java/org/apache/helix/metaclient/impl/zk/TestZkMetaClientCache.java b/meta-client/src/test/java/org/apache/helix/metaclient/impl/zk/TestZkMetaClientCache.java
new file mode 100644
index 000000000..a3a5b4eee
--- /dev/null
+++ b/meta-client/src/test/java/org/apache/helix/metaclient/impl/zk/TestZkMetaClientCache.java
@@ -0,0 +1,57 @@
+package org.apache.helix.metaclient.impl.zk;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import org.apache.helix.metaclient.factories.MetaClientCacheConfig;
+import org.apache.helix.metaclient.impl.zk.factory.ZkMetaClientConfig;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class TestZkMetaClientCache extends ZkMetaClientTestBase {
+ private static final String PATH = "/Cache";
+
+ @Test
+ public void testCreateClient() {
+ final String key = "/TestZkMetaClientCache_testCreate";
+ try (ZkMetaClient<String> zkMetaClientCache = createZkMetaClientCache()) {
+ zkMetaClientCache.connect();
+ // Perform some random non-read operation
+ zkMetaClientCache.create(key, ENTRY_STRING_VALUE);
+
+ try {
+ //Perform some read operation - should fail.
+ // TODO: Remove this once implemented.
+ zkMetaClientCache.get(key);
+ Assert.fail("Should have failed with non implemented yet.");
+ } catch (Exception ignored) {
+ }
+ }
+ }
+
+ protected static ZkMetaClientCache<String> createZkMetaClientCache() {
+ ZkMetaClientConfig config =
+ new ZkMetaClientConfig.ZkMetaClientConfigBuilder().setConnectionAddress(ZK_ADDR)
+ //.setZkSerializer(new TestStringSerializer())
+ .build();
+ MetaClientCacheConfig cacheConfig = new MetaClientCacheConfig(PATH, true, true, true);
+ return new ZkMetaClientCache<>(config, cacheConfig);
+ }
+}
[helix] 02/04: Skip one time listener re-register for exists for ZkClient - MetaClient usage. (#2637)
Posted by xy...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xyuanlu pushed a commit to branch metaclient
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 2649e6e6a27d932b0c5d9c6efdc46982de6d0b44
Author: xyuanlu <xy...@gmail.com>
AuthorDate: Tue Sep 26 23:54:37 2023 -0700
Skip one time listener re-register for exists for ZkClient - MetaClient usage. (#2637)
---
.../org/apache/helix/metaclient/impl/zk/TestZkMetaClient.java | 10 +++++++++-
.../java/org/apache/helix/zookeeper/zkclient/ZkClient.java | 2 +-
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/meta-client/src/test/java/org/apache/helix/metaclient/impl/zk/TestZkMetaClient.java b/meta-client/src/test/java/org/apache/helix/metaclient/impl/zk/TestZkMetaClient.java
index 19f21977b..86edbf8eb 100644
--- a/meta-client/src/test/java/org/apache/helix/metaclient/impl/zk/TestZkMetaClient.java
+++ b/meta-client/src/test/java/org/apache/helix/metaclient/impl/zk/TestZkMetaClient.java
@@ -517,12 +517,20 @@ public class TestZkMetaClient extends ZkMetaClientTestBase{
throws Exception {
}
};
- zkMetaClient.subscribeDataChange(basePath, listener, false);
+ DirectChildChangeListener cldListener = new DirectChildChangeListener() {
+
+ @Override
+ public void handleDirectChildChange(String key) throws Exception {
+
+ }
+ };
+ zkMetaClient.subscribeDataChange(basePath, listener, true);
zkMetaClient.create(basePath, "");
zkMetaClient.get(basePath);
zkMetaClient.exists(basePath);
zkMetaClient.getDataAndStat(basePath);
zkMetaClient.getDirectChildrenKeys(basePath);
+ zkMetaClient.subscribeDirectChildChange(basePath, cldListener, true);
}
}
diff --git a/zookeeper-api/src/main/java/org/apache/helix/zookeeper/zkclient/ZkClient.java b/zookeeper-api/src/main/java/org/apache/helix/zookeeper/zkclient/ZkClient.java
index 2a06158d0..110587e09 100644
--- a/zookeeper-api/src/main/java/org/apache/helix/zookeeper/zkclient/ZkClient.java
+++ b/zookeeper-api/src/main/java/org/apache/helix/zookeeper/zkclient/ZkClient.java
@@ -1474,7 +1474,7 @@ public class ZkClient implements Watcher {
}
public boolean exists(final String path) {
- return exists(path, hasChildOrDataListeners(path));
+ return exists(path, (!_usePersistWatcher) && hasChildOrDataListeners(path));
}
protected boolean exists(final String path, final boolean watch) {
[helix] 03/04: Lattice cache - caching just data implementation (#2619)
Posted by xy...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xyuanlu pushed a commit to branch metaclient
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 827ca822d2dff97f936608a033c30e252cedd1f1
Author: Marcos Rico Peng <55...@users.noreply.github.com>
AuthorDate: Wed Sep 27 18:32:25 2023 -0700
Lattice cache - caching just data implementation (#2619)
Lattice cache - caching just data implementation
---------
Co-authored-by: mapeng <ma...@linkedin.com>
---
.../metaclient/api/MetaClientCacheInterface.java | 2 +-
.../factories/MetaClientCacheConfig.java | 5 -
.../metaclient/factories/MetaClientFactory.java | 25 ++--
.../metaclient/impl/zk/ZkMetaClientCache.java | 153 ++++++++++++++++++---
.../metaclient/impl/zk/TestZkMetaClientCache.java | 85 ++++++++++--
5 files changed, 220 insertions(+), 50 deletions(-)
diff --git a/meta-client/src/main/java/org/apache/helix/metaclient/api/MetaClientCacheInterface.java b/meta-client/src/main/java/org/apache/helix/metaclient/api/MetaClientCacheInterface.java
index 348bd0929..6449970bb 100644
--- a/meta-client/src/main/java/org/apache/helix/metaclient/api/MetaClientCacheInterface.java
+++ b/meta-client/src/main/java/org/apache/helix/metaclient/api/MetaClientCacheInterface.java
@@ -36,7 +36,7 @@ public interface MetaClientCacheInterface<T> extends MetaClientInterface<T> {
private final String _nodeKey;
- TrieNode(String path, String nodeKey) {
+ public TrieNode(String path, String nodeKey) {
_path = path;
_nodeKey = nodeKey;
_children = new HashMap<>();
diff --git a/meta-client/src/main/java/org/apache/helix/metaclient/factories/MetaClientCacheConfig.java b/meta-client/src/main/java/org/apache/helix/metaclient/factories/MetaClientCacheConfig.java
index 9e0323601..07972945a 100644
--- a/meta-client/src/main/java/org/apache/helix/metaclient/factories/MetaClientCacheConfig.java
+++ b/meta-client/src/main/java/org/apache/helix/metaclient/factories/MetaClientCacheConfig.java
@@ -25,13 +25,11 @@ public class MetaClientCacheConfig {
private final String _rootEntry;
private boolean _cacheData = false;
private boolean _cacheChildren = false;
- private boolean _lazyCaching = true;
public MetaClientCacheConfig(String rootEntry, boolean cacheData, boolean cacheChildren, boolean lazyCaching) {
_rootEntry = rootEntry;
_cacheData = cacheData;
_cacheChildren = cacheChildren;
- _lazyCaching = lazyCaching;
}
public String getRootEntry() {
@@ -46,7 +44,4 @@ public class MetaClientCacheConfig {
return _cacheChildren;
}
- public boolean getLazyCaching() {
- return _lazyCaching;
- }
}
diff --git a/meta-client/src/main/java/org/apache/helix/metaclient/factories/MetaClientFactory.java b/meta-client/src/main/java/org/apache/helix/metaclient/factories/MetaClientFactory.java
index ebb4549da..7cc86a8a9 100644
--- a/meta-client/src/main/java/org/apache/helix/metaclient/factories/MetaClientFactory.java
+++ b/meta-client/src/main/java/org/apache/helix/metaclient/factories/MetaClientFactory.java
@@ -40,13 +40,7 @@ public class MetaClientFactory {
throw new IllegalArgumentException("MetaClientConfig cannot be null.");
}
if (MetaClientConfig.StoreType.ZOOKEEPER.equals(config.getStoreType())) {
- ZkMetaClientConfig zkMetaClientConfig = new ZkMetaClientConfig.ZkMetaClientConfigBuilder().
- setConnectionAddress(config.getConnectionAddress())
- .setMetaClientReconnectPolicy(config.getMetaClientReconnectPolicy())
- .setConnectionInitTimeoutInMillis(config.getConnectionInitTimeoutInMillis())
- .setSessionTimeoutInMillis(config.getSessionTimeoutInMillis())
- .build();
- return new ZkMetaClientFactory().getMetaClient(zkMetaClientConfig);
+ return new ZkMetaClientFactory().getMetaClient(createZkMetaClientConfig(config));
}
return null;
}
@@ -56,14 +50,17 @@ public class MetaClientFactory {
throw new IllegalArgumentException("MetaClientConfig cannot be null.");
}
if (MetaClientConfig.StoreType.ZOOKEEPER.equals(config.getStoreType())) {
- ZkMetaClientConfig zkMetaClientConfig = new ZkMetaClientConfig.ZkMetaClientConfigBuilder().
- setConnectionAddress(config.getConnectionAddress())
- .setMetaClientReconnectPolicy(config.getMetaClientReconnectPolicy())
- .setConnectionInitTimeoutInMillis(config.getConnectionInitTimeoutInMillis())
- .setSessionTimeoutInMillis(config.getSessionTimeoutInMillis())
- .build();
- return new ZkMetaClientFactory().getMetaClientCache(zkMetaClientConfig, cacheConfig);
+ return new ZkMetaClientFactory().getMetaClientCache(createZkMetaClientConfig(config), cacheConfig);
}
return null;
}
+
+ private ZkMetaClientConfig createZkMetaClientConfig(MetaClientConfig config) {
+ return new ZkMetaClientConfig.ZkMetaClientConfigBuilder().
+ setConnectionAddress(config.getConnectionAddress())
+ .setMetaClientReconnectPolicy(config.getMetaClientReconnectPolicy())
+ .setConnectionInitTimeoutInMillis(config.getConnectionInitTimeoutInMillis())
+ .setSessionTimeoutInMillis(config.getSessionTimeoutInMillis())
+ .build();
+ }
}
diff --git a/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/ZkMetaClientCache.java b/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/ZkMetaClientCache.java
index af1c9d791..66eb6f91e 100644
--- a/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/ZkMetaClientCache.java
+++ b/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/ZkMetaClientCache.java
@@ -21,31 +21,38 @@ package org.apache.helix.metaclient.impl.zk;
import org.apache.helix.metaclient.api.ChildChangeListener;
import org.apache.helix.metaclient.api.MetaClientCacheInterface;
-import org.apache.helix.metaclient.datamodel.DataRecord;
import org.apache.helix.metaclient.exception.MetaClientException;
import org.apache.helix.metaclient.factories.MetaClientCacheConfig;
-import org.apache.helix.metaclient.factories.MetaClientConfig;
+import org.apache.helix.metaclient.impl.zk.adapter.ChildListenerAdapter;
import org.apache.helix.metaclient.impl.zk.factory.ZkMetaClientConfig;
-import org.apache.helix.metaclient.impl.zk.factory.ZkMetaClientFactory;
-import org.apache.helix.metaclient.recipes.lock.LockInfoSerializer;
import org.apache.helix.zookeeper.zkclient.ZkClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
+import java.util.Queue;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
public class ZkMetaClientCache<T> extends ZkMetaClient<T> implements MetaClientCacheInterface<T> {
- private Map<String, DataRecord> _dataCacheMap;
+ private ConcurrentHashMap<String, T> _dataCacheMap;
private final String _rootEntry;
private TrieNode _childrenCacheTree;
private ChildChangeListener _eventListener;
private boolean _cacheData;
private boolean _cacheChildren;
- private boolean _lazyCaching;
private static final Logger LOG = LoggerFactory.getLogger(ZkMetaClientCache.class);
private ZkClient _cacheClient;
+ private ExecutorService executor;
+
+ // TODO: Look into using conditional variable instead of latch.
+ private final CountDownLatch _initializedCache = new CountDownLatch(1);
/**
* Constructor for ZkMetaClientCache.
@@ -56,19 +63,43 @@ public class ZkMetaClientCache<T> extends ZkMetaClient<T> implements MetaClientC
super(config);
_cacheClient = getZkClient();
_rootEntry = cacheConfig.getRootEntry();
- _lazyCaching = cacheConfig.getLazyCaching();
_cacheData = cacheConfig.getCacheData();
_cacheChildren = cacheConfig.getCacheChildren();
+
+ if (_cacheData) {
+ _dataCacheMap = new ConcurrentHashMap<>();
+ }
+ if (_cacheChildren) {
+ _childrenCacheTree = new TrieNode(_rootEntry, null);
+ }
}
+ /**
+ * Get data for a given key.
+ * If datacache is enabled, will fetch for cache. If it doesn't exist
+ * returns null (for when initial populating cache is in progress).
+ * @param key key to identify the entry
+ * @return data for the key
+ */
@Override
- public Stat exists(String key) {
- throw new MetaClientException("Not implemented yet.");
+ public T get(final String key) {
+ if (_cacheData) {
+ T data = getDataCacheMap().get(key);
+ if (data == null) {
+ LOG.debug("Data not found in cache for key: {}. This could be because the cache is still being populated.", key);
+ }
+ return data;
+ }
+ return super.get(key);
}
@Override
- public T get(final String key) {
- throw new MetaClientException("Not implemented yet.");
+ public List<T> get(List<String> keys) {
+ List<T> dataList = new ArrayList<>();
+ for (String key : keys) {
+ dataList.add(get(key));
+ }
+ return dataList;
}
@Override
@@ -81,14 +112,100 @@ public class ZkMetaClientCache<T> extends ZkMetaClient<T> implements MetaClientC
throw new MetaClientException("Not implemented yet.");
}
- @Override
- public List<T> get(List<String> keys) {
- throw new MetaClientException("Not implemented yet.");
+ private void populateAllCache() {
+ // TODO: Concurrently populate children and data cache.
+ if (!_cacheClient.exists(_rootEntry)) {
+ LOG.warn("Root entry: {} does not exist.", _rootEntry);
+ // Let the other threads know that the cache is populated.
+ _initializedCache.countDown();
+ return;
+ }
+
+ Queue<String> queue = new ArrayDeque<>();
+ queue.add(_rootEntry);
+
+ while (!queue.isEmpty()) {
+ String node = queue.poll();
+ if (_cacheData) {
+ T dataRecord = _cacheClient.readData(node, true);
+ _dataCacheMap.put(node, dataRecord);
+ }
+ queue.addAll(_cacheClient.getChildren(node));
+ }
+ // Let the other threads know that the cache is populated.
+ _initializedCache.countDown();
}
- @Override
- public List<Stat> exists(List<String> keys) {
- throw new MetaClientException("Not implemented yet.");
+ private class CacheUpdateRunnable implements Runnable {
+ private final String path;
+ private final ChildChangeListener.ChangeType changeType;
+
+ public CacheUpdateRunnable(String path, ChildChangeListener.ChangeType changeType) {
+ this.path = path;
+ this.changeType = changeType;
+ }
+
+ @Override
+ public void run() {
+ waitForPopulateAllCache();
+ // TODO: HANDLE DEDUP EVENT CHANGES
+ switch (changeType) {
+ case ENTRY_CREATED:
+ // Not implemented yet.
+ modifyDataInCache(path, false);
+ break;
+ case ENTRY_DELETED:
+ // Not implemented yet.
+ modifyDataInCache(path, true);
+ break;
+ case ENTRY_DATA_CHANGE:
+ modifyDataInCache(path, false);
+ break;
+ default:
+ LOG.error("Unknown change type: " + changeType);
+ }
+ }
+ }
+
+ private void waitForPopulateAllCache() {
+ try {
+ _initializedCache.await();
+ } catch (InterruptedException e) {
+ throw new MetaClientException("Interrupted while waiting for cache to populate.", e);
+ }
+ }
+
+ private void modifyDataInCache(String path, Boolean isDelete) {
+ if (_cacheData) {
+ if (isDelete) {
+ getDataCacheMap().remove(path);
+ } else {
+ T dataRecord = _cacheClient.readData(path, true);
+ getDataCacheMap().put(path, dataRecord);
+ }
+ }
}
+ public ConcurrentHashMap<String, T> getDataCacheMap() {
+ return _dataCacheMap;
+ }
+
+ public TrieNode getChildrenCacheTree() {
+ return _childrenCacheTree;
+ }
+
+ /**
+ * Connect to the underlying ZkClient.
+ */
+ @Override
+ public void connect() {
+ super.connect();
+ _eventListener = (path, changeType) -> {
+ Runnable cacheUpdateRunnable = new CacheUpdateRunnable(path, changeType);
+ executor.execute(cacheUpdateRunnable);
+ };
+ executor = Executors.newSingleThreadExecutor();
+ _cacheClient.subscribePersistRecursiveListener(_rootEntry, new ChildListenerAdapter(_eventListener));
+ populateAllCache();
+ }
}
diff --git a/meta-client/src/test/java/org/apache/helix/metaclient/impl/zk/TestZkMetaClientCache.java b/meta-client/src/test/java/org/apache/helix/metaclient/impl/zk/TestZkMetaClientCache.java
index a3a5b4eee..30a0a729d 100644
--- a/meta-client/src/test/java/org/apache/helix/metaclient/impl/zk/TestZkMetaClientCache.java
+++ b/meta-client/src/test/java/org/apache/helix/metaclient/impl/zk/TestZkMetaClientCache.java
@@ -19,39 +19,100 @@ package org.apache.helix.metaclient.impl.zk;
* under the License.
*/
-
import org.apache.helix.metaclient.factories.MetaClientCacheConfig;
import org.apache.helix.metaclient.impl.zk.factory.ZkMetaClientConfig;
import org.testng.Assert;
import org.testng.annotations.Test;
+import java.util.ArrayList;
+import java.util.List;
+
public class TestZkMetaClientCache extends ZkMetaClientTestBase {
- private static final String PATH = "/Cache";
+ private static final String DATA_PATH = "/data";
+ private static final String DATA_VALUE = "testData";
@Test
public void testCreateClient() {
- final String key = "/TestZkMetaClientCache_testCreate";
- try (ZkMetaClient<String> zkMetaClientCache = createZkMetaClientCache()) {
+ final String key = "/testCreate";
+ try (ZkMetaClientCache<String> zkMetaClientCache = createZkMetaClientCacheLazyCaching(key)) {
zkMetaClientCache.connect();
// Perform some random non-read operation
zkMetaClientCache.create(key, ENTRY_STRING_VALUE);
+ }
+ }
+
+ @Test
+ public void testCacheDataUpdates() {
+ final String key = "/testCacheDataUpdates";
+ try (ZkMetaClientCache<String> zkMetaClientCache = createZkMetaClientCacheLazyCaching(key)) {
+ zkMetaClientCache.connect();
+ zkMetaClientCache.create(key, "test");
+ zkMetaClientCache.create(key + DATA_PATH, DATA_VALUE);
+
+ // Get data for DATA_PATH and cache it
+ String data = zkMetaClientCache.get(key + DATA_PATH);
+ Assert.assertEquals(data, zkMetaClientCache.getDataCacheMap().get(key + DATA_PATH));
+
+ // Update data for DATA_PATH
+ String newData = zkMetaClientCache.update(key + DATA_PATH, currentData -> currentData + "1");
+
+ // Verify that cached data is updated. Might take some time
+ for (int i = 0; i < 10; i++) {
+ if (zkMetaClientCache.getDataCacheMap().get(key + DATA_PATH).equals(newData)) {
+ break;
+ }
+ Thread.sleep(1000);
+ }
+ Assert.assertEquals(newData, zkMetaClientCache.getDataCacheMap().get(key + DATA_PATH));
+
+ zkMetaClientCache.delete(key + DATA_PATH);
+ // Verify that cached data is updated. Might take some time
+ for (int i = 0; i < 10; i++) {
+ if (zkMetaClientCache.getDataCacheMap().get(key + DATA_PATH) == null) {
+ break;
+ }
+ Thread.sleep(1000);
+ }
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Test
+ public void testBatchGet() {
+ final String key = "/testBatchGet";
+ try (ZkMetaClientCache<String> zkMetaClientCache = createZkMetaClientCacheLazyCaching(key)) {
+ zkMetaClientCache.connect();
+ zkMetaClientCache.create(key, "test");
+ zkMetaClientCache.create(key + DATA_PATH, DATA_VALUE);
+
+ ArrayList<String> keys = new ArrayList<>();
+ keys.add(key);
+ keys.add(key + DATA_PATH);
+
+ ArrayList<String> values = new ArrayList<>();
+ values.add("test");
+ values.add(DATA_VALUE);
- try {
- //Perform some read operation - should fail.
- // TODO: Remove this once implemented.
- zkMetaClientCache.get(key);
- Assert.fail("Should have failed with non implemented yet.");
- } catch (Exception ignored) {
+ for (int i = 0; i < 10; i++) {
+ // Get data for DATA_PATH and cache it
+ List<String> data = zkMetaClientCache.get(keys);
+ if (data.equals(values)) {
+ break;
+ }
+ Thread.sleep(1000);
}
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
}
}
- protected static ZkMetaClientCache<String> createZkMetaClientCache() {
+ protected static ZkMetaClientCache<String> createZkMetaClientCacheLazyCaching(String rootPath) {
ZkMetaClientConfig config =
new ZkMetaClientConfig.ZkMetaClientConfigBuilder().setConnectionAddress(ZK_ADDR)
//.setZkSerializer(new TestStringSerializer())
.build();
- MetaClientCacheConfig cacheConfig = new MetaClientCacheConfig(PATH, true, true, true);
+ MetaClientCacheConfig cacheConfig = new MetaClientCacheConfig(rootPath, true, true, true);
return new ZkMetaClientCache<>(config, cacheConfig);
}
}
[helix] 04/04: Add recursiveCreate functionality to metaclient (#2607)
Posted by xy...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xyuanlu pushed a commit to branch metaclient
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 6bd97a98c47e7caee76d3a704ec55aa4732413b5
Author: Grant Paláu Spencer <80...@users.noreply.github.com>
AuthorDate: Wed Oct 4 13:26:43 2023 -0700
Add recursiveCreate functionality to metaclient (#2607)
Co-authored-by: Grant Palau Spencer <gs...@linkedin.com>
---
.../helix/metaclient/api/MetaClientInterface.java | 15 +++++
.../helix/metaclient/impl/zk/ZkMetaClient.java | 65 +++++++++++++++++++++-
.../metaclient/impl/zk/util/ZkMetaClientUtil.java | 25 +++++++++
.../metaclient/impl/zk/TestStressZkClient.java | 30 ++++++++++
.../helix/metaclient/impl/zk/TestZkMetaClient.java | 46 ++++++++++++++-
5 files changed, 179 insertions(+), 2 deletions(-)
diff --git a/meta-client/src/main/java/org/apache/helix/metaclient/api/MetaClientInterface.java b/meta-client/src/main/java/org/apache/helix/metaclient/api/MetaClientInterface.java
index 5b26896a9..5403e7ca4 100644
--- a/meta-client/src/main/java/org/apache/helix/metaclient/api/MetaClientInterface.java
+++ b/meta-client/src/main/java/org/apache/helix/metaclient/api/MetaClientInterface.java
@@ -152,6 +152,21 @@ public interface MetaClientInterface<T> {
*/
void create(final String key, final T data, final EntryMode mode);
+ /**
+ * Create an entry of given EntryMode with given key and data. If any parent node in the node
+ * hierarchy does not exist, then the parent node will attempt to be created. The entry will not
+ * be created if there is an existing entry with the same full key. Ephemeral nodes cannot have
+ * children, so only the final child in the created path will be ephemeral.
+ */
+ void recursiveCreate(final String key, final T Data, final EntryMode mode);
+
+ /**
+ * Create a TTL entry with given key, data, and expiry time (ttl). If any parent node in the node
+ * hierarchy does not exist, then the parent node will attempt to be created. The entry will not be created if
+ * there is an existing entry with the same full key.
+ */
+ void recursiveCreateWithTTL(String key, T data, long ttl);
+
/**
* Create an entry of given EntryMode with given key, data, and expiry time (ttl).
* The entry will automatically purge when reached expiry time and has no children.
diff --git a/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/ZkMetaClient.java b/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/ZkMetaClient.java
index 8753747f3..16d28c6d7 100644
--- a/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/ZkMetaClient.java
+++ b/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/ZkMetaClient.java
@@ -41,6 +41,7 @@ import org.apache.helix.metaclient.api.Op;
import org.apache.helix.metaclient.api.OpResult;
import org.apache.helix.metaclient.exception.MetaClientException;
import org.apache.helix.metaclient.exception.MetaClientNoNodeException;
+import org.apache.helix.metaclient.exception.MetaClientNodeExistsException;
import org.apache.helix.metaclient.impl.zk.adapter.ChildListenerAdapter;
import org.apache.helix.metaclient.impl.zk.adapter.DataListenerAdapter;
import org.apache.helix.metaclient.impl.zk.adapter.DirectChildListenerAdapter;
@@ -63,6 +64,7 @@ import org.apache.zookeeper.Watcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.helix.metaclient.impl.zk.util.ZkMetaClientUtil.separateIntoUniqueNodePaths;
import static org.apache.helix.metaclient.impl.zk.util.ZkMetaClientUtil.translateZkExceptionToMetaclientException;
@@ -104,7 +106,7 @@ public class ZkMetaClient<T> implements MetaClientInterface<T>, AutoCloseable {
}
@Override
- public void create(String key, Object data, MetaClientInterface.EntryMode mode) {
+ public void create(String key, Object data, EntryMode mode) {
try {
_zkClient.create(key, data, ZkMetaClientUtil.convertMetaClientMode(mode));
@@ -115,6 +117,67 @@ public class ZkMetaClient<T> implements MetaClientInterface<T>, AutoCloseable {
}
}
+ @Override
+ public void recursiveCreate(String key, T data, EntryMode mode) {
+ // Function named recursiveCreate to match naming scheme, but actual work is iterative
+ iterativeCreate(key, data, mode, -1);
+ }
+
+ @Override
+ public void recursiveCreateWithTTL(String key, T data, long ttl) {
+ iterativeCreate(key, data, EntryMode.TTL, ttl);
+ }
+
+ private void iterativeCreate(String key, T data, EntryMode mode, long ttl) {
+ List<String> nodePaths = separateIntoUniqueNodePaths(key);
+ int i = 0;
+ // Ephemeral nodes cant have children, so change mode when creating parents
+ EntryMode parentMode = (EntryMode.EPHEMERAL.equals(mode) ?
+ EntryMode.PERSISTENT : mode);
+
+ // Iterate over paths, starting with full key then attempting each successive parent
+ // Try /a/b/c, if parent /a/b, does not exist, then try to create parent, etc..
+ while (i < nodePaths.size()) {
+ // If parent exists or there is no parent node, then try to create the node
+ // and break out of loop on successful create
+ if (i == nodePaths.size() - 1 || _zkClient.exists(nodePaths.get(i+1))) {
+ try {
+ if (EntryMode.TTL.equals(mode)) {
+ createWithTTL(nodePaths.get(i), data, ttl);
+ } else {
+ create(nodePaths.get(i), data, i == 0 ? mode : parentMode);
+ }
+ // Race condition may occur where a node is created by another thread in between loops.
+ // We should not throw error if this occurs for parent nodes, only for the full node path.
+ } catch (MetaClientNodeExistsException e) {
+ if (i == 0) {
+ throw e;
+ }
+ }
+ break;
+ // Else try to create parent in next loop iteration
+ } else {
+ i++;
+ }
+ }
+
+ // Reattempt creation of children that failed due to parent not existing
+ while (--i >= 0) {
+ try {
+ if (EntryMode.TTL.equals(mode)) {
+ createWithTTL(nodePaths.get(i), data, ttl);
+ } else {
+ create(nodePaths.get(i), data, i == 0 ? mode : parentMode);
+ }
+ // Catch same race condition as above
+ } catch (MetaClientNodeExistsException e) {
+ if (i == 0) {
+ throw e;
+ }
+ }
+ }
+ }
+
@Override
public void createWithTTL(String key, T data, long ttl) {
try {
diff --git a/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/util/ZkMetaClientUtil.java b/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/util/ZkMetaClientUtil.java
index f93e98919..aee0c698a 100644
--- a/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/util/ZkMetaClientUtil.java
+++ b/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/util/ZkMetaClientUtil.java
@@ -20,6 +20,7 @@ package org.apache.helix.metaclient.impl.zk.util;
*/
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.EnumMap;
import java.util.HashMap;
@@ -358,4 +359,28 @@ public class ZkMetaClientUtil {
return MetaClientException.ReturnCode.DB_USER_ERROR;
}
}
+
+ // Returns null if no parent path
+ public static String getZkParentPath(String path) {
+ int idx = path.lastIndexOf('/');
+ return idx == 0 ? null : path.substring(0, idx);
+ }
+
+ // Splits a path into the paths for each node along the way.
+ // /a/b/c --> /a/b/c, /a/b, /a
+ public static List<String> separateIntoUniqueNodePaths(String path) {
+ if (path == null || "/".equals(path)) {
+ return null;
+ }
+
+ String[] subPath = path.split("/");
+ String[] nodePaths = new String[subPath.length-1];
+ StringBuilder tempPath = new StringBuilder();
+ for (int i = 1; i < subPath.length; i++) {
+ tempPath.append( "/");
+ tempPath.append(subPath[i]);
+ nodePaths[subPath.length - 1 - i] = tempPath.toString();
+ }
+ return Arrays.asList(nodePaths);
+ }
}
diff --git a/meta-client/src/test/java/org/apache/helix/metaclient/impl/zk/TestStressZkClient.java b/meta-client/src/test/java/org/apache/helix/metaclient/impl/zk/TestStressZkClient.java
index 6f358f0e8..284ce3645 100644
--- a/meta-client/src/test/java/org/apache/helix/metaclient/impl/zk/TestStressZkClient.java
+++ b/meta-client/src/test/java/org/apache/helix/metaclient/impl/zk/TestStressZkClient.java
@@ -28,6 +28,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.helix.metaclient.api.*;
import org.apache.helix.metaclient.datamodel.DataRecord;
import org.apache.helix.metaclient.exception.MetaClientException;
+import org.apache.helix.metaclient.exception.MetaClientNodeExistsException;
import org.apache.helix.metaclient.impl.zk.factory.ZkMetaClientConfig;
import org.apache.helix.metaclient.recipes.lock.DataRecordSerializer;
import org.testng.Assert;
@@ -129,6 +130,35 @@ public class TestStressZkClient extends ZkMetaClientTestBase {
Assert.assertEquals(_zkMetaClient.countDirectChildren(zkParentKey), 0);
}
+ @Test
+ public void testRecursiveCreate() {
+ final String zkParentKey = "/stressZk_testRecursiveCreate";
+ _zkMetaClient.create(zkParentKey, ENTRY_STRING_VALUE);
+
+ int count = (int) Math.pow(TEST_ITERATION_COUNT, 1/3d);
+ for (int i = 0; i < count; i++) {
+
+ for (int j = 0; j < count; j++) {
+
+ for (int k = 0; k < count; k++) {
+ String key = zkParentKey + "/" + i + "/" + j + "/" + k;
+ _zkMetaClient.recursiveCreate(key, String.valueOf(k), PERSISTENT);
+ Assert.assertEquals(String.valueOf(k), _zkMetaClient.get(key));
+ }
+ }
+ try {
+ _zkMetaClient.recursiveCreate(zkParentKey + "/" + i, "should_fail", PERSISTENT);
+ Assert.fail("Should have failed due to node existing");
+ } catch (MetaClientNodeExistsException ignoredException) {
+ }
+ }
+
+ // cleanup
+ _zkMetaClient.recursiveDelete(zkParentKey);
+ Assert.assertEquals(_zkMetaClient.countDirectChildren(zkParentKey), 0);
+
+ }
+
@Test
public void testGet() {
final String zkParentKey = "/stressZk_testGet";
diff --git a/meta-client/src/test/java/org/apache/helix/metaclient/impl/zk/TestZkMetaClient.java b/meta-client/src/test/java/org/apache/helix/metaclient/impl/zk/TestZkMetaClient.java
index 86edbf8eb..a5da69f2f 100644
--- a/meta-client/src/test/java/org/apache/helix/metaclient/impl/zk/TestZkMetaClient.java
+++ b/meta-client/src/test/java/org/apache/helix/metaclient/impl/zk/TestZkMetaClient.java
@@ -26,6 +26,8 @@ import org.apache.helix.metaclient.api.MetaClientInterface;
import org.apache.helix.metaclient.exception.MetaClientException;
import org.apache.helix.metaclient.api.DirectChildChangeListener;
+import java.io.StringWriter;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
@@ -38,7 +40,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.lang3.NotImplementedException;
-import org.apache.helix.metaclient.api.ConnectStateChangeListener;
import org.apache.helix.metaclient.api.DataChangeListener;
import org.apache.helix.metaclient.api.Op;
import org.apache.helix.metaclient.api.OpResult;
@@ -50,6 +51,7 @@ import org.testng.annotations.Test;
import static org.apache.helix.metaclient.api.DataChangeListener.ChangeType.ENTRY_UPDATE;
import static org.apache.helix.metaclient.api.MetaClientInterface.EntryMode.CONTAINER;
+import static org.apache.helix.metaclient.api.MetaClientInterface.EntryMode.EPHEMERAL;
import static org.apache.helix.metaclient.api.MetaClientInterface.EntryMode.PERSISTENT;
@@ -97,6 +99,48 @@ public class TestZkMetaClient extends ZkMetaClientTestBase{
}
}
+ @Test
+ public void testRecursiveCreate() {
+ final String path = "/Test/ZkMetaClient/_fullPath";
+
+
+ try (ZkMetaClient<String> zkMetaClient = createZkMetaClient()) {
+ zkMetaClient.connect();
+ MetaClientInterface.EntryMode mode = EPHEMERAL;
+
+ // Should succeed even if one of the parent nodes exists
+ String extendedPath = "/A" + path;
+ zkMetaClient.create("/A", ENTRY_STRING_VALUE, PERSISTENT);
+ zkMetaClient.recursiveCreate(extendedPath, ENTRY_STRING_VALUE, mode);
+ Assert.assertNotNull(zkMetaClient.exists(extendedPath));
+
+ // Should succeed if no parent nodes exist
+ zkMetaClient.recursiveCreate(path, ENTRY_STRING_VALUE, mode);
+ Assert.assertNotNull(zkMetaClient.exists(path));
+ Assert.assertEquals(zkMetaClient.getDataAndStat("/Test").getRight().getEntryType(), PERSISTENT);
+ Assert.assertEquals(zkMetaClient.getDataAndStat(path).getRight().getEntryType(), mode);
+
+ // Should throw NodeExistsException if child node exists
+ zkMetaClient.recursiveCreate(path, ENTRY_STRING_VALUE, mode);
+ Assert.fail("Should have failed due to node already created");
+ } catch (MetaClientException e) {
+ Assert.assertEquals(e.getMessage(), "org.apache.helix.zookeeper.zkclient.exception.ZkNodeExistsException: org.apache.zookeeper.KeeperException$NodeExistsException: KeeperErrorCode = NodeExists for /Test/ZkMetaClient/_fullPath");
+ System.out.println(e.getMessage());
+ }
+
+ }
+
+ @Test
+ public void testRecursiveCreateWithTTL() {
+ final String path = "/Test/ZkMetaClient/_fullPath/withTTL";
+
+ try (ZkMetaClient<String> zkMetaClient = createZkMetaClient()) {
+ zkMetaClient.connect();
+ zkMetaClient.recursiveCreateWithTTL(path, ENTRY_STRING_VALUE, 1000);
+ Assert.assertNotNull(zkMetaClient.exists(path));
+ }
+ }
+
@Test
public void testRenewTTL() {
final String key = "/TestZkMetaClient_testRenewTTL_1";