You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by il...@apache.org on 2024/01/23 04:01:45 UTC
(solr) branch jira/solr-17125-zero-replicas created (now a713415c0f8)
This is an automated email from the ASF dual-hosted git repository.
ilan pushed a change to branch jira/solr-17125-zero-replicas
in repository https://gitbox.apache.org/repos/asf/solr.git
at a713415c0f8 Initial code drop for ZERO replicas. SIP-20 and SOLR-17125
This branch includes the following new commits:
new a713415c0f8 Initial code drop for ZERO replicas. SIP-20 and SOLR-17125
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(solr) 01/01: Initial code drop for ZERO replicas. SIP-20 and SOLR-17125
Posted by il...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ilan pushed a commit to branch jira/solr-17125-zero-replicas
in repository https://gitbox.apache.org/repos/asf/solr.git
commit a713415c0f8bc8efff301a478c6baf21701621d5
Author: Ilan Ginzburg <ig...@salesforce.com>
AuthorDate: Tue Jan 23 04:57:18 2024 +0100
Initial code drop for ZERO replicas. SIP-20 and SOLR-17125
Co-authored-by: Andy Vuong <an...@gmail.com>
Co-authored-by: Bilal Waheed <mb...@users.noreply.github.com>
Co-authored-by: Megan Carey <me...@gmail.com>
Co-authored-by: Olivier Galizzi <og...@salesforce.com>
Co-authored-by: Pierre Salagnac <pi...@gmail.com>
Co-authored-by: Andy Throgmorton <an...@andythrogmorton.com>
Co-authored-by: Yonik Seeley <yo...@apache.org>
Co-authored-by: Paul McArthur <pm...@salesforce.com>
Co-authored-by: David Wayne Smiley <ds...@apache.org>
Co-authored-by: Vincent P <vi...@gmail.com>
Co-authored-by: Yohann Callea <yo...@gmail.com>
Co-authored-by: gbellaton <66...@users.noreply.github.com>
Co-authored-by: Bruno Roustant <br...@gmail.com>
Co-authored-by: Mathieu Marie <mm...@salesforce.com>
Co-authored-by: Yannis Hector <yh...@salesforce.com>
Co-authored-by: Haythem <ha...@yahoo.fr>
Co-authored-by: Dallas Dias <dd...@salesforce.com>
Co-authored-by: Chockalingam Viswanathan <c....@salesforce.com>
Co-authored-by: Prabhdeep Singh Gill <pr...@salesforce.com>
Co-authored-by: Ilan Ginzburg <il...@murblanc.org>
---
solr/ZERO-REPLICAS.adoc | 890 +++++++++++++++++++++
.../api/model/CreateCollectionRequestBody.java | 5 +
.../client/api/model/CreateShardRequestBody.java | 5 +
solr/bin/solr | 16 +
.../solr/cloud/ShardLeaderElectionContext.java | 161 ++--
.../java/org/apache/solr/cloud/ZkController.java | 161 ++--
.../solr/cloud/api/collections/AddReplicaCmd.java | 32 +-
.../apache/solr/cloud/api/collections/Assign.java | 34 +-
.../api/collections/CollectionHandlingUtils.java | 18 +-
.../cloud/api/collections/CreateCollectionCmd.java | 52 +-
.../solr/cloud/api/collections/CreateShardCmd.java | 11 +-
.../cloud/api/collections/DeleteCollectionCmd.java | 9 +
.../solr/cloud/api/collections/DeleteShardCmd.java | 6 +
.../solr/cloud/api/collections/MigrateCmd.java | 9 +-
.../api/collections/ReindexCollectionCmd.java | 7 +-
.../solr/cloud/api/collections/RestoreCmd.java | 37 +-
.../solr/cloud/api/collections/SplitShardCmd.java | 20 +-
.../solr/cloud/overseer/ClusterStateMutator.java | 12 +-
.../solr/cloud/overseer/CollectionMutator.java | 4 +-
.../src/java/org/apache/solr/cluster/Replica.java | 8 +-
.../impl/SimpleClusterAbstractionsImpl.java | 4 +
.../java/org/apache/solr/core/CoreContainer.java | 59 +-
.../src/java/org/apache/solr/core/NodeConfig.java | 16 +-
.../java/org/apache/solr/core/SolrXmlConfig.java | 27 +
.../src/java/org/apache/solr/core/ZeroConfig.java | 277 +++++++
.../apache/solr/handler/ReplicationHandler.java | 13 +
.../solr/handler/admin/CollectionsHandler.java | 5 +-
.../org/apache/solr/handler/admin/SplitOp.java | 56 +-
.../solr/handler/admin/api/CreateCollection.java | 49 +-
.../handler/admin/api/RestoreCollectionAPI.java | 4 +-
.../apache/solr/handler/admin/api/RestoreCore.java | 9 +
.../solr/handler/component/SearchHandler.java | 19 +
.../java/org/apache/solr/servlet/HttpSolrCall.java | 74 +-
.../apache/solr/update/DefaultSolrCoreState.java | 7 +
.../src/java/org/apache/solr/update/UpdateLog.java | 22 +-
.../DistributedUpdateProcessorFactory.java | 11 +-
.../processor/DistributedZkUpdateProcessor.java | 67 +-
.../update/processor/ZeroStoreUpdateProcessor.java | 222 +++++
.../java/org/apache/solr/zero/client/ZeroFile.java | 248 ++++++
.../apache/solr/zero/client/ZeroStoreClient.java | 501 ++++++++++++
.../solr/zero/client/ZeroStoreClientFactory.java | 93 +++
.../org/apache/solr/zero/client/package-info.java | 19 +
.../zero/exception/CorruptedCoreException.java | 25 +
.../apache/solr/zero/exception/ZeroException.java | 35 +
.../solr/zero/exception/ZeroLockException.java | 27 +
.../apache/solr/zero/exception/package-info.java | 19 +
.../solr/zero/metadata/LocalCoreMetadata.java | 256 ++++++
.../solr/zero/metadata/MetadataCacheManager.java | 455 +++++++++++
.../zero/metadata/MetadataComparisonResult.java | 195 +++++
.../solr/zero/metadata/ZeroMetadataController.java | 504 ++++++++++++
.../solr/zero/metadata/ZeroMetadataVersion.java | 67 ++
.../solr/zero/metadata/ZeroStoreShardMetadata.java | 190 +++++
.../apache/solr/zero/metadata/package-info.java | 19 +
.../solr/zero/process/CollectionDeletionTask.java | 70 ++
.../apache/solr/zero/process/CorePullStatus.java | 65 ++
.../org/apache/solr/zero/process/CorePuller.java | 800 ++++++++++++++++++
.../solr/zero/process/CorePullerBlockingQueue.java | 232 ++++++
.../org/apache/solr/zero/process/CorePusher.java | 501 ++++++++++++
.../solr/zero/process/CorePusherExecutionInfo.java | 151 ++++
.../apache/solr/zero/process/DeleteProcessor.java | 196 +++++
.../org/apache/solr/zero/process/DeleterTask.java | 181 +++++
.../solr/zero/process/FilesDeletionTask.java | 60 ++
.../solr/zero/process/ShardDeletionTask.java | 95 +++
.../apache/solr/zero/process/ZeroAccessLocks.java | 192 +++++
.../process/ZeroCoreIndexingBatchProcessor.java | 337 ++++++++
.../apache/solr/zero/process/ZeroStoreManager.java | 692 ++++++++++++++++
.../org/apache/solr/zero/process/package-info.java | 19 +
.../apache/solr/zero/util/DeduplicatingList.java | 198 +++++
.../apache/solr/zero/util/FileTransferCounter.java | 52 ++
.../apache/solr/zero/util/IndexInputStream.java | 110 +++
.../java/org/apache/solr/zero/util/ToFromJson.java | 46 ++
.../org/apache/solr/zero/util/package-info.java | 19 +
.../configsets/cloud-minimal-zero/conf/schema.xml | 29 +
.../cloud-minimal-zero/conf/solrconfig.xml | 53 ++
.../zero-distrib-indexing/conf/schema.xml | 29 +
.../zero-distrib-indexing/conf/solrconfig.xml | 57 ++
.../solr/cloud/api/collections/ShardSplitTest.java | 5 +
.../plugins/AffinityPlacementFactoryTest.java | 134 ++--
.../plugins/MinimizeCoresPlacementFactoryTest.java | 4 +-
.../solr/zero/client/ZeroStoreClientTest.java | 460 +++++++++++
.../ZeroCollectionBackupRestoreTest.java | 289 +++++++
.../ZeroStoreShardMetadataEvictionTest.java | 174 ++++
.../ZeroStoreShardMetadataTest.java | 161 ++++
.../ZeroStoreSimpleCollectionTest.java | 194 +++++
.../zero/cloudapicollections/package-info.java | 22 +
.../solr/zero/metadata/FileDeleteStrategyTest.java | 172 ++++
.../solr/zero/metadata/LocalCoreMetadataTest.java | 412 ++++++++++
.../ZeroMetadataVersionControllerTest.java | 229 ++++++
.../zero/metadata/ZeroStoreShardMetadataTest.java | 138 ++++
.../solr/zero/process/CoreCorruptionTest.java | 435 ++++++++++
.../apache/solr/zero/process/CorePullerTest.java | 537 +++++++++++++
.../apache/solr/zero/process/CorePusherTest.java | 186 +++++
.../solr/zero/process/DeleteProcessorTest.java | 628 +++++++++++++++
.../zero/process/PullMergeDeduplicationTest.java | 132 +++
.../process/SimpleZeroStoreEndToEndPullTest.java | 144 ++++
.../process/SimpleZeroStoreEndToEndPushTest.java | 115 +++
.../solr/zero/process/ZeroCollectionTestUtil.java | 111 +++
.../solr/zero/process/ZeroCoreConcurrencyTest.java | 882 ++++++++++++++++++++
.../solr/zero/process/ZeroCoreDiscoveryTest.java | 349 ++++++++
.../ZeroCoreIndexingBatchProcessorTest.java | 376 +++++++++
.../zero/process/ZeroStoreDeletionProcessTest.java | 355 ++++++++
.../process/ZeroStoreDistributedIndexingTest.java | 358 +++++++++
.../solr/zero/process/ZeroStoreManagerTest.java | 84 ++
.../zero/process/ZeroStoreMissingCoreTest.java | 414 ++++++++++
.../zero/process/ZeroStoreReplicaRecoveryTest.java | 65 ++
.../zero/process/ZeroStoreSolrCloudTestCase.java | 368 +++++++++
.../solr/zero/process/ZeroStoreSplitTest.java | 632 +++++++++++++++
.../solr/zero/util/DeduplicationListTest.java | 233 ++++++
.../solr/zero/util/IndexInputStreamTest.java | 93 +++
.../client/solrj/cloud/DistribStateManager.java | 3 +
.../client/solrj/impl/ZkDistribStateManager.java | 20 +
.../apache/solr/common/cloud/ZkStateReader.java | 2 +
.../solrj/request/CollectionAdminRequest.java | 74 +-
.../apache/solr/common/cloud/DocCollection.java | 28 +
.../java/org/apache/solr/common/cloud/Replica.java | 16 +-
.../org/apache/solr/common/cloud/ReplicaCount.java | 71 +-
.../solr/common/params/CollectionAdminParams.java | 3 +
.../apache/solr/common/cloud/ReplicaCountTest.java | 37 +-
.../apache/solr/cloud/MiniSolrCloudCluster.java | 20 +-
.../org/apache/solr/cloud/SolrCloudTestCase.java | 11 +
.../apache/solr/cluster/placement/Builders.java | 19 +-
.../solr/cluster/placement/BuildersTest.java | 7 +-
.../web/js/angular/controllers/collections.js | 4 +-
solr/webapp/web/partials/collection_overview.html | 5 +
solr/webapp/web/partials/collections.html | 18 +-
125 files changed, 18181 insertions(+), 323 deletions(-)
diff --git a/solr/ZERO-REPLICAS.adoc b/solr/ZERO-REPLICAS.adoc
new file mode 100644
index 00000000000..c829cdfd82c
--- /dev/null
+++ b/solr/ZERO-REPLICAS.adoc
@@ -0,0 +1,890 @@
+////
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+////
+
+= Zero replica type and storage
+:toc: macro
+:toclevels: 3
+
+January 2024
+
+toc::[]
+
+== Introduction
+
+The code in this branch https://github.com/apache/solr/tree/jira/solr-17125-zero-replicas[[.underline]#jira/solr-17125-zero-replicas#] is an initial and work in progress contribution
+proposal allowing for a separation of compute and storage in SolrCloud,
+as introduced by
+https://cwiki.apache.org/confluence/display/SOLR/SIP-20%3A+Separation+of+Compute+and+Storage+in+SolrCloud[[.underline]#SIP-20#]
+and referenced in Jira ticket
+https://issues.apache.org/jira/browse/SOLR-17125[[.underline]#SOLR-17125#]
+- _SIP-20 ZERO replicas, separation of compute and storage_.
+
+See <<Past plans>> on previous attempts at
+implementing this (or something similar).
+
+== Very high level overview
+
+The proposal is architected around:
+
+* Introduction of a new replica type called `ZERO` (`Replica.Type.ZERO`)
+augmenting the existing set of `NRT`, `TLOG` and `PULL` types, and a way to
+create collections based on this new replica type.
+* A shared repository called the Zero store in which the segments of each
+shard with `ZERO` replicas are persisted (each shard is persisted only
+once).
+* Local SolrCloud node disks used as caches (a better wording might be
+“non-persistent disks”). No data loss happens if a node is shut down at
+any given time or if all nodes are shut down at any given time and
+restart with empty disks
+* A guarantee that there is no data loss if two nodes consider themselves
+leaders of the same shard and try to update the Zero store concurrently.
+
+`ZERO` replicas use a local disk like other replicas usually do, push
+(write) data to the Zero store after indexing and pull (read) data from
+the Zero store when the local disk is not up to date (for serving
+queries, for indexing or any other need). This enables the local disk
+used by `ZERO` replicas to be ephemeral (i.e. can start empty on each node
+restart). By extension, a `ZERO` shard is a shard having replicas of type
+`ZERO` (and then they are all of type ZERO, mix and match with standard
+replica types is not supported) and being part of a `ZERO` collection (a
+collection that again has only `ZERO` shards).
+
+The principles governing how that new replica type is used and the
+design of the Zero store integration:
+
+* `ZERO` replicas read from and write to the Zero store (e.g. AWS S3, Google
+GCS etc.) and do not directly communicate with each other like standard
+SolrCloud replicas do. There is no Solr-to-Solr replication for these
+replicas.
+* *The Zero store content is the source of truth for index (segment)
+data*. No node specific state is ever needed to recover an index,
+regardless of how the cluster or the node was shut down or crashed.
+Nodes are stateless.
+* There is a single image on the Zero store per shard (one image and zero
+copies, hence the naming, and it also sounded cool), as opposed to an
+image per replica which implies data duplication, and in the SolrCloud
+context requires peer to peer communication between replicas.
+
+In a nutshell, the cores (replicas) that are expected on each node are
+identified by looking at ZooKeeper collection states. Replicas of type
+`ZERO` that are assigned to a given node but do not exist on its disk are
+created empty at startup (the node might have started with an empty
+disk). SolrCloud is then happy and can do the usual core opening, shard
+leader election etc. Actual data for the replica is fetched (pulled)
+from the Zero store when and if needed.
+
+Pulls may happen asynchronously when queries are served by non leader
+replicas (to refresh the locally cached state of the replica) and
+synchronously when a leader replica processes an indexing batch if that
+replica is not up to date: the replica first gets itself up to date with
+the Zero store if needed, processes locally the indexing work, pushes
+the updated segments and metadata to the Zero store and only then
+acknowledges the batch to the client.
+
+This is a current limitation whereas each indexing batch is committed
+then pushed to the Zero store before being acknowledged to the client.
+Work is planned to remedy this, see <<Future plans>>.
+
+Because in a SolrCloud cluster two replicas can consider themselves
+shard leaders at the same time (a replica might still be executing code
+assuming leadership after the quorum has voted it out, this is
+unavoidable unless leadership change waits for the outgoing leader to
+acknowledge, which does not work if the outgoing leader is really gone),
+a strategy was put in place for writes in which two concurrent replicas
+writing to the Zero store for a given shard do not overwrite each other
+and only one succeeds. This strategy uses unique filenames on the Zero
+store (no opportunity for overwrites at that level) and uses a compare
+and set operation in ZooKeeper (a.k.a. conditional update) to pick which
+one wins the race. The non winner reports a failure to the client that
+would then retry. This is expected to be a rare occurrence.
+
+== The Zero store
+
+The Zero store is a shared store accessible from all the nodes and
+stores for each `ZERO` shard the different files that constitute the
+corresponding core (segments files and `segments_N` file).
+
+The `BackupRepository` implementation is reused (it should be renamed if
+this code is merged since it would no longer be backup specific).
+Various implementations already exist in Solr code, for example for AWS
+S3 (`S3BackupRepository`), Google GCS (`GCSBackupRepository`) or a local
+file system implementation (`LocalFileSystemRepository`) for testing.
+
+The storage structure for each shard is described in
+<<The Zero store shard storage structure>>.
+
+== The Zero implementation
+
+An attempt was made to isolate Zero code in specific classes to make
+integration easier. This section describes the new classes introduced by
+the Zero implementation.
+
+These classes (and tests for that matter) are in subpackages of
+`org.apache.solr.zero` (in module `core`). This is a new package that does
+not exist in stock SolrCloud.
+
+The only new class being elsewhere is `*ZeroConfig*` which is in
+`org.apache.solr.core` like other configuration classes. It allows
+tweaking parameters of the Zero implementation and defining the
+`BackupRepository` to use for the Zero store.
+
+* In `org.apache.solr.zero.*client*`:
+** `*ZeroStoreClient*` is an “internal” client for Zero operations used
+within the Zero code.
+** `*ZeroStoreClientFactory*` this class instantiates `ZeroStoreClient` and
+also builds the `BackupRepository` used for the Zero store.
+** `*ZeroFile*` represents a file on the Zero store, possibly with a mapping
+to its local cached copy.
+* In `org.apache.solr.zero.*exception*`: Zero operations related exceptions.
+** `*CorruptedCoreException*`: used when a core pulled from the Zero store
+can’t be opened
+** `*ZeroException*`: generic exception for Zero implementation issues
+** `*ZeroLockException*`: inability to acquire a lock
+* In `org.apache.solr.zero.*metadata*`: classes dealing with Zero store and
+local metadata of a core.
+** `*ZeroStoreShardMetadata*` is a class serialized into (or deserialized
+from) a file stored in the Zero store that is the “root” file for
+finding the structure and content of a shard. This metadata includes the
+list of all files (with their Zero store names and local disk names)
+that make up the shard and can be used to create a replica of the
+shard. +
+This class also contains a list of files from prior commit points of the
+shard still present in the Zero store but no longer used and that should
+be deleted (they eventually are).
+** `*ServerSideMetadata*` the Solr node view of a replica (core) index.
+Whereas the Zero equivalent class `ZeroStoreShardMetadata` is “pure”
+metadata class, `ServerSideMetadata` also contains utility methods to
+build an instance from a local core and other helpers.
+** `*ZeroMetadataController*` deals with tracking shard metadata in ZooKeeper
+(needed in addition to actual data store in the Zero store) and provides
+comparison methods for instances of `ServerSideMetadata` and of
+`ZeroStoreShardMetadata` to compute which files need to be pulled from the
+Zero store (`diffMetadataForPull()`) or pushed from the local core to the
+Zero store (`diffMetadataForPush()`).
+** `*ZeroMetadataVersion*` a very simple class capturing the metadata suffix
+value and ZooKeeper node version, used for tracking in ZooKeeper which
+Zero store file (representing `ZeroStoreShardMetadata`) is the current
+version of the shard.
+** `*CoreMetadataComparisonResult*` captures the result of the comparison of
+the local server core and the remote Zero store metadata version of that
+shard.
+** `*MetadataCacheManager*` manages the per shard metadata cache for the
+cores (i.e. replicas) present on a node as well as some local core
+lifecycle wrt the Zero store.
+* In `org.apache.solr.zero.*process*`:
+** [red]#`*ZeroStoreManager*` is the entry point for interactions of Solr code with
+the Zero store. It is the class accessed from existing SolrCloud classes
+to do Zero store related operations.#
+** `*CorePuller*` deals with pulling a core from the Zero store and creating
+or updating a local copy.
+** `*CorePullStatus*`: an enum of the different ways a pull can end (happily
+or not).
+** `*CorePullerBlockingQueue*` provides a wrapper around a `DeduplicatingList`
+to be used in an `Executor` executing the pulls by calling `CorePuller`.
+This allows enqueuing (too) many pulls for a core without redundantly
+executing all of them. Pulls are currently enqueued from `HttpSolrCall`
+(for query requests) using
+`ZeroStoreManager.enqueueCorePullFromZeroStore`.
+** `*CorePusher*` deals with pushing a local core (or parts of it) from the
+local file system to the Zero store.
+** `*CorePusherExecutionInfo*`: captures the outcome of a core push
+** `*DeleteProcessor*` deals with deleting files on the Zero store. It
+enqueues for asynchronous execution delete tasks for different types of
+Zero file removal. Steady state file removal uses `deleteFiles()`,
+Overseer command execution uses `deleteShard()` and `deleteCollection()`.
+** `*DeleterTask*`, `*FilesDeletionTask*`, `*ShardDeletionTask*` and
+`*CollectionDeletionTask*` are different strategies to delete files by the
+`DeleteProcessor`.
+** `*ZeroAccessLocks*`: the locking logic to protect pull and push operations
+as well as indexing
+** `*ZeroCoreIndexingBatchProcessor*`: methods used at different stages of
+indexing into Zero replicas.
+* In `org.apache.solr.zero.*util*`:
+** `*DeduplicatingList*` a list-like data structure that deduplicates
+inserted entries by merging with existing entries.
+** `*FileTransferCounter*`: stats for core push and pulls
+** `*IndexInputStream*`: the stream used to push a local file to the Zero
+store
+** `*ToFromJson*`: the class converting `ZeroStoreShardMetadata` instances to
+files and vice versa
+
+== Configuring SolrCloud to support `ZERO` collections
+
+A SolrCloud cluster can support normal replica types (`NRT`, `TLOG` and
+`PULL`) and also Zero store based collections using replicas of type
+`Replica.Type.ZERO` (`ZERO` replicas for short). By extension, a collection
+based on `ZERO` replicas is referred to as a `ZERO` collection.
+
+SolrCloud needs to be configured with a Zero store to enable `ZERO`
+replicas. This is done by setting system property `zeroStoreEnabled=true`.
+During initialization of `CoreContainer` in `loadInternal()`, if that
+property is `true` the Zero implementation is initialized by creating an
+instance of `ZeroStoreManager`. The decision to enable or not the Zero
+implementation is only done at startup.
+
+In order to enable the Zero store, a repository must also be configured
+for it. This is using the configuration section `<repository>` in section
+`<repositories>` in section `<zero>` of `solr.xml`. That configuration
+contains the class of the repository implementation as well as
+additional parameters. See the backup-restore section of the deployment
+guide for details.
+
+== The Zero store shard storage structure
+
+All files of a shard, for the lifetime of that shard storage on the Zero
+store, are stored under a single “repository” (path or directory).
+
+All files corresponding to the current (latest) acknowledged commit
+point of that shard are present (using different names than the local
+file names) as well as a file that is specific to the Zero store called
+`shard.metadata._xxx_` (the `_xxx_` is a random UUID suffix) that contains
+the serialization of an instance of `ZeroStoreShardMetadata` for the
+shard. This structure contains the mapping between the Zero store names
+of the shard files and the local file system names of these files as
+expected by Solr.
+
+The Zero store file names are the Solr file names (for example
+`_y45.cfs`) to which a random suffix is added so that two nodes pushing
+Solr files with identical names do not collide or overwrite each other.
+The Zero store file name for the above file could for example be
+`_y45.cfs.84d9443a-62f8-45a7-9015-64dff429c6ca`.
+
+The suffix is the same type of suffix used for the `shard.metadata` file.
+Even though all such suffixes could be different, to make debugging
+easier, all files pushed to the Zero store in a single push “session”
+are using the same random suffix.
+
+This means that `shard.metadata.84d9443a-62f8-45a7-9015-64dff429c6ca`
+corresponds to the version of the index in which
+`_y45.cfs.84d9443a-62f8-45a7-9015-64dff429c6ca` was initially
+introduced. Of course, each `shard.metadata.xxx` can (and usually does
+except the first one for a shard) contain references to files having
+been initially added in different push sessions and therefore having
+different suffixes.
+
+The role of the suffix is to guarantee correct behavior when (unlikely
+and rare but does happen) two nodes concurrently update the same shard.
+See section
+<<Locking and managing concurrent Zero activity>>.
+
+== Creating `ZERO` collections
+
+The admin UI is changed to allow the creation of a `ZERO` collection which
+is a collection of shards having only `ZERO` replicas. A `zeroIndex`
+attribute (defaulting to `false`) is added to the `showAddCollection`
+function in `collections.js` which is referenced from the `collections.html`
+Solr admin interface.
+
+`Create` (in `CollectionAdminRequest`) is invoked by the `CollectionsHandler`
+when creating a collection. A flag is added indicating if the collection
+is `ZERO` and the number of `ZERO` replicas it should have. Collections
+backed by the Zero store can only have `ZERO` replicas.
+
+`CreateCollectionCmd`, `CreateShardCmd` and `AddReplicaCmd` (the Overseer side
+of Collection API commands) are modified accordingly to deal with the
+new replica type and attributes.
+
+The path to the data stored on the Zero store is composed of the
+collection name and the shard name. No replica or core specific names
+given the storage is per shard.
+See method `ZeroFile.getShardDirectoryURI()` that delegates part of the
+work to the underlying `BackupRepository` (the actual name/path is
+therefore implementation dependent).
+
+Note that the first push to the Zero store after collection creation is
+done when processing the first indexing batch for the shard. Remains to
+be verified if the cluster correctly deals with shards never pushed to
+the Zero store (in case of node restart, or backup being done etc).
+
+== Node startup
+
+When a node starts, it might not contain the data of its `ZERO` replicas
+and might not even contain the directory structure where these
+cores/replicas would go. The replicas assigned to a node are listed in
+ZooKeeper (in the various collections’ `state.json` files) and the
+corresponding data is available in the Zero store for `ZERO` replicas
+(after some resolution process involving Zero related metadata stored in
+ZooKeeper, see <<Access to Zero store>>).
+
+Given SolrCloud opens all cores (a replica is
+[line-through]#materialized by# a core) at startup and makes the
+replicas participate in the shard leader election for their shard, in
+order not to overwhelm the system and load large amount of data from the
+Zero store then open many “real” cores (i.e. cores with data) that might
+never get used, a tradeoff was implemented. All cores for `ZERO` replicas
+(or a shorthand: `ZERO` cores) are created empty locally and opened. That
+way they can participate in shard leader election as expected. These
+replicas are marked `ACTIVE`, even though initially they are empty! See
+also section <<Querying>>.
+
+When/if replicas are accessed and require the actual data, that data
+will be fetched (pulled) from the Zero store. Note that if the local
+disk already has data for a `ZERO` core (from a previous run of the Solr
+JVM if the volume is persistent) then the core will be open as is. It
+will eventually get refreshed to the data corresponding to it in the
+Zero store (the state of the Zero store for a shard is the source of
+truth, not the state on the local disk) if it is queried or indexed.
+
+Method `CoreContainer.discoverAdditionalCoreDescriptorsForZeroReplicas()`
+is called from `CoreContainer.loadInternal()`, it iterates over all the
+collections of the cluster and for `ZERO` collections creates a
+`CoreDescriptor` for each replica (if any) that should be on the local
+node (unless that replica was already found locally), adds it to the set
+of cores discovered locally (returned by
+`CorePropertiesLocator.discover()`) and creates the `core.properties` file
+locally at the appropriate path (this file is a standard Solr file that
+is unrelated to the `shard.metadata.xxx` file used by `ZERO` replicas).
+
+At a later stage of `CoreContainer.loadInternal()`, all cores (found
+locally and added for `ZERO` replicas) are opened (if so configured using
+`loadOnStartup` defaulting to `true`, with `false` not a recommended setting
+in SolrCloud mode).
+
+== Access to Zero store
+
+Writes to the Zero store are done by the leader replica of a shard for
+pushing newly/recently indexed documents. It is possible to have more
+than one replica considering itself as a shard leader (for the same
+shard) as leadership changes. In such a case there’s a need to both
+protect the consistency of data in the Zero store (to avoid corruption
+due to a mix and match of incompatible core files written by different
+nodes) and also make sure no data is lost due to being overwritten (e.g.
+node A write a segment, node B then overwrites that segment with an
+identically named segment, causing the data written by node A to be
+lost, even though node A might have acknowledged to the client the
+received indexing batch).
+
+In a nutshell (see
+<<The Zero store shard storage structure>>), writes to the Zero store use unique names
+and never overwrite any file (the uniqueness is guaranteed by adding a
+random suffix to the file names). The `shard.metadata._suffix_` file
+contains the list of all files on the Zero store that constitute the
+core (it references files previously pushed and still needed as well as
+the new set of files just written). Once all these writes (the segment
+files and `shard.metadata._suffix_`) have successfully completed, an
+update is made to a shard specific node in ZooKeeper to capture the
+`suffix` of the latest version of the `shard.metadata` file. That update is
+made by the indexing thread as a conditional update (checking that the
+version of the pre update ZooKeeper node hasn’t change from when
+indexing started), to make sure no update from another node sneaked in
+between the moment the indexing thread pulled the latest content from
+the Zero store before starting indexing and when it pushes its updates
+back to the Zero store after finishing indexing.
+
+If the conditional update succeeds, the indexing content and changes to
+the core have been correctly persisted and will not get overwritten. If
+the conditional update to ZooKeeper fails, the indexing batch fails. It
+will have to be executed again after having pulled the latest copy of
+the core from the Zero store. This means some other update from another
+node sneaked in and made an update to that shard on the Zero store, and
+given we don’t want to lose (overwrite) that update, the new indexing
+batch has to be redone on top of that update (the retry is not automatic
+although it could be made to be, the client gets a failure return for
+its indexing batch and has to resubmit).
+
+The above protects concurrent updates from different SolrCloud nodes to
+a shard on the Zero store. This should not happen often, but it does
+happen when a shard leader loses leadership while processing an indexing
+batch and continues processing while the new leader starts processing
+another indexing batch.
+
+Management of the ZooKeeper `metadataSuffix` node for the shard is done in
+`ZeroMetadataController`. The actual metadata is stored in
+`ZeroMetadataVersion` (the suffix itself and the corresponding ZooKeeper
+version for the conditional update), a cache of the data is maintained
+in `MetadataCacheManager`.
+
+In addition to the correctness guarantees based on the above, we don’t
+want inefficiency in the access to the Zero store from within a single
+Solr node, which is the most common case.
+
+For example when two indexing batches are executed concurrently on a
+given replica, we don’t want both to try to push data concurrently
+knowing one of the batches will have to fail. Also when two concurrent
+requests arrive for the same replica that both need to pull data from
+the Zero store, we don’t want to pull files twice.
+
+Therefore we need synchronization on pulls and pushes that happen for a
+replica on a given node. This is described in the next section.
+
+Synchronization with or more precisely protection against what happens
+through a different replica of the same shard on another node is taken
+care of by the mechanism described above.
+
+== Per core Zero related metadata
+
+`MetadataCacheManager` is the class managing the state of a core (replica)
+related to Zero store interactions as seen from a SolrCloud node. It
+tracks failures pulling a core from the Zero store (to avoid retrying
+forever). For each core it caches an instance of static inner class
+`MetadataCacheEntry`.
+
+For each local replica, the last known `metadataSuffix` and associated
+ZooKeeper node `version` are tracked. The version will be used for
+conditional updates when writes (pushes) are done to the Zero store in
+order to detect if the Zero store content has changed (in which case the
+conditional update fails). A boolean `cacheLikelyUpToDate` is a hint
+indicating the server considers that the cached `metadataSuffix` and
+version of MetadataCacheEntry are current (to skip re-reading them from
+ZooKeeper). If the values end up being not current, a conditional update
+failure will trigger a refresh from ZooKeeper and when needed retrieving
+latest content from the Zero store..
+
+=== Locking and managing concurrent Zero activity
+
+==== Overview
+
+Locks must be acquired to manipulate a replica. Locks are defined and
+used via class `ZeroAccessLocks`. An inner class `CloseableLock` wrapping
+locks allows acquiring the lock in a try-with-resources to have it
+released automatically (this is not always possible so locks are also
+acquired and released in a more usual way).
+
+There are 3 locks: for pulling, for pushing and for indexing. Internally
+these are implemented using a read-write lock for indexing and pull, and
+a simple lock for push.
+
+The idea is to allow parallel indexing batches (multiple threads can
+acquire the indexing lock concurrently), but disallow any pull while
+indexing takes place. Also only allow a single push at a time.
+
+Follows a summary of the ways different operations interact when they
+happen at the same time on the same or different nodes. The following
+sections then get into more details.
+
+Locking within a node:
+
+[width="100%",cols="26%,37%,37%",]
+|===
+| |Pull |Indexing
+
+|Pull a|
+For efficiency, serialize pulls (to avoid pulling twice the same files).
+
+_→ Pulls use a lock for mutual exclusion_
+
+a|
+No pulls while indexing is in progress to allow indexing to rely on the
+cached `metadataSuffix` node version.
+
+_→ Indexing uses a lock that allows other indexing but prohibits pulls_
+
+|Indexing >|Same as top right ↗️ a|
+Actual indexing can proceed in parallel (like stock SolrCloud), pushes
+are serialized.
+
+_→ Indexing uses a lock that allows other indexing but prohibits pulls_
+
+_→ The push to Zero store phase is done under *another mutual exclusion*
+lock_
+
+|===
+
+Interactions between two different nodes:
+
+[width="100%",cols="26%,37%,37%",]
+|===
+| |Pull on node B |Indexing on node B
+
+|Pull on node A |No limitations, can proceed in parallel |No
+limitations, can proceed in parallel
+
+|Indexing on node A >|Same as top right ↗️ |Edge case of shard leader tenure overlap. One
+batch succeeds, the other one fails. No data loss.
+|===
+
+==== Locking for pushing
+
+For pushes, acquiring the push lock guarantees there is no more than a
+single push at a time. If multiple pushes were allowed in parallel they
+would be pushing the same data and only one of them would succeed
+anyway.
+
+This lock exists for efficiency, not correctness.
+
+==== Locking for indexing and pulling
+
+Locking for indexing allows parallel indexing but prevents pulling while
+indexing is in progress (think indexing locking acquires a read lock and
+pulling acquires the write lock). If new shard content on the Zero store
+is becoming available from another node while a node is indexing (should
+be relatively rare), the indexing batch will eventually fail: the
+`metadataSuffix` node in ZooKeeper will not have the expected version due
+to the update done by another node.
+
+If pulling was allowed during indexing and updated the `version` of the
+cached `metadataSuffix` node, the ZooKeeper conditional update at the end
+of indexing could succeed. There would be a risk of data loss or
+conflict between the pulled content and locally written content (locally
+generated segment with same number as pulled segment). If pulling was
+allowed without updating the cached `version`, indexing would fail anyway.
+That’s why pulling is not permitted while indexing is in progress. The
+case this is protecting against is rare to very rare (two leaders for
+the same shard writing at more or less the same time) so no lock
+contention expected here.
+
+A naive solution to that issue would be to have each indexing batch
+record the version of the `metadataSuffix` node at the start of indexing
+(representing the version of the shard on top of which the indexing is
+done), and at the end of indexing do the conditional update using this
+version, regardless of pulls that might have occurred in the meantime on
+that node. This would work, and in case of a remote update to the Zero
+store shard data, the indexing batch would fail as expected.
+
+But this approach would be causing local inefficiencies and is not
+realistic if concurrent indexing batches on a replica are allowed: when
+two such batches run (starting from the same initial state of the
+`metadataSuffix` node), the first to finish would succeed and update the
+`metadataSuffix` node and the second would systematically fail. In
+practice this would mean no concurrent indexing.
+
+This is the reason that multiple indexing batches can proceed in
+parallel but no pulling from Zero store is allowed while an indexing is
+in progress.
+
+The indexing lock must be continuously held from before indexing starts
+to after the push that occurs at the end of indexing has completed.
+Before the push starts, the push lock must be acquired.
+
+If the indexing lock was released at the end of local indexing and
+commit and before the push to Zero store started, a pull could then
+happen. The pulled data could conflict with local core data, and
+regardless of the resolution choice (keeping the local segments or
+remote ones coming from the Zero store) the subsequent push - if not
+made to fail - would be causing some loss of data.
+
+Therefore, for correctness it is required to either prevent pulls from
+happening during indexing or making sure indexing fails if a pull does
+happen during indexing. The proposed implementation picked the first
+option but both are equivalent really (the indexing batch would fail in
+both if there was a need to do a pull, whether or not that pull was
+done).
+
+==== Indexing lifecycle with `ZERO` replicas
+
+The way concurrent indexing works with the Zero store is that
+Solr/Lucene is managing segment numbering and writes to the local index
+in the normal way. When a commit completes and the first of concurrent
+indexing batches completes, local files are pushed to the Zero store
+using unique filenames and a conditional update to the `metadataSuffix`
+ZooKeeper node is done with the `version` cached on this Solr node. This
+causes an increase of the `metadataSuffix` node version (`metadataSuffix` is
+always updated at the end of a push, this is what makes the pushed data
+visible - note this means each indexing batch with `ZERO` replicas does a
+write to ZooKeeper, this is not the case with existing replica types). A
+second indexing thread that has completed and committed waits for the
+push lock and for the first thread to complete the push and release the
+lock. The second thread then pushes to the Zero store the new content
+(diff between local index commit point and Zero store contents) and
+updates again the `metadataSuffix` node. It’s quite possible and even
+probable that the push at the end of the first batch did write to the
+Zero store the updates done by both batches (which might have resulted
+in one or more segments on disk), in which case the second thread, after
+waiting for the first push to finish, will discover that no additional
+files need to be pushed, will quickly complete and acknowledge the
+second batch.
+
+Note that when two different nodes push to the Zero store at the same
+time, only one batch succeeds and the other one fails, no data is lost
+nor overwritten. Current implementation will return a failure to the
+client (in the future the batch could be retried after refreshing the
+local replica state so it succeeds).
+
+==== Locking summary
+
+To sum it up, locking on a given SolrCloud node allows using updated
+versions of the `metadataSuffix` ZooKeeper node by indexing batches and
+therefore allows concurrent indexing on a node.
+
+Writing files with unique names to the Zero store and validating the
+actual write by a conditional update to the ZooKeeper `metadataSuffix`
+node allows managing consistency and guaranteeing no data loss in the
+Zero store when multiple nodes attempt to update the same shard on the
+Zero store (rare but not impossible, has been observed in production).
+
+Here’s a summary of the ways different operations interact when they
+happen at the same time on the same or different nodes.
+
+== Querying
+
+As described in <<Node startup>>, cores
+might start their life empty (local disk is a cache, source of truth is
+the Zero store). Currently, loading a core from the Zero store is
+triggered by either a query or an indexing request addressed to the
+core.
+
+Pulls triggered by queries are done asynchronously. When receiving a
+request which is not an `/update`, a pull is enqueued. This is done very
+early in the request lifecycle in `HttpSolrCall.init()` (this should be
+changed and moved to the handlers/components that actually need the core
+content).
+
+In `SearchHandler.handleRequestBody()`, there’s a stopgap check to ensure
+the core was pulled from Zero store at least once. If it wasn’t, the
+query is rejected right away. This ensures we never answer to a query
+with no results, mostly because we create empty cores at node startup
+for all the replicas that are discovered from ZooKeeper. An outright
+rejection causes other layers of SolrCloud to query (for a while)
+replicas on other nodes, letting the node with missing data refresh
+itself and be able to then handle queries. Rolling cluster restarts need
+somewhat more work to guarantee that there’s always at least one up to
+date replica to serve queries (if all nodes restart in quick sequence,
+all replicas of a shard might not have refreshed themselves from the
+Zero store, causing an outage). Code handling this case is not too
+complex and can be contributed later.
+
+This does not ensure the local core on a non-leader replica is always
+up-to-date. We may still process a query without having the very recent
+updates to the replica if they were done in another node. This is not so
+different from what happens with `TLOG` or `PULL` replicas, where
+replication between Solr nodes is not real time.
+
+In the Zero store case though, the update could be much older. For
+example the non leader replica got updated a week ago when it was
+leader. It will be non empty from that point on, but the quality of
+results it will be able to serve might not be very high. If queries
+target all replicas and there is a constant stream, then non leader
+replicas will always be more or less up to date. If there’s only
+indexing and no queries, the first queries on the non leader replica
+might be served based on old replica contents.
+
+Pull requests are enqueued after each query (this should be improved),
+but as they’re waiting in a queue to be executed, new requests are
+deduplicated. This means there is at most a single outstanding pull
+request for a replica, a subsequent pull request will only be enqueued
+if a new query arrives.
+
+== Indexing
+
+The first location in code where we have custom code for indexing for
+`ZERO` replicas is class `DistributedUpdateProcessorFactory`. For ZERO
+replicas, instead of using the standard `DistributedZkUpdateProcessor`,
+subclass `ZeroStoreUpdateProcessor` is used. This is mostly a hook to make
+sure the core is pulled from Zero store before processing the batch, and
+then pushed back to Zero store when the batch is complete. This happens
+respectively:
+
+[arabic]
+. In `postSetupHook()` (added in the parent class and overridden in the
+subclass) that invokes
+`ZeroCoreIndexingBatchProcessor.addOrDeleteGoingToBeIndexedLocally()`.
+. In `processCommit()` (standard method overridden from the parent class)
+that invokes
+`ZeroCoreIndexingBatchProcessor.hardCommitCompletedLocally()`.
+
+
+There is also code specific to `ZERO` replica in class `HttpSolrCall` to
+always add the `commit=true` parameter to the indexing batch. This code
+will be removed once the Transaction Log is adapted to work at the shard
+level with `ZERO` replicas. See <<Future plans>>.
+
+Indexing itself proceeds normally all the way to the commit. Push to
+Zero store happens after the new segments have been created on the local
+file system.
+
+== Shard leader elections
+
+The differences in leader election for `ZERO` replicas are minimal, and
+reside in whether we allow the replica that wins the election to
+actually become the leader replica (normal upstream leader election
+first elects a replica *then* decides if it is fit to become the leader
+based on shard terms) and as a consequence how long/if a replica waits
+for other replicas to join the election (with `ZERO` replicas no need to
+wait).
+
+When indexing to a `ZERO` replica, the latest available data will always
+be available since the source of truth (Zero store) is available to all
+replicas. Therefore the `ZERO` replica that wins the leader election is
+always legitimate to actually be the leader. In the method
+`ShardLeaderElectionContext.runLeaderProcess()`, we skip checks related to
+shard terms for `ZERO` replicas (and given that unlike `NRT` and `TLOG` there
+is no forwarding of indexing batches with `ZERO` replicas, shard terms are
+not even maintained, so it’s good news they’re not needed).
+
+Strictly speaking, for correctness and durability there is no need to
+even do shard leader elections for `ZERO` replicas. The protections around
+how the Zero store can be updated guarantees that any replica doing
+indexing will end up doing the right thing: reject an indexing request
+if it doesn’t have the last data of the shard and update itself with the
+latest data to be able to process subsequent indexing requests. But
+given shard leader election is so pervasive in SolrCloud, the ZERO
+replica implementation at least in its current state conforms to the
+general logic with only small adjustments as described above.
+
+== Admin operations
+
+=== Collection creation
+
+Collection creation in `CreateCollectionCmd` creates the `metadataSuffix`
+ZooKeeper node when creating a `ZERO` collection.
+
+=== Restore
+
+`RestoreCmd` can restore a "normal" collection as `ZERO` (this can be used
+for testing the `ZERO` feature) and vice versa.
+
+There aren’t many changes to the restore code overall to support `ZERO`.
+`RestoreCore` does push the restored core to the Zero store though.
+
+=== Shard Splits
+
+Core (shard) split requires the index. Before processing the split for a
+`ZERO` replica, the index is pulled from the Zero store in a similar way
+to pulls done before processing an indexing batch. In `SplitOp`, a wrapper
+method is executed before calling the split code, makes sure the index
+is up to date from the Zero store, then grabs the `ZERO` indexing lock on
+the parent core and executes the standard split method.
+
+That split method got slightly modified to return the list of created
+subcores (and not close them) so that the wrapper method can push these
+new cores to the Zero store then close them.
+
+Indexing batches received (and pushed to the Zero store) while a split
+is in progress might be lost if they update the Zero store after its
+contents were fetched in `SplitOp` on the node where the split happens. In
+order to prevent this from happening, indexing is rejected while a split
+is in progress (remains to be seen if that can be changed once the
+Transaction Log is adapted to the `ZERO` replicas). The rejection is
+implemented by checking if there is a splitting “lock” in ZooKeeper (a
+path of the form `/collections/_collectionName_/_shardName_-splitting`).
+The method that performs the check is
+`SplitShardCmd.shardSplitLockHeld()`.
+
+The `...-splitting` path is created in `SplitShardCmd.lockForSplit` before
+the `SplitOp` is called on the node.
+
+Note there is a race window here (a small one) in which an indexing
+batch is not rejected because no split is in progress when the check is
+done, but assuming the indexing is slow to process, by the time it
+updates the Zero store image of the shard, the split has started and
+that batch is going to be lost (not present on any of the children). A
+possible fix would be to update the `version` of the `metadataSuffix` node
+before pulling the contents of the core for the split, and making sure
+rejection during indexing in case of a split happens after the indexing
+node has pulled the core (the increased `version` would make sure the
+indexing batch ends up failing). This would have to be reconsidered once
+`ZERO` replicas use the transaction log (see
+<<Future plans>>).
+
+== Random implementation notes
+
+Listing current limitations and/or work to do.
+
+=== CollectionsHandler
+
+`MODIFYCOLLECTION` does not work with `ZERO` collections
+
+=== CoreCorruptionTest
+
+`testIndexFilePullFailure()` and `testCorruptedIndexFile()` seem to fail
+sometimes.
+
+=== CorePusherTest
+
+The cluster is shutdown in `@After` but is built at the start of
+individual tests, and not all of them...
+
+=== CreateCollectionCmd
+
+In `populateShardNames()` there’s a change (with `TODO`) that likely needs
+to be contributed to the `main` branch independently of the Zero code
+implementation (it is a general fix).
+
+=== CreateCollectionRequestBody
+
+`TODO` regarding other places related to v2 API that might need changes
+
+=== CreateShardRequestBody
+
+`TODO` regarding other places related to v2 API that might need changes
+
+=== DocCollection
+
+Checks for collection vs. replica type might be redundant
+
+=== HttpSolrCall
+
+Hard coding of the request paths that trigger a pull. This should be
+changed and delegated to the specific handlers/components that require a
+pull.
+
+=== MigrateCmd
+
+`MIGRATE` does not work with `ZERO` replicas
+
+=== ReindexCollectionCmd
+
+`REINDEXCOLLECTION` does not work with `ZERO` replicas
+
+=== SearchHandler
+
+Rejecting query if core never pulled from Zero store. Need something
+more elegant
+
+=== SplitShardCmd
+
+Method `shardSplitLockHeld()` checks the presence of a lock using a
+`ZkController` whereas existing methods `lockForSplit()` and
+`unlockForSplit()` use a `SolrCloudManager`. Unfortunately the caller of
+`shardSplitLockHeld()` does not have an instance of `SolrCloudManager`.
+
+=== ZeroCoreConcurrencyTest
+
+Flapping (and disabled) test `todoTestIndexingQueriesDeletesFailovers()`.
+
+=== ZeroCoreIndexingBatchProcessor
+
+Indexing does happen on `INACTIVE` shards when split happens, this should
+not be the case. See `ZeroCoreIndexingBatchProcessor` and run tests in
+`ZeroStoreSplitTest`.
+
+== Future plans
+
+The `ZERO` code as of this writing (January 2024) has known limitations
+forcing commits with each indexing batch and basically bypassing
+anything that relies on the transaction log (nodes are stateless, the
+transaction log is assumed to not survive a restart). Moreover (but
+relatively minor drawback), each such indexing batch also includes a
+write to a ZooKeeper node (updating `metadataSuffix`).
+
+The next step is to adapt the transaction log from its current per
+replica state and make it a per shard abstraction that is stored in the
+Zero store in a way that makes any `ZERO` replica able to add to it and to
+consume it (like any `ZERO` replica can update itself from the Zero
+store). Once this is available, some aspects of the current
+implementation of `ZERO` replicas would have to be reconsidered.
+
+== Past plans
+
+First found trace of an intention to implement something similar to the
+proposal in this branch dates back to July 2014.
+https://issues.apache.org/jira/browse/SOLR-6237[[.underline]#SOLR-6237#]
+- _An option to have only leaders write and replicas read when using a
+shared file system with SolrCloud_.
+
+A more recent attempt to contribute the code that evolved into the
+current `ZERO` replicas proposal was made in January 2019. The new replica
+type was then called SHARED. The umbrella ticket is
+https://issues.apache.org/jira/browse/SOLR-13101[[.underline]#SOLR-13101#]
+- _Shared storage via a new SHARED replica type,_ and its (long
+abandoned) branch is
+https://github.com/apache/lucene-solr/tree/jira/SOLR-13101[[.underline]#lucene-solr
+jira/SOLR-13101#].
+
+The two tickets are marked as “Unresolved” and “Won't Fix”.
+
+Hopefully history doesn’t *always* repeat itself.
diff --git a/solr/api/src/java/org/apache/solr/client/api/model/CreateCollectionRequestBody.java b/solr/api/src/java/org/apache/solr/client/api/model/CreateCollectionRequestBody.java
index 5dce5840102..018cb8be0cb 100644
--- a/solr/api/src/java/org/apache/solr/client/api/model/CreateCollectionRequestBody.java
+++ b/solr/api/src/java/org/apache/solr/client/api/model/CreateCollectionRequestBody.java
@@ -38,6 +38,11 @@ public class CreateCollectionRequestBody {
@JsonProperty public Integer nrtReplicas;
+ @JsonProperty public Integer zeroReplicas;
+ // TODO other "v2" places where ZERO needs to be added?
+
+ @JsonProperty public Boolean isZeroIndex;
+
@JsonProperty public Boolean waitForFinalState;
@JsonProperty public Boolean perReplicaState;
diff --git a/solr/api/src/java/org/apache/solr/client/api/model/CreateShardRequestBody.java b/solr/api/src/java/org/apache/solr/client/api/model/CreateShardRequestBody.java
index ea871998cca..081ba454c7b 100644
--- a/solr/api/src/java/org/apache/solr/client/api/model/CreateShardRequestBody.java
+++ b/solr/api/src/java/org/apache/solr/client/api/model/CreateShardRequestBody.java
@@ -34,6 +34,11 @@ public class CreateShardRequestBody {
@JsonProperty public Integer pullReplicas;
+ @JsonProperty public Integer zeroReplicas;
+ // TODO other "v2" places where ZERO needs to be added?
+
+ @JsonProperty public Boolean isZeroIndex;
+
@JsonProperty("createReplicas")
public Boolean createReplicas;
diff --git a/solr/bin/solr b/solr/bin/solr
index 6b8270d66e0..b931de3c5de 100755
--- a/solr/bin/solr
+++ b/solr/bin/solr
@@ -1669,6 +1669,22 @@ if [ "${SOLR_MODE:-}" == 'solrcloud' ]; then
CLOUD_MODE_OPTS+=("-DcreateZkChroot=$ZK_CREATE_CHROOT")
fi
+ if [ -n "$ZERO_STORE_ENABLED" ]; then
+ CLOUD_MODE_OPTS+=("-DzeroStoreEnabled=$ZERO_STORE_ENABLED")
+ fi
+
+ if [ -n "$ZERO_STORE_NUM_FILE_PUSHER_THREADS" ]; then
+ CLOUD_MODE_OPTS+=("-DzeroStoreNumFilePusherThreads=$ZERO_STORE_NUM_FILE_PUSHER_THREADS")
+ fi
+
+ if [ -n "$ZERO_STORE_NUM_FILE_PULLER_THREADS" ]; then
+ CLOUD_MODE_OPTS+=("-DzeroStoreNumFilePullerThreads=$ZERO_STORE_NUM_FILE_PULLER_THREADS")
+ fi
+
+ if [ -n "$ZERO_STORE_MAX_ASYNC_CORE_PULLS" ]; then
+ CLOUD_MODE_OPTS+=("-DzeroStoreNumCorePullerThreads=$ZERO_STORE_MUM_CORE_PULLER_THREADS")
+ fi
+
# and if collection1 needs to be bootstrapped
if [ -e "$SOLR_HOME/collection1/core.properties" ]; then
CLOUD_MODE_OPTS+=('-Dbootstrap_confdir=./solr/collection1/conf' '-Dcollection.configName=myconf' '-DnumShards=1')
diff --git a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java
index e1768e0cd37..4ae57220de7 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java
@@ -107,6 +107,7 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
throws KeeperException, InterruptedException {
String coreName = leaderProps.getStr(ZkStateReader.CORE_NAME_PROP);
ActionThrottle lt;
+ boolean isZeroReplica;
try (SolrCore core = cc.getCore(coreName)) {
if (core == null) {
// shutdown or removed
@@ -114,6 +115,8 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
}
MDCLoggingContext.setCore(core);
lt = core.getUpdateHandler().getSolrCoreState().getLeaderThrottle();
+ isZeroReplica =
+ core.getCoreDescriptor().getCloudDescriptor().getReplicaType() == Replica.Type.ZERO;
}
try {
@@ -151,7 +154,9 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
}
}
- if (!weAreReplacement) {
+ // Do not wait for other replicas when type is ZERO.
+ // Since we can pull data from Zero store, we don't require an up-to-date leader
+ if (!isZeroReplica && !weAreReplacement) {
waitForReplicasToComeUp(leaderVoteWait);
}
@@ -173,98 +178,104 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
replicaType = core.getCoreDescriptor().getCloudDescriptor().getReplicaType();
coreNodeName = core.getCoreDescriptor().getCloudDescriptor().getCoreNodeName();
- // should I be leader?
- ZkShardTerms zkShardTerms = zkController.getShardTerms(collection, shardId);
- if (zkShardTerms.registered(coreNodeName) && !zkShardTerms.canBecomeLeader(coreNodeName)) {
- if (!waitForEligibleBecomeLeaderAfterTimeout(
- zkShardTerms, coreNodeName, leaderVoteWait)) {
- rejoinLeaderElection(core);
+
+ // if ZERO replica, skip sync and recovery stages. a ZERO replica that is not up-to-date
+ // can still become leader; it will sync the latest from Zero store with the next request.
+ if (replicaType != Replica.Type.ZERO) {
+ // should I be leader?
+ ZkShardTerms zkShardTerms = zkController.getShardTerms(collection, shardId);
+ if (zkShardTerms.registered(coreNodeName)
+ && !zkShardTerms.canBecomeLeader(coreNodeName)) {
+ if (!waitForEligibleBecomeLeaderAfterTimeout(
+ zkShardTerms, coreNodeName, leaderVoteWait)) {
+ rejoinLeaderElection(core);
+ return;
+ } else {
+ // only log an error if this replica win the election
+ setTermToMax = true;
+ }
+ }
+
+ if (isClosed) {
return;
- } else {
- // only log an error if this replica win the election
- setTermToMax = true;
}
- }
- if (isClosed) {
- return;
- }
+ log.info("I may be the new leader - try and sync");
- log.info("I may be the new leader - try and sync");
+ // we are going to attempt to be the leader
+ // first cancel any current recovery
+ core.getUpdateHandler().getSolrCoreState().cancelRecovery();
- // we are going to attempt to be the leader
- // first cancel any current recovery
- core.getUpdateHandler().getSolrCoreState().cancelRecovery();
+ if (weAreReplacement) {
+ // wait a moment for any floating updates to finish
+ try {
+ Thread.sleep(2500);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, e);
+ }
+ }
- if (weAreReplacement) {
- // wait a moment for any floating updates to finish
+ PeerSync.PeerSyncResult result = null;
+ boolean success = false;
try {
- Thread.sleep(2500);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, e);
+ result = syncStrategy.sync(zkController, core, leaderProps, weAreReplacement);
+ success = result.isSuccess();
+ } catch (Exception e) {
+ log.error("Exception while trying to sync", e);
+ result = PeerSync.PeerSyncResult.failure();
}
- }
- PeerSync.PeerSyncResult result = null;
- boolean success = false;
- try {
- result = syncStrategy.sync(zkController, core, leaderProps, weAreReplacement);
- success = result.isSuccess();
- } catch (Exception e) {
- log.error("Exception while trying to sync", e);
- result = PeerSync.PeerSyncResult.failure();
- }
+ UpdateLog ulog = core.getUpdateHandler().getUpdateLog();
- UpdateLog ulog = core.getUpdateHandler().getUpdateLog();
-
- if (!success) {
- boolean hasRecentUpdates = false;
- if (ulog != null) {
- // TODO: we could optimize this if necessary
- try (UpdateLog.RecentUpdates recentUpdates = ulog.getRecentUpdates()) {
- hasRecentUpdates = !recentUpdates.getVersions(1).isEmpty();
+ if (!success) {
+ boolean hasRecentUpdates = false;
+ if (ulog != null) {
+ // TODO: we could optimize this if necessary
+ try (UpdateLog.RecentUpdates recentUpdates = ulog.getRecentUpdates()) {
+ hasRecentUpdates = !recentUpdates.getVersions(1).isEmpty();
+ }
}
- }
- if (!hasRecentUpdates) {
- // we failed sync, but we have no versions - we can't sync in that case
- // - we were active
- // before, so become leader anyway if no one else has any versions either
- if (result.getOtherHasVersions().orElse(false)) {
- log.info(
- "We failed sync, but we have no versions - we can't sync in that case. But others have some versions, so we should not become leader");
- success = false;
- } else {
- log.info(
- "We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway");
- success = true;
+ if (!hasRecentUpdates) {
+ // we failed sync, but we have no versions - we can't sync in that case
+ // - we were active
+ // before, so become leader anyway if no one else has any versions either
+ if (result.getOtherHasVersions().orElse(false)) {
+ log.info(
+ "We failed sync, but we have no versions - we can't sync in that case. But others have some versions, so we should not become leader");
+ success = false;
+ } else {
+ log.info(
+ "We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway");
+ success = true;
+ }
}
}
- }
- // solrcloud_debug
- if (log.isDebugEnabled()) {
- try {
- RefCounted<SolrIndexSearcher> searchHolder = core.getNewestSearcher(false);
- SolrIndexSearcher searcher = searchHolder.get();
+ // solrcloud_debug
+ if (log.isDebugEnabled()) {
try {
- if (log.isDebugEnabled()) {
- log.debug(
- "{} synched {}",
- core.getCoreContainer().getZkController().getNodeName(),
- searcher.count(new MatchAllDocsQuery()));
+ RefCounted<SolrIndexSearcher> searchHolder = core.getNewestSearcher(false);
+ SolrIndexSearcher searcher = searchHolder.get();
+ try {
+ if (log.isDebugEnabled()) {
+ log.debug(
+ "{} synched {}",
+ core.getCoreContainer().getZkController().getNodeName(),
+ searcher.count(new MatchAllDocsQuery()));
+ }
+ } finally {
+ searchHolder.decref();
}
- } finally {
- searchHolder.decref();
+ } catch (Exception e) {
+ log.error("Error in solrcloud_debug block", e);
}
- } catch (Exception e) {
- log.error("Error in solrcloud_debug block", e);
}
- }
- if (!success) {
- rejoinLeaderElection(core);
- return;
+ if (!success) {
+ rejoinLeaderElection(core);
+ return;
+ }
}
}
@@ -463,6 +474,8 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
}
// on startup and after connection timeout, wait for all known shards
+ // We also wait here for Replica.Type.ZERO but none will appear since we don't mix ZERO
+ // with other replica types, and we don't execute that method when current replica is ZERO
if (found >= slices.getReplicas(leaderEligibleReplicaTypes).size()) {
log.info("Enough replicas found to continue.");
return true;
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index 51aa6e024c8..a7690e0a6a1 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -1326,87 +1326,102 @@ public class ZkController implements Closeable {
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
}
- // in this case, we want to wait for the leader as long as the leader might
- // wait for a vote, at least - but also long enough that a large cluster has
- // time to get its act together
- String leaderUrl = getLeader(cloudDesc, leaderVoteWait + 600000);
-
- String ourUrl = ZkCoreNodeProps.getCoreUrl(baseUrl, coreName);
- log.debug("We are {} and leader is {}", ourUrl, leaderUrl);
- boolean isLeader = leaderUrl.equals(ourUrl);
- assert !isLeader || replica.getType().leaderEligible
- : replica.getType().name() + " replica became leader!";
-
- try (SolrCore core = cc.getCore(desc.getName())) {
-
- // recover from local transaction log and wait for it to complete before
- // going active
- // TODO: should this be moved to another thread? To recoveryStrat?
- // TODO: should this actually be done earlier, before (or as part of)
- // leader election perhaps?
-
- if (core == null) {
- throw new SolrException(
- ErrorCode.SERVICE_UNAVAILABLE, "SolrCore is no longer available to register");
- }
+ if (replica.getType() == Type.ZERO) {
+ // ZERO replicas: leadership is only needed during indexing.
+ // Replicas need to register for potential leadership on startup but that's it.
+ // As any ZERO replica can become leader at any time, no need to wait in an election
+ // ZERO replicas do not use the update log.
+ publish(desc, Replica.State.ACTIVE);
+ } else {
+ // Non ZERO replica types (NRT, TLOG, PULL): leadership is important. On startup,
+ // a replica needs to be designated as the leader or to synchronize with the leader.
+ // Until either happen, a replica can't be considered ACTIVE.
+
+ // in this case, we want to wait for the leader as long as the leader might
+ // wait for a vote, at least - but also long enough that a large cluster has
+ // time to get its act together
+ String leaderUrl = getLeader(cloudDesc, leaderVoteWait + 600000);
+
+ String ourUrl = ZkCoreNodeProps.getCoreUrl(baseUrl, coreName);
+ log.debug("We are {} and leader is {}", ourUrl, leaderUrl);
+ boolean isLeader = leaderUrl.equals(ourUrl);
+ assert !isLeader || replica.getType().leaderEligible
+ : replica.getType().name() + " replica became leader!";
+
+ try (SolrCore core = cc.getCore(desc.getName())) {
+
+ // recover from local transaction log and wait for it to complete before
+ // going active
+ // TODO: should this be moved to another thread? To recoveryStrat?
+ // TODO: should this actually be done earlier, before (or as part of)
+ // leader election perhaps?
+
+ if (core == null) {
+ throw new SolrException(
+ ErrorCode.SERVICE_UNAVAILABLE, "SolrCore is no longer available to register");
+ }
- UpdateLog ulog = core.getUpdateHandler().getUpdateLog();
- boolean isTlogReplicaAndNotLeader = replica.getType() == Replica.Type.TLOG && !isLeader;
- if (isTlogReplicaAndNotLeader) {
- String commitVersion = ReplicateFromLeader.getCommitVersion(core);
- if (commitVersion != null) {
- ulog.copyOverOldUpdates(Long.parseLong(commitVersion));
+ UpdateLog ulog = core.getUpdateHandler().getUpdateLog();
+ boolean isTlogReplicaAndNotLeader = replica.getType() == Replica.Type.TLOG && !isLeader;
+ if (isTlogReplicaAndNotLeader) {
+ String commitVersion = ReplicateFromLeader.getCommitVersion(core);
+ if (commitVersion != null) {
+ ulog.copyOverOldUpdates(Long.parseLong(commitVersion));
+ }
}
- }
- // we will call register again after zk expiration and on reload
- if (!afterExpiration && !core.isReloaded() && ulog != null && !isTlogReplicaAndNotLeader) {
- // disable recovery in case shard is in construction state (for shard splits)
- Slice slice = getClusterState().getCollection(collection).getSlice(shardId);
- if (slice.getState() != Slice.State.CONSTRUCTION || !isLeader) {
- Future<UpdateLog.RecoveryInfo> recoveryFuture =
- core.getUpdateHandler().getUpdateLog().recoverFromLog();
- if (recoveryFuture != null) {
- log.info(
- "Replaying tlog for {} during startup... NOTE: This can take a while.", ourUrl);
- recoveryFuture.get(); // NOTE: this could potentially block for
- // minutes or more!
- // TODO: public as recovering in the mean time?
- // TODO: in the future we could do peersync in parallel with recoverFromLog
- } else {
- if (log.isDebugEnabled()) {
- log.debug("No LogReplay needed for core={} baseURL={}", core.getName(), baseUrl);
+ // we will call register again after zk expiration and on reload
+ if (!afterExpiration
+ && !core.isReloaded()
+ && ulog != null
+ && !isTlogReplicaAndNotLeader) {
+ // disable recovery in case shard is in construction state (for shard splits)
+ Slice slice = getClusterState().getCollection(collection).getSlice(shardId);
+ if (slice.getState() != Slice.State.CONSTRUCTION || !isLeader) {
+ Future<UpdateLog.RecoveryInfo> recoveryFuture =
+ core.getUpdateHandler().getUpdateLog().recoverFromLog();
+ if (recoveryFuture != null) {
+ log.info(
+ "Replaying tlog for {} during startup... NOTE: This can take a while.", ourUrl);
+ recoveryFuture.get(); // NOTE: this could potentially block for
+ // minutes or more!
+ // TODO: public as recovering in the mean time?
+ // TODO: in the future we could do peersync in parallel with recoverFromLog
+ } else {
+ if (log.isDebugEnabled()) {
+ log.debug("No LogReplay needed for core={} baseURL={}", core.getName(), baseUrl);
+ }
}
}
}
- }
- boolean didRecovery =
- checkRecovery(
- recoverReloadedCores,
- isLeader,
- skipRecovery,
- collection,
- coreZkNodeName,
- shardId,
- core,
- cc,
- afterExpiration);
- if (!didRecovery) {
- if (isTlogReplicaAndNotLeader) {
- startReplicationFromLeader(coreName, true);
+ boolean didRecovery =
+ checkRecovery(
+ recoverReloadedCores,
+ isLeader,
+ skipRecovery,
+ collection,
+ coreZkNodeName,
+ shardId,
+ core,
+ cc,
+ afterExpiration);
+ if (!didRecovery) {
+ if (isTlogReplicaAndNotLeader) {
+ startReplicationFromLeader(coreName, true);
+ }
+ publish(desc, Replica.State.ACTIVE);
}
- publish(desc, Replica.State.ACTIVE);
- }
- if (replica.getType().leaderEligible) {
- // the watcher is added to a set so multiple calls of this method will left only one
- // watcher
- shardTerms.addListener(
- new RecoveringCoreTermWatcher(core.getCoreDescriptor(), getCoreContainer()));
+ if (replica.getType().leaderEligible) {
+ // the watcher is added to a set so multiple calls of this method will left only one
+ // watcher
+ shardTerms.addListener(
+ new RecoveringCoreTermWatcher(core.getCoreDescriptor(), getCoreContainer()));
+ }
+ core.getCoreDescriptor().getCloudDescriptor().setHasRegistered(true);
+ } catch (Exception e) {
+ unregister(coreName, desc, false);
+ throw e;
}
- core.getCoreDescriptor().getCloudDescriptor().setHasRegistered(true);
- } catch (Exception e) {
- unregister(coreName, desc, false);
- throw e;
}
// make sure we have an update cluster state right away
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java
index b7ed4954fbf..bb61b706344 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java
@@ -131,7 +131,9 @@ public class AddReplicaCmd implements CollApiCmds.CollectionApiCommand {
int timeout = message.getInt(TIMEOUT, 10 * 60); // 10 minutes
boolean parallel = message.getBool("parallel", false);
- ReplicaCount numReplicas = ReplicaCount.fromMessage(message, null, 1);
+ // Default replica type is NRT in SolrCloud, but if collection is Zero store based,
+ // default (and only acceptable type) is Replica.Type.ZERO
+ ReplicaCount numReplicas = ReplicaCount.fromMessage(message, null, 1, coll.isZeroIndex());
int totalReplicas = numReplicas.total();
if (totalReplicas > 1) {
@@ -236,6 +238,18 @@ public class AddReplicaCmd implements CollApiCmds.CollectionApiCommand {
boolean skipCreateReplicaInClusterState,
CreateReplica createReplica)
throws InterruptedException, KeeperException {
+ // Fail fast if adding wrong replicas to a collection
+ if (!coll.isZeroIndex() && createReplica.replicaType == Replica.Type.ZERO) {
+ throw new RuntimeException(
+ "Can't add a Replica.Type.ZERO to a collection not backed by Zero store");
+ }
+ if (coll.isZeroIndex() && createReplica.replicaType != Replica.Type.ZERO) {
+ throw new RuntimeException(
+ "Can't add a "
+ + createReplica.replicaType
+ + " replica to a collection backed by Zero store");
+ }
+
if (!skipCreateReplicaInClusterState) {
Map<String, Object> replicaProps =
Utils.makeMap(
@@ -397,7 +411,22 @@ public class AddReplicaCmd implements CollApiCmds.CollectionApiCommand {
boolean skipNodeAssignment = message.getBool(CollectionAdminParams.SKIP_NODE_ASSIGNMENT, false);
String sliceName = message.getStr(SHARD_ID_PROP);
DocCollection collection = clusterState.getCollection(collectionName);
+
int totalReplicas = numReplicas.total();
+ int numZeroReplicas = numReplicas.get(Replica.Type.ZERO);
+ if (numZeroReplicas != 0 && numZeroReplicas != totalReplicas) {
+ throw new RuntimeException(
+ "ZERO and non ZERO replica types specified for collection "
+ + collectionName
+ + ". ZERO: "
+ + numZeroReplicas
+ + " NRT: "
+ + numReplicas.get(Replica.Type.NRT)
+ + " PULL: "
+ + numReplicas.get(Replica.Type.PULL)
+ + " TLOG: "
+ + numReplicas.get(Replica.Type.TLOG));
+ }
String node = message.getStr(CoreAdminParams.NODE);
Object createNodeSetStr = message.get(CollectionHandlingUtils.CREATE_NODE_SET);
@@ -417,6 +446,7 @@ public class AddReplicaCmd implements CollApiCmds.CollectionApiCommand {
collection.getName(),
sliceName,
numReplicas,
+ collection.isZeroIndex(),
createNodeSetStr,
cloudManager,
coreContainer);
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java
index 1d7d3a4087d..b6d7fbd5f07 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java
@@ -324,6 +324,7 @@ public class Assign {
String collectionName,
String shard,
ReplicaCount numReplicas,
+ boolean isZeroIndex,
Object createNodeSet,
SolrCloudManager cloudManager,
CoreContainer coreContainer)
@@ -358,6 +359,7 @@ public class Assign {
.forCollection(collectionName)
.forShard(Collections.singletonList(shard))
.assignReplicas(numReplicas)
+ .setZeroIndex(isZeroIndex)
.onNodes(createNodeList)
.build();
AssignStrategy assignStrategy = createAssignStrategy(coreContainer);
@@ -496,16 +498,38 @@ public class Assign {
public final List<String> shardNames;
public final List<String> nodes;
public final ReplicaCount numReplicas;
+ public final boolean zeroIndex;
public AssignRequest(
String collectionName,
List<String> shardNames,
List<String> nodes,
- ReplicaCount numReplicas) {
+ ReplicaCount numReplicas,
+ boolean zeroIndex) {
this.collectionName = collectionName;
this.shardNames = shardNames;
this.nodes = nodes;
this.numReplicas = numReplicas;
+ this.zeroIndex = zeroIndex;
+
+ // ZERO replica related verifications
+ int totalReplicas = numReplicas.total();
+ int numZeroReplicas = numReplicas.get(Replica.Type.ZERO);
+ if (numZeroReplicas != 0 && numZeroReplicas != totalReplicas) {
+ throw new RuntimeException(
+ "Illegal combination of ZERO ("
+ + numZeroReplicas
+ + ") and non ZERO ("
+ + (totalReplicas - numZeroReplicas)
+ + ") replicas for collection "
+ + collectionName);
+ }
+ if (!zeroIndex && numZeroReplicas != 0) {
+ throw new RuntimeException(
+ "Can't specify ZERO replicas for collection "
+ + collectionName
+ + " non backed by Zero store");
+ }
}
}
@@ -514,6 +538,7 @@ public class Assign {
private List<String> shardNames;
private List<String> nodes;
private ReplicaCount numReplicas;
+ private boolean zeroIndex;
public AssignRequestBuilder() {
this.numReplicas = ReplicaCount.empty();
@@ -539,10 +564,15 @@ public class Assign {
return this;
}
+ public AssignRequestBuilder setZeroIndex(boolean zeroIndex) {
+ this.zeroIndex = zeroIndex;
+ return this;
+ }
+
public AssignRequest build() {
Objects.requireNonNull(collectionName, "The collectionName cannot be null");
Objects.requireNonNull(shardNames, "The shard names cannot be null");
- return new AssignRequest(collectionName, shardNames, nodes, numReplicas);
+ return new AssignRequest(collectionName, shardNames, nodes, numReplicas, zeroIndex);
}
}
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java
index d73f6f4e395..cc9511d7554 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java
@@ -55,6 +55,7 @@ import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.DocCollection.CollectionStateProps;
import org.apache.solr.common.cloud.DocRouter;
import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Replica.Type;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
@@ -109,9 +110,16 @@ public class CollectionHandlingUtils {
// Immutable Maps are null-hostile, so build our own
public static final Map<String, Object> COLLECTION_PROPS_AND_DEFAULTS =
- Collections.unmodifiableMap(makeCollectionPropsAndDefaults());
+ Collections.unmodifiableMap(makeCollectionPropsAndDefaults(false));
- private static Map<String, Object> makeCollectionPropsAndDefaults() {
+ /**
+ * Default values for a collection based on Zero store ({@link DocCollection#isZeroIndex()}
+ * returns {@code true}, replicas are {@link Type#ZERO}).
+ */
+ public static final Map<String, Object> ZERO_INDEX_COLLECTION_PROPS_AND_DEFAULTS =
+ Collections.unmodifiableMap(makeCollectionPropsAndDefaults(true));
+
+ private static Map<String, Object> makeCollectionPropsAndDefaults(boolean zeroIndex) {
Map<String, Object> propsAndDefaults =
Utils.makeMap(
CollectionStateProps.DOC_ROUTER,
@@ -119,11 +127,13 @@ public class CollectionHandlingUtils {
CollectionStateProps.REPLICATION_FACTOR,
"1",
CollectionStateProps.PER_REPLICA_STATE,
- null);
+ null,
+ ZkStateReader.ZERO_INDEX,
+ Boolean.toString(zeroIndex));
for (Replica.Type replicaType : Replica.Type.values()) {
propsAndDefaults.put(
replicaType.numReplicasPropertyName,
- replicaType == Replica.Type.defaultType() ? "1" : "0");
+ replicaType == Replica.Type.defaultType(zeroIndex) ? "1" : "0");
}
return propsAndDefaults;
}
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
index 020146cfdc7..344117c3a67 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
@@ -17,6 +17,7 @@
package org.apache.solr.cloud.api.collections;
+import static org.apache.solr.common.cloud.ZkStateReader.ZERO_INDEX;
import static org.apache.solr.common.params.CollectionAdminParams.ALIAS;
import static org.apache.solr.common.params.CollectionAdminParams.COLL_CONF;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
@@ -39,6 +40,7 @@ import java.util.NoSuchElementException;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
import org.apache.solr.client.solrj.cloud.AlreadyExistsException;
import org.apache.solr.client.solrj.cloud.BadVersionException;
import org.apache.solr.client.solrj.cloud.DelegatingCloudManager;
@@ -104,11 +106,20 @@ public class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {
if (ccc.getZkStateReader().aliasesManager != null) { // not a mock ZkStateReader
ccc.getZkStateReader().aliasesManager.update();
}
+ final boolean isZeroIndex = message.getBool(ZERO_INDEX, false);
final Aliases aliases = ccc.getZkStateReader().getAliases();
final String collectionName = message.getStr(NAME);
final boolean waitForFinalState = message.getBool(WAIT_FOR_FINAL_STATE, false);
final String alias = message.getStr(ALIAS, collectionName);
log.info("Create collection {}", collectionName);
+
+ if (isZeroIndex && !ccc.getCoreContainer().isZeroStoreEnabled()) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ "Collection "
+ + collectionName
+ + " cannot be created because Zero store is not enabled on this cluster.");
+ }
final boolean isPRS = message.getBool(CollectionStateProps.PER_REPLICA_STATE, false);
if (clusterState.hasCollection(collectionName)) {
throw new SolrException(
@@ -134,7 +145,7 @@ public class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {
// fail fast if parameters are wrong or incomplete
List<String> shardNames = populateShardNames(message, router);
- ReplicaCount numReplicas = getNumReplicas(message);
+ ReplicaCount numReplicas = getNumReplicas(message, isZeroIndex);
DocCollection newColl = null;
final String collectionPath = DocCollection.getCollectionPath(collectionName);
@@ -223,6 +234,15 @@ public class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {
"Could not fully create collection: " + collectionName);
}
+ if (isZeroIndex) {
+ for (String shardName : shardNames) {
+ ccc.getCoreContainer()
+ .getZeroStoreManager()
+ .getZeroMetadataController()
+ .createMetadataNode(collectionName, shardName);
+ }
+ }
+
// refresh cluster state (value read below comes from Zookeeper watch firing following the
// update done previously, be it by Overseer or by this thread when updates are distributed)
clusterState = ccc.getSolrCloudManager().getClusterState();
@@ -521,10 +541,16 @@ public class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {
"It's unusual to run two replica of the same slice on the same Solr-instance.");
}
+ // true if the collection is backed by Zero store.
+ // In which case only org.apache.solr.common.cloud.Replica.Type#Zero
+ // replicas matter and no other replicas should be defined.
+ final boolean zeroIndex = message.getBool(ZERO_INDEX, false);
+
Assign.AssignRequest assignRequest =
new Assign.AssignRequestBuilder()
.forCollection(collectionName)
.forShard(shardNames)
+ .setZeroIndex(zeroIndex)
.assignReplicas(numReplicas)
.onNodes(nodeList)
.build();
@@ -580,18 +606,34 @@ public class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {
throw new SolrException(
ErrorCode.BAD_REQUEST, CollectionHandlingUtils.NUM_SLICES + " must be > 0");
}
+
+ // TODO unclear if the following bloc of code should be here (belongs directly in main?)
+ // When specified, retrieve shard names from the request
+ // If the list length does not match the parameter for number of shards, fallback to the
+ // default names
+ // This is mostly used when creating a collection for a restore
+ Object shards = message.get(CollectionHandlingUtils.SHARDS_PROP);
+ if (shards instanceof Map) {
+ List<String> providedNames =
+ ((Map<?, ?>) shards)
+ .keySet().stream().map(Object::toString).collect(Collectors.toList());
+ if (providedNames.size() == numSlices) {
+ return providedNames;
+ }
+ }
+
ClusterStateMutator.getShardNames(numSlices, shardNames);
}
return shardNames;
}
- private ReplicaCount getNumReplicas(ZkNodeProps message) {
- ReplicaCount numReplicas = ReplicaCount.fromMessage(message);
+ private ReplicaCount getNumReplicas(ZkNodeProps message, boolean isZeroIndex) {
+ ReplicaCount numReplicas = ReplicaCount.fromMessage(message, isZeroIndex);
boolean hasLeaderEligibleReplica = numReplicas.hasLeaderReplica();
- if (!hasLeaderEligibleReplica && !numReplicas.contains(Replica.Type.defaultType())) {
+ if (!hasLeaderEligibleReplica && !numReplicas.contains(Replica.Type.defaultType(isZeroIndex))) {
// Ensure that there is at least one replica that can become leader if the user did
// not force a replica count.
- numReplicas.put(Replica.Type.defaultType(), 1);
+ numReplicas.put(Replica.Type.defaultType(isZeroIndex), 1);
} else if (!hasLeaderEligibleReplica) {
// This can still fail if the user manually forced "0" replica counts.
throw new SolrException(
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateShardCmd.java
index b76250cfd8d..9c59fd923f8 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateShardCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateShardCmd.java
@@ -66,7 +66,8 @@ public class CreateShardCmd implements CollApiCmds.CollectionApiCommand {
}
DocCollection collection = clusterState.getCollection(collectionName);
- ReplicaCount numReplicas = ReplicaCount.fromMessage(message, collection, 1);
+ boolean isZeroIndex = collection.isZeroIndex();
+ ReplicaCount numReplicas = ReplicaCount.fromMessage(message, collection, 1, isZeroIndex);
if (!numReplicas.hasLeaderReplica()) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
@@ -75,6 +76,14 @@ public class CreateShardCmd implements CollApiCmds.CollectionApiCommand {
+ "), there must be at least one leader-eligible replica");
}
+ if (isZeroIndex) {
+ // create the shard metadataSuffix znode
+ ccc.getCoreContainer()
+ .getZeroStoreManager()
+ .getZeroMetadataController()
+ .createMetadataNode(extCollectionName, sliceName);
+ }
+
if (ccc.getDistributedClusterStateUpdater().isDistributedStateUpdate()) {
// The message has been crafted by CollectionsHandler.CollectionOperation.CREATESHARD_OP and
// defines the QUEUE_OPERATION to be CollectionParams.CollectionAction.CREATESHARD. Likely a
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java
index a1e9012aaf0..7c8081345d7 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java
@@ -47,6 +47,7 @@ import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.snapshots.SolrSnapshotManager;
import org.apache.solr.handler.admin.ConfigSetsHandler;
+import org.apache.solr.zero.process.ZeroStoreManager;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -139,6 +140,14 @@ public class DeleteCollectionCmd implements CollApiCmds.CollectionApiCommand {
}
}
+ // Delete the collection files from Zero store. We want to delete all the files before we
+ // delete the collection state from ZooKeeper.
+ DocCollection docCollection = zkStateReader.getClusterState().getCollectionOrNull(collection);
+ if (docCollection != null && docCollection.isZeroIndex()) {
+ ZeroStoreManager zeroStoreManager = ccc.getCoreContainer().getZeroStoreManager();
+ zeroStoreManager.deleteCollection(collection);
+ }
+
ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, DELETE.toLower(), NAME, collection);
if (ccc.getDistributedClusterStateUpdater().isDistributedStateUpdate()) {
ccc.getDistributedClusterStateUpdater()
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteShardCmd.java
index 72c93c02c62..cc942801607 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteShardCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteShardCmd.java
@@ -43,6 +43,7 @@ import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.zero.process.ZeroStoreManager;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -182,6 +183,11 @@ public class DeleteShardCmd implements CollApiCmds.CollectionApiCommand {
log.debug("Waiting for delete shard action to complete");
cleanupLatch.await(1, TimeUnit.MINUTES);
+ if (clusterState.getCollection(collectionName).isZeroIndex()) {
+ ZeroStoreManager zeroStoreManager = ccc.getCoreContainer().getZeroStoreManager();
+ zeroStoreManager.deleteShard(collectionName, slice.getName());
+ }
+
ZkNodeProps m =
new ZkNodeProps(
Overseer.QUEUE_OPERATION,
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java
index cdcb3c27449..3ffc061cbfd 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java
@@ -56,6 +56,11 @@ import org.apache.solr.util.TimeOut;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+/**
+ * TODO Migration is NOT working with Zero store based collections (having replicas of type {@link
+ * org.apache.solr.common.cloud.Replica.Type#ZERO}). <br>
+ * If/once we make it work, delete this comment.
+ */
public class MigrateCmd implements CollApiCmds.CollectionApiCommand {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final CollectionCommandContext ccc;
@@ -307,7 +312,7 @@ public class MigrateCmd implements CollApiCmds.CollectionApiCommand {
CREATE.toLower(),
NAME,
tempSourceCollectionName,
- Replica.Type.defaultType().numReplicasPropertyName,
+ Replica.Type.defaultType(false).numReplicasPropertyName, // TODO does not work for ZERO
1,
CollectionHandlingUtils.NUM_SLICES,
1,
@@ -393,7 +398,7 @@ public class MigrateCmd implements CollApiCmds.CollectionApiCommand {
ccc.getSolrCloudManager().getDistribStateManager(),
zkStateReader.getClusterState().getCollection(tempSourceCollectionName),
tempSourceSlice.getName(),
- Replica.Type.defaultType());
+ Replica.Type.defaultType(false)); // TODO does not work for ZERO
props = new HashMap<>();
props.put(Overseer.QUEUE_OPERATION, ADDREPLICA.toLower());
props.put(COLLECTION_PROP, tempSourceCollectionName);
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java
index 91299b3c259..a65a5f8c73d 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java
@@ -66,6 +66,10 @@ import org.slf4j.LoggerFactory;
/**
* Reindex a collection, usually in order to change the index schema.
*
+ * <p><b>TODO Reindex is NOT working with Zero store based collections (having replicas of type
+ * {@link org.apache.solr.common.cloud.Replica.Type#ZERO}). <br>
+ * If/once we make it work, delete this comment.</b>
+ *
* <p>WARNING: Reindexing is potentially a lossy operation - some indexed data that is not available
* as stored fields may be irretrievably lost, so users should use this command with caution,
* evaluating the potential impact by using different source and target collection names first, and
@@ -249,7 +253,8 @@ public class ReindexCollectionCmd implements CollApiCmds.CollectionApiCommand {
String query = message.getStr(CommonParams.Q, "*:*");
String fl = message.getStr(CommonParams.FL, "*");
Integer rf = message.getInt(ZkStateReader.REPLICATION_FACTOR, coll.getReplicationFactor());
- ReplicaCount numReplicas = ReplicaCount.fromMessage(message, coll);
+ // TODO REINDEXCOLLECTION currently doesn't work with ZERO collections
+ ReplicaCount numReplicas = ReplicaCount.fromMessage(message, coll, false);
int numShards = message.getInt(ZkStateReader.NUM_SHARDS_PROP, coll.getActiveSlices().size());
DocRouter router = coll.getRouter();
if (router == null) {
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java
index b774663b1e2..8b6a7f079ed 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java
@@ -21,6 +21,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR;
import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_TYPE;
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.ZERO_INDEX;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATESHARD;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
@@ -214,11 +215,24 @@ public class RestoreCmd implements CollApiCmds.CollectionApiCommand {
*/
private class RestoreOnANewCollection {
private final ReplicaCount numReplicas;
+ private final boolean zeroIndex;
private final ZkNodeProps message;
private RestoreOnANewCollection(ZkNodeProps message, DocCollection backupCollectionState) {
this.message = message;
- this.numReplicas = ReplicaCount.fromMessage(message, backupCollectionState);
+
+ this.zeroIndex = message.getBool(CollectionAdminParams.ZERO_INDEX, false);
+ if (zeroIndex == backupCollectionState.isZeroIndex()) {
+ // Use the replica counts specified in the message and for those not present, in the backup
+ this.numReplicas = ReplicaCount.fromMessage(message, backupCollectionState, zeroIndex);
+ } else {
+ // Restoring a backed up Zero collection into a non Zero collection or the other way around.
+ // Can't use the replica counts in the backed up collection, they are not compatible with
+ // the collection being created.
+ // Use whatever the message asks for and if the message says nothing, ask for 2 replicas
+ // for the default type.
+ this.numReplicas = ReplicaCount.fromMessage(message, null, 2, zeroIndex);
+ }
}
public void process(NamedList<Object> results, RestoreContext rc) throws Exception {
@@ -318,11 +332,25 @@ public class RestoreCmd implements CollApiCmds.CollectionApiCommand {
propMap.put(Overseer.QUEUE_OPERATION, CREATE.toString());
// mostly true. Prevents autoCreated=true in the collection state.
propMap.put("fromApi", "true");
- propMap.put(REPLICATION_FACTOR, numReplicas.get(Replica.Type.defaultType()));
- numReplicas.writeProps(propMap);
+ propMap.put(REPLICATION_FACTOR, numReplicas.get(Replica.Type.defaultType(zeroIndex)));
+
+ if (zeroIndex == backupCollectionState.isZeroIndex()) {
+ // When restoring into same type of collection, inherit from backed up collection replica
+ // counts not specified in the message (forces specifying zero counts for undesired replica)
+ numReplicas.writeProps(propMap);
+ } else {
+ // When restoring a ZERO collection into a non ZERO collection and vice versa, can't use
+ // incompatible defaults coming from the backed up collection as done in the loop below.
+ propMap.put(ZERO_INDEX, zeroIndex);
+ numReplicas.writePropsOrZeros(propMap);
+ }
// inherit settings from input API, defaulting to the backup's setting. Ex: replicationFactor
- for (String collProp : CollectionHandlingUtils.COLLECTION_PROPS_AND_DEFAULTS.keySet()) {
+ for (String collProp :
+ (zeroIndex
+ ? CollectionHandlingUtils.ZERO_INDEX_COLLECTION_PROPS_AND_DEFAULTS
+ : CollectionHandlingUtils.COLLECTION_PROPS_AND_DEFAULTS)
+ .keySet()) {
Object val =
message.getProperties().getOrDefault(collProp, backupCollectionState.get(collProp));
if (val != null && propMap.get(collProp) == null) {
@@ -401,6 +429,7 @@ public class RestoreCmd implements CollApiCmds.CollectionApiCommand {
.forCollection(restoreCollection)
.forShard(sliceNames)
.assignReplicas(numReplicas)
+ .setZeroIndex(zeroIndex)
.onNodes(nodeList)
.build();
Assign.AssignStrategy assignStrategy = Assign.createAssignStrategy(ccc.getCoreContainer());
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
index 92c93e18b0e..7e11f405bec 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
@@ -50,6 +50,7 @@ import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.cloud.DistributedClusterStateUpdater;
import org.apache.solr.cloud.Overseer;
+import org.apache.solr.cloud.ZkController;
import org.apache.solr.cloud.api.collections.CollectionHandlingUtils.ShardRequestTracker;
import org.apache.solr.cloud.overseer.OverseerAction;
import org.apache.solr.common.LinkedHashMapWriter;
@@ -274,6 +275,7 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
params.set(CoreAdminParams.GET_RANGES, "true");
params.set(CommonAdminParams.SPLIT_METHOD, splitMethod.toLower());
params.set(CoreAdminParams.CORE, parentShardLeader.getStr("core"));
+ params.set(CoreAdminParams.REPLICA_TYPE, leaderReplicaType.toString());
// only 2 sub-shards are currently supported
// int numSubShards = message.getInt(NUM_SUB_SHARDS, DEFAULT_NUM_SUB_SHARDS);
// params.set(NUM_SUB_SHARDS, Integer.toString(numSubShards));
@@ -370,7 +372,6 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
String subSlice = subSlices.get(i);
String subShardName = subShardNames.get(i);
DocRouter.Range subRange = subRanges.get(i);
-
log.debug("Creating slice {} of collection {} on {}", subSlice, collectionName, nodeName);
LinkedHashMapWriter<Object> propMap = new LinkedHashMapWriter<>();
@@ -383,6 +384,13 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
propMap.put("shard_parent_node", nodeName);
propMap.put("shard_parent_zk_session", leaderZnodeStat.getEphemeralOwner());
+ if (collection.isZeroIndex()) {
+ ccc.getCoreContainer()
+ .getZeroStoreManager()
+ .getZeroMetadataController()
+ .createMetadataNode(collectionName, subSlice);
+ }
+
if (ccc.getDistributedClusterStateUpdater().isDistributedStateUpdate()) {
ccc.getDistributedClusterStateUpdater()
.doSingleStateUpdate(
@@ -488,6 +496,7 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.SPLIT.toString());
params.set(CommonAdminParams.SPLIT_METHOD, splitMethod.toLower());
params.set(CoreAdminParams.CORE, parentShardLeader.getStr("core"));
+ params.set(CoreAdminParams.REPLICA_TYPE, leaderReplicaType.toString());
for (int i = 0; i < subShardNames.size(); i++) {
String subShardName = subShardNames.get(i);
params.add(CoreAdminParams.TARGET_CORE, subShardName);
@@ -551,6 +560,7 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
.forCollection(collectionName)
.forShard(subSlices)
.assignReplicas(numReplicas)
+ .setZeroIndex(collection.isZeroIndex())
.onNodes(
Assign.getLiveOrLiveAndCreateNodeSetList(
clusterState.getLiveNodes(),
@@ -1242,4 +1252,12 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
}
}
}
+
+ public static boolean shardSplitLockHeld(
+ ZkController zkController, String collection, String shard) throws Exception {
+ // Would have been better to use a SolrCloudManager instance for the check rather than a
+ // ZkController but caller doesn't have such a reference
+ String path = ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection + "/" + shard + "-splitting";
+ return zkController.pathExists(path);
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/ClusterStateMutator.java b/solr/core/src/java/org/apache/solr/cloud/overseer/ClusterStateMutator.java
index 11b3795745b..e7081f850a5 100644
--- a/solr/core/src/java/org/apache/solr/cloud/overseer/ClusterStateMutator.java
+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/ClusterStateMutator.java
@@ -71,6 +71,8 @@ public class ClusterStateMutator {
Object messageShardsObj = message.get("shards");
+ boolean isZeroIndex = message.getBool(ZkStateReader.ZERO_INDEX, false);
+
Map<String, Slice> slices;
// we are being explicitly told the slice data (e.g. coll restore)
if (messageShardsObj instanceof Map) {
@@ -104,11 +106,15 @@ public class ClusterStateMutator {
Map<String, Object> collectionProps = new HashMap<>();
- for (Map.Entry<String, Object> e :
- CollectionHandlingUtils.COLLECTION_PROPS_AND_DEFAULTS.entrySet()) {
+ Map<String, Object> collectionDefaults =
+ isZeroIndex
+ ? CollectionHandlingUtils.ZERO_INDEX_COLLECTION_PROPS_AND_DEFAULTS
+ : CollectionHandlingUtils.COLLECTION_PROPS_AND_DEFAULTS;
+
+ for (Map.Entry<String, Object> e : collectionDefaults.entrySet()) {
Object val = message.get(e.getKey());
if (val == null) {
- val = CollectionHandlingUtils.COLLECTION_PROPS_AND_DEFAULTS.get(e.getKey());
+ val = collectionDefaults.get(e.getKey());
}
if (val != null) collectionProps.put(e.getKey(), val);
}
diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/CollectionMutator.java b/solr/core/src/java/org/apache/solr/cloud/overseer/CollectionMutator.java
index 088c4f8fed7..397cb288db8 100644
--- a/solr/core/src/java/org/apache/solr/cloud/overseer/CollectionMutator.java
+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/CollectionMutator.java
@@ -62,6 +62,7 @@ public class CollectionMutator {
String shardId = message.getStr(ZkStateReader.SHARD_ID_PROP);
DocCollection collection = clusterState.getCollection(collectionName);
Slice slice = collection.getSlice(shardId);
+
if (slice == null) {
Map<String, Replica> replicas = Collections.emptyMap();
Map<String, Object> sliceProps = new HashMap<>();
@@ -149,7 +150,8 @@ public class CollectionMutator {
// SOLR-11676 : keep NRT_REPLICAS and REPLICATION_FACTOR in sync
if (prop.equals(REPLICATION_FACTOR)) {
props.put(
- Replica.Type.defaultType().numReplicasPropertyName, message.get(REPLICATION_FACTOR));
+ Replica.Type.defaultType(coll.isZeroIndex()).numReplicasPropertyName,
+ message.get(REPLICATION_FACTOR));
}
}
}
diff --git a/solr/core/src/java/org/apache/solr/cluster/Replica.java b/solr/core/src/java/org/apache/solr/cluster/Replica.java
index 2d6978c63f1..14a4b4a1d2c 100644
--- a/solr/core/src/java/org/apache/solr/cluster/Replica.java
+++ b/solr/core/src/java/org/apache/solr/cluster/Replica.java
@@ -36,11 +36,15 @@ public interface Replica {
/** {@link Node} on which this {@link Replica} is located. */
Node getNode();
- /** The order of this enum is important from the most to least "important" replica type. */
+ /**
+ * The order of this enum is important from the most to least "important" replica type. {@link
+ * ReplicaType#ZERO} is different though, its position in the enum does not matter.
+ */
enum ReplicaType {
NRT('n'),
TLOG('t'),
- PULL('p');
+ PULL('p'),
+ ZERO('z');
private final char suffixChar;
diff --git a/solr/core/src/java/org/apache/solr/cluster/placement/impl/SimpleClusterAbstractionsImpl.java b/solr/core/src/java/org/apache/solr/cluster/placement/impl/SimpleClusterAbstractionsImpl.java
index cb86dc304e5..cfae7cdb0fd 100644
--- a/solr/core/src/java/org/apache/solr/cluster/placement/impl/SimpleClusterAbstractionsImpl.java
+++ b/solr/core/src/java/org/apache/solr/cluster/placement/impl/SimpleClusterAbstractionsImpl.java
@@ -392,6 +392,8 @@ class SimpleClusterAbstractionsImpl {
return Replica.ReplicaType.TLOG;
case PULL:
return Replica.ReplicaType.PULL;
+ case ZERO:
+ return Replica.ReplicaType.ZERO;
default:
throw new RuntimeException("Unexpected " + type);
}
@@ -456,6 +458,8 @@ class SimpleClusterAbstractionsImpl {
return org.apache.solr.common.cloud.Replica.Type.TLOG;
case PULL:
return org.apache.solr.common.cloud.Replica.Type.PULL;
+ case ZERO:
+ return org.apache.solr.common.cloud.Replica.Type.ZERO;
default:
throw new IllegalArgumentException("Unknown " + type);
}
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 1da0802cf08..3455721c061 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -37,6 +37,7 @@ import java.lang.invoke.MethodHandles;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
@@ -152,6 +153,7 @@ import org.apache.solr.util.OrderedExecutor;
import org.apache.solr.util.RefCounted;
import org.apache.solr.util.StartupLoggingUtils;
import org.apache.solr.util.stats.MetricUtils;
+import org.apache.solr.zero.process.ZeroStoreManager;
import org.apache.zookeeper.KeeperException;
import org.glassfish.hk2.utilities.binding.AbstractBinder;
import org.glassfish.jersey.server.ApplicationHandler;
@@ -270,6 +272,8 @@ public class CoreContainer {
protected MetricsHandler metricsHandler;
+ protected ZeroStoreManager zeroStoreManager;
+
private volatile SolrClientCache solrClientCache;
private volatile Map<String, SolrCache<?, ?>> caches;
@@ -1009,6 +1013,12 @@ public class CoreContainer {
if (isZooKeeperAware()) {
metricManager.loadClusterReporters(metricReporters, this);
+
+ ZeroConfig zeroConfig = getNodeConfig().getZeroConfig();
+ if (zeroConfig.isZeroStoreEnabled()) {
+ log.info("Zero store is enabled in Solr Cloud. Building ZeroStoreManager.");
+ zeroStoreManager = new ZeroStoreManager(this, zeroConfig);
+ }
}
// setup executor to load cores in parallel
@@ -1031,6 +1041,23 @@ public class CoreContainer {
new Object[] {this});
try {
List<CoreDescriptor> cds = coresLocator.discover(this);
+ if (isZeroStoreEnabled()) {
+ // We can start on a clean node and might be missing cores of ZERO replicas that belong to
+ // this node.
+ // We will make sure we create those missing core descriptors (if any) and add to the list
+ // of discovered core descriptors.
+ // Later in this method the cores will be created for those additional core descriptors
+ // because isLoadOnStartup is set to true for them
+ List<CoreDescriptor> additionalCoreDescriptors =
+ getZeroStoreManager().discoverAdditionalCoreDescriptorsForZeroReplicas(cds, this);
+
+ if (!additionalCoreDescriptors.isEmpty()) {
+ // enhance the list of discovered cores with the additional ones we just
+ // discovered/created
+ cds = new ArrayList<>(cds);
+ cds.addAll(additionalCoreDescriptors);
+ }
+ }
cds = coreSorter.sort(cds);
checkForDuplicateCoreNames(cds);
status |= CORE_DISCOVERY_COMPLETE;
@@ -1340,6 +1367,10 @@ public class CoreContainer {
metricManager.closeReporters(
SolrMetricManager.getRegistryName(SolrInfoBean.Group.cluster));
}
+
+ if (zeroStoreManager != null) {
+ zeroStoreManager.shutdown();
+ }
}
try {
@@ -1740,6 +1771,11 @@ public class CoreContainer {
throw e;
}
solrCores.removeCoreDescriptor(dcore);
+
+ if (isZeroStoreEnabled()) {
+ zeroStoreManager.evictCoreZeroMetadata(dcore);
+ }
+
final SolrException solrException =
new SolrException(
ErrorCode.SERVER_ERROR, "Unable to create core [" + dcore.getName() + "]", e);
@@ -1753,6 +1789,11 @@ public class CoreContainer {
t);
coreInitFailures.put(dcore.getName(), new CoreLoadFailure(dcore, e));
solrCores.removeCoreDescriptor(dcore);
+
+ if (isZeroStoreEnabled()) {
+ zeroStoreManager.evictCoreZeroMetadata(dcore);
+ }
+
if (core != null && !core.isClosed()) IOUtils.closeQuietly(core);
throw t;
} finally {
@@ -2122,7 +2163,6 @@ public class CoreContainer {
*/
public void unload(
String name, boolean deleteIndexDir, boolean deleteDataDir, boolean deleteInstanceDir) {
-
CoreDescriptor cd = solrCores.getCoreDescriptor(name);
if (name != null) {
@@ -2136,6 +2176,9 @@ public class CoreContainer {
// If last time around we didn't successfully load, make sure that all traces of the
// coreDescriptor are gone.
if (cd != null) {
+ if (isZeroStoreEnabled()) {
+ zeroStoreManager.evictCoreZeroMetadata(cd);
+ }
solrCores.removeCoreDescriptor(cd);
coresLocator.delete(this, cd);
}
@@ -2152,6 +2195,9 @@ public class CoreContainer {
boolean close = solrCores.isLoadedNotPendingClose(name);
SolrCore core = solrCores.remove(name);
+ if (isZeroStoreEnabled()) {
+ zeroStoreManager.evictCoreZeroMetadata(cd);
+ }
solrCores.removeCoreDescriptor(cd);
coresLocator.delete(this, cd);
if (core == null) {
@@ -2205,6 +2251,9 @@ public class CoreContainer {
// The old coreDescriptor is obsolete, so remove it. registerCore will put it back.
CoreDescriptor cd = core.getCoreDescriptor();
solrCores.removeCoreDescriptor(cd);
+ if (isZeroStoreEnabled()) {
+ zeroStoreManager.evictCoreZeroMetadata(cd);
+ }
cd.setProperty("name", toName);
solrCores.addCoreDescriptor(cd);
core.setName(toName);
@@ -2414,6 +2463,10 @@ public class CoreContainer {
return zkSys.getZkController();
}
+ public ZeroStoreManager getZeroStoreManager() {
+ return zeroStoreManager;
+ }
+
public NodeConfig getConfig() {
return cfg;
}
@@ -2507,6 +2560,10 @@ public class CoreContainer {
return tragicException != null;
}
+ public boolean isZeroStoreEnabled() {
+ return getZeroStoreManager() != null;
+ }
+
public ContainerPluginsRegistry getContainerPluginsRegistry() {
return containerPluginsRegistry;
}
diff --git a/solr/core/src/java/org/apache/solr/core/NodeConfig.java b/solr/core/src/java/org/apache/solr/core/NodeConfig.java
index ef1cbbf2dfd..e2af4c81d19 100644
--- a/solr/core/src/java/org/apache/solr/core/NodeConfig.java
+++ b/solr/core/src/java/org/apache/solr/core/NodeConfig.java
@@ -112,7 +112,7 @@ public class NodeConfig {
private final String managementPath;
private final PluginInfo[] backupRepositoryPlugins;
-
+ private final ZeroConfig zeroConfig;
private final MetricsConfig metricsConfig;
private final Map<String, CacheConfig> cachesConfig;
@@ -151,6 +151,7 @@ public class NodeConfig {
SolrResourceLoader loader,
Properties solrProperties,
PluginInfo[] backupRepositoryPlugins,
+ ZeroConfig zeroConfig,
MetricsConfig metricsConfig,
Map<String, CacheConfig> cachesConfig,
PluginInfo tracerConfig,
@@ -190,6 +191,7 @@ public class NodeConfig {
this.loader = loader;
this.solrProperties = solrProperties;
this.backupRepositoryPlugins = backupRepositoryPlugins;
+ this.zeroConfig = zeroConfig;
this.metricsConfig = metricsConfig;
this.cachesConfig = cachesConfig == null ? Collections.emptyMap() : cachesConfig;
this.tracerConfig = tracerConfig;
@@ -416,6 +418,10 @@ public class NodeConfig {
return backupRepositoryPlugins;
}
+ public ZeroConfig getZeroConfig() {
+ return zeroConfig;
+ }
+
public MetricsConfig getMetricsConfig() {
return metricsConfig;
}
@@ -602,6 +608,7 @@ public class NodeConfig {
private String managementPath;
private Properties solrProperties = new Properties();
private PluginInfo[] backupRepositoryPlugins;
+ private ZeroConfig zeroConfig;
private MetricsConfig metricsConfig;
private Map<String, CacheConfig> cachesConfig;
private PluginInfo tracerConfig;
@@ -653,6 +660,7 @@ public class NodeConfig {
setSolrDataHome(System.getProperty(SolrXmlConfig.SOLR_DATA_HOME));
setConfigSetBaseDirectory("configsets");
this.metricsConfig = new MetricsConfig.MetricsConfigBuilder().build();
+ this.zeroConfig = new ZeroConfig.ZeroConfigBuilder().build();
}
public NodeConfigBuilder setCoreRootDirectory(String coreRootDirectory) {
@@ -783,6 +791,11 @@ public class NodeConfig {
return this;
}
+ public NodeConfigBuilder setZeroConfig(ZeroConfig zeroConfig) {
+ this.zeroConfig = zeroConfig;
+ return this;
+ }
+
public NodeConfigBuilder setMetricsConfig(MetricsConfig metricsConfig) {
this.metricsConfig = metricsConfig;
return this;
@@ -911,6 +924,7 @@ public class NodeConfig {
loader,
solrProperties,
backupRepositoryPlugins,
+ zeroConfig,
metricsConfig,
cachesConfig,
tracerConfig,
diff --git a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
index 4c15cd9b7a9..9097901cb3f 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
@@ -166,6 +166,7 @@ public class SolrXmlConfig {
if (cloudConfig != null) configBuilder.setCloudConfig(cloudConfig);
configBuilder.setBackupRepositoryPlugins(
getBackupRepositoryPluginInfos(root.get("backup").getAll("repository")));
+ configBuilder.setZeroConfig(getZeroConfig(root.get("zero")));
configBuilder.setClusterPlugins(getClusterPlugins(loader, root));
// <metrics><hiddenSysProps></metrics> will be removed in Solr 10, but until then, use it if a
// <hiddenSysProps> is not provided under <solr>.
@@ -253,6 +254,7 @@ public class SolrXmlConfig {
assertSingleInstance(root.getAll("logging"), "logging");
assertSingleInstance(root.get("logging").getAll("watcher"), "logging/watcher");
assertSingleInstance(root.getAll("backup"), "backup");
+ assertSingleInstance(root.getAll("zero"), "zero");
assertSingleInstance(root.getAll("coreAdminHandlerActions"), "coreAdminHandlerActions");
}
@@ -649,6 +651,22 @@ public class SolrXmlConfig {
return configs;
}
+ private static ZeroConfig getZeroConfig(ConfigNode zero) {
+ ZeroConfig.ZeroConfigBuilder builder = new ZeroConfig.ZeroConfigBuilder();
+ boolean enabled = zero.boolAttr("enabled", true);
+ builder.setEnabled(enabled);
+ if (enabled) {
+ builder.setBackupRepositoryPlugins(
+ getPluginsInfo(zero.get("repositories").getAll("repository")));
+ } else {
+ if (log.isInfoEnabled()) {
+ log.info("Zero store (ZERO replica) type is disabled.");
+ }
+ }
+
+ return builder.build();
+ }
+
private static PluginInfo[] getClusterPlugins(SolrResourceLoader loader, ConfigNode root) {
List<PluginInfo> clusterPlugins = new ArrayList<>();
@@ -827,4 +845,13 @@ public class SolrXmlConfig {
if (cfg == null || !cfg.exists()) return null;
return new PluginInfo(cfg, cfg.name(), false, true);
}
+
+ private static PluginInfo[] getPluginsInfo(List<ConfigNode> cfgs) {
+ List<PluginInfo> configs = new ArrayList<>();
+ for (ConfigNode cfg : cfgs) {
+ PluginInfo pluginInfo = getPluginInfo(cfg);
+ if (pluginInfo != null) configs.add(pluginInfo);
+ }
+ return configs.toArray(new PluginInfo[0]);
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/core/ZeroConfig.java b/solr/core/src/java/org/apache/solr/core/ZeroConfig.java
new file mode 100644
index 00000000000..1f4f87f792d
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/core/ZeroConfig.java
@@ -0,0 +1,277 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.core;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.util.concurrent.TimeUnit;
+import org.apache.solr.zero.process.CorePuller;
+
+/**
+ * In memory representation of the {@code <zero> ... </zero>} section of solr.xml configuration file
+ * For now contains only the description of a BackupRepository to be used as an abstraction over
+ * desired Zero store. Multiple repositories can be defined, but only one will be used : - the
+ * enabled one - the first of the enabled ones if multiple are enabled - the first one if none are
+ * enabled the zero section must contain enabled = true for the repositories configuration to be
+ * loaded
+ */
+public class ZeroConfig {
+
+ private final PluginInfo[] repositories;
+ private final boolean enabled;
+
+ @VisibleForTesting
+ public ZeroConfig() {
+ this(false, new PluginInfo[0]);
+ }
+
+ private ZeroConfig(boolean enabled, PluginInfo[] repositories) {
+ this.enabled = enabled;
+ this.repositories = repositories;
+ }
+
+ public boolean isEnabled() {
+ return enabled;
+ }
+
+ private static final PluginInfo[] NO_OP_REPOSITORIES = new PluginInfo[0];
+
+ public PluginInfo[] getBackupRepositoryPlugins() {
+ if (enabled) {
+ return repositories;
+ } else {
+ return NO_OP_REPOSITORIES;
+ }
+ }
+
+ /**
+ * Limit to the number of Zero store files to delete accepted on the delete queue (and lost in
+ * case of server crash). When the queue reaches that size, no more deletes are accepted (will be
+ * retried later for a core, next time it is pushed).
+ */
+ private final int almostMaxDeleterQueueSize = 200;
+
+ private final int deleterThreadPoolSize = 5;
+
+ private final int maxDeleteAttempts = 50;
+ private final long deleteSleepMsFailedAttempt = TimeUnit.SECONDS.toMillis(10);
+
+ /**
+ * After a successful core push has marked a file to be deleted, wait at least this long before
+ * actually deleting it from the Zero store. This is in case another node is still fetching files
+ * from the Zero store based on an older version of shard.properties that includes that file in
+ * the commit point.
+ *
+ * <p>Note the time at which a file is marked deleted is when it got added to the new
+ * shard.metadata file (or actually to the {@link
+ * org.apache.solr.zero.metadata.ZeroStoreShardMetadata}), not when the push finished in success.
+ * If the push took a very long time, by the time the new shard.metadata is visible, it's possible
+ * that all newly added files to delete are already ripe for deleting (which is likely ok, and
+ * even if that makes another node's pull to fail, upon retry that node will be able to pull the
+ * core correctly).
+ */
+ private final long deleteDelayMs = TimeUnit.MINUTES.toMillis(3);
+
+ /** Enables/disables Zero store for a Solr node. */
+ private final boolean zeroStoreEnabled =
+ Boolean.getBoolean(ZeroSystemProperty.ZeroStoreEnabled.getPropertyName());
+
+ /**
+ * Thread pool size for the executor that pushes files to Zero store. Defaults to 30 threads for
+ * now, to support higher indexing throughput.
+ */
+ private final int numFilePusherThreads =
+ Integer.getInteger(ZeroSystemProperty.numFilePusherThreads.getPropertyName(), 30);
+
+ /**
+ * Thread pool size for the executor that pulls files from Zero store. Defaults to 100 threads.
+ */
+ private final int numFilePullerThreads =
+ Integer.getInteger(ZeroSystemProperty.numFilePullerThreads.getPropertyName(), 100);
+
+ /** Max number of cores being concurrenty pulled async. Defaults to 10 cores. */
+ private final int numCorePullerThreads =
+ Integer.getInteger(ZeroSystemProperty.numCorePullerThreads.getPropertyName(), 10);
+
+ /**
+ * Number of automatic retries (after a transient IO error) to pull a core before giving up. This
+ * is the number of retries in addition to the initial pull attempt. The automatic retries are
+ * only triggered when the error is considered as transient. Defaults to 10. Attempts are spaced
+ * by at least {@link #corePullRetryDelayMs}), which means we'll retry for at least 180 seconds
+ * before giving up. This is something to adjust as the implementation of the delay between
+ * retries is cleaned up, see {@link CorePuller}.
+ */
+ private final int numCorePullAutoRetries =
+ Integer.getInteger(ZeroSystemProperty.numCorePullAutoRetries.getPropertyName(), 10);
+
+ /**
+ * Retry delay, in milliseconds, after a core pull failure when loading a core. Defaults to 20s.
+ */
+ private final long corePullRetryDelayMs =
+ TimeUnit.SECONDS.toMillis(
+ Integer.getInteger(ZeroSystemProperty.corePullRetryDelayS.getPropertyName(), 20));
+
+ /**
+ * Base delay, in milliseconds, between two attempts to pull core index files when starting an
+ * index update. Then the effective delay is computed by multiplying this base delay with the
+ * number of previous consecutive failed attempts. Defaults to 3s.
+ */
+ private final long corePullAttemptDelayMs =
+ TimeUnit.SECONDS.toMillis(
+ Integer.getInteger(ZeroSystemProperty.corePullAttemptDelayS.getPropertyName(), 3));
+
+ /**
+ * Maximum number of consecutive failed attempts to pull a core. The number of attempts takes into
+ * account the automatic retries, if any, and the request-based attempts. For example, if a core
+ * is corrupted on the Zero store side, then after this maximum number of attempts to pull it and
+ * open the index, no more pulls will be done and any attempt will fail with an exception.
+ *
+ * <p>TODO : not used as intended / seems to act as a duplicate to numCorePullAutoRetries
+ */
+ private final int maxFailedCorePullAttempts =
+ Integer.getInteger(ZeroSystemProperty.maxFailedCorePullAttempts.getPropertyName(), 20);
+
+ /**
+ * Delay, in milliseconds, before allowing another attempts when all pull attempts have been
+ * exhausted. Defaults to 5 min.
+ */
+ private final long allAttemptsExhaustedRetryDelayMs =
+ TimeUnit.MINUTES.toMillis(
+ Integer.getInteger(
+ ZeroSystemProperty.allAttemptsExhaustedRetryDelayMin.getPropertyName(), 5));
+
+ public boolean isZeroStoreEnabled() {
+ return zeroStoreEnabled;
+ }
+
+ /** Thread pool size for the executor that pushes files to Zero store. */
+ public int getNumFilePusherThreads() {
+ return numFilePusherThreads;
+ }
+
+ /** Thread pool size for the executor that pulls files from Zero store. */
+ public int getNumFilePullerThreads() {
+ return numFilePullerThreads;
+ }
+
+ /** Max number of cores being concurrenty pulled async */
+ public int getNumCorePullerThreads() {
+ return numCorePullerThreads;
+ }
+
+ /**
+ * Number of automatic retries to pull a core before giving up. This is the number of retries in
+ * addition to the initial pull attempt. The automatic retries are only triggered when the error
+ * is considered as transient.
+ *
+ * <p>TODO requires a review as this configuration is only used from tests ;(
+ */
+ public int getNumCorePullAutoRetries() {
+ return numCorePullAutoRetries;
+ }
+
+ /** Retry delay, in milliseconds, after a core pull failure when loading a core. */
+ public long getCorePullRetryDelay() {
+ return corePullRetryDelayMs;
+ }
+
+ /**
+ * Base delay, in milliseconds, between two attempts to pull core index files when starting an
+ * index update. Then the effective delay is computed by multiplying this base delay with the
+ * number of previous consecutive failed attempts.
+ */
+ public long getCorePullAttemptDelay() {
+ return corePullAttemptDelayMs;
+ }
+
+ /** Maximum number of consecutive failed attempts to pull a core. */
+ public int getMaxFailedCorePullAttempts() {
+ return maxFailedCorePullAttempts;
+ }
+
+ public long getDeleteDelayMs() {
+ return deleteDelayMs;
+ }
+
+ public int getAlmostMaxDeleterQueueSize() {
+ return almostMaxDeleterQueueSize;
+ }
+
+ public int getDeleterThreadPoolSize() {
+ return deleterThreadPoolSize;
+ }
+
+ public int getMaxDeleteAttempts() {
+ return maxDeleteAttempts;
+ }
+
+ public long getDeleteSleepMsFailedAttempt() {
+ return deleteSleepMsFailedAttempt;
+ }
+
+ /**
+ * Delay, in milliseconds, before allowing another attempts when all pull attempts have been
+ * exhausted.
+ */
+ public long getAllAttemptsExhaustedRetryDelay() {
+ return allAttemptsExhaustedRetryDelayMs;
+ }
+
+ /** Enum of constants representing the system properties used by the Zero store feature */
+ public enum ZeroSystemProperty {
+ ZeroStoreEnabled("zeroStoreEnabled"),
+ numFilePusherThreads("zeroStoreNumFilePusherThreads"),
+ numFilePullerThreads("zeroStoreNumFilePullerThreads"),
+ numCorePullerThreads("zeroStoreNumCorePullerThreads"),
+ numCorePullAutoRetries("zeroStoreNumCorePullAutoRetries"),
+ corePullRetryDelayS("zeroStoreCorePullRetryDelayS"),
+ corePullAttemptDelayS("zeroStoreCorePullAttemptDelayS"),
+ maxFailedCorePullAttempts("zeroStoreMaxFailedCorePullAttempts"),
+ allAttemptsExhaustedRetryDelayMin("zeroStoreAllAttemptsExhaustedRetryDelayMin"),
+ ;
+
+ private final String name;
+
+ ZeroSystemProperty(String propName) {
+ name = propName;
+ }
+
+ public String getPropertyName() {
+ return name;
+ }
+ }
+
+ public static class ZeroConfigBuilder {
+ private PluginInfo[] repositoryPlugins = new PluginInfo[0];
+ private boolean enabled = true;
+
+ public ZeroConfigBuilder() {}
+
+ public ZeroConfigBuilder setEnabled(boolean enabled) {
+ this.enabled = enabled;
+ return this;
+ }
+
+ public ZeroConfigBuilder setBackupRepositoryPlugins(PluginInfo[] repositoryPlugins) {
+ this.repositoryPlugins = repositoryPlugins != null ? repositoryPlugins : new PluginInfo[0];
+ return this;
+ }
+
+ public ZeroConfig build() {
+ return new ZeroConfig(enabled, repositoryPlugins);
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
index a5451dc1b2b..3f09ffec0d6 100644
--- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
@@ -69,6 +69,7 @@ import org.apache.solr.api.JerseyResource;
import org.apache.solr.client.api.model.SolrJerseyResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.ModifiableSolrParams;
@@ -82,6 +83,7 @@ import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.SuppressForbidden;
import org.apache.solr.core.CloseHook;
import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.core.IndexDeletionPolicyWrapper;
import org.apache.solr.core.SolrCore;
@@ -266,6 +268,10 @@ public class ReplicationHandler extends RequestHandlerBase
final SolrParams solrParams = req.getParams();
String command = solrParams.required().get(COMMAND);
+ if (isZeroReplica(core)) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "ZERO replicas should never peer replicate!");
+ }
+
// This command does not give the current index version of the leader
// It gives the current 'replicateable' index version
if (command.equals(CMD_INDEX_VERSION)) {
@@ -312,6 +318,13 @@ public class ReplicationHandler extends RequestHandlerBase
}
}
+ private boolean isZeroReplica(SolrCore core) {
+ CoreDescriptor cd = core.getCoreDescriptor();
+ return cd != null
+ && cd.getCloudDescriptor() != null
+ && cd.getCloudDescriptor().getReplicaType() == Replica.Type.ZERO;
+ }
+
static boolean getBoolWithBackwardCompatibility(
SolrParams params, String preferredKey, String alternativeKey, boolean defaultValue) {
Boolean value = params.getBool(preferredKey);
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index 485670a4421..7429c1b7314 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -1053,7 +1053,10 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
DocCollection.verifyProp(m, prop);
}
if (m.get(REPLICATION_FACTOR) != null) {
- m.put(Replica.Type.defaultType().numReplicasPropertyName, m.get(REPLICATION_FACTOR));
+ // TODO assuming a non Zero index collection here! MODIFYCOLLECTION doesn't work on ZERO
+ // yet
+ m.put(
+ Replica.Type.defaultType(false).numReplicasPropertyName, m.get(REPLICATION_FACTOR));
}
return m;
}),
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/SplitOp.java b/solr/core/src/java/org/apache/solr/handler/admin/SplitOp.java
index fd29d31fcd0..c970d2db6c5 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/SplitOp.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/SplitOp.java
@@ -43,10 +43,12 @@ import org.apache.solr.common.cloud.CompositeIdRouter;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.DocCollection.CollectionStateProps;
import org.apache.solr.common.cloud.DocRouter;
+import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.params.CommonAdminParams;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
@@ -56,6 +58,7 @@ import org.apache.solr.update.SplitIndexCommand;
import org.apache.solr.update.UpdateHandler;
import org.apache.solr.util.RTimer;
import org.apache.solr.util.RefCounted;
+import org.apache.solr.zero.process.ZeroStoreManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -75,8 +78,51 @@ class SplitOp implements CoreAdminHandler.CoreAdminOp {
return true;
}
+ /**
+ * This method delegates the actual split work to {@link
+ * #executeInternal(CoreAdminHandler.CallInfo)} and does additional work for ZERO replicas.
+ */
+ @SuppressWarnings("try")
@Override
public void execute(CoreAdminHandler.CallInfo it) throws Exception {
+ SolrParams params = it.req.getParams();
+ List<SolrCore> newCores = null;
+ try {
+ if (!Replica.Type.ZERO.name().equals(params.get(CoreAdminParams.REPLICA_TYPE))) {
+ // Not a ZERO replica? Or not even SolrCloud? delegate without doing anything else
+ newCores = executeInternal(it);
+ } else {
+ // ZERO replica: need to pull from Zero store first and hold the indexing lock during the
+ // whole process
+ CoreContainer coreContainer = it.handler.getCoreContainer();
+ try (SolrCore core = coreContainer.getCore(params.get(CoreAdminParams.CORE))) {
+ ZeroStoreManager storeManager = coreContainer.getZeroStoreManager();
+ storeManager.pullCoreFromZeroStore(core);
+ try (AutoCloseable ignore = storeManager.acquireIndexingLockForSplit(core)) {
+ newCores = executeInternal(it);
+ }
+ // Now push the new cores to Zero store (if parent is ZERO, so are the children)
+ if (newCores != null) {
+ for (SolrCore newCore : newCores) {
+ newCore.getCoreContainer().getZeroStoreManager().initialCorePushToZeroStore(newCore);
+ }
+ }
+ }
+ }
+ } finally {
+ if (newCores != null) {
+ for (SolrCore newCore : newCores) {
+ newCore.close();
+ }
+ }
+ }
+ }
+
+ /**
+ * @return the list of created {@link SolrCore} or possibly <code>null</code>. If returns
+ * non-null, caller must close all the cores.
+ */
+ private List<SolrCore> executeInternal(CoreAdminHandler.CallInfo it) throws Exception {
SolrParams params = it.req.getParams();
String splitKey = params.get("split.key");
String[] newCoreNames = params.getParams("targetCore");
@@ -86,7 +132,7 @@ class SplitOp implements CoreAdminHandler.CoreAdminOp {
// to calculate the prefix ranges, and do the actual split in a separate request
if (params.getBool(GET_RANGES, false)) {
handleGetRanges(it, cname);
- return;
+ return null;
}
// if not using splitByPrefix, determine split partitions
@@ -231,12 +277,10 @@ class SplitOp implements CoreAdminHandler.CoreAdminOp {
} finally {
if (req != null) req.close();
if (parentCore != null) parentCore.close();
- if (newCores != null) {
- for (SolrCore newCore : newCores) {
- newCore.close();
- }
- }
+ // closing newCores is caller's responsibility
}
+
+ return newCores;
}
/**
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollection.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollection.java
index 4042d303be1..50893d6c444 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollection.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollection.java
@@ -33,6 +33,8 @@ import static org.apache.solr.common.params.CollectionAdminParams.PROPERTY_PREFI
import static org.apache.solr.common.params.CollectionAdminParams.PULL_REPLICAS;
import static org.apache.solr.common.params.CollectionAdminParams.REPLICATION_FACTOR;
import static org.apache.solr.common.params.CollectionAdminParams.TLOG_REPLICAS;
+import static org.apache.solr.common.params.CollectionAdminParams.ZERO_INDEX;
+import static org.apache.solr.common.params.CollectionAdminParams.ZERO_REPLICAS;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE;
import static org.apache.solr.common.params.CoreAdminParams.NAME;
@@ -198,16 +200,31 @@ public class CreateCollection extends AdminAPIBase implements CreateCollectionAp
} else {
rawProperties.put(CREATE_NODE_SET, "EMPTY");
}
- // 'nrtReplicas' and 'replicationFactor' are both set on the remote message, despite being
- // functionally equivalent.
- if (reqBody.replicationFactor != null) {
- rawProperties.put(REPLICATION_FACTOR, reqBody.replicationFactor);
- if (reqBody.nrtReplicas == null) rawProperties.put(NRT_REPLICAS, reqBody.replicationFactor);
- }
- if (reqBody.nrtReplicas != null) {
- rawProperties.put(NRT_REPLICAS, reqBody.nrtReplicas);
- if (reqBody.replicationFactor == null)
- rawProperties.put(REPLICATION_FACTOR, reqBody.nrtReplicas);
+ if (reqBody.isZeroIndex == null || !reqBody.isZeroIndex) {
+ // 'nrtReplicas' and 'replicationFactor' are both set on the remote message, despite being
+ // functionally equivalent.
+ if (reqBody.replicationFactor != null) {
+ rawProperties.put(REPLICATION_FACTOR, reqBody.replicationFactor);
+ if (reqBody.nrtReplicas == null) rawProperties.put(NRT_REPLICAS, reqBody.replicationFactor);
+ }
+ if (reqBody.nrtReplicas != null) {
+ rawProperties.put(NRT_REPLICAS, reqBody.nrtReplicas);
+ if (reqBody.replicationFactor == null)
+ rawProperties.put(REPLICATION_FACTOR, reqBody.nrtReplicas);
+ }
+ } else {
+ rawProperties.put(ZERO_INDEX, true);
+ // Zero Index, there are only ZERO replicas
+ if (reqBody.replicationFactor != null) {
+ rawProperties.put(REPLICATION_FACTOR, reqBody.replicationFactor);
+ if (reqBody.zeroReplicas == null)
+ rawProperties.put(ZERO_REPLICAS, reqBody.replicationFactor);
+ }
+ if (reqBody.zeroReplicas != null) {
+ rawProperties.put(ZERO_REPLICAS, reqBody.zeroReplicas);
+ if (reqBody.replicationFactor == null)
+ rawProperties.put(REPLICATION_FACTOR, reqBody.zeroReplicas);
+ }
}
if (reqBody.properties != null) {
@@ -314,6 +331,10 @@ public class CreateCollection extends AdminAPIBase implements CreateCollectionAp
requestBody.tlogReplicas = params.getInt(ZkStateReader.TLOG_REPLICAS);
requestBody.pullReplicas = params.getInt(ZkStateReader.PULL_REPLICAS);
requestBody.nrtReplicas = params.getInt(ZkStateReader.NRT_REPLICAS);
+ requestBody.zeroReplicas = params.getInt(ZkStateReader.ZERO_REPLICAS);
+ // At collection creation (only) we can decide if it is Zero store based (i.e. having only
+ // ZERO replicas)
+ requestBody.isZeroIndex = params.getBool(ZkStateReader.ZERO_INDEX);
requestBody.waitForFinalState = params.getBool(WAIT_FOR_FINAL_STATE);
requestBody.perReplicaState = params.getBool(PER_REPLICA_STATE);
requestBody.alias = params.get(ALIAS);
@@ -339,6 +360,14 @@ public class CreateCollection extends AdminAPIBase implements CreateCollectionAp
"Cannot specify both replicationFactor and nrtReplicas as they mean the same thing");
}
+ if (requestBody.replicationFactor != null
+ && requestBody.zeroReplicas != null
+ && (!requestBody.replicationFactor.equals(requestBody.zeroReplicas))) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ "Cannot specify both replicationFactor and zeroReplicas as they mean the same thing");
+ }
+
SolrIdentifierValidator.validateCollectionName(requestBody.name);
if (requestBody.shardNames != null && !requestBody.shardNames.isEmpty()) {
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollectionAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollectionAPI.java
index e1e071e6418..9d8f5dd0612 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollectionAPI.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollectionAPI.java
@@ -26,6 +26,7 @@ import static org.apache.solr.common.params.CollectionAdminParams.CREATE_NODE_SE
import static org.apache.solr.common.params.CollectionAdminParams.CREATE_NODE_SET_SHUFFLE_PARAM;
import static org.apache.solr.common.params.CollectionAdminParams.PROPERTY_PREFIX;
import static org.apache.solr.common.params.CollectionAdminParams.REPLICATION_FACTOR;
+import static org.apache.solr.common.params.CollectionAdminParams.ZERO_INDEX;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
import static org.apache.solr.common.params.CommonParams.NAME;
import static org.apache.solr.common.params.CoreAdminParams.BACKUP_ID;
@@ -77,7 +78,8 @@ public class RestoreCollectionAPI extends BackupAPIBase {
COLL_CONF,
REPLICATION_FACTOR,
CREATE_NODE_SET_PARAM,
- CREATE_NODE_SET_SHUFFLE_PARAM))
+ CREATE_NODE_SET_SHUFFLE_PARAM,
+ ZERO_INDEX))
.collect(Collectors.toUnmodifiableSet());
@Inject
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCore.java b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCore.java
index 3997b1971b4..eaf988aa44c 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCore.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCore.java
@@ -25,6 +25,7 @@ import org.apache.solr.client.api.model.RestoreCoreRequestBody;
import org.apache.solr.client.api.model.SolrJerseyResponse;
import org.apache.solr.cloud.CloudDescriptor;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore;
@@ -144,6 +145,14 @@ public class RestoreCore extends CoreAdminAPIBase implements RestoreCoreApi {
if (updateLog != null) {
updateLog.applyBufferedUpdates();
}
+
+ // Now push the core to the Zero store if applicable
+ if (core.getCoreDescriptor()
+ .getCloudDescriptor()
+ .getReplicaType()
+ .equals(Replica.Type.ZERO)) {
+ core.getCoreContainer().getZeroStoreManager().initialCorePushToZeroStore(core);
+ }
}
}
diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
index 92b8f090dfc..97e61d6b942 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
@@ -73,6 +73,7 @@ import org.apache.solr.util.circuitbreaker.CircuitBreaker;
import org.apache.solr.util.circuitbreaker.CircuitBreakerRegistry;
import org.apache.solr.util.plugin.PluginInfoInitialized;
import org.apache.solr.util.plugin.SolrCoreAware;
+import org.apache.solr.zero.process.ZeroStoreManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
@@ -460,6 +461,24 @@ public class SearchHandler extends RequestHandlerBase
SolrQueryTimeoutImpl.set(req);
try {
+ /*
+ * Stopgap: This is not an end-to-end query freshness solution for ZERO replica. Rather,
+ * it is just taking care of a special case: if a core has never synced with the Zero
+ * store then it should fail the query.
+ *
+ * This stopgap is only taking care of the query path that goes through SearchHandler. This
+ * is not a generic enough place.
+ * A cleaner solution would have to find a more central place to take care of different
+ * entry points where we care about freshness.
+ *
+ * See also comment in HttpSolrCall.init()
+ */
+ CoreContainer coreContainer = core.getCoreContainer();
+ if (coreContainer.isZeroStoreEnabled()) {
+ ZeroStoreManager zeroStoreManager = coreContainer.getZeroStoreManager();
+ zeroStoreManager.ensureZeroCoreFreshness(core);
+ }
+
// The semantics of debugging vs not debugging are different enough that
// it makes sense to have two control loops
if (!rb.isDebug()) {
diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
index e93c412676e..afd31b8e554 100644
--- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
+++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
@@ -88,6 +88,7 @@ import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.common.util.CommandOperation;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.JsonSchemaValidator;
@@ -331,6 +332,27 @@ public class HttpSolrCall {
solrReq = parser.parse(core, path, req);
}
+ if (cores.isZooKeeperAware()) {
+ String collectionName = core.getCoreDescriptor().getCloudDescriptor().getCollectionName();
+ DocCollection collection = getCollection(collectionName);
+ if (collection != null && collection.isZeroIndex()) {
+ // TODO this is not clean, the decision to pull from Zero store should not be made here
+ // Needs to be moved to the appropriate handlers/components (enqueue pull by default
+ // from all handlers and allow handlers to disable that enqueue, in which case allow
+ // components in that handler to explicitly pull)
+ if (doesPathContainUpdate()) {
+ // Zero collection only supports hard commits therefore we always ensure one
+ addCommitIfAbsent();
+ // don't enqueue a pull on updates as those will already trigger their own synchronous
+ // pulls
+ } else {
+ if (pathRequiresZeroStorePulling(path)) {
+ cores.getZeroStoreManager().enqueueCorePullFromZeroStore(core);
+ }
+ }
+ }
+ }
+
invalidStates =
checkStateVersionsAreValid(solrReq.getParams().get(CloudSolrClient.STATE_VERSION));
@@ -345,6 +367,46 @@ public class HttpSolrCall {
action = PASSTHROUGH;
}
+ private void addCommitIfAbsent() {
+ Boolean currentValue = solrReq.getParams().getBool(UpdateParams.COMMIT);
+ if (currentValue == null || !currentValue) {
+ ModifiableSolrParams params = new ModifiableSolrParams(solrReq.getParams());
+ params.set(UpdateParams.COMMIT, "true");
+ // normally commits wait for a searcher, but we only need durability here so don't wait
+ if (params.get(UpdateParams.WAIT_SEARCHER) == null) {
+ params.set(UpdateParams.WAIT_SEARCHER, false);
+ }
+
+ solrReq.setParams(params);
+ }
+ }
+
+ /** TODO this is ugly we shouldn't hard code a path that could change */
+ protected boolean doesPathContainUpdate() {
+ return path.endsWith("/update") || path.contains("/update/");
+ }
+
+ /**
+ * Find out if our request should trigger a pull from Zero store when local core is stale.
+ *
+ * <p>TODO this is ugly we shouldn't hard code a path that could change
+ */
+ protected boolean pathRequiresZeroStorePulling(String servletPath) {
+ // get the request handler from the path (taken from SolrDispatchFilter)
+ int idx = servletPath.indexOf('/');
+ if (idx != -1) {
+ String action = servletPath.substring(idx);
+ return action.startsWith("/select")
+ || action.startsWith("/spellcheck")
+ || action.startsWith("/result_promotion")
+ || action.startsWith("/indexLookup")
+ || action.startsWith("/highlight")
+ || action.startsWith("/backup");
+ } else {
+ return false;
+ }
+ }
+
protected void autoCreateSystemColl(String corename) throws Exception {
if (core == null
&& SYSTEM_COLL.equals(corename)
@@ -1030,16 +1092,20 @@ public class HttpSolrCall {
return result;
}
- protected SolrCore getCoreByCollection(String collectionName, boolean isPreferLeader) {
+ protected DocCollection getCollection(String collectionName) {
ZkStateReader zkStateReader = cores.getZkController().getZkStateReader();
-
ClusterState clusterState = zkStateReader.getClusterState();
- DocCollection collection = clusterState.getCollectionOrNull(collectionName, true);
+
+ return clusterState.getCollectionOrNull(collectionName, true);
+ }
+
+ protected SolrCore getCoreByCollection(String collectionName, boolean isPreferLeader) {
+ DocCollection collection = getCollection(collectionName);
if (collection == null) {
return null;
}
- Set<String> liveNodes = clusterState.getLiveNodes();
+ Set<String> liveNodes = cores.getZkController().getClusterState().getLiveNodes();
if (isPreferLeader) {
List<Replica> leaderReplicas =
diff --git a/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java b/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
index 732304af8bb..da70e0c1406 100644
--- a/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
+++ b/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
@@ -30,9 +30,11 @@ import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.MergePolicy;
import org.apache.lucene.search.Sort;
import org.apache.solr.cloud.ActionThrottle;
+import org.apache.solr.cloud.CloudDescriptor;
import org.apache.solr.cloud.RecoveryStrategy;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.cloud.Replica;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.DirectoryFactory;
@@ -295,6 +297,11 @@ public final class DefaultSolrCoreState extends SolrCoreState
@Override
public void doRecovery(CoreContainer cc, CoreDescriptor cd) {
+ CloudDescriptor cloudDescriptor = cd.getCloudDescriptor();
+ if (cloudDescriptor != null && cloudDescriptor.getReplicaType() == Replica.Type.ZERO) {
+ log.info("Skipping recovery because ZERO replicas don't need it.");
+ return;
+ }
Runnable recoveryTask =
new Runnable() {
diff --git a/solr/core/src/java/org/apache/solr/update/UpdateLog.java b/solr/core/src/java/org/apache/solr/update/UpdateLog.java
index c9301356a0c..088dfa1baf9 100644
--- a/solr/core/src/java/org/apache/solr/update/UpdateLog.java
+++ b/solr/core/src/java/org/apache/solr/update/UpdateLog.java
@@ -105,6 +105,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
private boolean debug = log.isDebugEnabled();
private boolean trace = log.isTraceEnabled();
private boolean usableForChildDocs;
+ private boolean rejectUpdatesWhileBuffering;
public enum SyncLevel {
NONE,
@@ -349,9 +350,12 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
dataDir = (String) info.initArgs.get("dir");
defaultSyncLevel = SyncLevel.getSyncLevel((String) info.initArgs.get("syncLevel"));
+ rejectUpdatesWhileBuffering =
+ Boolean.TRUE.equals(info.initArgs.getBooleanArg("rejectUpdatesWhileBuffering"));
numRecordsToKeep = objToInt(info.initArgs.get("numRecordsToKeep"), 100);
maxNumLogsToKeep = objToInt(info.initArgs.get("maxNumLogsToKeep"), 10);
numVersionBuckets = objToInt(info.initArgs.get("numVersionBuckets"), 65536);
+
if (numVersionBuckets <= 0)
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR,
@@ -591,6 +595,18 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
return Long.parseLong(last.substring(TLOG_NAME.length() + 1));
}
+ /**
+ * Throw an exception if rejectUpdatesWhileBuffering=true. Method called while in BUFFERING state.
+ */
+ private void throwIfRejectUpdatesWhileBuffering() {
+ if (rejectUpdatesWhileBuffering) {
+ throw new SolrException(
+ ErrorCode.SERVICE_UNAVAILABLE,
+ "UpdateLog initialized with "
+ + "rejectUpdatesWhileBuffering=true, and we are in BUFFERING state. Rejecting update.");
+ }
+ }
+
public void add(AddUpdateCommand cmd) {
add(cmd, false);
}
@@ -602,6 +618,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
synchronized (this) {
if ((cmd.getFlags() & UpdateCommand.BUFFERING) != 0) {
+ throwIfRejectUpdatesWhileBuffering();
ensureBufferTlog();
bufferTlog.write(cmd);
return;
@@ -671,6 +688,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
synchronized (this) {
if ((cmd.getFlags() & UpdateCommand.BUFFERING) != 0) {
+ throwIfRejectUpdatesWhileBuffering();
ensureBufferTlog();
bufferTlog.writeDelete(cmd);
return;
@@ -700,6 +718,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
public void deleteByQuery(DeleteUpdateCommand cmd) {
synchronized (this) {
if ((cmd.getFlags() & UpdateCommand.BUFFERING) != 0) {
+ throwIfRejectUpdatesWhileBuffering();
ensureBufferTlog();
bufferTlog.writeDeleteByQuery(cmd);
return;
@@ -1740,7 +1759,8 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
recoveryInfo = new RecoveryInfo();
if (log.isInfoEnabled()) {
- log.info("Starting to buffer updates. {}", this);
+ log.info(
+ "Starting to {} updates. {}", rejectUpdatesWhileBuffering ? "reject" : "buffer", this);
}
state = State.BUFFERING;
diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessorFactory.java
index 141ee7ff0b8..71787e9cd55 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessorFactory.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessorFactory.java
@@ -19,6 +19,7 @@ package org.apache.solr.update.processor;
import java.util.Collections;
import java.util.Set;
import java.util.TreeSet;
+import org.apache.solr.common.cloud.Replica;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
@@ -56,9 +57,17 @@ public class DistributedUpdateProcessorFactory extends UpdateRequestProcessorFac
DistributedUpdateProcessor distribUpdateProcessor =
isZkAware
- ? new DistributedZkUpdateProcessor(req, rsp, next)
+ ? getZkAwareInstance(req, rsp, next)
: new DistributedUpdateProcessor(req, rsp, next);
// note: will sometimes return DURP (no overhead) instead of wrapping
return RoutedAliasUpdateProcessor.wrap(req, distribUpdateProcessor);
}
+
+ private static DistributedUpdateProcessor getZkAwareInstance(
+ SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) {
+ return req.getCore().getCoreDescriptor().getCloudDescriptor().getReplicaType()
+ == Replica.Type.ZERO
+ ? new ZeroStoreUpdateProcessor(req, rsp, next)
+ : new DistributedZkUpdateProcessor(req, rsp, next);
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
index 35d96a7c14a..a4c09049bce 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
@@ -76,12 +76,12 @@ import org.slf4j.LoggerFactory;
public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
- private final CloudDescriptor cloudDesc;
- private final ZkController zkController;
+ protected final CloudDescriptor cloudDesc;
+ protected final ZkController zkController;
private final SolrCmdDistributor cmdDistrib;
protected List<SolrCmdDistributor.Node> nodes;
private Set<String> skippedCoreNodeNames;
- private final String collection;
+ protected final String collection;
private boolean readOnlyCollection = false;
private boolean broadcastDeleteById = false;
@@ -127,7 +127,7 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
}
}
- private boolean isReadOnly() {
+ protected boolean isReadOnly() {
return readOnlyCollection || req.getCore().readOnly;
}
@@ -270,6 +270,8 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
// replicationTracker to null if we aren't the leader or subShardLeader
checkReplicationTracker(cmd);
+ postSetupHook();
+
super.processAdd(cmd);
}
@@ -366,6 +368,8 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
// replicationTracker to null if we aren't the leader or subShardLeader
checkReplicationTracker(cmd);
+ postSetupHook();
+
super.doDeleteById(cmd);
}
@@ -459,6 +463,9 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
// check if client has requested minimum replication factor information. will set
// replicationTracker to null if we aren't the leader or subShardLeader
checkReplicationTracker(cmd);
+
+ postSetupHook();
+
super.doDeleteByQuery(cmd, replicas, coll);
}
@@ -712,27 +719,7 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
clusterState = zkController.getClusterState();
DocCollection coll = clusterState.getCollection(collection);
- Slice slice = coll.getRouter().getTargetSlice(id, doc, route, req.getParams(), coll);
-
- if (slice == null) {
- // No slice found. Most strict routers will have already thrown an exception, so a null
- // return is a signal to use the slice of this core.
- // TODO: what if this core is not in the targeted collection?
- String shardId = cloudDesc.getShardId();
- slice = coll.getSlice(shardId);
- if (slice == null) {
- throw new SolrException(
- SolrException.ErrorCode.BAD_REQUEST, "No shard " + shardId + " in " + coll);
- }
- // if doc == null, then this is a DeleteById request with missing route, flag for forwarding
- // to all shard leaders
- if (doc == null
- && coll.getRouter() instanceof CompositeIdRouter
- && coll.getActiveSlicesMap().size() > 1) {
- broadcastDeleteById = true;
- }
- }
-
+ Slice slice = getTargetSlice(coll, id, doc, route);
DistribPhase phase = DistribPhase.parseParam(req.getParams().get(DISTRIB_UPDATE_PARAM));
if (DistribPhase.FROMLEADER == phase && !couldIbeSubShardLeader(coll)) {
@@ -800,6 +787,36 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
}
}
+ protected void postSetupHook() {}
+
+ protected Slice getTargetSlice(
+ DocCollection coll, String id, SolrInputDocument doc, String route) {
+ Slice slice = coll.getRouter().getTargetSlice(id, doc, route, req.getParams(), coll);
+
+ if (slice == null) {
+ // No slice found. Most strict routers will have already thrown an exception, so a null
+ // return is a signal to use the slice of this core.
+ // TODO: what if this core is not in the targeted collection?
+ String shardId = cloudDesc.getShardId();
+ slice = coll.getSlice(shardId);
+
+ if (slice == null) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST, "No shard " + shardId + " in " + coll);
+ }
+
+ // if doc == null, then this is a DeleteById request with missing route, flag for forwarding
+ // to all shard leaders
+ if (doc == null
+ && coll.getRouter() instanceof CompositeIdRouter
+ && coll.getActiveSlicesMap().size() > 1) {
+ broadcastDeleteById = true;
+ }
+ }
+
+ return slice;
+ }
+
@Override
protected boolean shouldCloneCmdDoc() {
boolean willDistrib = isLeader && nodes != null && nodes.size() > 0;
diff --git a/solr/core/src/java/org/apache/solr/update/processor/ZeroStoreUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/ZeroStoreUpdateProcessor.java
new file mode 100644
index 00000000000..97706486382
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/update/processor/ZeroStoreUpdateProcessor.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.update.processor;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.solr.cloud.api.collections.SplitShardCmd;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Replica.Type;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.update.CommitUpdateCommand;
+import org.apache.solr.update.DeleteUpdateCommand;
+import org.apache.solr.update.SolrCmdDistributor;
+import org.apache.solr.update.UpdateCommand;
+import org.apache.solr.zero.process.ZeroCoreIndexingBatchProcessor;
+
+/**
+ * This class is extending and replacing {@link DistributedZkUpdateProcessor} to be used in {@link
+ * DistributedUpdateProcessorFactory} when dealing with cores for {@link
+ * org.apache.solr.common.cloud.Replica.Type#ZERO} replicas.
+ *
+ * <p>It the glue for adding Zero related code to normal SolrCloud processing: Rejection if shard
+ * splits are in progress is done in this class, and actual work delegated to {@link
+ * ZeroCoreIndexingBatchProcessor} (this only happens if the request is to be processed locally as
+ * opposed to being forwarded to other nodes/replicas).
+ *
+ * <p>Delegated work is calling {@link
+ * ZeroCoreIndexingBatchProcessor#addOrDeleteGoingToBeIndexedLocally} when an indexing batch starts
+ * and {@link ZeroCoreIndexingBatchProcessor#hardCommitCompletedLocally} after local hard commit was
+ * done.
+ *
+ * <p>Plugging into the {@code updateRequestProcessorChain} of {@code solrconfig.xml} is not ideal,
+ * it assumes {@link DistributedUpdateProcessorFactory} is configured there in the right place.
+ * Would be nice to find another way of plugging the Zero processing into SolrCloud (pulling before
+ * data is needed, pushing after indexing has completed).
+ *
+ * <p>Given the locking in use, it is assumed that indexing of a given batch (from the first doc to
+ * the hard commit, sent by the client or added automatically in {@link
+ * org.apache.solr.servlet.HttpSolrCall#addCommitIfAbsent}) is done by a single thread.
+ */
+public class ZeroStoreUpdateProcessor extends DistributedZkUpdateProcessor {
+ private ZeroCoreIndexingBatchProcessor zeroCoreIndexingBatchProcessor;
+
+ public ZeroStoreUpdateProcessor(
+ SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) {
+ super(req, rsp, next);
+ }
+
+ @Override
+ public void processCommit(CommitUpdateCommand cmd) throws IOException {
+
+ if (isReadOnly()) {
+ throw new SolrException(
+ SolrException.ErrorCode.FORBIDDEN, "Collection " + collection + " is read-only.");
+ }
+
+ if (!cmd.softCommit) {
+ doLocalCommit(cmd);
+ getZeroCoreIndexingBatchProcessor().hardCommitCompletedLocally();
+ }
+ }
+
+ /**
+ * Override the behavior of setupRequest so that it rejects an update if the target shard is
+ * splitting
+ */
+ @Override
+ protected List<SolrCmdDistributor.Node> setupRequest(
+ String id, SolrInputDocument doc, String route, UpdateCommand cmd) {
+ DocCollection coll = clusterState.getCollection(cloudDesc.getCollectionName());
+ Slice slice = getTargetSlice(coll, id, doc, route);
+
+ rejectUpdateIfSplitting(coll, slice);
+
+ return super.setupRequest(id, doc, route, cmd);
+ }
+
+ @Override
+ protected void postSetupHook() {
+ // this should be called after setupRequest and before the doc is indexed locally
+ // because it needs to know whether current replica is leader or subShardLeader for the doc or
+ // not
+ if (willBeIndexedLocally()) {
+ getZeroCoreIndexingBatchProcessor().addOrDeleteGoingToBeIndexedLocally();
+ }
+ }
+
+ /**
+ * Override the behavior of doDeleteByQuery so that it rejects an update if any shard in the
+ * collection is splitting
+ */
+ @Override
+ public void doDeleteByQuery(DeleteUpdateCommand cmd) throws IOException {
+ DocCollection coll = clusterState.getCollection(cloudDesc.getCollectionName());
+
+ // This will reject the update if any shard in the collection is currently being split
+ rejectIfConstructing(coll.getSlices());
+ super.doDeleteByQuery(cmd);
+ }
+
+ /** Reject the update if the target shard is splitting. */
+ private void rejectUpdateIfSplitting(DocCollection coll, Slice shard) {
+ // Find the sub shards, if any
+ Set<Slice> subShards =
+ coll.getSlices().stream()
+ .filter(s -> (s.getParent() != null && s.getParent().equals(shard.getName())))
+ .collect(Collectors.toSet());
+ if (!subShards.isEmpty()) {
+ // Reject if they are in construction or recovery state
+ rejectIfConstructing(subShards);
+ }
+ }
+
+ /**
+ * Reject the update if any of the shards in the provided collection are in construction or
+ * recovery state, indicating that the parent is splitting. This is because the transaction log is
+ * not trusted to work (nodes are stateless) so we avoid using it.
+ *
+ * <p>Do note though that there is a related issue, see comment at the end of constructor of
+ * {@link ZeroCoreIndexingBatchProcessor#ZeroCoreIndexingBatchProcessor}
+ *
+ * @param shards The collection of shards to check for construction or recovery state
+ */
+ private void rejectIfConstructing(Collection<Slice> shards) {
+
+ Set<String> splittingShards =
+ shards.stream()
+ .filter(
+ s ->
+ (s.getState() == Slice.State.CONSTRUCTION
+ || s.getState() == Slice.State.RECOVERY))
+ .map(s -> s.getParent())
+ .collect(Collectors.toSet());
+
+ for (String splittingShard : splittingShards) {
+ // To avoid potential deadlock in the case of catastrophic Overseer failure during a split, it
+ // is necessary
+ // to verify that the split lock exists, and that the split is actually "in progress"
+ try {
+ if (SplitShardCmd.shardSplitLockHeld(
+ zkController, cloudDesc.getCollectionName(), splittingShard)) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVICE_UNAVAILABLE,
+ "Rejecting updates to shard " + splittingShard + " while a split is in progress.");
+ }
+ } catch (SolrException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVICE_UNAVAILABLE,
+ "Rejecting updates to shard "
+ + splittingShard
+ + ", can't check if a split is in progress.",
+ e);
+ }
+ }
+ }
+
+ @Override
+ protected void doClose() {
+ if (zeroCoreIndexingBatchProcessor != null) {
+ zeroCoreIndexingBatchProcessor.close();
+ }
+ super.doClose();
+ }
+
+ private boolean willBeIndexedLocally() {
+ // forwardToLeader: if true, then the update is going to be forwarded to its rightful leader.
+ // The doc being added or deleted might not even belong to the current core's (req.getCore())
+ // shard.
+ // isLeader: if true, then the current core (req.getCore()) is the leader of the shard to which
+ // the doc being added or deleted belongs to.
+ // For ZERO replicas only leader replicas do local indexing. Follower ZERO replicas do not do
+ // any local indexing and forward the add/delete updates to the leader replica.
+ // isSubShardLeader: if true, then the current core (req.getCore()) is the leader of a sub shard
+ // being built (why is this rejected then? Should be rejected if the subshard is not active yet
+ // but logically it would end up being indexed locally).
+ //
+ // Therefore, only the leader replicas of the ZERO active shards will process docs locally and
+ // thus need to pull/push from the Zero store during indexing.
+ return !forwardToLeader && isLeader && !isSubShardLeader;
+ }
+
+ private ZeroCoreIndexingBatchProcessor getZeroCoreIndexingBatchProcessor() {
+ assert Type.ZERO.equals(replicaType);
+
+ if (zeroCoreIndexingBatchProcessor == null) {
+ updateCoreIndexingBatchProcessor(
+ new ZeroCoreIndexingBatchProcessor(req.getCore(), clusterState, rsp));
+ }
+ return zeroCoreIndexingBatchProcessor;
+ }
+
+ @VisibleForTesting
+ public void updateCoreIndexingBatchProcessor(
+ ZeroCoreIndexingBatchProcessor zeroCoreIndexingBatchProcessor) {
+ this.zeroCoreIndexingBatchProcessor = zeroCoreIndexingBatchProcessor;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/zero/client/ZeroFile.java b/solr/core/src/java/org/apache/solr/zero/client/ZeroFile.java
new file mode 100644
index 00000000000..035fb543f9b
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/zero/client/ZeroFile.java
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.zero.client;
+
+import com.fasterxml.jackson.annotation.JsonGetter;
+import com.google.common.annotations.VisibleForTesting;
+import java.net.URI;
+import java.time.Instant;
+import java.util.Objects;
+import org.apache.solr.core.backup.repository.BackupRepository;
+import org.apache.solr.zero.util.ToFromJson;
+
+/**
+ * The parent representation of a file on the Zero store. Contains the information required to
+ * locate and name that file in the store, but nothing else. This class is instantiated directly
+ * when listing files on the Zero store.
+ *
+ * <p>ZeroFile instances (including existing subclasses) are equal if they represent the same Zero
+ * store file, even if other parameters of the instance are different. This makes it easier to
+ * compare for example {@link ZeroFile} files coming from a directory listing of the Zero store and
+ * {@link WithLocal} files of a core that are present both locally and on the Zero store.
+ */
+public class ZeroFile {
+ protected final String collectionName;
+ protected final String shardName;
+ protected final String zeroFileName;
+
+ /** For Jackson deserialization. See {@link ToFromJson}. */
+ protected ZeroFile() {
+ collectionName = shardName = zeroFileName = null;
+ }
+
+ public ZeroFile(String collectionName, String shardName, String zeroFileName) {
+ this.collectionName = collectionName;
+ this.shardName = shardName;
+ this.zeroFileName = zeroFileName;
+ }
+
+ protected ZeroFile(ZeroFile f) {
+ this.collectionName = f.collectionName;
+ this.shardName = f.shardName;
+ this.zeroFileName = f.zeroFileName;
+ }
+
+ public String getCollectionName() {
+ return collectionName;
+ }
+
+ public String getShardName() {
+ return shardName;
+ }
+
+ public String getZeroFileName() {
+ return zeroFileName;
+ }
+
+ public URI getFileURI(BackupRepository repo) {
+ URI pathURI = getShardDirectoryURI(repo);
+ return repo.resolve(pathURI, getZeroFileName());
+ }
+
+ public URI getShardDirectoryURI(BackupRepository repo) {
+ URI collectionURI = getCollectionDirectoryURI(repo);
+ return repo.resolveDirectory(collectionURI, shardName);
+ }
+
+ public URI getCollectionDirectoryURI(BackupRepository repo) {
+ URI baseURI = repo.createURI(repo.getBackupLocation(null));
+ return repo.resolveDirectory(baseURI, collectionName);
+ }
+
+ public boolean pathEmpty() {
+ return (collectionName == null
+ || collectionName.isEmpty()
+ || shardName == null
+ || shardName.isEmpty());
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof ZeroFile) {
+ ZeroFile other = (ZeroFile) obj;
+ return Objects.equals(collectionName, other.collectionName)
+ && Objects.equals(shardName, other.shardName)
+ && Objects.equals(zeroFileName, other.zeroFileName);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(collectionName, shardName, zeroFileName);
+ }
+
+ /**
+ * A file stored in the Zero store with its matching local copy name. The names of the two are
+ * totally independent of each other, even though in general the Zero store name is built using
+ * the local Solr name plus a random suffix.
+ *
+ * <p>This class relies on {@link ZeroFile#equals(Object)}. Two local files representing the same
+ * Zero store file but having different Solr names are considered equal. In practice this should
+ * never happen, but relying on the superclass comparison allows for example comparing for
+ * equality files that are part of an index and files coming from a listing of files on the Zero
+ * store.
+ */
+ public static class WithLocal extends ZeroFile {
+ protected final String solrFileName;
+
+ private final boolean checksumPresent;
+
+ /**
+ * Name the file should have on a Solr server retrieving it, not including the core specific
+ * part of the filename (i.e. the path)
+ */
+ private final long fileSize;
+
+ /**
+ * Lucene generated checksum of the file. It is used in addition to file size to compare local
+ * and Zero store files.
+ */
+ private final long checksum;
+
+ /** For Jackson deserialization. See {@link ToFromJson}. */
+ protected WithLocal() {
+ solrFileName = null;
+ checksumPresent = false;
+ fileSize = checksum = 0L;
+ }
+
+ @VisibleForTesting
+ WithLocal(String collectionName, String shardName, String solrFileName, String zeroFileName) {
+ this(collectionName, shardName, solrFileName, zeroFileName, false, -1L, -1L);
+ }
+
+ public WithLocal(
+ String collectionName,
+ String shardName,
+ String solrFileName,
+ String zeroFileName,
+ long fileSize,
+ long checksum) {
+ this(collectionName, shardName, solrFileName, zeroFileName, true, fileSize, checksum);
+ }
+
+ private WithLocal(
+ String collectionName,
+ String shardName,
+ String solrFileName,
+ String zeroFileName,
+ boolean checksumPresent,
+ long fileSize,
+ long checksum) {
+ super(collectionName, shardName, zeroFileName);
+ this.solrFileName = solrFileName;
+ this.checksumPresent = checksumPresent;
+ this.fileSize = fileSize;
+ this.checksum = checksum;
+ }
+
+ public String getSolrFileName() {
+ return solrFileName;
+ }
+
+ public boolean isChecksumPresent() {
+ return checksumPresent;
+ }
+
+ public long getFileSize() {
+ return this.fileSize;
+ }
+
+ public long getChecksum() {
+ return this.checksum;
+ }
+ }
+
+ /**
+ * A file stored in the Zero store that is no longer used and should be deleted.
+ *
+ * <p>Deletes on the Zero store do not need to know the local Solr file name, therefore this class
+ * does not extend {@link WithLocal}.
+ *
+ * <p>This class relies on {@link ZeroFile#equals(Object)}. Two files to delete representing the
+ * same Zero store file but having different {@link #deletedAt} values are considered equal.
+ */
+ public static class ToDelete extends ZeroFile {
+
+ // Even though deletes are only done when it is clear the file is no longer needed (a successful
+ // push updated the Zero store commit point), it is possible that a slow pull on another node is
+ // still pulling files using an older commit point. Even if this happens qnd the pull fails, a
+ // subsequent pull would likely succeed. Maybe this timestamp is overdesign.
+ private Instant deletedAt;
+
+ /** For easy Jackson serialization of Instant type. See {@link ToFromJson}. */
+ @JsonGetter("deletedAt")
+ public long getMyInstantValEpoch() {
+ return deletedAt.toEpochMilli();
+ }
+
+ /** For Jackson deserialization of Instant type. See {@link ToFromJson}. */
+ @JsonGetter("deletedAt")
+ public void setMyInstantValEpoch(long epochMilli) {
+ this.deletedAt = Instant.ofEpochMilli(epochMilli);
+ }
+
+ /** For Jackson deserialization. See {@link ToFromJson}. */
+ protected ToDelete() {
+ deletedAt = null;
+ }
+
+ public ToDelete(ZeroFile zeroFile, Instant deletedAt) {
+ super(zeroFile);
+ this.deletedAt = deletedAt;
+ }
+
+ public ToDelete(String collectionName, String shardName, String zeroFileName) {
+ this(collectionName, shardName, zeroFileName, Instant.now());
+ }
+
+ public ToDelete(
+ String collectionName, String shardName, String zeroFileName, Instant deletedAt) {
+ super(collectionName, shardName, zeroFileName);
+ this.deletedAt = deletedAt;
+ }
+
+ /**
+ * @return time in milliseconds (converted from nanotime) that file was marked as deleted
+ */
+ public Instant getDeletedAt() {
+ return this.deletedAt;
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/zero/client/ZeroStoreClient.java b/solr/core/src/java/org/apache/solr/zero/client/ZeroStoreClient.java
new file mode 100644
index 00000000000..8dadcc52231
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/zero/client/ZeroStoreClient.java
@@ -0,0 +1,501 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.zero.client;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.InstrumentedExecutorService;
+import com.codahale.metrics.MetricRegistry;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.lang.invoke.MethodHandles;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileAlreadyExistsException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.stream.Collectors;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.NamedThreadFactory;
+import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.core.DirectoryFactory;
+import org.apache.solr.core.SolrInfoBean;
+import org.apache.solr.core.backup.repository.BackupRepository;
+import org.apache.solr.core.backup.repository.LocalFileSystemRepository;
+import org.apache.solr.metrics.SolrMetricManager;
+import org.apache.solr.metrics.SolrMetricProducer;
+import org.apache.solr.metrics.SolrMetricsContext;
+import org.apache.solr.zero.exception.ZeroException;
+import org.apache.solr.zero.metadata.ZeroStoreShardMetadata;
+import org.apache.solr.zero.util.IndexInputStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class is a wrapper around BackupRepository for easy reading and writing to an arbitrary
+ * underlying Zero store.
+ */
+public class ZeroStoreClient {
+
+ static final int CHUNK_SIZE = 16 * 1024 * 1024; // 16 MBs
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ /**
+ * Using {@link BackupRepository} to instantiate the Zero store. That class will have to be
+ * eventually renamed as with this change its role extends beyond backups (keeping it as is to
+ * simplify looking at the Zero branch).
+ */
+ protected final BackupRepository zeroRepository;
+
+ private static final String METRIC_BASE_NAME = "ZeroStore";
+
+ /**
+ * Identifiers for file pusher/puller thread pools that run on all Solr nodes containing Zero
+ * collections
+ */
+ private static final String ZERO_STORE_FILE_PUSHER_THREAD_POOL = "ZeroStoreFilePusherThreadPool";
+
+ private static final String ZERO_STORE_FILE_PULLER_THREAD_POOL = "ZeroStoreFilePullerThreadPool";
+ private final ExecutorService filePushExecutor;
+ private final ExecutorService filePullExecutor;
+
+ private final Counter bytesPushed;
+ private final Counter numFilesSuccessfullyPushed;
+ private final Counter bytesPulled;
+ private final Counter numFilesSuccessfullyPulled;
+
+ public ZeroStoreClient(
+ BackupRepository zeroRepository,
+ SolrMetricManager metricManager,
+ final int numFilePusherThreads,
+ final int numFilePullerThreads) {
+ this.zeroRepository = zeroRepository;
+
+ String registryName = SolrMetricManager.getRegistryName(SolrInfoBean.Group.node);
+ MetricRegistry registry = metricManager.registry(registryName);
+
+ // Executor for pushing individual files to Zero store
+ ExecutorService unwrappedFilePushExecutor =
+ ExecutorUtil.newMDCAwareFixedThreadPool(
+ numFilePusherThreads, new NamedThreadFactory(ZERO_STORE_FILE_PUSHER_THREAD_POOL));
+ this.filePushExecutor =
+ new InstrumentedExecutorService(
+ unwrappedFilePushExecutor, registry, METRIC_BASE_NAME + ".push.execution");
+
+ // Executor for pulling individual files from Zero store
+ ExecutorService unwrappedFilePullExecutor =
+ ExecutorUtil.newMDCAwareFixedThreadPool(
+ numFilePullerThreads, new NamedThreadFactory(ZERO_STORE_FILE_PULLER_THREAD_POOL));
+ this.filePullExecutor =
+ new InstrumentedExecutorService(
+ unwrappedFilePullExecutor, registry, METRIC_BASE_NAME + ".pull.execution");
+
+ SolrMetricsContext solrMetricsContext =
+ new SolrMetricsContext(
+ metricManager, registryName, SolrMetricProducer.getUniqueMetricTag(this, null));
+
+ bytesPushed = solrMetricsContext.counter("push.numBytes", METRIC_BASE_NAME);
+ numFilesSuccessfullyPushed =
+ solrMetricsContext.counter("push.numFilesSuccessful", METRIC_BASE_NAME);
+ solrMetricsContext.gauge(() -> numFilePusherThreads, true, "push.numThreads", METRIC_BASE_NAME);
+
+ bytesPulled = solrMetricsContext.counter("pull.numBytes", METRIC_BASE_NAME);
+ numFilesSuccessfullyPulled =
+ solrMetricsContext.counter("pull.numFilesSuccessful", METRIC_BASE_NAME);
+ solrMetricsContext.gauge(() -> numFilePullerThreads, true, "pull.numThreads", METRIC_BASE_NAME);
+ }
+
+ /**
+ * Replaces the special SHARD_METADATA_ZERO_FILENAME file on the Zero store for the core by a new
+ * version passed as a {@link ZeroStoreShardMetadata} instance.
+ *
+ * @param zeroShardMetadataFile description of the Zero shard index data for which to write the
+ * metadata file
+ * @param shardMetadata Zero shard metadata content to be serialized and written to the Zero store
+ */
+ public void pushShardMetadata(
+ ZeroFile.WithLocal zeroShardMetadataFile, ZeroStoreShardMetadata shardMetadata)
+ throws ZeroException {
+ if (zeroShardMetadataFile.pathEmpty())
+ throw new ZeroException("Can't write shard metadata file to empty path");
+ try {
+ createCoreStorage(zeroShardMetadataFile);
+ String zcmJson = shardMetadata.toJson();
+ OutputStream os =
+ zeroRepository.createOutput(zeroShardMetadataFile.getFileURI(zeroRepository));
+ os.write(zcmJson.getBytes(StandardCharsets.UTF_8));
+ os.close();
+ } catch (IOException e) {
+ throw handleIOException(e);
+ } catch (Exception ex) {
+ throw new ZeroException(ex);
+ }
+ }
+
+ /**
+ * Reads the special CORE_METADATA_ZERO_FILENAME file from the Zero store for the core and returns
+ * the corresponding {@link ZeroStoreShardMetadata} object.
+ *
+ * @param zeroShardMetadataFile description of the Zero shard index data to get metadata for
+ * @return <code>null</code> if the core does not exist on the Zero store or method {@link
+ * #pushShardMetadata} was never called for it. Otherwise returns the latest value written
+ * using {@link #pushShardMetadata} ("latest" here based on the consistency model of the
+ * underlying store, in practice the last value written by any server given the strong
+ * consistency of the Salesforce S3 implementation).
+ */
+ public ZeroStoreShardMetadata pullShardMetadata(ZeroFile zeroShardMetadataFile)
+ throws ZeroException {
+ try {
+
+ if (!shardMetadataExists(zeroShardMetadataFile)) {
+ return null;
+ }
+ try (IndexInput ii = pullStream(zeroShardMetadataFile)) {
+ OutputStream os = new ByteArrayOutputStream();
+ readIndexInput(ii, os);
+ String decodedJson = os.toString();
+ os.close();
+ return ZeroStoreShardMetadata.fromJson(decodedJson);
+ }
+ } catch (IOException e) {
+ throw handleIOException(e);
+ } catch (Exception ex) {
+ throw new ZeroException(ex);
+ }
+ }
+
+ /**
+ * Returns an input stream for the given Zero store file. The caller must close the stream when
+ * done.
+ *
+ * @param zeroFile description of the Zero shard index data to get a stream from
+ * @return the file's input stream
+ */
+ public IndexInput pullStream(ZeroFile zeroFile) throws ZeroException {
+ if (zeroFile.pathEmpty()) throw new ZeroException("Can't read from empty Zero store path");
+ try {
+ return zeroRepository.openInput(
+ zeroFile.getShardDirectoryURI(zeroRepository),
+ zeroFile.getZeroFileName(),
+ IOContext.DEFAULT);
+ } catch (IOException e) {
+ throw handleIOException(e);
+ } catch (Exception ex) {
+ throw new ZeroException(ex);
+ }
+ }
+
+ /**
+ * There are two interfaces to push a Zero core (shard) to the Zero store. In any case writes are
+ * done file by file. This interface writes to the external Zero store using an input stream for
+ * the given Zero store file.
+ *
+ * @param zeroFile description of the Zero store file to push data to
+ * @param is input stream of the file to push, which will be closed by this method after use
+ */
+ public void pushStream(ZeroFile zeroFile, InputStream is) throws ZeroException {
+ if (zeroFile.pathEmpty()) throw new ZeroException("Can't write to empty Zero store path");
+ try {
+ createCoreStorage(zeroFile);
+ OutputStream os = zeroRepository.createOutput(zeroFile.getFileURI(zeroRepository));
+ is.transferTo(os);
+ is.close();
+ os.close();
+ } catch (IOException e) {
+ throw handleIOException(e);
+ } catch (Exception ex) {
+ throw new ZeroException(ex);
+ }
+ }
+
+ /**
+ * Checks if the shard index data with the given Zero store shard metadata file exists. This
+ * method could actually be used to check the existence of any Zero store file, but it happens to
+ * only be used for shard metadata files.
+ *
+ * @param zeroShardMetadataFile description of the Zero store core (shard) metadata file to check
+ * @return true if the metadata file exists on the Zero store
+ */
+ public boolean shardMetadataExists(ZeroFile zeroShardMetadataFile) throws ZeroException {
+ try {
+ return zeroRepository.exists(zeroShardMetadataFile.getFileURI(zeroRepository));
+ } catch (IOException e) {
+ throw handleIOException(e);
+ } catch (Exception ex) {
+ throw new ZeroException(ex);
+ }
+ }
+
+ /**
+ * Batch delete files from the Zero store. Any file path that specifies a non-existent file will
+ * not be treated as an error and should return success.
+ *
+ * @param zeroFiles list of Zero file to be deleted
+ */
+ public void deleteZeroFiles(Collection<ZeroFile> zeroFiles) {
+ // TODO do we need to do something specific around deleting directories?
+ var files =
+ zeroFiles.stream()
+ .collect(Collectors.groupingBy(f -> f.getShardDirectoryURI(zeroRepository)));
+
+ for (Map.Entry<URI, List<ZeroFile>> entry : files.entrySet()) {
+ URI baseURI = entry.getKey();
+ List<String> filesName =
+ entry.getValue().stream().map(ZeroFile::getZeroFileName).collect(Collectors.toList());
+ try {
+ zeroRepository.delete(baseURI, filesName);
+ } catch (IOException e) {
+ if (log.isErrorEnabled()) {
+ log.error(
+ "Could not delete one of the following files in directory {} : {}",
+ baseURI.toString(),
+ StrUtils.join(filesName, ','),
+ e);
+ }
+ }
+ }
+ }
+
+ /** Lists all file names within the given path */
+ public Set<ZeroFile> listShardZeroFiles(String collectionName, String shardName) {
+ try {
+ URI shardURI = getShardURI(collectionName, shardName);
+ return Arrays.stream(zeroRepository.listAll(shardURI))
+ .map(zeroFileName -> new ZeroFile(collectionName, shardName, zeroFileName))
+ .collect(Collectors.toSet());
+ } catch (Exception e) {
+ log.error(
+ "Error while listing file for collection={} shard={}", collectionName, shardName, e);
+ return new LinkedHashSet<>();
+ }
+ }
+
+ public Set<ZeroFile> listCollectionZeroFiles(String collectionName) {
+ try {
+ URI collectionURI = getCollectionURI(collectionName);
+ return Arrays.stream(zeroRepository.listAll(collectionURI))
+ .flatMap(shardName -> listShardZeroFiles(collectionName, shardName).stream())
+ .collect(Collectors.toSet());
+ } catch (Exception e) {
+ log.error("Error while listing file for collection={}", collectionName, e);
+ return new LinkedHashSet<>();
+ }
+ }
+
+ /** Closes any resources used by the client */
+ public void shutdown() {
+ if (log.isInfoEnabled()) {
+ log.info("ZeroStoreClient is shutting down");
+ }
+
+ ExecutorUtil.shutdownAndAwaitTermination(filePushExecutor);
+ if (log.isInfoEnabled()) {
+ log.info("{} has shutdown", ZERO_STORE_FILE_PUSHER_THREAD_POOL);
+ }
+ ExecutorUtil.shutdownAndAwaitTermination(filePullExecutor);
+ if (log.isInfoEnabled()) {
+ log.info("{} has shutdown", ZERO_STORE_FILE_PULLER_THREAD_POOL);
+ }
+
+ try {
+ zeroRepository.close();
+ log.info("Zero store repository has shutdown");
+ } catch (IOException e) {
+ log.error("Zero store repository couldn't close properly");
+ }
+ }
+
+ /**
+ * Helper function to wrap a generic IOException into a ZeroException
+ *
+ * @param ioe exception to wrap
+ * @return wrapping exception
+ */
+ private ZeroException handleIOException(IOException ioe) {
+ String errMessage =
+ String.format(Locale.ROOT, "A Zero store IOException was thrown! %s", ioe.toString());
+ return new ZeroException(errMessage, ioe);
+ }
+
+ /**
+ * Required for some {@link BackupRepository} like {@link LocalFileSystemRepository}. Others do
+ * not need that like S3 since there are no directories, just keys with a prefix meaning no need
+ * to create the empty prefix before creating the file method is synchronized to avoid multi
+ * threading issues
+ *
+ * @param zeroFile file for which we want to be sure the directory hierarchy exists
+ */
+ private synchronized void createCoreStorage(ZeroFile zeroFile) throws IOException {
+ try {
+ URI collectionDirURI = zeroFile.getCollectionDirectoryURI(zeroRepository);
+ try {
+ // TODO the collection directory should be created once when collection created
+ if (!zeroRepository.exists(collectionDirURI)) {
+ zeroRepository.createDirectory(collectionDirURI);
+ }
+ } catch (FileAlreadyExistsException e) {
+ log.warn(
+ "Could not create collection directory as it already exists, collection={}",
+ zeroFile.getCollectionName(),
+ e);
+ }
+
+ // Maybe the collection directory was already created by another Solr node, but we anyway
+ // need to try creating the shard directory.
+ URI shardDirURI = zeroFile.getShardDirectoryURI(zeroRepository);
+ try {
+ if (!zeroRepository.exists(shardDirURI)) {
+ zeroRepository.createDirectory(shardDirURI);
+ }
+ } catch (FileAlreadyExistsException e) {
+ log.warn(
+ "Could not create shard directory as it already exists, collection={} shard={}",
+ zeroFile.getCollectionName(),
+ zeroFile.getShardName(),
+ e);
+ }
+ } catch (IOException e) {
+ log.warn(
+ "Error while creating directory for collection={} shard={}",
+ zeroFile.getCollectionName(),
+ zeroFile.getShardName(),
+ e);
+ throw e;
+ }
+ }
+
+ public URI getCollectionURI(String collectionName) {
+ URI baseURI = zeroRepository.createURI(zeroRepository.getBackupLocation(null));
+ return zeroRepository.resolveDirectory(baseURI, collectionName);
+ }
+
+ public URI getShardURI(String collectionName, String shardName) {
+ URI baseURI = zeroRepository.createURI(zeroRepository.getBackupLocation(null));
+ return zeroRepository.resolveDirectory(baseURI, collectionName, shardName);
+ }
+
+ public static void readIndexInput(IndexInput ii, OutputStream os) throws IOException {
+ byte[] buffer = new byte[CHUNK_SIZE];
+ int bufferLen;
+ long remaining = ii.length();
+ while (remaining > 0) {
+ bufferLen = remaining >= CHUNK_SIZE ? CHUNK_SIZE : (int) remaining;
+ ii.readBytes(buffer, 0, bufferLen);
+ os.write(buffer, 0, bufferLen);
+ remaining -= bufferLen;
+ }
+ }
+
+ public void deleteShardDirectory(String collectionName, String shardName) throws IOException {
+ URI shardURI = getShardURI(collectionName, shardName);
+ zeroRepository.deleteDirectory(shardURI);
+ }
+
+ public void deleteCollectionDirectory(String collectionName) throws IOException {
+ URI collectionURI = getCollectionURI(collectionName);
+ List<String> shardDirectoryName =
+ new ArrayList<>(Arrays.asList(zeroRepository.listAll(collectionURI)));
+ for (String d : shardDirectoryName) {
+ deleteShardDirectory(collectionName, d);
+ }
+ zeroRepository.deleteDirectory(collectionURI);
+ }
+
+ /** Asynchronously push shard.metadata to Zero store, using the push thread pool. */
+ public CompletableFuture<Void> pushShardMetadataAsync(
+ ZeroFile.WithLocal zeroShardMetadataFile, ZeroStoreShardMetadata shardMetadata) {
+ return CompletableFuture.runAsync(
+ () -> {
+ try {
+ pushShardMetadata(zeroShardMetadataFile, shardMetadata);
+ } catch (ZeroException e) {
+ throw new RuntimeException(
+ "Exception while pushing " + zeroShardMetadataFile.getZeroFileName(), e);
+ }
+ },
+ filePushExecutor);
+ }
+
+ /** Asynchronously push a local file to Zero store, using the push thread pool. */
+ public CompletableFuture<Void> pushFileAsync(Directory dir, ZeroFile.WithLocal localZeroFile) {
+ return CompletableFuture.runAsync(
+ () -> pushFileToZeroStore(dir, localZeroFile), filePushExecutor);
+ }
+
+ private void pushFileToZeroStore(Directory dir, ZeroFile.WithLocal localZeroFile) {
+ // This method pushes the normal segment files (as opposed to shard.metadata) and they should
+ // have file size and checksum set
+ assert localZeroFile.isChecksumPresent();
+ // Use Lucene-level IndexInput to lock on file lifecycle; don't want files deleted from
+ // underneath us
+ try (IndexInput ii = dir.openInput(localZeroFile.getSolrFileName(), IOContext.READONCE);
+ InputStream is = new IndexInputStream(ii)) {
+
+ pushStream(localZeroFile, is);
+
+ numFilesSuccessfullyPushed.inc();
+ long pushedFileLength = dir.fileLength(localZeroFile.getSolrFileName());
+ bytesPushed.inc(pushedFileLength);
+ assert pushedFileLength == localZeroFile.getFileSize();
+ } catch (Exception e) {
+ throw new RuntimeException(
+ "Exception while pushing file: " + localZeroFile.getSolrFileName(), e);
+ }
+ }
+
+ /** Asynchronously pull a local file from Zero store, using the pull thread pool, */
+ public CompletableFuture<Void> pullFileAsync(Directory dir, ZeroFile.WithLocal fileToDownload) {
+ return CompletableFuture.runAsync(
+ () -> pullFileFromZeroStore(dir, fileToDownload), filePullExecutor);
+ }
+
+ private void pullFileFromZeroStore(Directory destDir, ZeroFile.WithLocal fileToDownload) {
+ try (IndexOutput io =
+ destDir.createOutput(
+ fileToDownload.getSolrFileName(), DirectoryFactory.IOCONTEXT_NO_CACHE);
+ IndexInput ii = pullStream(fileToDownload)) {
+
+ if (log.isDebugEnabled()) {
+ log.debug(
+ "Copying {} from Zero store",
+ fileToDownload
+ .getSolrFileName()); // Logline has MDC, no need to add core/shard/replica info
+ }
+ io.copyBytes(ii, ii.length());
+ numFilesSuccessfullyPulled.inc();
+ bytesPulled.inc(fileToDownload.getFileSize());
+ } catch (ZeroException | IOException ex) {
+ throw new RuntimeException(
+ "Exception while pulling file: " + fileToDownload.getSolrFileName(), ex);
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/zero/client/ZeroStoreClientFactory.java b/solr/core/src/java/org/apache/solr/zero/client/ZeroStoreClientFactory.java
new file mode 100644
index 00000000000..c32501a29d1
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/zero/client/ZeroStoreClientFactory.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.zero.client;
+
+import java.lang.invoke.MethodHandles;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.core.NodeConfig;
+import org.apache.solr.core.PluginInfo;
+import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.core.ZeroConfig;
+import org.apache.solr.core.backup.repository.BackupRepository;
+import org.apache.solr.core.backup.repository.BackupRepositoryFactory;
+import org.apache.solr.metrics.SolrMetricManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Used to create a new ZeroStoreClient with underlying {@link BackupRepository}. BackupRepository
+ * properties are stored in solr.xml in the {@code <zero> ... </zero>} section
+ */
+public class ZeroStoreClientFactory {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public static ZeroStoreClient newInstance(NodeConfig cfg, SolrMetricManager metricManager) {
+
+ return new ZeroStoreClient(
+ newZeroRepository(cfg),
+ metricManager,
+ cfg.getZeroConfig().getNumFilePusherThreads(),
+ cfg.getZeroConfig().getNumFilePullerThreads());
+ }
+
+ public static BackupRepository newZeroRepository(NodeConfig cfg) {
+
+ SolrResourceLoader ressourceLoader = cfg.getSolrResourceLoader();
+ ZeroConfig zeroConfig = cfg.getZeroConfig();
+ PluginInfo[] repoPlugins = zeroConfig.getBackupRepositoryPlugins();
+ BackupRepositoryFactory zeroRepositoryFactory = new BackupRepositoryFactory(repoPlugins);
+
+ BackupRepository repository;
+
+ List<String> enabledRepos =
+ Arrays.stream(repoPlugins)
+ .filter(PluginInfo::isEnabled)
+ .map(p -> p.name)
+ .collect(Collectors.toList());
+ if (enabledRepos.isEmpty()) {
+ List<String> allRepos =
+ Arrays.stream(repoPlugins).map(p -> p.name).collect(Collectors.toList());
+ if (allRepos.isEmpty()) {
+ log.warn("No Zero store BackupRepository defined, using default");
+ repository = zeroRepositoryFactory.newInstance(ressourceLoader);
+ } else if (allRepos.size() == 1) {
+ repository = zeroRepositoryFactory.newInstance(ressourceLoader, allRepos.get(0));
+ } else {
+ log.warn(
+ "Multiple not enabled Zero store BackupRepository defined ({}), using first: {}",
+ StrUtils.join(allRepos, ','),
+ allRepos.get(0));
+ repository = zeroRepositoryFactory.newInstance(ressourceLoader, allRepos.get(0));
+ }
+ } else if (enabledRepos.size() == 1) {
+ repository = zeroRepositoryFactory.newInstance(ressourceLoader, enabledRepos.get(0));
+ } else {
+ log.warn(
+ "Multiple enabled Zero store BackupRepository defined ({}), using first: {}",
+ StrUtils.join(enabledRepos, ','),
+ enabledRepos.get(0));
+ repository = zeroRepositoryFactory.newInstance(ressourceLoader, enabledRepos.get(0));
+ }
+
+ return repository;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/zero/client/package-info.java b/solr/core/src/java/org/apache/solr/zero/client/package-info.java
new file mode 100644
index 00000000000..2608726223e
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/zero/client/package-info.java
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** Zero store client implementations, utils, and exceptions */
+package org.apache.solr.zero.client;
diff --git a/solr/core/src/java/org/apache/solr/zero/exception/CorruptedCoreException.java b/solr/core/src/java/org/apache/solr/zero/exception/CorruptedCoreException.java
new file mode 100644
index 00000000000..8ec96a3963d
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/zero/exception/CorruptedCoreException.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.zero.exception;
+
+/** Exception thrown when a core is corrupted locally. */
+public class CorruptedCoreException extends Exception {
+
+ public CorruptedCoreException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/zero/exception/ZeroException.java b/solr/core/src/java/org/apache/solr/zero/exception/ZeroException.java
new file mode 100644
index 00000000000..e6964066262
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/zero/exception/ZeroException.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.zero.exception;
+
+/**
+ * Parent class of Zero store related issues. Likely to change and maybe disappear but good enough
+ * for a PoC.
+ */
+public class ZeroException extends Exception {
+ public ZeroException(Throwable cause) {
+ super(cause);
+ }
+
+ public ZeroException(String message) {
+ super(message);
+ }
+
+ public ZeroException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/zero/exception/ZeroLockException.java b/solr/core/src/java/org/apache/solr/zero/exception/ZeroLockException.java
new file mode 100644
index 00000000000..27cccff9a5d
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/zero/exception/ZeroLockException.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.zero.exception;
+
+import java.util.Locale;
+
+public class ZeroLockException extends Exception {
+
+ public ZeroLockException(String format, Object... args) {
+ super(String.format(Locale.ROOT, format, args));
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/zero/exception/package-info.java b/solr/core/src/java/org/apache/solr/zero/exception/package-info.java
new file mode 100644
index 00000000000..fcd31c4be76
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/zero/exception/package-info.java
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** Zero store exception types */
+package org.apache.solr.zero.exception;
diff --git a/solr/core/src/java/org/apache/solr/zero/metadata/LocalCoreMetadata.java b/solr/core/src/java/org/apache/solr/zero/metadata/LocalCoreMetadata.java
new file mode 100644
index 00000000000..e4adf58f451
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/zero/metadata/LocalCoreMetadata.java
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.zero.metadata;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.NoSuchFileException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.lucene.index.IndexCommit;
+import org.apache.lucene.store.Directory;
+import org.apache.solr.core.IndexDeletionPolicyWrapper;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.zero.exception.ZeroException;
+import org.apache.solr.zero.process.CorePuller;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Object capturing the metadata of a Solr core on a Solr node. */
+public class LocalCoreMetadata {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ /**
+ * When an instance of this class is created for pushing data to the Zero store, reserve the
+ * commit point for a short while to give the caller time to save it while it works on it. Can't
+ * save it here directly as it would be awkward to try to release it on all execution paths.
+ */
+ private static final long RESERVE_COMMIT_DURATION = TimeUnit.SECONDS.toMillis(1L);
+
+ private static final int MAX_ATTEMPTS_TO_CAPTURE_COMMIT_POINT = 5;
+
+ /**
+ * Files composing the core. They are referenced from the core's current commit point's segments_N
+ * file which is ALSO included in this collection.
+ */
+ private Collection<String> latestCommitFiles;
+
+ /**
+ * Names of all files (all of them, no exception) in the local index directory. These files do not
+ * matter when pushing contents to Zeo store but they do matter if Zero store content being pulled
+ * conflicts with them.
+ */
+ private Set<String> allFiles;
+
+ /**
+ * Hash of the directory content used to make sure the content doesn't change as we proceed to
+ * pull new files from the Zero store (if we need to pull new files from there)
+ */
+ private String directoryHash;
+
+ /**
+ * Generation number of the local index. This generation number is only meant to identify a
+ * scenario where local index generation number is higher than what we have in Zero store. In that
+ * scenario we would switch index to a new directory when pulling contents from the Zero store.
+ * Because in the presence of higher generation number locally, simply pulling the content of the
+ * Zero store for the core does not work (Solr favors the highest generation locally...). It is
+ * also used for saving (reserving) local commit while doing pushes.
+ */
+ private long generation;
+
+ protected final SolrCore core;
+
+ /** Given a core name, builds the local metadata */
+ public LocalCoreMetadata(SolrCore core) {
+
+ this.core = core;
+ }
+
+ public void readMetadata(boolean reserveCommit, boolean captureDirHash) throws Exception {
+
+ Directory coreDir = getCoreDirectory();
+ if (coreDir == null) return;
+
+ try {
+ Set<String> latestCommitBuilder;
+ IndexCommit latestCommit;
+ int attempt = 1;
+ // we don't have an atomic way of capturing a commit point or taking a snapshot of the
+ // entire index directory to compute a directory hash. In the commit point case, there
+ // is a slight chance of losing files between getting a latest commit and reserving it.
+ // Likewise, there is a chance of losing files between the time we list all file names
+ // in the directory and computing the directory hash. Therefore, we try to capture the
+ // commit point and compute the directory hash in a loop with maximum number of attempts.
+ String hash = null;
+ String[] fileNames;
+ while (true) {
+ try {
+ // Work around possible bug returning same file multiple times by using a set here
+ // See org.apache.solr.handler.ReplicationHandler.getFileList()
+ latestCommitBuilder = new HashSet<>();
+ latestCommit = tryCapturingLatestCommit(latestCommitBuilder, reserveCommit);
+
+ // Capture now the hash and verify again after files have been pulled and before the
+ // directory is updated (or before the index is switched to use a new directory) to
+ // make sure there are no local changes at the same time that might lead to a
+ // corruption in case of interaction with the download or might be a sign of other
+ // problems (it is not expected that indexing can happen on a local directory of a
+ // ZERO replica if that replica is not up-to-date with the Zero store version).
+ fileNames = coreDir.listAll();
+ if (captureDirHash) {
+ hash = getSolrDirectoryHash(coreDir, fileNames);
+ }
+ break;
+ } catch (FileNotFoundException | NoSuchFileException ex) {
+ attempt++;
+ if (attempt > MAX_ATTEMPTS_TO_CAPTURE_COMMIT_POINT) {
+ throw ex;
+ }
+ String reason = ex.getMessage();
+ log.error(
+ "Failed to capture directory snapshot for either commit point or entire directory listing: core={} attempt={} reason={}",
+ getCoreName(),
+ attempt,
+ reason);
+ }
+ }
+
+ generation = latestCommit.getGeneration();
+ latestCommitFiles = Collections.unmodifiableSet(latestCommitBuilder);
+ directoryHash = hash;
+
+ // Need to inventory all local files in case files that need to be pulled from Zero store
+ // conflict
+ // with them.
+ allFiles = Set.of(fileNames);
+ } finally {
+ releaseCoreDirectory(coreDir);
+ }
+ }
+
+ private IndexCommit tryCapturingLatestCommit(
+ Set<String> latestCommitBuilder, boolean reserveCommit) throws ZeroException, IOException {
+ IndexDeletionPolicyWrapper deletionPolicy = core.getDeletionPolicy();
+ IndexCommit latestCommit = deletionPolicy.getLatestCommit();
+ if (latestCommit == null) {
+ throw new ZeroException("Core " + getCoreName() + " has no available commit point");
+ }
+
+ if (reserveCommit) {
+ // Caller will save the commit point shortly. See CorePushPull.pushFilesToZeroStore()
+ deletionPolicy.setReserveDuration(latestCommit.getGeneration(), RESERVE_COMMIT_DURATION);
+ }
+ // Note we add here all segment related files as well as the commit point's segments_N file
+ // Commit points do not contain lock (write.lock) files.
+ latestCommitBuilder.addAll(latestCommit.getFileNames());
+ return latestCommit;
+ }
+
+ /**
+ * Computes a hash of a Solr Directory in order to make sure the directory doesn't change as we
+ * pull content into it (if we need to pull content into it)
+ */
+ private String getSolrDirectoryHash(Directory coreDir, String[] filesNames)
+ throws NoSuchAlgorithmException, IOException {
+ MessageDigest digest =
+ MessageDigest.getInstance("sha1"); // not sure MD5 is available in Solr jars
+
+ // Computing the hash requires items to be submitted in the same order...
+ Arrays.sort(filesNames);
+
+ for (String fileName : filesNames) {
+ // .lock files come and go. Ignore them (we're closing the Index Writer before adding any
+ // pulled files to the Core)
+ if (!fileName.endsWith(".lock")) {
+ // Hash the file name and file size so we can tell if any file has changed (or files
+ // appeared or vanished)
+ digest.update(fileName.getBytes(StandardCharsets.UTF_8));
+ try {
+ digest.update(
+ Long.toString(coreDir.fileLength(fileName)).getBytes(StandardCharsets.UTF_8));
+ } catch (FileNotFoundException | NoSuchFileException fnf) {
+ // The file was deleted between the listAll() and the check, use an impossible size to not
+ // match a digest
+ // for which the file is completely present or completely absent (which will cause this
+ // hash to never match that directory again).
+ digest.update(Long.toString(-42).getBytes(StandardCharsets.UTF_8));
+ }
+ }
+ }
+
+ return new String(Hex.encodeHex(digest.digest()));
+ }
+
+ protected String getCoreName() {
+ return this.core.getName();
+ }
+
+ protected String getCollectionName() {
+ return core.getCoreDescriptor().getCollectionName();
+ }
+
+ protected String getShardName() {
+ return core.getCoreDescriptor().getCloudDescriptor().getShardId();
+ }
+
+ public long getGeneration() {
+ return this.generation;
+ }
+
+ public String getDirectoryHash() {
+ return directoryHash;
+ }
+
+ public Set<String> getAllFiles() {
+ return allFiles;
+ }
+
+ public Collection<String> getLatestCommitFiles() {
+ return latestCommitFiles;
+ }
+
+ @Override
+ public String toString() {
+ return "collectionName="
+ + getCollectionName()
+ + " shardName="
+ + getShardName()
+ + " coreName="
+ + getCoreName()
+ + " generation="
+ + generation;
+ }
+
+ public Directory getCoreDirectory() throws IOException {
+ return CorePuller.getDirectory(core, core.getIndexDir());
+ }
+
+ public void releaseCoreDirectory(Directory coreDir) throws IOException {
+ if (coreDir != null) core.getDirectoryFactory().release(coreDir);
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/zero/metadata/MetadataCacheManager.java b/solr/core/src/java/org/apache/solr/zero/metadata/MetadataCacheManager.java
new file mode 100644
index 00000000000..7cffbb7e555
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/zero/metadata/MetadataCacheManager.java
@@ -0,0 +1,455 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.zero.metadata;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.lang.invoke.MethodHandles;
+import java.util.Locale;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.ZeroConfig;
+import org.apache.solr.zero.process.CorePuller;
+import org.apache.solr.zero.process.ZeroAccessLocks;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class maintains a cache of Zero store shard metadata for local cores of ZERO replicas. For
+ * each local core, an instance of {@link MetadataCacheEntry} is stored. The data comes in part from
+ * the Zero store, in part from ZooKeeper.
+ */
+public class MetadataCacheManager {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ /**
+ * Value for a newly initialized MetadataCacheEntry instance. Not expected to make it to ZK,
+ * unlike real UUID's or the default value {@link
+ * ZeroMetadataController#METADATA_NODE_DEFAULT_VALUE} written at the very beginning of the ZERO
+ * shard's life cycle
+ */
+ private static final String METADATA_SUFFIX_CACHE_INITIAL_VALUE = "iNiTiAl";
+
+ private final ZeroConfig config;
+ private final CoreContainer coreContainer;
+
+ /**
+ * This map maintains for a local core information about Zero store shard metadata, corresponding
+ * metadataSuffix and version, as well as the locks for Zero store operations done on this node
+ */
+ private final ConcurrentHashMap<String, MetadataCacheEntry> coresMetadata;
+
+ /**
+ * Counters for the failed core pull attempts. Key is the core name, value is the counter which
+ * also includes the last attempt time.
+ */
+ private final ConcurrentHashMap<String, PullAttempts> coresPullAttempts;
+
+ /** Used by tests to provide their own {@link PullAttempts} factory. */
+ private final Supplier<PullAttempts> pullAttemptsFactory;
+
+ public MetadataCacheManager(ZeroConfig config, CoreContainer coreContainer) {
+ this(config, coreContainer, PullAttempts::new);
+ }
+
+ public MetadataCacheManager(
+ ZeroConfig config, CoreContainer coreContainer, Supplier<PullAttempts> pullAttemptsFactory) {
+ this.config = config;
+ this.coreContainer = coreContainer;
+ this.pullAttemptsFactory = pullAttemptsFactory;
+ coresMetadata = buildMetadataCache();
+ coresPullAttempts = new ConcurrentHashMap<>();
+ }
+
+ /**
+ * Logs the current {@link ZeroCoreStage} a core is at. Increments the pull attempts ({@link
+ * ZeroCoreStage#PULL_STARTED}) to abort pulling if there were more than {@link
+ * ZeroConfig#getMaxFailedCorePullAttempts()} unsuccessful pulls (without the corresponding {@link
+ * ZeroCoreStage#PULL_SUCCEEDED}).
+ */
+ public void recordState(SolrCore core, ZeroCoreStage stage) {
+ String coreName = getCoreName(core);
+ String collectionName = getCollectionName(core);
+ String shardName = getShardName(core);
+ log.info(
+ "RecordZeroCoreStage: collection={} shard={} core={} stage={}",
+ collectionName,
+ shardName,
+ coreName,
+ stage);
+ switch (stage) {
+ case PULL_STARTED:
+ coresPullAttempts.compute(
+ coreName,
+ (__, attempts) -> {
+ if (attempts == null) {
+ attempts = pullAttemptsFactory.get();
+ }
+ return attempts.checkAndIncrement(coreName, config);
+ });
+ break;
+ case PULL_SUCCEEDED:
+ coresPullAttempts.remove(coreName);
+ break;
+ case PULL_FAILED:
+ coresPullAttempts.computeIfPresent(coreName, (__, attempts) -> attempts.onFailedAttempt());
+ break;
+ case PULL_FAILED_WITH_CORRUPTION:
+ // We only want to retry once to pull a corrupted core.
+ coresPullAttempts.compute(
+ coreName,
+ (__, attempts) -> {
+ if (attempts == null) {
+ attempts = pullAttemptsFactory.get();
+ }
+ if (!attempts.onFailedAttempt().exhaustAllButOne(config)) {
+ // Done with attempts to pull the corrupted core. Set the replica state to
+ // RECOVERY_FAILED
+ // to prevent updates and queries on it, and to alert.
+ setReplicaStateToRecoveryFailed(collectionName, shardName, coreName);
+ }
+ return attempts;
+ });
+ break;
+ default:
+ break;
+ }
+ }
+
+ /**
+ * Sets the replica state to RECOVERY_FAILED for a corrupted core that cannot be pulled from the
+ * Zero store. This state will prevent any update or query on the replica, and it will be
+ * monitored with a metric.
+ */
+ private void setReplicaStateToRecoveryFailed(
+ String collectionName, String shardName, String coreName) {
+ try (SolrCore core = coreContainer.getCore(coreName)) {
+ try {
+ coreContainer
+ .getZkController()
+ .publish(core.getCoreDescriptor(), Replica.State.RECOVERY_FAILED);
+ } catch (Exception e) {
+ log.error(
+ "Cannot set replica state to {} for core {} shard {} collection {}",
+ Replica.State.RECOVERY_FAILED,
+ coreName,
+ shardName,
+ collectionName);
+ }
+ }
+ }
+
+ /**
+ * Updates the {@link MetadataCacheEntry} for the core with passed in {@link ZeroMetadataVersion},
+ * {@link ZeroStoreShardMetadata} and {@code cacheLikelyUpToDate}
+ */
+ public void updateCoreMetadata(
+ String coreName,
+ ZeroMetadataVersion shardMetadataVersion,
+ ZeroStoreShardMetadata zeroStoreShardMetadata,
+ boolean cacheLikelyUpToDate) {
+ MetadataCacheEntry currentMetadata = getOrCreateCoreMetadata(coreName);
+ MetadataCacheEntry updatedMetadata =
+ currentMetadata.updatedOf(
+ shardMetadataVersion, zeroStoreShardMetadata, cacheLikelyUpToDate);
+ updateCoreMetadata(coreName, currentMetadata, updatedMetadata);
+ }
+
+ /** Updates {@link MetadataCacheEntry} for the core with passed in {@code cacheLikelyUpToDate} */
+ public void updateCoreMetadata(String coreName, boolean cacheLikelyUpToDate) {
+ MetadataCacheEntry currentMetadata = getOrCreateCoreMetadata(coreName);
+ MetadataCacheEntry updatedMetadata = currentMetadata.updatedOf(cacheLikelyUpToDate);
+ updateCoreMetadata(coreName, currentMetadata, updatedMetadata);
+ }
+
+ /**
+ * Evicts an entry from the {@link #coresMetadata} if one exists for the given core name.
+ *
+ * @return {@code true} if an entry was actually removed
+ */
+ public boolean removeCoreMetadata(String coreName) {
+ coresPullAttempts.remove(coreName);
+ return coresMetadata.remove(coreName) != null;
+ }
+
+ private void updateCoreMetadata(
+ String coreName, MetadataCacheEntry currentMetadata, MetadataCacheEntry updatedMetadata) {
+ if (currentMetadata.getZeroAccessLocks().canUpdateCoreMetadata()) {
+ if (log.isInfoEnabled()) {
+ log.info(
+ "updateCoreMetadata: core={} current={} updated={}",
+ coreName,
+ currentMetadata,
+ updatedMetadata);
+ }
+ coresMetadata.put(coreName, updatedMetadata);
+ } else {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "Error updating the MetadataCacheEntry because the proper "
+ + "locks are not held by the current thread. "
+ + currentMetadata.locksHeldToString()
+ + " core="
+ + coreName);
+ }
+ }
+
+ /**
+ * Returns the cached {@link MetadataCacheEntry} representing Zero store metadata of the core or
+ * creates a new entry for the core if none exists.
+ */
+ public MetadataCacheEntry getOrCreateCoreMetadata(String coreName) {
+ return coresMetadata.computeIfAbsent(coreName, k -> new MetadataCacheEntry());
+ }
+
+ protected ConcurrentHashMap<String, MetadataCacheEntry> buildMetadataCache() {
+ return new ConcurrentHashMap<>();
+ }
+
+ public boolean hasInitialCacheSuffixValue(MetadataCacheEntry coreMetadata) {
+ return MetadataCacheManager.METADATA_SUFFIX_CACHE_INITIAL_VALUE.equals(
+ coreMetadata.getMetadataVersion().getMetadataSuffix());
+ }
+
+ protected String getCoreName(SolrCore solrCore) {
+ return solrCore.getName();
+ }
+
+ protected String getShardName(SolrCore solrCore) {
+ return solrCore.getCoreDescriptor().getCloudDescriptor().getShardId();
+ }
+
+ protected String getCollectionName(SolrCore solrCore) {
+ return solrCore.getCoreDescriptor().getCollectionName();
+ }
+
+ /**
+ * This represents metadata that needs to be cached for a core of a Zero collection {@link
+ * DocCollection#isZeroIndex()} so that it can be properly synchronized for concurrent indexing,
+ * pushes and pulls.
+ */
+ public static class MetadataCacheEntry {
+ /**
+ * Value originating from a ZooKeeper node used to handle conditionally and safely update the
+ * shard.metadata file written to the Zero store.
+ */
+ private final ZeroMetadataVersion metadataVersion;
+
+ /**
+ * {@link ZeroStoreShardMetadata} representing the state corresponding to {@link
+ * #metadataVersion}
+ */
+ private final ZeroStoreShardMetadata zeroStoreShardMetadata;
+
+ /**
+ * Whether cache likely up to date with {@link ZeroMetadataVersion} of the shard. In steady
+ * state this is set for leader cores when they push and pull {@link
+ * CorePuller#pullCoreWithRetries(boolean, long, long)} since followers cannot index. In
+ * presence of this flag we can skip consulting zookeeper before processing an indexing batch
+ * (if the flag proved wrong, indexing will fail and the cached state will get updated).
+ */
+ private final boolean cacheLikelyUpToDate;
+
+ /**
+ * The locks only need to be initialized once. When the metadata changes (and a new instance of
+ * {@link MetadataCacheEntry} created), the locks from the previous one must be reused.
+ */
+ private final ZeroAccessLocks locks;
+
+ /** Constructor for initial insertion of an entry into the cache. */
+ public MetadataCacheEntry() {
+ // Metadata suffix that will never be actually used in the Zero store
+ // Start with version 0 since when we use this entry to update the ZooKeeper node, it's
+ // the first update after that node got created.
+ this.metadataVersion = new ZeroMetadataVersion(METADATA_SUFFIX_CACHE_INITIAL_VALUE, 0);
+ this.zeroStoreShardMetadata = new ZeroStoreShardMetadata();
+ this.cacheLikelyUpToDate = false;
+ this.locks = new ZeroAccessLocks();
+ }
+
+ /**
+ * Used by {@link #updatedOf(boolean)} and {@link #updatedOf(ZeroMetadataVersion,
+ * ZeroStoreShardMetadata, boolean)} in this class. Visible outside the clas only for tests.
+ */
+ @VisibleForTesting
+ public MetadataCacheEntry(
+ ZeroMetadataVersion metadataVersion,
+ ZeroStoreShardMetadata zeroStoreShardMetadata,
+ boolean cacheLikelyUpToDate,
+ ZeroAccessLocks locks) {
+ this.metadataVersion = metadataVersion;
+ this.zeroStoreShardMetadata = zeroStoreShardMetadata;
+ this.cacheLikelyUpToDate = cacheLikelyUpToDate;
+ this.locks = locks;
+ }
+
+ public ZeroMetadataVersion getMetadataVersion() {
+ return metadataVersion;
+ }
+
+ public ZeroStoreShardMetadata getZeroShardMetadata() {
+ return zeroStoreShardMetadata;
+ }
+
+ public boolean isCacheLikelyUpToDate() {
+ return cacheLikelyUpToDate;
+ }
+
+ public ZeroAccessLocks getZeroAccessLocks() {
+ return locks;
+ }
+
+ private MetadataCacheEntry updatedOf(
+ ZeroMetadataVersion metadataVersion,
+ ZeroStoreShardMetadata zeroStoreShardMetadata,
+ boolean cacheLikelyUpToDate) {
+ return new MetadataCacheEntry(
+ metadataVersion, zeroStoreShardMetadata, cacheLikelyUpToDate, locks);
+ }
+
+ private MetadataCacheEntry updatedOf(boolean cacheLikelyUpToDate) {
+ return new MetadataCacheEntry(
+ metadataVersion, zeroStoreShardMetadata, cacheLikelyUpToDate, locks);
+ }
+
+ public String locksHeldToString() {
+ return String.format(
+ Locale.ROOT, "%s cacheLikelyUpToDate=%s", metadataVersion, cacheLikelyUpToDate);
+ }
+ }
+
+ /**
+ * Various stages a core of a Zero collection {@link DocCollection#isZeroIndex()} might go through
+ * during indexing and querying.
+ */
+ public enum ZeroCoreStage {
+ /** Necessary locks have been acquired and we have started to pull from the Zero store. */
+ PULL_STARTED,
+ /** Pull has ended and succeeded, and we are about to release the necessary locks. */
+ PULL_SUCCEEDED,
+ /** Pull has ended but failed, and we are about to release the necessary locks. */
+ PULL_FAILED,
+ /** Pull has ended but failed because of a corrupted core index. */
+ PULL_FAILED_WITH_CORRUPTION,
+ /** We have received an indexing batch but necessary locks have not been acquired yet. */
+ INDEXING_BATCH_RECEIVED,
+ /**
+ * We have passed the Zero store pull stage (if applicable) and are in sync with Zero store. Now
+ * we will proceed with local indexing.
+ */
+ LOCAL_INDEXING_STARTED,
+ /** Local indexing finished but has not been pushed to Zero store. */
+ LOCAL_INDEXING_FINISHED,
+ /** Necessary locks have been acquired and push to Zero store has started. */
+ PUSH_STARTED,
+ /** Files have been pushed to Zero store. */
+ FILE_PUSHED,
+ /** Zookeeper has been successfully updated with new metadata. */
+ ZK_UPDATE_FINISHED,
+ /** Local cache {@link #coresMetadata} has been successfully updated with new metadata. */
+ LOCAL_CACHE_UPDATE_FINISHED,
+ /**
+ * Push (either successful or failed) has ended and we are about to release the necessary locks.
+ */
+ PUSH_FINISHED,
+ /**
+ * Indexing batch (either successful or failed) has ended and we are about to release the
+ * necessary locks.
+ */
+ INDEXING_BATCH_FINISHED
+ }
+
+ /** Records the number of failed pull attempts and the last attempt time. */
+ @VisibleForTesting
+ public static class PullAttempts {
+ private int count;
+ private boolean pullAllowed;
+ private long lastAttemptTimeNs;
+
+ /**
+ * Checks the pull attempts count and either increments it, or throws a {@link SolrException} if
+ * the pull is not allowed until some delay elapses since the previous attempt. Sets
+ * lastAttemptTimeNs to the current time if the attempt is allowed.
+ */
+ synchronized PullAttempts checkAndIncrement(String coreName, ZeroConfig config) {
+ long timeNs = nanoTime();
+ long retryInNs = computeRemainingTimeNs(timeNs, config);
+ if (retryInNs > 0) {
+ throw new SolrException(
+ SolrException.ErrorCode.INVALID_STATE,
+ coreName
+ + " will not be pulled from the Zero store; canceling pull attempt due to "
+ + count
+ + " failed attempts, next attempt allowed in "
+ + TimeUnit.NANOSECONDS.toSeconds(retryInNs)
+ + " s");
+ }
+ count++;
+ pullAllowed = true;
+ // Resetting lastAttemptTimeNs here is not required, just in case onFailedAttempt() is not
+ // called.
+ lastAttemptTimeNs = timeNs;
+ return this;
+ }
+
+ synchronized PullAttempts onFailedAttempt() {
+ // Only reset lastAttemptTimeNs if the pull was allowed and failed.
+ if (pullAllowed) {
+ lastAttemptTimeNs = nanoTime();
+ pullAllowed = false;
+ }
+ return this;
+ }
+
+ private long computeRemainingTimeNs(long timeNs, ZeroConfig config) {
+ long remainingTimeNs;
+ if (count == 0) {
+ remainingTimeNs = 0;
+ } else {
+ long delayMs =
+ count >= config.getMaxFailedCorePullAttempts()
+ ?
+ // All pull attempts are exhausted. Still retry once periodically.
+ config.getAllAttemptsExhaustedRetryDelay()
+ :
+ // Allow the next pull attempt only if the last attempt was long enough ago.
+ count * config.getCorePullAttemptDelay();
+ remainingTimeNs = TimeUnit.MILLISECONDS.toNanos(delayMs) - (timeNs - lastAttemptTimeNs);
+ }
+ return remainingTimeNs;
+ }
+
+ synchronized boolean exhaustAllButOne(ZeroConfig config) {
+ if (count >= config.getMaxFailedCorePullAttempts()) {
+ return false;
+ }
+ count = config.getMaxFailedCorePullAttempts() - 1;
+ return true;
+ }
+
+ @VisibleForTesting
+ protected long nanoTime() {
+ return System.nanoTime();
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/zero/metadata/MetadataComparisonResult.java b/solr/core/src/java/org/apache/solr/zero/metadata/MetadataComparisonResult.java
new file mode 100644
index 00000000000..252e8ba8f46
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/zero/metadata/MetadataComparisonResult.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.zero.metadata;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.NoSuchFileException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.lucene.store.Directory;
+import org.apache.solr.zero.client.ZeroFile;
+
+/**
+ * Class capturing the differences between a local Solr core metadata {@link LocalCoreMetadata} and
+ * the Zero store metadata of the corresponding shard {@link ZeroStoreShardMetadata}.
+ */
+public class MetadataComparisonResult {
+
+ private final String metadataSuffix;
+
+ /** See {@link LocalCoreMetadata#generation} */
+ private final long localGeneration;
+
+ private final long distantGeneration;
+
+ /** Zero store files that need to be pulled */
+ private final Collection<ZeroFile.WithLocal> filesToPull;
+
+ /** Zero store files that need to be pushed */
+ private final Collection<ZeroFile.WithLocal> filesToPush;
+
+ /** Zero store files that need to be deleted */
+ private final Collection<ZeroFile.WithLocal> filesToDelete;
+
+ /**
+ * True if the local index contents conflict with contents to be pulled from the Zero store. If
+ * they conflict the core will be moved to a new index dir when pulling Zero store contents Two
+ * cases that result in a conflict: 1. local index is at higher generation number than Zero store
+ * generation number 2. index file with given name exists in both places but with a different size
+ * or checksum
+ */
+ private final boolean localConflictingWithZero;
+
+ /**
+ * Hash of the directory content used to make sure the content doesn't change as we proceed to
+ * pull new files from Zero store (if we need to pull new files from Zero store)
+ */
+ private final String directoryHash;
+
+ /**
+ * Result of a comparison of a local core and the Zero store version of the corresponding shard to
+ * decide how to align the two. Used when the goal is to pull newer content from the Zero store.
+ *
+ * @param localConflictingWithZero when true, some local files and Zero store files conflict.
+ * After pulling, the core will be switched to a new directory.
+ */
+ public MetadataComparisonResult(
+ long localGeneration,
+ long distantGeneration,
+ String directoryHash,
+ boolean localConflictingWithZero,
+ Collection<ZeroFile.WithLocal> filesToPull) {
+ this.localGeneration = localGeneration;
+ this.distantGeneration = distantGeneration;
+ this.directoryHash = directoryHash;
+ this.filesToPush = Collections.emptySet();
+ this.filesToDelete = Collections.emptySet();
+ this.filesToPull = filesToPull;
+ this.localConflictingWithZero = localConflictingWithZero;
+ this.metadataSuffix = null;
+ }
+
+ /**
+ * Result of a comparison of a local core and the Zero store version of the corresponding shard to
+ * decide how to align the two. Used when the goal is to push local updates to the Zero store.
+ */
+ public MetadataComparisonResult(
+ long localGeneration,
+ long distantGeneration,
+ String directoryHash,
+ String metadataSuffix,
+ Collection<ZeroFile.WithLocal> filesToPush,
+ Collection<ZeroFile.WithLocal> filesToDelete) {
+ this.localGeneration = localGeneration;
+ this.distantGeneration = distantGeneration;
+ this.directoryHash = directoryHash;
+ this.metadataSuffix = metadataSuffix;
+ this.filesToPush = filesToPush;
+ this.filesToDelete = filesToDelete;
+ this.filesToPull = Collections.emptySet();
+ this.localConflictingWithZero = false;
+ }
+
+ public Collection<ZeroFile.WithLocal> getFilesToPush() {
+ return filesToPush;
+ }
+
+ public Collection<ZeroFile.WithLocal> getFilesToPull() {
+ return filesToPull;
+ }
+
+ public Collection<ZeroFile.WithLocal> getFilesToDelete() {
+ return filesToDelete;
+ }
+
+ public boolean isLocalConflictingWithZero() {
+ return localConflictingWithZero;
+ }
+
+ public long getLocalGeneration() {
+ return this.localGeneration;
+ }
+
+ public long getDistantGeneration() {
+ return this.distantGeneration;
+ }
+
+ public String getMetadataSuffix() {
+ return metadataSuffix;
+ }
+
+ /**
+ * Returns {@code true} if the contents of the directory passed into this method is identical to
+ * the contents of the directory of the Solr core of this instance, taken at instance creation
+ * time. If the directory hash was not computed at the instance creation time, then we throw an
+ * IllegalStateException indicating a programming error.
+ *
+ * <p>Passing in the Directory (expected to be the directory of the same core used during
+ * construction) because it seems safer than trying to get it again here...
+ *
+ * @throws IllegalStateException if this instance was not created with a computed directoryHash
+ */
+ public boolean isSameDirectoryContent(Directory coreDir)
+ throws NoSuchAlgorithmException, IOException {
+ if (directoryHash == null) {
+ throw new IllegalStateException("Directory hash was not computed for the given core");
+ }
+ return directoryHash.equals(getSolrDirectoryHash(coreDir, coreDir.listAll()));
+ }
+
+ /**
+ * Computes a hash of a Solr Directory in order to make sure the directory doesn't change as we
+ * pull content into it (if we need to pull content into it)
+ */
+ private String getSolrDirectoryHash(Directory coreDir, String[] filesNames)
+ throws NoSuchAlgorithmException, IOException {
+ MessageDigest digest =
+ MessageDigest.getInstance("sha1"); // not sure MD5 is available in Solr jars
+
+ // Computing the hash requires items to be submitted in the same order...
+ Arrays.sort(filesNames);
+
+ for (String fileName : filesNames) {
+ // .lock files come and go. Ignore them (we're closing the Index Writer before adding any
+ // pulled files to the Core)
+ if (!fileName.endsWith(".lock")) {
+ // Hash the file name and file size so we can tell if any file has changed (or files
+ // appeared or vanished)
+ digest.update(fileName.getBytes(StandardCharsets.UTF_8));
+ try {
+ digest.update(
+ Long.toString(coreDir.fileLength(fileName)).getBytes(StandardCharsets.UTF_8));
+ } catch (FileNotFoundException | NoSuchFileException fnf) {
+ // The file was deleted between the listAll() and the check, use an impossible size to not
+ // match a digest
+ // for which the file is completely present or completely absent (which will cause this
+ // hash to never match that directory again).
+ digest.update(Long.toString(-42).getBytes(StandardCharsets.UTF_8));
+ }
+ }
+ }
+
+ return new String(Hex.encodeHex(digest.digest()));
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/zero/metadata/ZeroMetadataController.java b/solr/core/src/java/org/apache/solr/zero/metadata/ZeroMetadataController.java
new file mode 100644
index 00000000000..a68540da405
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/zero/metadata/ZeroMetadataController.java
@@ -0,0 +1,504 @@
+/*
+
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.solr.zero.metadata;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.solr.client.solrj.cloud.AlreadyExistsException;
+import org.apache.solr.client.solrj.cloud.BadVersionException;
+import org.apache.solr.client.solrj.cloud.SolrCloudManager;
+import org.apache.solr.client.solrj.cloud.VersionedData;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.zero.client.ZeroFile;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Class that manages metadata for Zero Index-based collections in Solr Cloud and ZooKeeper. */
+public class ZeroMetadataController {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ /**
+ * This is the initial value written to ZooKeeper for the {@link #SUFFIX_NODE_NAME} for a shard.
+ * See also {@link MetadataCacheManager#METADATA_SUFFIX_CACHE_INITIAL_VALUE} used as the initial
+ * value in the cache but not written to ZK.
+ */
+ public static final String METADATA_NODE_DEFAULT_VALUE = "DeFauLT";
+
+ public static final String SUFFIX_NODE_NAME = "metadataSuffix";
+
+ /**
+ * This file captures the shard metadata stored in the Zero store with each shard. When present
+ * locally (before being pushed to the Zero store), that file represents the metadata of the local
+ * core. Core vs shard, there is some naming ambiguity but eventually the two mean the same thing
+ * in the context of this file.
+ */
+ public static final String SHARD_METADATA_ZERO_FILENAME = "shard.metadata";
+
+ private static final String SEGMENTS_N_PREFIX = "segments_";
+
+ private final SolrCloudManager cloudManager;
+
+ public ZeroMetadataController(SolrCloudManager cloudManager) {
+ this.cloudManager = cloudManager;
+ }
+
+ public boolean hasDefaultNodeSuffix(ZeroMetadataVersion shardMetadataVersion) {
+ return METADATA_NODE_DEFAULT_VALUE.equals(shardMetadataVersion.getMetadataSuffix());
+ }
+
+ public boolean hasDefaultNodeSuffix(MetadataCacheManager.MetadataCacheEntry coreMetadata) {
+ return METADATA_NODE_DEFAULT_VALUE.equals(
+ coreMetadata.getMetadataVersion().getMetadataSuffix());
+ }
+
+ /**
+ * Creates a new metadataSuffix node if it doesn't exist for the shard and overwrites it if it
+ * does. This is only used when creating a shard, not when updating it (after indexing)
+ */
+ public void createMetadataNode(String collectionName, String shardName) {
+ ClusterState clusterState = cloudManager.getClusterStateProvider().getClusterState();
+ DocCollection collection = clusterState.getCollection(collectionName);
+ if (!collection.isZeroIndex()) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "Can't create a metadataNode for collection "
+ + collectionName
+ + " that is not"
+ + " of type Zero for shard "
+ + shardName);
+ }
+
+ String metadataPath = getMetadataBasePath(collectionName, shardName) + "/" + SUFFIX_NODE_NAME;
+ Map<String, Object> nodeProps = new HashMap<>();
+ nodeProps.put(SUFFIX_NODE_NAME, METADATA_NODE_DEFAULT_VALUE);
+
+ createPersistentNode(metadataPath, Utils.toJSON(nodeProps));
+ }
+
+ /**
+ * If the update is successful, the returned {@link ZeroMetadataVersion} will contain the new
+ * version as well as the value of the data just written.
+ *
+ * @param value the value to be written to ZooKeeper
+ * @param version the ZooKeeper node version to conditionally update on. Except in tests, this is
+ * never -1 (that allows the update to succeed regardless of existing node version)
+ */
+ public ZeroMetadataVersion updateMetadataValueWithVersion(
+ String collectionName, String shardName, String value, int version) {
+ String metadataPath = getMetadataBasePath(collectionName, shardName) + "/" + SUFFIX_NODE_NAME;
+ try {
+ Map<String, Object> nodeProps = new HashMap<>();
+ nodeProps.put(SUFFIX_NODE_NAME, value);
+
+ VersionedData data =
+ cloudManager
+ .getDistribStateManager()
+ .setAndGetResult(metadataPath, Utils.toJSON(nodeProps), version);
+ Map<?, ?> nodeUserData = (Map<?, ?>) Utils.fromJSON(data.getData());
+ String metadataSuffix = (String) nodeUserData.get(ZeroMetadataController.SUFFIX_NODE_NAME);
+ return new ZeroMetadataVersion(metadataSuffix, data.getVersion());
+ } catch (BadVersionException e) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "Error updating path: " + metadataPath + " due to mismatching versions",
+ e);
+ } catch (NoSuchElementException | KeeperException e) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "Error updating path: " + metadataPath + " in ZooKeeper",
+ e);
+ }
+ }
+
+ /** Reads the {@link ZeroMetadataVersion} for the shard from ZooKeeper. */
+ public ZeroMetadataVersion readMetadataValue(String collectionName, String shardName) {
+ String metadataPath = getMetadataBasePath(collectionName, shardName) + "/" + SUFFIX_NODE_NAME;
+ try {
+ VersionedData data = cloudManager.getDistribStateManager().getData(metadataPath, null);
+ Map<?, ?> nodeUserData = (Map<?, ?>) Utils.fromJSON(data.getData());
+ String metadataSuffix = (String) nodeUserData.get(ZeroMetadataController.SUFFIX_NODE_NAME);
+ return new ZeroMetadataVersion(metadataSuffix, data.getVersion());
+ } catch (IOException | NoSuchElementException | KeeperException e) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "Error reading data from path: " + metadataPath + " in ZooKeeper",
+ e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "Error reading data from path: " + metadataPath + " in ZooKeeper",
+ e);
+ }
+ }
+
+ /** Removes the metadata node on zookeeper for the given collection and shard name. */
+ public void cleanUpMetadataNodes(String collectionName, String shardName) {
+ String metadataPath = getMetadataBasePath(collectionName, shardName) + "/" + SUFFIX_NODE_NAME;
+ try {
+ cloudManager.getDistribStateManager().removeRecursively(metadataPath, true, true);
+ } catch (Exception e) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "Error deleting path " + metadataPath + " in Zookeeper",
+ e);
+ }
+ }
+
+ private void createPersistentNode(String path, byte[] data) {
+ try {
+ // if path already exists, overwrite it
+ cloudManager.getDistribStateManager().makePath(path, data, CreateMode.PERSISTENT, false);
+ } catch (InterruptedException e) {
+ Thread.interrupted();
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR, "Error creating path " + path + " in Zookeeper", e);
+ } catch (IOException | AlreadyExistsException | KeeperException e) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR, "Error creating path " + path + " in Zookeeper", e);
+ }
+ }
+
+ protected String getMetadataBasePath(String collectionName, String shardName) {
+ return ZkStateReader.COLLECTIONS_ZKNODE
+ + "/"
+ + collectionName
+ + "/"
+ + ZkStateReader.SHARD_LEADERS_ZKNODE
+ + "/"
+ + shardName;
+ }
+
+ public MetadataComparisonResult diffMetadataForPush(
+ LocalCoreMetadata localMetadata, ZeroStoreShardMetadata distantMetadata, String suffix)
+ throws Exception {
+ Directory coreDir = localMetadata.getCoreDirectory();
+ try {
+ checkSegmentFilesIntegrity(localMetadata, distantMetadata);
+ return diffMetadataForPush(coreDir, localMetadata, distantMetadata, suffix);
+ } finally {
+ localMetadata.releaseCoreDirectory(coreDir);
+ }
+ }
+
+ public MetadataComparisonResult diffMetadataforPull(
+ LocalCoreMetadata localMetadata, ZeroStoreShardMetadata distantMetadata) throws Exception {
+ Directory coreDir = localMetadata.getCoreDirectory();
+ try {
+ checkSegmentFilesIntegrity(localMetadata, distantMetadata);
+ return diffMetadataForPull(coreDir, localMetadata, distantMetadata);
+ } finally {
+ localMetadata.releaseCoreDirectory(coreDir);
+ }
+ }
+
+ private MetadataComparisonResult diffMetadataForPull(
+ Directory coreDir, LocalCoreMetadata localMetadata, ZeroStoreShardMetadata distantMetadata) {
+
+ boolean localConflictingWithZero = false;
+
+ Map<String, ZeroFile.WithLocal> zeroFilesMissingLocally =
+ getZeroFilesMissingLocally(localMetadata, distantMetadata);
+ Map<String, ZeroFile.WithLocal> latestCommitFilesAlreadyOnZeroButDifferent =
+ getLatestCommitFilesAlreadyOnZeroButDifferent(coreDir, localMetadata, distantMetadata);
+ if (!latestCommitFilesAlreadyOnZeroButDifferent.isEmpty()) {
+ localConflictingWithZero = true;
+ zeroFilesMissingLocally = distantMetadata.getZeroFilesAsMap();
+ }
+
+ Map<String, ZeroFile.WithLocal> filesOnBothSideButNotInLatestCommit =
+ getFilesOnBothSideButNotInLatestCommit(localMetadata, zeroFilesMissingLocally);
+ if (!filesOnBothSideButNotInLatestCommit.isEmpty()) {
+ // We are resolving metadata for a PULL, it means we expect the local core to be behind or
+ // at the Zero version. We therefore do not expect local files not in the commit point that
+ // are still in the Zero store commit point. It might mean the local core did some indexing
+ // and merged segments then dropped from the commit point.
+ //
+ // This can happen if the core was the leader, indexed locally but then failed to push to the
+ // Zero store, and is now pulling again after another replica did a push. In these case we do
+ // want to align on the Zero store version.
+ localConflictingWithZero = true;
+ zeroFilesMissingLocally = distantMetadata.getZeroFilesAsMap();
+
+ filesOnBothSideButNotInLatestCommit
+ .values()
+ .forEach(
+ bf -> {
+ if (log.isInfoEnabled()) {
+ log.info(
+ "File exists locally outside of current commit point. collectionName={} shardName={}"
+ + " coreName={} solrFileName={} zeroFileName={}",
+ localMetadata.getCollectionName(),
+ localMetadata.getShardName(),
+ localMetadata.getCoreName(),
+ bf.getSolrFileName(),
+ bf.getZeroFileName());
+ }
+ });
+ }
+
+ // If local index generation is higher than Zero's we will download to new directory because
+ // if both segments_N files are present in same directory Solr will open the higher one (and
+ // we really want the lower one from the Zero store as we're currently pulling content from
+ // there).
+ if (!distantMetadata.getZeroFiles().isEmpty()
+ && localMetadata.getGeneration() > distantMetadata.getGeneration()) {
+ localConflictingWithZero = true;
+ zeroFilesMissingLocally = distantMetadata.getZeroFilesAsMap();
+ if (log.isInfoEnabled()) {
+ log.info(
+ "local generation higher than Zero store. coreName={} localGeneration={} zeroGeneration={}",
+ localMetadata.getCoreName(),
+ localMetadata.getGeneration(),
+ distantMetadata.getGeneration());
+ }
+ }
+
+ return new MetadataComparisonResult(
+ localMetadata.getGeneration(),
+ distantMetadata.getGeneration(),
+ localMetadata.getDirectoryHash(),
+ localConflictingWithZero,
+ zeroFilesMissingLocally.values());
+ }
+
+ private MetadataComparisonResult diffMetadataForPush(
+ Directory coreDir,
+ LocalCoreMetadata localMetadata,
+ ZeroStoreShardMetadata distantMetadata,
+ String newSuffix) {
+
+ Collection<ZeroFile.WithLocal> filesToPush =
+ getLocalFilesMissingOnZero(localMetadata, distantMetadata).stream()
+ .map(
+ f ->
+ getNewZeroLocalFile(
+ coreDir,
+ f,
+ localMetadata.getCollectionName(),
+ localMetadata.getShardName(),
+ newSuffix))
+ .collect(Collectors.toSet());
+ Collection<ZeroFile.WithLocal> filesToDelete =
+ getZeroFilesMissingLocally(localMetadata, distantMetadata).values();
+ return new MetadataComparisonResult(
+ localMetadata.getGeneration(),
+ distantMetadata.getGeneration(),
+ localMetadata.getDirectoryHash(),
+ newSuffix,
+ filesToPush,
+ filesToDelete);
+ }
+
+ /**
+ * @return intersection(all local files, distant files - latest committed files)
+ */
+ private Map<String, ZeroFile.WithLocal> getFilesOnBothSideButNotInLatestCommit(
+ LocalCoreMetadata localMetadata, Map<String, ZeroFile.WithLocal> zeroFilesMissingLocally) {
+
+ Map<String, ZeroFile.WithLocal> filesOnBothSideButNotInLatestCommit =
+ new HashMap<>(zeroFilesMissingLocally);
+ filesOnBothSideButNotInLatestCommit.keySet().retainAll(localMetadata.getAllFiles());
+ return filesOnBothSideButNotInLatestCommit;
+ }
+
+ /**
+ * @return intersection(distant files, latest committed files).filter(size or checksum different)
+ */
+ private Map<String, ZeroFile.WithLocal> getLatestCommitFilesAlreadyOnZeroButDifferent(
+ Directory coreDir, LocalCoreMetadata localMetadata, ZeroStoreShardMetadata distantZeroFiles) {
+ Map<String, ZeroFile.WithLocal> latestCommitFilesAlreadyOnZero =
+ distantZeroFiles.getZeroFilesAsMap();
+ latestCommitFilesAlreadyOnZero.keySet().retainAll(localMetadata.getLatestCommitFiles());
+ return latestCommitFilesAlreadyOnZero.entrySet().stream()
+ .filter(
+ f ->
+ areDistantAndLocalFileDifferent(
+ coreDir, f.getValue(), f.getKey(), localMetadata.getCoreName()))
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+ }
+
+ /**
+ * @return distant files - latest committed files
+ */
+ private Map<String, ZeroFile.WithLocal> getZeroFilesMissingLocally(
+ LocalCoreMetadata localMetadata, ZeroStoreShardMetadata distantZeroFiles) {
+
+ Map<String, ZeroFile.WithLocal> zeroFilesMissingLocally = distantZeroFiles.getZeroFilesAsMap();
+ zeroFilesMissingLocally.keySet().removeAll(localMetadata.getLatestCommitFiles());
+ return zeroFilesMissingLocally;
+ }
+
+ /**
+ * @return latest committed files - distant files
+ */
+ private Set<String> getLocalFilesMissingOnZero(
+ LocalCoreMetadata localMetadata, ZeroStoreShardMetadata distantZeroFiles) {
+ Set<String> localFileNamesMissingOnZero = new HashSet<>(localMetadata.getLatestCommitFiles());
+ localFileNamesMissingOnZero.removeAll(distantZeroFiles.getZeroFileNames());
+ return localFileNamesMissingOnZero;
+ }
+
+ private void checkSegmentFilesIntegrity(
+ LocalCoreMetadata localMetadata, ZeroStoreShardMetadata distantMetadata)
+ throws SolrException {
+ List<ZeroFile.WithLocal> segmentFiles =
+ distantMetadata.getZeroFiles().stream()
+ .filter(this::isSegmentsNFilename)
+ .limit(2)
+ .collect(Collectors.toList());
+ if (segmentFiles.size() == 2) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ String.format(
+ Locale.ROOT,
+ "Zero store for collection %s and shard %s has conflicting files %s and %s",
+ localMetadata.getCollectionName(),
+ localMetadata.getShardName(),
+ segmentFiles.get(0),
+ segmentFiles.get(1)));
+ } else if (segmentFiles.isEmpty() && !distantMetadata.getZeroFiles().isEmpty()) {
+ // TODO - for now just log and propagate the error up, this class shouldn't do corruption
+ // checking now
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR, "Zero store has missing segments file");
+ }
+ }
+
+ protected ZeroFile.WithLocal getNewZeroLocalFile(
+ Directory coreDir, String fileName, String collectionName, String shardName, String suffix) {
+ long fileSize = 0L;
+ long fileChecksum = 0L;
+ try (final IndexInput indexInput = coreDir.openInput(fileName, IOContext.READONCE)) {
+ fileSize = indexInput.length();
+ fileChecksum = CodecUtil.retrieveChecksum(indexInput);
+ } catch (IOException e) {
+ log.warn("Failed to get file metadata {}", fileName, e);
+ // IO error when opening the file (probably not found, for example it could be wiped
+ // in CorePushPull if a corruption was detected).
+ // Consider the file empty so that it is pulled again from the Zero store.
+ }
+ return new ZeroFile.WithLocal(
+ collectionName,
+ shardName,
+ fileName,
+ buildFileName(fileName, suffix),
+ fileSize,
+ fileChecksum);
+ }
+
+ protected boolean areDistantAndLocalFileDifferent(
+ Directory coreDir, ZeroFile.WithLocal distantFile, String localFileName, String coreName) {
+ long localFileSize = 0L;
+ long localFileChecksum = 0L;
+ try (final IndexInput indexInput = coreDir.openInput(localFileName, IOContext.READONCE)) {
+ localFileSize = indexInput.length();
+ localFileChecksum = CodecUtil.retrieveChecksum(indexInput);
+ } catch (IOException e) {
+ log.warn("Error getting file metadata {}", localFileName, e);
+ // IO error when opening the file (probably not found, for example it could be wiped
+ // in CorePushPull if a corruption was detected).
+ // Consider the file empty so that it is pulled again from the Zero store.
+ }
+
+ boolean different =
+ (localFileSize != distantFile.getFileSize()
+ || localFileChecksum != distantFile.getChecksum());
+
+ if (different) {
+ String message =
+ String.format(
+ Locale.ROOT,
+ "Size/Checksum conflicts. collectionName=%s shardName=%s coreName=%s fileName=%s zeroStoreFileName=%s"
+ + " localSize=%s zeroStoreSize=%s localChecksum=%s zeroStoreChecksum=%s",
+ distantFile.getCollectionName(),
+ distantFile.getShardName(),
+ coreName,
+ localFileName,
+ distantFile.getZeroFileName(),
+ localFileSize,
+ distantFile.getFileSize(),
+ localFileChecksum,
+ distantFile.getChecksum());
+ log.info(message);
+ }
+ return different;
+ }
+
+ /** Identify the segments_N file in local Zero files. */
+ public boolean isSegmentsNFilename(ZeroFile.WithLocal file) {
+ return file.getSolrFileName().startsWith(SEGMENTS_N_PREFIX);
+ }
+
+ public ZeroFile.WithLocal newShardMetadataZeroFile(
+ String collectionName, String shardName, String suffix) {
+ return new ZeroFile.WithLocal(
+ collectionName,
+ shardName,
+ SHARD_METADATA_ZERO_FILENAME,
+ buildFileName(ZeroMetadataController.SHARD_METADATA_ZERO_FILENAME, suffix),
+ ZeroStoreShardMetadata.FAKE_CORE_METADATA_LENGTH,
+ ZeroStoreShardMetadata.FAKE_CORE_METADATA_CHECKSUM);
+ }
+
+ /**
+ * This method implements the naming convention for files on the Zero store.
+ *
+ * <p>There really are two naming conventions, one for the {@link #SHARD_METADATA_ZERO_FILENAME},
+ * and one for all other files. For simplicity the two conventions are identical.
+ *
+ * <ol>
+ * <li>{@link #SHARD_METADATA_ZERO_FILENAME} (<code>shard.metadata</code>) is named on the Zero
+ * store <code>shard.metadata.randomSuffix</code>, the random suffix is stored in Zookeeper
+ * and that's how the current <code>shard.metadata</code> file is identified (there might be
+ * older such files, they will have a different suffix from the one stored in Zookeeper)
+ * <li>All other files (composing a Solr core) can have any name, as long as it's unique so no
+ * chance of overwriting an existing file on the Zero store. The mapping from Zero store
+ * file name to local file name is maintained in the <code>shard.metadata</code> file so
+ * there's no need for a specific naming convention (same convention used nonetheless).
+ * </ol>
+ */
+ public static String buildFileName(String prefix, String suffix) {
+ return prefix + "." + suffix;
+ }
+
+ public ZeroFile.ToDelete newShardMetadataZeroFileToDelete(
+ String collectionName, String shardName, String metadataSuffix) {
+ return new ZeroFile.ToDelete(
+ collectionName, shardName, buildFileName(SHARD_METADATA_ZERO_FILENAME, metadataSuffix));
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/zero/metadata/ZeroMetadataVersion.java b/solr/core/src/java/org/apache/solr/zero/metadata/ZeroMetadataVersion.java
new file mode 100644
index 00000000000..f0d26e71ab9
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/zero/metadata/ZeroMetadataVersion.java
@@ -0,0 +1,67 @@
+/*
+
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.solr.zero.metadata;
+
+import org.apache.solr.common.cloud.DocCollection;
+
+/**
+ * This represents the metadata suffix and the corresponding Zookeeper version of the node for a
+ * shard of a Zero collection {@link DocCollection#isZeroIndex()}
+ */
+public class ZeroMetadataVersion {
+ /** The metadataSuffix for the shard stored in the Zero store. */
+ private final String metadataSuffix;
+
+ /** version of zookeeper node maintaining the metadata */
+ private final int version;
+
+ public ZeroMetadataVersion(String metadataSuffix, int version) {
+ assert metadataSuffix != null;
+ this.metadataSuffix = metadataSuffix;
+ this.version = version;
+ }
+
+ public String getMetadataSuffix() {
+ return metadataSuffix;
+ }
+
+ public int getVersion() {
+ return version;
+ }
+
+ @Override
+ public String toString() {
+ return "metadataSuffix=" + metadataSuffix + " version=" + version;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null || obj.getClass() != this.getClass()) {
+ return false;
+ }
+
+ final ZeroMetadataVersion other = (ZeroMetadataVersion) obj;
+ return this.version == other.version && this.metadataSuffix.equals(other.metadataSuffix);
+ }
+
+ @Override
+ public int hashCode() {
+ // This object is never inserted into a collection
+ return 0;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/zero/metadata/ZeroStoreShardMetadata.java b/solr/core/src/java/org/apache/solr/zero/metadata/ZeroStoreShardMetadata.java
new file mode 100644
index 00000000000..caecea1fc77
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/zero/metadata/ZeroStoreShardMetadata.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.zero.metadata;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.UUID;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import org.apache.solr.zero.client.ZeroFile;
+import org.apache.solr.zero.process.CorePusher;
+import org.apache.solr.zero.util.ToFromJson;
+
+/**
+ * Object defining metadata stored in Zero store for a Zero Collection shard and its builders
+ * (storage in Zero store is a single copy per shard and not per core or replica as usual in
+ * SolrCloud). This metadata includes all actual segment files as well as the segments_N file of the
+ * commit point.
+ *
+ * <p>This object is serialized to/from Json and stored in the Zero store as a file named <code>
+ * shard.metadata</code> (with a random suffix).
+ *
+ * <p>Although in the Zero store storage is per shard, each Solr node manages this file for its
+ * specific core (replica) of the shard. The update strategy via unique filenames on the Zero store
+ * and a tiebreaker using zookeeper guarantees consistency at the shard level (a replica doesn't
+ * overwrite an update to the shard in the Zero store done by another replica on another node).
+ */
+public class ZeroStoreShardMetadata {
+
+ /** shard.metadata file doesn't have its length or checksum compared so using fake values */
+ public static final long FAKE_CORE_METADATA_LENGTH = -10;
+
+ public static final long FAKE_CORE_METADATA_CHECKSUM = -10;
+ public static final long UNDEFINED_GENERATION = -1L;
+
+ /**
+ * Generation number of index represented by this metadata. Generation numbers across replicas
+ * don't carry any meaning since each replica can be doing its own indexing. We are only using
+ * this generation for optimizations in the context of a single replica. It should not be used
+ * beyond following stated purposes. If its mere existence is popping it into consideration for
+ * solving unintended use cases, then we can remove it and implement the suggested alternatives
+ * for following usages. 1. identify a scenario where local index generation number is higher than
+ * what we have in Zero store. In that scenario we would switch index to a new directory when
+ * pulling contents from Zero store. Because in the presence of higher generation number locally,
+ * Zero store contents cannot establish their legitimacy. Storing this number is just an
+ * optimization. We can always infer that number from segment_N file and get rid of this usage.
+ * {@link ZeroMetadataController#diffMetadataforPull(LocalCoreMetadata, ZeroStoreShardMetadata)}
+ * ()} 2. {@link CorePusher#endToEndPushCoreToZeroStore()} piggy backs on cached
+ * ZeroStoreShardMetadata's generation number instead of carrying a separate lastGenerationPushed
+ * property in the cache. This is also an optimization, we can always add lastGenerationPushed
+ * property to cache and get rid of this usage.
+ */
+ private final long generation;
+
+ /**
+ * The array of files that constitute the current commit point of the core (as known by the Zero
+ * store). This array is not ordered! There are no duplicate entries in it either
+ */
+ private final Set<ZeroFile.WithLocal> zeroFiles;
+
+ /**
+ * Files marked for delete but not yet removed from the Zero store. Files are only actually
+ * deleted after a new version of shard.metadata has been successfully written to the Zero store
+ * and "validated" by the ZooKeeper metadataSuffix update.
+ */
+ private final Set<ZeroFile.ToDelete> zeroFilesToDelete;
+
+ /** For Jackson deserialization. See {@link ToFromJson}. */
+ public ZeroStoreShardMetadata() {
+ this(UNDEFINED_GENERATION);
+ }
+
+ /** Always builds non "isCorrupt" and non "isDeleted" metadata. */
+ public ZeroStoreShardMetadata(
+ Set<ZeroFile.WithLocal> zeroFiles,
+ Set<ZeroFile.ToDelete> zeroFilesToDelete,
+ long generation) {
+ this.zeroFiles = zeroFiles;
+ this.zeroFilesToDelete = zeroFilesToDelete;
+ this.generation = generation;
+ }
+
+ @VisibleForTesting
+ public ZeroStoreShardMetadata(long generation) {
+ this.zeroFiles = new HashSet<>();
+ this.zeroFilesToDelete = new HashSet<>();
+ this.generation = generation;
+ }
+
+ /** Adds a file to the set of "active" files listed in the metadata */
+ public void addFile(ZeroFile.WithLocal f) {
+ this.zeroFiles.add(f);
+ }
+
+ /** Removes a file from the set of "active" files listed in the metadata */
+ public void removeFile(ZeroFile.WithLocal f) {
+ boolean removed = this.zeroFiles.remove(f);
+ assert removed; // If we remove things that are not there, likely a bug in our code
+ }
+
+ /**
+ * Adds a file to the set of files to delete listed in the metadata
+ *
+ * <p>This method should always be called with {@link #removeFile(ZeroFile.WithLocal)} above
+ * (except when adding for delete the current shard.metadata file).
+ */
+ public void addFileToDelete(ZeroFile.ToDelete f) {
+ this.zeroFilesToDelete.add(f);
+ }
+
+ public long getGeneration() {
+ return this.generation;
+ }
+
+ public Set<ZeroFile.WithLocal> getZeroFiles() {
+ return zeroFiles;
+ }
+
+ public Set<ZeroFile.ToDelete> getZeroFilesToDelete() {
+ return zeroFilesToDelete;
+ }
+
+ public Map<String, ZeroFile.WithLocal> getZeroFilesAsMap() {
+ return zeroFiles.stream()
+ .collect(Collectors.toMap(ZeroFile.WithLocal::getSolrFileName, Function.identity()));
+ }
+
+ public Set<String> getZeroFileNames() {
+ return zeroFiles.stream().map(ZeroFile.WithLocal::getSolrFileName).collect(Collectors.toSet());
+ }
+
+ /** Removes a file from the set of "deleted" files listed in the metadata */
+ public void removeFilesFromDeleted(Set<ZeroFile.ToDelete> files) {
+ int originalSize = this.zeroFilesToDelete.size();
+ boolean removed = this.zeroFilesToDelete.removeAll(files);
+ int totalRemoved = originalSize - this.zeroFilesToDelete.size();
+
+ // If we remove things that are not there, likely a bug in our code
+ assert removed && (totalRemoved == files.size());
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof ZeroStoreShardMetadata)) return false;
+
+ ZeroStoreShardMetadata that = (ZeroStoreShardMetadata) o;
+
+ if (this.generation != that.generation) return false;
+ if (!this.zeroFiles.equals(that.zeroFiles)) return false;
+ return this.zeroFilesToDelete.equals(that.zeroFilesToDelete);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(generation, this.zeroFiles, this.zeroFilesToDelete);
+ }
+
+ public static String generateMetadataSuffix() {
+ return UUID.randomUUID().toString();
+ }
+
+ public String toJson() throws Exception {
+ ToFromJson<ZeroStoreShardMetadata> converter = new ToFromJson<>();
+ return converter.toJson(this);
+ }
+
+ public static ZeroStoreShardMetadata fromJson(String json) throws Exception {
+ ToFromJson<ZeroStoreShardMetadata> converter = new ToFromJson<>();
+ return converter.fromJson(json, ZeroStoreShardMetadata.class);
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/zero/metadata/package-info.java b/solr/core/src/java/org/apache/solr/zero/metadata/package-info.java
new file mode 100644
index 00000000000..623c6d818a2
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/zero/metadata/package-info.java
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** APIs for handling Zero store metadata */
+package org.apache.solr.zero.metadata;
diff --git a/solr/core/src/java/org/apache/solr/zero/process/CollectionDeletionTask.java b/solr/core/src/java/org/apache/solr/zero/process/CollectionDeletionTask.java
new file mode 100644
index 00000000000..00c709893a6
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/zero/process/CollectionDeletionTask.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.zero.process;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.file.NoSuchFileException;
+import org.apache.solr.zero.client.ZeroStoreClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CollectionDeletionTask extends FilesDeletionTask {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public CollectionDeletionTask(
+ ZeroStoreClient zeroStoreClient,
+ String collectionName,
+ boolean allowRetry,
+ int maxRetryAttempt) {
+ super(
+ zeroStoreClient,
+ collectionName,
+ zeroStoreClient.listCollectionZeroFiles(collectionName),
+ allowRetry,
+ maxRetryAttempt);
+ }
+
+ @Override
+ public String getActionName() {
+ return "DELETE_COLLECTION_FILES";
+ }
+
+ @Override
+ public DeleterTask.Result call() {
+ DeleterTask.Result result = super.call();
+ if (result.isSuccess()) {
+ try {
+ zeroStoreClient.deleteCollectionDirectory(collectionName);
+ } catch (NoSuchFileException ex) {
+ if (log.isWarnEnabled())
+ log.warn(
+ "Could not delete Zero store directory for collection={} as it does not exists",
+ collectionName);
+ } catch (IOException ex) {
+ result.updateSuccess(false);
+ if (log.isWarnEnabled())
+ log.warn(
+ "Could not delete Zero store directory for collection={} after all files have been deleted",
+ collectionName);
+ }
+ }
+ return result;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/zero/process/CorePullStatus.java b/solr/core/src/java/org/apache/solr/zero/process/CorePullStatus.java
new file mode 100644
index 00000000000..557ca3d1a1f
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/zero/process/CorePullStatus.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.zero.process;
+
+/** Enumerates all possible outcomes of core pull tasks */
+public enum CorePullStatus {
+ /** Core pulled successfully from Zero store */
+ SUCCESS(true, false),
+ /** There was no need to pull the core from the Zero store */
+ NOT_NEEDED(true, false),
+ /** The pull task was merged with another duplicate task in the queue (deduplicated) */
+ DUPLICATE_REQUEST(true, false),
+
+ /** Core failed to pull from the Zero store, unknown cause */
+ FAILURE(false, true),
+ /** Pulling process was interrupted */
+ INTERRUPTED(false, true),
+ /**
+ * Core not pulled from Zero store because corresponding shard.metadata file was not found there
+ */
+ ZERO_METADATA_MISSING(false, false),
+ /** The pull lock was not acquired immediately, task can be retried */
+ FAILED_TO_ACQUIRE_LOCK(false, true);
+
+ private final boolean isTransientError;
+ private final boolean isSuccess;
+
+ CorePullStatus(boolean isSuccess, boolean isTransientError) {
+ assert !(isSuccess && isTransientError);
+ this.isSuccess = isSuccess;
+ this.isTransientError = isTransientError;
+ }
+
+ /**
+ * Only defined for a status when ({@link #isSuccess()} returns {@code false}).
+ *
+ * @return {@code true} when it makes sense to retry (a few times) running the same task again,
+ * i.e. when the cause of the failure might disappear.
+ */
+ public boolean isTransientError() {
+ return isTransientError;
+ }
+
+ /**
+ * @return {@code true} if content was successfully synced from Zero store OR if no sync was
+ * required.
+ */
+ public boolean isSuccess() {
+ return isSuccess;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/zero/process/CorePuller.java b/solr/core/src/java/org/apache/solr/zero/process/CorePuller.java
new file mode 100644
index 00000000000..456da6c3214
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/zero/process/CorePuller.java
@@ -0,0 +1,800 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.zero.process;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.lang.reflect.Array;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Locale;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BiConsumer;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.solr.core.DirectoryFactory;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.logging.MDCLoggingContext;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.zero.client.ZeroFile;
+import org.apache.solr.zero.client.ZeroStoreClient;
+import org.apache.solr.zero.exception.CorruptedCoreException;
+import org.apache.solr.zero.exception.ZeroException;
+import org.apache.solr.zero.exception.ZeroLockException;
+import org.apache.solr.zero.metadata.LocalCoreMetadata;
+import org.apache.solr.zero.metadata.MetadataCacheManager;
+import org.apache.solr.zero.metadata.MetadataComparisonResult;
+import org.apache.solr.zero.metadata.ZeroMetadataController;
+import org.apache.solr.zero.metadata.ZeroMetadataVersion;
+import org.apache.solr.zero.metadata.ZeroStoreShardMetadata;
+import org.apache.solr.zero.util.DeduplicatingList;
+import org.apache.solr.zero.util.FileTransferCounter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Code for pulling updates for a specific core from the Zero store. */
+public class CorePuller implements DeduplicatingList.Deduplicatable<String> {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private final ZeroStoreClient zeroStoreClient;
+ private final SolrCore solrCore;
+ private final MetadataCacheManager metadataCacheManager;
+ private final ZeroMetadataController metadataController;
+ private final Instant queuedTime;
+ private final AtomicInteger currentPullAttempt;
+ private final int maxPullAttempts;
+ private final Instant lastPullAttemptTime;
+
+ /**
+ * A callback method for end pull notification for async pulls (queries). Will be a no-op in
+ * production but some tests do set it. Note no notification will be received when the pull throws
+ * an exception rather than return a failure status code, which is the case for most though not
+ * all errors.
+ */
+ private final BiConsumer<CorePuller, CorePullStatus> pullEndNotification;
+
+ CorePuller(
+ SolrCore solrCore,
+ ZeroStoreClient zeroStoreClient,
+ MetadataCacheManager metadataCacheManager,
+ ZeroMetadataController metadataController,
+ Instant queuedTime,
+ int currentPullAttempt,
+ int maxPullAttempts,
+ Instant lastPullAttemptTime,
+ BiConsumer<CorePuller, CorePullStatus> pullEndNotification) {
+ this.zeroStoreClient = zeroStoreClient;
+ this.metadataCacheManager = metadataCacheManager;
+ this.metadataController = metadataController;
+ this.solrCore = solrCore;
+ this.queuedTime = queuedTime;
+ this.currentPullAttempt = new AtomicInteger(currentPullAttempt);
+ this.lastPullAttemptTime = lastPullAttemptTime;
+ this.maxPullAttempts = maxPullAttempts;
+ this.pullEndNotification = pullEndNotification != null ? pullEndNotification : (u, v) -> {};
+ }
+
+ public CorePuller(
+ SolrCore solrCore,
+ ZeroStoreClient zeroStoreClient,
+ MetadataCacheManager metadataCacheManager,
+ ZeroMetadataController metadataController,
+ int maxPullAttempts,
+ BiConsumer<CorePuller, CorePullStatus> pullEndNotification) {
+ this(
+ solrCore,
+ zeroStoreClient,
+ metadataCacheManager,
+ metadataController,
+ Instant.now(),
+ 0,
+ maxPullAttempts,
+ Instant.EPOCH,
+ pullEndNotification);
+ }
+
+ /** Needed for the {@link CorePuller} to be used in a {@link DeduplicatingList}. */
+ @Override
+ public String getDedupeKey() {
+ return this.getCoreName();
+ }
+
+ public static DeduplicatingList<String, CorePuller> getDeduplicatingList(int maxCapacity) {
+ return new DeduplicatingList<>(maxCapacity, new CorePuller.CorePullerMerger());
+ }
+
+ public void incCurrentPullAttempts() {
+ currentPullAttempt.incrementAndGet();
+ }
+
+ public int getCurrentPullAttempt() {
+ return currentPullAttempt.get();
+ }
+
+ public int getMaxPullAttempts() {
+ return maxPullAttempts;
+ }
+
+ private CorePuller newUpdatedPuller(
+ Instant queuedTime, int attempts, int maxAttempts, Instant lastAttemptTime) {
+
+ return new CorePuller(
+ solrCore,
+ zeroStoreClient,
+ metadataCacheManager,
+ metadataController,
+ queuedTime,
+ attempts,
+ maxAttempts,
+ lastAttemptTime,
+ pullEndNotification);
+ }
+
+ /**
+ * This method is only used in this class for now because the "re-enqueue with delay"
+ * implementation is imperfect. Longer term, such a re-enqueue should be handled outside this
+ * class.
+ */
+ public Instant getLastPullAttemptTime() {
+ return lastPullAttemptTime;
+ }
+
+ public Instant getQueuedTime() {
+ return this.queuedTime;
+ }
+
+ /**
+ * Manages multiple retries calling {@link #lockAndPullCore} to pull a core from Zero store when
+ * transient failures are encountered.
+ */
+ @SuppressWarnings("try")
+ public CorePullStatus pullCoreWithRetries(
+ boolean isLeaderPulling, long secondsToWaitPullLock, long retryDelay) throws ZeroException {
+ MDCLoggingContext.setCore(getCore());
+ try {
+ if (skipPullIfShardAlreadySynced(isLeaderPulling)) return CorePullStatus.NOT_NEEDED;
+
+ // Initialization needed if we don't pass in the loop even once (tests...). Then
+ // CorePullStatus.FAILURE
+ // being a transient error, the log will be correct that all retries exhausted.
+ CorePullStatus pullStatus = CorePullStatus.FAILURE;
+
+ while (getCurrentPullAttempt() < maxPullAttempts) {
+ // Sleep a bit if we're retrying, to let the previous error disappear naturally
+ try {
+ sleepIfNeeded(retryDelay);
+ } catch (InterruptedException ie) {
+ // No retry if the wait is interrupted (no timeout here)
+ Thread.currentThread().interrupt();
+ String message = "Wait done before core pull got interrupted, likely shutdown?";
+ log.error(message, ie);
+ throw new ZeroException(message);
+ }
+ // Do the actual pull
+ pullStatus = lockAndPullCore(isLeaderPulling, secondsToWaitPullLock);
+ // Check the return code and throw exceptions for most errors
+ incCurrentPullAttempts();
+ if (pullStatus.isSuccess()) {
+ if (log.isInfoEnabled()) {
+ log.info(
+ String.format(
+ Locale.ROOT,
+ "Pulling core for collection %s shard %s succeeded. Last status=%s attempts=%s",
+ getCollectionName(),
+ getShardName(),
+ pullStatus,
+ getCurrentPullAttempt()));
+ }
+ notifyPullEnd(pullStatus);
+ return pullStatus;
+ } else if (!pullStatus.isTransientError()) {
+ break;
+ }
+ if (log.isInfoEnabled()) {
+ log.info(
+ String.format(
+ Locale.ROOT,
+ "Pulling core for collection %s shard %s failed with transient error. Retrying. Last status=%s attempts=%s",
+ getCollectionName(),
+ getShardName(),
+ pullStatus,
+ getCurrentPullAttempt()));
+ }
+ }
+ // Give up if unrecoverable or attempts exhausted
+ if (log.isWarnEnabled()) {
+ log.warn(
+ String.format(
+ Locale.ROOT,
+ "Pulling core for collection %s shard %s failed %s. Giving up. Last status=%s attempts=%s",
+ getCollectionName(),
+ getShardName(),
+ pullStatus.isTransientError()
+ ? "after max attempts reached"
+ : "with non transient error",
+ pullStatus,
+ getCurrentPullAttempt()));
+ }
+ throw new ZeroException(pullStatus.name());
+ } finally {
+ MDCLoggingContext.clear();
+ }
+ }
+
+ @VisibleForTesting
+ void notifyPullEnd(CorePullStatus pullStatus) {
+ // Tests count down on a latch to observe the end of the pull
+ pullEndNotification.accept(this, pullStatus);
+ }
+
+ /**
+ * Acquires the required locks and calls {@link #pullCoreFilesFromZeroStore(boolean)} to do the
+ * actual pulling work. Also checks a few conditions to skip the pull completely when not needed.
+ * No pull completion notification ({@link #notifyPullEnd(CorePullStatus)}) done from this method
+ * because if there are retries, we'd only notify after multiple failures (or a success).
+ */
+ @SuppressWarnings("try")
+ public CorePullStatus lockAndPullCore(boolean isLeaderPulling, long secondsToWaitPullLock) {
+ if (skipPullIfShardAlreadySynced(isLeaderPulling)) return CorePullStatus.NOT_NEEDED;
+ try (ZeroAccessLocks.NoThrowAutoCloseable ignore =
+ metadataCacheManager
+ .getOrCreateCoreMetadata(solrCore.getName())
+ .getZeroAccessLocks()
+ .acquirePullLock(secondsToWaitPullLock)) {
+ if (skipPullIfShardHasDefaultSuffix(isLeaderPulling)) {
+ return CorePullStatus.NOT_NEEDED;
+ }
+ return pullCoreFilesFromZeroStore(isLeaderPulling);
+ } catch (ZeroLockException ex) {
+ log.warn("Failed to acquire lock for pull", ex);
+ return CorePullStatus.FAILED_TO_ACQUIRE_LOCK;
+ } catch (ZeroException ex) {
+ log.warn("Failure pulling core", ex);
+ return CorePullStatus.FAILURE;
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ log.warn("Pull interrupted", ex);
+ return CorePullStatus.INTERRUPTED;
+ }
+ }
+
+ private String getTaskDescription() {
+ return String.format(
+ Locale.ROOT,
+ "(col=%s,shard=%s,name=%s,attempt=%d,thread=%s)",
+ getCollectionName(),
+ getShardName(),
+ getCoreName(),
+ getCurrentPullAttempt() + 1,
+ Thread.currentThread().getName());
+ }
+
+ /**
+ * This method is used to quickly ends core pulling if shard has never been updated since creation
+ * Requires read lock to be taken otherwise fails
+ */
+ private boolean skipPullIfShardHasDefaultSuffix(boolean isLeaderPulling) throws ZeroException {
+ ZeroMetadataVersion shardMetadataVersion = getShardMetadataVersion();
+ if (metadataController.hasDefaultNodeSuffix(shardMetadataVersion)) {
+ ZeroStoreShardMetadata emptyZeroStoreShardMetadata = new ZeroStoreShardMetadata();
+ metadataCacheManager.updateCoreMetadata(
+ solrCore.getName(), shardMetadataVersion, emptyZeroStoreShardMetadata, isLeaderPulling);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * This method is used to quickly end core pulling if server side and remote shard have the same
+ * version
+ */
+ private boolean skipPullIfShardAlreadySynced(boolean isLeaderPulling) {
+ MetadataCacheManager.MetadataCacheEntry coreMetadata =
+ metadataCacheManager.getOrCreateCoreMetadata(solrCore.getName());
+ ZeroMetadataVersion shardMetadataVersion = getShardMetadataVersion();
+ // If leader is pulling, compare metadata to cache only if cache assumed up to date
+ return (coreMetadata.isCacheLikelyUpToDate() || !isLeaderPulling)
+ && coreMetadata.getMetadataVersion().equals(shardMetadataVersion);
+ }
+
+ /**
+ * Method that sleeps for a while depending on the number of pull attempts already done (in order
+ * not to retry pulling in short sequence and likely hit the same transient errors)
+ *
+ * @param retryDelay milliseconds to let elapse since previous pull attempt, by sleeping if needed
+ */
+ private void sleepIfNeeded(long retryDelay) throws InterruptedException {
+ if (getCurrentPullAttempt() != 0) {
+ long sleepTime =
+ retryDelay - getLastPullAttemptTime().until(Instant.now(), ChronoUnit.MILLIS);
+ if (sleepTime > 0) {
+ Thread.sleep(sleepTime);
+ }
+ }
+ }
+
+ /**
+ * Pulls required files representing the core update from the Zero store and returns the success
+ * or failure status. Pull lock ({@link ZeroAccessLocks#acquirePullLock(long)}) must be held.
+ */
+ private CorePullStatus pullCoreFilesFromZeroStore(boolean isLeaderPulling) throws ZeroException {
+ if (log.isInfoEnabled()) {
+ log.info("Initiating core pull {}", getTaskDescription());
+ }
+
+ ZeroMetadataVersion shardMetadataVersion = getShardMetadataVersion();
+
+ MetadataCacheManager.ZeroCoreStage stage = null;
+ ZeroStoreShardMetadata zeroMetadata;
+ CorePullStatus corePullStatus = CorePullStatus.FAILURE;
+
+ try {
+ metadataCacheManager.recordState(solrCore, MetadataCacheManager.ZeroCoreStage.PULL_STARTED);
+
+ // Get Zero store shard metadata
+ ZeroFile.WithLocal localZeroFile =
+ metadataController.newShardMetadataZeroFile(
+ getCollectionName(), getShardName(), shardMetadataVersion.getMetadataSuffix());
+ zeroMetadata = zeroStoreClient.pullShardMetadata(localZeroFile);
+
+ // Handle callback
+ if (zeroMetadata == null) {
+ return CorePullStatus.ZERO_METADATA_MISSING;
+ }
+
+ // Diff local metadata against Zero store metadata. Given we're doing a pull, don't
+ // need to reserve the commit point.
+ // We do need to compute a directory hash to verify after pulling or before switching index
+ // dirs that no local changes occurred concurrently
+ LocalCoreMetadata localCoreMetadata = new LocalCoreMetadata(solrCore);
+ localCoreMetadata.readMetadata(false, true);
+ MetadataComparisonResult metadataComparisonResult =
+ metadataController.diffMetadataforPull(localCoreMetadata, zeroMetadata);
+
+ // If there is nothing to pull, we should report SUCCESS_EQUIVALENT and do nothing.
+ if (!metadataComparisonResult.getFilesToPull().isEmpty()) {
+ pullUpdateFromZero(metadataComparisonResult, true);
+ corePullStatus = CorePullStatus.SUCCESS;
+ } else if (getOpenTimeForSearcher().isBefore(getLastPullAttemptTime())) {
+ // If the searcher's opened ts is less than the last time this task ran, we assume it failed
+ // to open successfully
+ log.warn(
+ "Looks like a searcher was never opened after the last pull from Zero store, doing that now. {} ",
+ getTaskDescription());
+ openSearcher(true);
+ corePullStatus = CorePullStatus.SUCCESS;
+ } else {
+ log.warn(
+ "There are no files to pull and we don't need to re-open a new searcher, even though we do not match the version in zk? {}",
+ getTaskDescription());
+ corePullStatus = CorePullStatus.NOT_NEEDED;
+ }
+
+ metadataCacheManager.updateCoreMetadata(
+ solrCore.getName(), shardMetadataVersion, zeroMetadata, isLeaderPulling);
+
+ } catch (InterruptedException e) {
+ corePullStatus = CorePullStatus.INTERRUPTED;
+ stage = MetadataCacheManager.ZeroCoreStage.PULL_FAILED;
+ throwZeroException(e, "Failed attempt to pull core");
+ } catch (CorruptedCoreException e) {
+ corePullStatus = CorePullStatus.FAILURE;
+ stage = MetadataCacheManager.ZeroCoreStage.PULL_FAILED_WITH_CORRUPTION;
+ throwZeroException(e, "Failed attempt to pull core");
+ } catch (Exception e) {
+ corePullStatus = CorePullStatus.FAILURE;
+ throwZeroException(e, "Failed attempt to pull core");
+ } finally {
+ if (stage == null) {
+ stage =
+ corePullStatus.isSuccess()
+ ? MetadataCacheManager.ZeroCoreStage.PULL_SUCCEEDED
+ : MetadataCacheManager.ZeroCoreStage.PULL_FAILED;
+ }
+ metadataCacheManager.recordState(solrCore, stage);
+ }
+ return corePullStatus;
+ }
+
+ /**
+ * We're doing here what replication does in {@link
+ * org.apache.solr.handler.IndexFetcher#fetchLatestIndex(boolean, boolean)}.
+ *
+ * <p>This method will work in 2 cases:
+ *
+ * <ol>
+ * <li>Local core needs to fetch an update from Zero store
+ * <li>Local core did not exist (was created empty before calling this method) and is fetched
+ * from Zero store
+ * </ol>
+ *
+ * @param waitForSearcher {@code true} if this call should wait until the index searcher is
+ * created (so that any query after the return from this method sees the new pulled content)
+ * or {@code false} if we request a new index searcher to be eventually created but do not
+ * wait for it to be created (a query following the return from this call might see the old
+ * core content).
+ * @throws CorruptedCoreException If the local core index cannot be opened after pulling it from
+ * the Zero store.
+ * @throws Exception We currently do not treat differently errors such as Zero store temporarily
+ * not available or network issues. We therefore consider all exceptions thrown by this method
+ * as a sign that it is durably not possible to pull the core from the Zero Store. TODO This
+ * has to be revisited at some point
+ */
+ void pullUpdateFromZero(
+ MetadataComparisonResult metadataComparisonResult, boolean waitForSearcher) throws Exception {
+ Instant startTime = Instant.now();
+ boolean isSuccessful = false;
+ FileTransferCounter counter = new FileTransferCounter();
+ try {
+ // if there is a conflict between local and Zero store contents we will move the core to a new
+ // index directory
+ boolean coreSwitchedToNewIndexDir = false;
+ // Create temp directory (within the core local folder).
+ // If we are moving index to a new directory because of conflict then this will be that new
+ // directory.
+ // Even if we are not moving to a newer directory we will first download files from the Zero
+ // store into this temp directory.
+ // Then we will move files from temp directory to index directory. This is to avoid leaving
+ // a download half done in case of failure as well as to limit the time during which
+ // we close then reopen the index writer to take into account the new files. In theory
+ // nothing should be changing the local directory as we pull files from the Zero store,
+ // but let's be defensive (we're checking further down that local dir hasn't changed
+ // in the meantime).
+ String tempIndexDirName = "index.pull." + System.nanoTime();
+ String tempIndexDirPath = solrCore.getDataDir() + tempIndexDirName;
+ Directory tempIndexDir = getDirectory(solrCore, tempIndexDirPath);
+ try {
+ String indexDirPath = solrCore.getIndexDir();
+ Collection<ZeroFile.WithLocal> filesToDownload;
+ if (metadataComparisonResult.isLocalConflictingWithZero()) {
+ // This is an optimization to not download everything from Zero store if possible
+ Directory indexDir = getDirectory(solrCore, indexDirPath);
+ try {
+ filesToDownload =
+ initializeNewIndexDirWithLocallyAvailableFiles(
+ indexDir, tempIndexDir, metadataComparisonResult.getFilesToPull());
+ } finally {
+ solrCore.getDirectoryFactory().release(indexDir);
+ }
+ } else {
+ filesToDownload = metadataComparisonResult.getFilesToPull();
+ }
+
+ pullZeroFiles(tempIndexDir, filesToDownload, counter);
+
+ Directory indexDir = getDirectory(solrCore, indexDirPath);
+ try {
+ if (!metadataComparisonResult.isLocalConflictingWithZero()) {
+ // TODO should we call solrCore.closeSearcher() here? IndexFetcher.fetchLatestIndex()
+ // does call it.
+ // Close the index writer to stop changes to this core
+ solrCore.getUpdateHandler().getSolrCoreState().closeIndexWriter(solrCore, true);
+ }
+
+ Exception thrownException = null;
+ try {
+ // Make sure Solr core directory content hasn't changed since we decided what we want
+ // to pull from the Zero store
+ if (!metadataComparisonResult.isSameDirectoryContent(indexDir)) {
+ // Maybe return something less aggressive than throwing an exception? TBD once we
+ // end up calling this method :)
+ throw new Exception(
+ "Local Directory content "
+ + indexDirPath
+ + " has changed since core pull from Zero store started. Aborting pull.");
+ }
+
+ if (metadataComparisonResult.isLocalConflictingWithZero()) {
+ // point index to the new directory.
+ solrCore.modifyIndexProps(tempIndexDirName);
+ coreSwitchedToNewIndexDir = true;
+ } else {
+ moveFilesFromTempToIndexDir(tempIndexDir, indexDir, metadataComparisonResult);
+ }
+ } catch (Exception e) {
+ // Do not mask the exception in the finally block below.
+ thrownException = e;
+ throw e;
+ } finally {
+ try {
+ openIndexWriter(
+ metadataComparisonResult.isLocalConflictingWithZero(), coreSwitchedToNewIndexDir);
+ } catch (Exception e) {
+ // We are not able to open an index writer on files we just wrote locally.
+ // Consider the index is corrupted and wipe it, so it will be pulled again.
+ wipeCorruptedIndex(indexDir);
+ wipeCorruptedIndex(tempIndexDir);
+ throw new CorruptedCoreException(
+ "Core " + getCoreName() + " is corrupted locally",
+ thrownException == null ? e : thrownException);
+ }
+ }
+ } finally {
+ try {
+ if (coreSwitchedToNewIndexDir) {
+ solrCore.getDirectoryFactory().doneWithDirectory(indexDir);
+ solrCore.getDirectoryFactory().remove(indexDir);
+ }
+ } catch (Exception e) {
+ log.warn("Cannot remove previous index directory {}", indexDir, e);
+ } finally {
+ solrCore.getDirectoryFactory().release(indexDir);
+ }
+ }
+ } finally {
+ try {
+ if (!coreSwitchedToNewIndexDir) {
+ solrCore.getDirectoryFactory().doneWithDirectory(tempIndexDir);
+ solrCore.getDirectoryFactory().remove(tempIndexDir);
+ }
+ } catch (Exception e) {
+ log.warn("Cannot remove temp directory {}", tempIndexDirPath, e);
+ } finally {
+ solrCore.getDirectoryFactory().release(tempIndexDir);
+ }
+ }
+ try {
+ openSearcher(waitForSearcher);
+ } catch (Exception e) {
+ // We are not able to open an index searcher on files we just wrote locally.
+ // Consider the index is corrupted and wipe it, so it will be pulled again.
+ Directory indexDir = getDirectory(solrCore, solrCore.getIndexDir());
+ try {
+ wipeCorruptedIndex(indexDir);
+ } finally {
+ solrCore.getDirectoryFactory().release(indexDir);
+ }
+ throw new CorruptedCoreException("Core " + getCoreName() + " is corrupted locally", e);
+ }
+ isSuccessful = true;
+ } finally {
+ if (log.isInfoEnabled()) {
+ log.info(
+ "PULL runTime={} bytesTransferred={} expectedFilesAffected={} actualFilesAffected={} isSuccessful={} "
+ + "localGeneration={} zeroGeneration={}",
+ startTime.until(Instant.now(), ChronoUnit.MILLIS),
+ counter.getBytesTransferred(),
+ counter.getExpectedFilesTransferred(),
+ counter.getActualFilesTransferred(),
+ isSuccessful,
+ metadataComparisonResult.getLocalGeneration(),
+ metadataComparisonResult.getDistantGeneration());
+ }
+ }
+ }
+
+ public static Directory getDirectory(SolrCore core, String path) throws IOException {
+ return core.getDirectoryFactory()
+ .get(path, DirectoryFactory.DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType);
+ }
+
+ @VisibleForTesting
+ protected void pullZeroFiles(
+ Directory tempIndexDir,
+ Collection<ZeroFile.WithLocal> filesToDownload,
+ FileTransferCounter counter)
+ throws InterruptedException, ExecutionException {
+ counter.setExpectedFilesTransferred(filesToDownload.size());
+
+ CompletableFuture<?>[] futures = new CompletableFuture<?>[filesToDownload.size()];
+ int i = 0;
+ for (ZeroFile.WithLocal bf : filesToDownload) {
+ futures[i++] = zeroStoreClient.pullFileAsync(tempIndexDir, bf);
+
+ counter.incrementActualFilesTransferred();
+ counter.incrementBytesTransferred(bf.getFileSize());
+ }
+
+ CompletableFuture.allOf(futures).get();
+ }
+
+ private Collection<ZeroFile.WithLocal> initializeNewIndexDirWithLocallyAvailableFiles(
+ Directory indexDir, Directory newIndexDir, Collection<ZeroFile.WithLocal> filesToPull) {
+ Collection<ZeroFile.WithLocal> filesToDownload = new HashSet<>();
+ for (ZeroFile.WithLocal zeroFile : filesToPull) {
+ assert zeroFile.isChecksumPresent();
+ try {
+ try (final IndexInput indexInput =
+ indexDir.openInput(zeroFile.getSolrFileName(), IOContext.READONCE)) {
+ if (indexInput.length() == zeroFile.getFileSize()
+ && CodecUtil.retrieveChecksum(indexInput) == zeroFile.getChecksum()) {
+ copyFileToDirectory(indexDir, zeroFile.getSolrFileName(), newIndexDir);
+ } else {
+ filesToDownload.add(zeroFile);
+ }
+ }
+ } catch (Exception ex) {
+ // Either file does not exist locally or copy failed, we will download from Zero store
+ filesToDownload.add(zeroFile);
+ }
+ }
+ return filesToDownload;
+ }
+
+ protected ZeroMetadataVersion getShardMetadataVersion() {
+ return metadataController.readMetadataValue(getCollectionName(), getShardName());
+ }
+
+ @VisibleForTesting
+ protected void openIndexWriter(boolean createNewIndexDir, boolean coreSwitchedToNewIndexDir)
+ throws Exception {
+ if (!createNewIndexDir) {
+ // The closed index writer must be opened back
+ // (in the finally block as stated in the javadoc of SolrCoreState.closeIndexWriter).
+ solrCore.getUpdateHandler().getSolrCoreState().openIndexWriter(solrCore);
+ } else if (coreSwitchedToNewIndexDir) {
+ solrCore.getUpdateHandler().newIndexWriter(true);
+ }
+ }
+
+ void openSearcher(boolean waitForSearcher) throws Exception {
+ if (waitForSearcher) {
+ // Open and register a new searcher, we don't need it but we wait for it to be open.
+ @SuppressWarnings("unchecked")
+ final Future<Void>[] waitSearcher = (Future<Void>[]) Array.newInstance(Future.class, 1);
+ solrCore.getSearcher(true, false, waitSearcher, true);
+ if (waitSearcher[0] == null) {
+ throw new Exception(
+ "Can't wait for index searcher to be created. Future queries might misbehave for core="
+ + solrCore.getName());
+ } else {
+ waitSearcher[0].get();
+ }
+ } else {
+ // Open and register a new searcher, but don't wait and we don't need it either.
+ solrCore.getSearcher(true, false, null, true);
+ }
+ }
+
+ /** Wipe the provided index directory. */
+ private void wipeCorruptedIndex(Directory dir) {
+ try {
+ // Delete the files individually instead of calling Directory.remove().
+ // The latter is not done immediately and may result in the files not being
+ // removed before the next pull attempt.
+ for (String file : dir.listAll()) {
+ dir.deleteFile(file);
+ }
+ } catch (Exception e) {
+ log.error("Cannot wipe the corrupted index in directory {}", dir, e);
+ }
+ }
+
+ /** Copies {@code fileName} from {@code fromDir} to {@code toDir} */
+ private void copyFileToDirectory(Directory fromDir, String fileName, Directory toDir)
+ throws IOException {
+ // TODO: Consider optimizing with org.apache.lucene.store.HardlinkCopyDirectoryWrapper
+ toDir.copyFrom(fromDir, fileName, fileName, DirectoryFactory.IOCONTEXT_NO_CACHE);
+ }
+
+ /** Moves {@code fileName} from {@code fromDir} to {@code toDir} */
+ private void moveFileToDirectory(Directory fromDir, String fileName, Directory toDir)
+ throws IOException {
+ // We don't need to keep the original files so we move them over.
+ // TODO: Consider optimizing with org.apache.lucene.store.HardlinkCopyDirectoryWrapper
+ solrCore
+ .getDirectoryFactory()
+ .move(fromDir, toDir, fileName, DirectoryFactory.IOCONTEXT_NO_CACHE);
+ }
+
+ private void moveFilesFromTempToIndexDir(
+ Directory tmpIndexDir, Directory dir, MetadataComparisonResult metadataComparisonResult)
+ throws IOException {
+ if (metadataComparisonResult.getFilesToPull().isEmpty()) {
+ return;
+ }
+
+ // Copy all files into the Solr directory
+ // Move the segments_N file last once all other are ok.
+ String segmentsN = null;
+ for (ZeroFile.WithLocal bf : metadataComparisonResult.getFilesToPull()) {
+ if (metadataController.isSegmentsNFilename(bf)) {
+ assert segmentsN == null;
+ segmentsN = bf.getSolrFileName();
+ } else {
+ // Copy all non segments_N files
+ moveFileToDirectory(tmpIndexDir, bf.getSolrFileName(), dir);
+ }
+ }
+ assert segmentsN != null;
+ // Copy segments_N file. From this point on the local core might be accessed and is up to date
+ // with Zero store content
+ moveFileToDirectory(tmpIndexDir, segmentsN, dir);
+ }
+
+ /**
+ * @return timestamp in millis when the core's searcher was opened, NOT wall-clock time but nano
+ * time, so that it can be compared against the last run timestamp of this job.
+ */
+ private Instant getOpenTimeForSearcher() throws IOException {
+ return solrCore == null
+ ? Instant.EPOCH
+ : Instant.ofEpochMilli(
+ TimeUnit.MILLISECONDS.convert(
+ solrCore.withSearcher(SolrIndexSearcher::getOpenTimeStamp).getTime(),
+ TimeUnit.NANOSECONDS));
+ }
+
+ public String getCoreName() {
+ return solrCore.getName();
+ }
+
+ public String getShardName() {
+ return solrCore.getCoreDescriptor().getCloudDescriptor().getShardId();
+ }
+
+ public String getCollectionName() {
+ return solrCore.getCoreDescriptor().getCollectionName();
+ }
+
+ private void throwZeroException(Exception ex, String format, Object... args)
+ throws ZeroException {
+ throw new ZeroException(
+ String.format(Locale.ROOT, format, args) + ": " + getTaskDescription(), ex);
+ }
+
+ private void throwZeroException(String format, Object... args) throws ZeroException {
+ throw new ZeroException(String.format(Locale.ROOT, format, args) + ": " + getTaskDescription());
+ }
+
+ public SolrCore getCore() {
+ return solrCore;
+ }
+
+ /** Needed for the {@link CorePuller} to be used in a {@link DeduplicatingList}. */
+ static class CorePullerMerger implements DeduplicatingList.Merger<String, CorePuller> {
+ /**
+ * Given two tasks (that have not yet started executing!) that target the same shard (and would
+ * basically do the same things were they both executed), returns a merged task that can replace
+ * both and that retains the oldest enqueue time and the smallest number of attempts, so we
+ * don't "lose" retries because of the merge yet we correctly report that tasks might have been
+ * waiting for execution for a long while.
+ *
+ * @return a merged {@link CorePuller} that can replace the two tasks passed as parameters.
+ */
+ @Override
+ public CorePuller merge(CorePuller puller1, CorePuller puller2) {
+ // We allow more opportunities to try as the core is changed again by Solr...
+ int mergedAttempts =
+ Math.min(puller1.getCurrentPullAttempt(), puller2.getCurrentPullAttempt());
+ int mergedMaxAttempts = Math.max(puller1.getMaxPullAttempts(), puller2.getMaxPullAttempts());
+
+ // ...and base the delay computation on the time of last attempt.
+ Instant mergedLastAttemptsTime =
+ puller1.getLastPullAttemptTime().isBefore(puller2.getLastPullAttemptTime())
+ ? puller2.getLastPullAttemptTime()
+ : puller1.getLastPullAttemptTime();
+
+ Instant minQueueTime =
+ puller1.getQueuedTime().isBefore(puller2.getQueuedTime())
+ ? puller1.getQueuedTime()
+ : puller2.getQueuedTime();
+ // We merge the tasks.
+ return puller1.newUpdatedPuller(
+ minQueueTime, mergedAttempts, mergedMaxAttempts, mergedLastAttemptsTime);
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/zero/process/CorePullerBlockingQueue.java b/solr/core/src/java/org/apache/solr/zero/process/CorePullerBlockingQueue.java
new file mode 100644
index 00000000000..db19e6acb8e
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/zero/process/CorePullerBlockingQueue.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.zero.process;
+
+import java.lang.invoke.MethodHandles;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import org.apache.solr.logging.MDCLoggingContext;
+import org.apache.solr.zero.util.DeduplicatingList;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A wrapper class for a {@link DeduplicatingList} so it appears to be a {@link BlockingQueue} of
+ * {@link Runnable} and can be used by the {@link ZeroStoreManager#asyncCorePullExecutor} {@link
+ * ThreadPoolExecutor}.
+ *
+ * <p>This is a bare-bones implementation. Only methods {@link #take()}, {@link #isEmpty()} and
+ * {@link #drainTo(Collection)} are implemented. This class is only meant to serve the single very
+ * specific use case mentioned above.
+ */
+public class CorePullerBlockingQueue implements BlockingQueue<Runnable> {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private final DeduplicatingList<String, CorePuller> pullTaskQueue;
+ private final ZeroStoreManager storeManager;
+
+ public CorePullerBlockingQueue(
+ DeduplicatingList<String, CorePuller> pullTaskQueue, ZeroStoreManager storeManager) {
+ this.pullTaskQueue = pullTaskQueue;
+ this.storeManager = storeManager;
+ }
+
+ /**
+ * Wrapper of {@link CorePuller} instantiated each time an object is {@link #take}-en from the
+ * {@link CorePullerBlockingQueue} so that the executor that consumes that queue ({@link
+ * ZeroStoreManager#asyncCorePullExecutor}) get a {@link Runnable} as expected.
+ */
+ static class RunnableAsyncCorePull implements Runnable {
+ private final CorePuller corePuller;
+ private final boolean isleaderPulling;
+ private final Instant enqueueTime = Instant.now();
+
+ public RunnableAsyncCorePull(CorePuller corePuller, boolean isleaderPulling) {
+ this.corePuller = corePuller;
+ this.isleaderPulling = isleaderPulling;
+ }
+
+ @Override
+ public void run() {
+ MDCLoggingContext.setCore(corePuller.getCore());
+ try {
+ CorePullStatus pullStatus = corePuller.lockAndPullCore(isleaderPulling, 0);
+ // Tests might count down on a latch to observe the end of the pull
+ corePuller.notifyPullEnd(pullStatus);
+ if (pullStatus.isSuccess()) {
+ if (log.isInfoEnabled()) {
+ log.info(
+ "Async Zero store pull finished successfully {} ms after being enqueued",
+ enqueueTime.until(Instant.now(), ChronoUnit.MILLIS));
+ }
+ } else {
+ // Just a log, no retries, the pull has failed. Future pulls might succeed
+ log.error(
+ "Async Zero store pull failed with status {} {} ms after being enqueued",
+ pullStatus,
+ enqueueTime.until(Instant.now(), ChronoUnit.MILLIS));
+ }
+ } finally {
+ MDCLoggingContext.clear();
+ }
+ }
+ }
+
+ @Override
+ public Runnable take() throws InterruptedException {
+ CorePuller cp = pullTaskQueue.removeFirst();
+ return new RunnableAsyncCorePull(cp, storeManager.isLeader(cp.getCore()));
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return pullTaskQueue.size() == 0;
+ }
+
+ @Override
+ public int drainTo(Collection<? super Runnable> c) {
+ int count = 0;
+ while (!isEmpty()) {
+ try {
+ c.add(take());
+ count++;
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ return count;
+ }
+
+ /*
+ * CorePullerBlockingQueue methods below this point throw UnsupportedOperationException
+ */
+
+ @Override
+ public boolean add(Runnable runnable) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean offer(Runnable runnable) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Runnable remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Runnable poll() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Runnable element() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Runnable peek() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void put(Runnable runnable) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean offer(Runnable runnable, long timeout, TimeUnit unit) throws InterruptedException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Runnable poll(long timeout, TimeUnit unit) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int remainingCapacity() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean remove(Object o) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean containsAll(Collection<?> c) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean addAll(Collection<? extends Runnable> c) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean removeAll(Collection<?> c) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean retainAll(Collection<?> c) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void clear() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int size() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean contains(Object o) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Iterator<Runnable> iterator() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Object[] toArray() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public <T> T[] toArray(T[] a) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int drainTo(Collection<? super Runnable> c, int maxElements) {
+ throw new UnsupportedOperationException();
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/zero/process/CorePusher.java b/solr/core/src/java/org/apache/solr/zero/process/CorePusher.java
new file mode 100644
index 00000000000..fa5017d16cb
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/zero/process/CorePusher.java
@@ -0,0 +1,501 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.zero.process;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.lang.invoke.MethodHandles;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Locale;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
+import org.apache.lucene.index.IndexCommit;
+import org.apache.lucene.store.Directory;
+import org.apache.solr.client.solrj.cloud.BadVersionException;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.logging.MDCLoggingContext;
+import org.apache.solr.zero.client.ZeroFile;
+import org.apache.solr.zero.client.ZeroStoreClient;
+import org.apache.solr.zero.exception.ZeroLockException;
+import org.apache.solr.zero.metadata.LocalCoreMetadata;
+import org.apache.solr.zero.metadata.MetadataCacheManager;
+import org.apache.solr.zero.metadata.MetadataCacheManager.MetadataCacheEntry;
+import org.apache.solr.zero.metadata.MetadataCacheManager.ZeroCoreStage;
+import org.apache.solr.zero.metadata.MetadataComparisonResult;
+import org.apache.solr.zero.metadata.ZeroMetadataController;
+import org.apache.solr.zero.metadata.ZeroMetadataVersion;
+import org.apache.solr.zero.metadata.ZeroStoreShardMetadata;
+import org.apache.solr.zero.util.FileTransferCounter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class executes synchronous pushes of core updates to the Zero store.
+ *
+ * <p>Pushes will be triggered at the end of an indexing batch when a shard's index data has changed
+ * locally and needs to be persisted in the Zero store, or when a core is created post split or
+ * restore.
+ *
+ * <p>Pushes to the Zero store do not happen during replica creation (including when part of
+ * collection creation) but only after the first indexing batch.
+ *
+ * <p>TODO This is most certainly an issue when a collection is created but not indexed right away,
+ * or when a collection is created with multiple shards and some of them are not indexed (possibly
+ * never indexed).
+ */
+public class CorePusher {
+
+ /**
+ * There should be no contention when acquiring these locks so a small timeout should be
+ * reasonable
+ */
+ private static final int LOCK_ACQUIRE_TIMEOUT = 5;
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ ZeroStoreClient zeroStoreClient;
+ DeleteProcessor deleteProcessor;
+ MetadataCacheManager metadataCacheManager;
+ ZeroMetadataController metadataController;
+ SolrCore solrCore;
+
+ public CorePusher(
+ SolrCore solrCore,
+ ZeroStoreClient zeroStoreClient,
+ DeleteProcessor deleteProcessor,
+ MetadataCacheManager metadataCacheManager,
+ ZeroMetadataController metadataController) {
+ this.zeroStoreClient = zeroStoreClient;
+ this.deleteProcessor = deleteProcessor;
+ this.metadataCacheManager = metadataCacheManager;
+ this.metadataController = metadataController;
+ this.solrCore = solrCore;
+ }
+
+ /**
+ * Pushes the local core updates to the Zero store and logs whether the push succeeded or failed.
+ * This is the end to end push method, comparing the local and distant states of the core (local
+ * core vs "distant" shard stored in Zero store), writing all required files to the Zero store and
+ * finally updating the metadata in ZooKeeper to refer to the new shard.metadata file.
+ */
+ @SuppressWarnings("try")
+ public CorePusherExecutionInfo endToEndPushCoreToZeroStore() throws SolrException {
+ try {
+ String collectionName = getCollectionName();
+ String shardName = getShardName();
+
+ MDCLoggingContext.setCore(solrCore);
+ if (log.isInfoEnabled()) {
+ log.info("Initiating push {}", getTaskDescription());
+ }
+
+ Instant startTime = Instant.now();
+ try (AutoCloseable ignore =
+ metadataCacheManager
+ .getOrCreateCoreMetadata(solrCore.getName())
+ .getZeroAccessLocks()
+ .acquirePushLock(LOCK_ACQUIRE_TIMEOUT)) {
+ long lockAcquisitionTime = startTime.until(Instant.now(), ChronoUnit.MILLIS);
+ try {
+ metadataCacheManager.recordState(solrCore, ZeroCoreStage.PUSH_STARTED);
+
+ IndexCommit latestCommit = solrCore.getDeletionPolicy().getLatestCommit();
+ if (latestCommit == null) {
+ throw new SolrException(
+ ErrorCode.SERVER_ERROR, "Core " + getCoreName() + " has no available commit point");
+ }
+
+ MetadataCacheEntry coreMetadata =
+ metadataCacheManager.getOrCreateCoreMetadata(solrCore.getName());
+ long zeroGeneration = coreMetadata.getZeroShardMetadata().getGeneration();
+ if (latestCommit.getGeneration() == zeroGeneration) {
+ // Everything up to latest commit point has already been pushed
+ // This can happen if another indexing batch comes in and acquires the push lock first
+ // and ends up pushing segments produced by this indexing batch.
+ // TODO could it be another node pushed a different segment of same generation?
+ if (log.isInfoEnabled())
+ log.info(
+ "Nothing to push, pushLockTime={}: {}",
+ lockAcquisitionTime,
+ getTaskDescription());
+ long totalTimeMs = startTime.until(Instant.now(), ChronoUnit.MILLIS);
+ return CorePusherExecutionInfo.localAlreadyUpToDate(
+ zeroGeneration, lockAcquisitionTime, totalTimeMs);
+ }
+
+ // Compute the differences (if any) between the local shard index data and shard index
+ // data on the Zero store. The names of new files to push to the Zero store will include
+ // a suffix that is generated here and passed to the comparison method. Even though not
+ // required, all files pushed together will have the same suffix (but different file
+ // names). This helps debug when trying to understand when a file appeared.
+ // Reserving the commit point, so it can be saved while pushing files to the Zero store.
+ // We don't need to compute a directory hash for the push scenario as we only need to
+ // verify local index does not change during pull.
+ LocalCoreMetadata localCoreMetadata = new LocalCoreMetadata(solrCore);
+ localCoreMetadata.readMetadata(true, false);
+ MetadataComparisonResult metadataComparisonResult =
+ metadataController.diffMetadataForPush(
+ localCoreMetadata,
+ coreMetadata.getZeroShardMetadata(),
+ ZeroStoreShardMetadata.generateMetadataSuffix());
+ if (metadataComparisonResult.getFilesToPush().isEmpty()) {
+ log.warn(
+ "Why there is nothing to push even when there is a newer commit point since last push pushLockTime={}: {}",
+ lockAcquisitionTime,
+ getTaskDescription());
+ long totalTimeMs = startTime.until(Instant.now(), ChronoUnit.MILLIS);
+ return CorePusherExecutionInfo.oddEmptyDiffCommitPoint(
+ zeroGeneration,
+ Collections.emptySet(),
+ metadataComparisonResult.getFilesToDelete(),
+ lockAcquisitionTime,
+ totalTimeMs);
+ }
+
+ // begin the push process
+ Instant pushStartTime = Instant.now();
+
+ ZeroStoreShardMetadata zeroStoreShardMetadata =
+ pushFilesToZeroStore(metadataComparisonResult);
+ metadataCacheManager.recordState(
+ solrCore, MetadataCacheManager.ZeroCoreStage.FILE_PUSHED);
+ long pushTimeMs = pushStartTime.until(Instant.now(), ChronoUnit.MILLIS);
+ // at this point we've pushed the new metadata file with the newMetadataSuffix and now
+ // need to write to zookeeper
+ Instant metadataUpdateStartTime = Instant.now();
+ ZeroMetadataVersion newShardMetadataVersion = null;
+ try {
+ newShardMetadataVersion =
+ metadataController.updateMetadataValueWithVersion(
+ collectionName,
+ shardName,
+ metadataComparisonResult.getMetadataSuffix(),
+ coreMetadata.getMetadataVersion().getVersion());
+ } catch (Exception ex) {
+ // conditional update of zookeeper failed, reset "cache likely up to date"
+ // regardless of the exception type.
+ // We observe that we can hit Keeper$ConnectionLoss while the suffix is actually updated
+ // That will make sure before processing next indexing batch, we sync with zookeeper and
+ // pull from the Zero store.
+ metadataCacheManager.updateCoreMetadata(solrCore.getName(), false);
+ throw ex;
+ }
+ metadataCacheManager.recordState(
+ solrCore, MetadataCacheManager.ZeroCoreStage.ZK_UPDATE_FINISHED);
+ long metadataUpdateTimeMs =
+ metadataUpdateStartTime.until(Instant.now(), ChronoUnit.MILLIS);
+
+ assert metadataComparisonResult
+ .getMetadataSuffix()
+ .equals(newShardMetadataVersion.getMetadataSuffix());
+ // after successful update to zookeeper, update cached core metadata with new info and
+ // consider cache likely up to date wrt the Zero store
+ metadataCacheManager.updateCoreMetadata(
+ solrCore.getName(), newShardMetadataVersion, zeroStoreShardMetadata, true);
+ metadataCacheManager.recordState(
+ solrCore, MetadataCacheManager.ZeroCoreStage.LOCAL_CACHE_UPDATE_FINISHED);
+ if (log.isInfoEnabled())
+ log.info(
+ "Successfully pushed to the Zero store, pushLockTime={}: {}",
+ lockAcquisitionTime,
+ getTaskDescription());
+ long totalTimeMs = startTime.until(Instant.now(), ChronoUnit.MILLIS);
+ return new CorePusherExecutionInfo(
+ true,
+ zeroGeneration,
+ metadataComparisonResult.getMetadataSuffix(),
+ metadataComparisonResult.getFilesToPush(),
+ metadataComparisonResult.getFilesToDelete(),
+ lockAcquisitionTime,
+ pushTimeMs,
+ metadataUpdateTimeMs,
+ totalTimeMs);
+ } finally {
+ metadataCacheManager.recordState(solrCore, ZeroCoreStage.PUSH_FINISHED);
+ }
+ }
+ // TODO - make error handling a little nicer?
+ } catch (InterruptedException e) {
+ throw new SolrException(
+ ErrorCode.SERVER_ERROR, "CorePusher was interrupted while pushing to the Zero store", e);
+ } catch (SolrException e) {
+ Throwable t = e.getCause();
+ if (t instanceof BadVersionException) {
+ throw new SolrException(
+ ErrorCode.SERVER_ERROR,
+ "CorePusher failed to push because the node "
+ + "version doesn't match, requestedVersion="
+ + ((BadVersionException) t).getRequested(),
+ t);
+ }
+ throw e;
+ } catch (Exception ex) {
+ // wrap every thrown exception in a solr exception
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "Error trying to push to the Zero store " + getTaskDescription(),
+ ex);
+ } finally {
+ MDCLoggingContext.clear();
+ }
+ }
+
+ /**
+ * This method calls the {@link #endToEndPushCoreToZeroStore()} after having acquired the indexing
+ * lock. It verifies that this is the first push of that shard to the Zero store.
+ *
+ * <p>Acquiring the indexing lock is slightly vain... This code works because it is called from a
+ * restore or post shard split, there are no competing cores elsewhere in the cluster that could
+ * push content to the Zero store while this core is trying to do its push.
+ *
+ * <p>TODO need to clarify the lifecycle of the MetadataCacheManager vs ZK stored metadataSuffix
+ */
+ @SuppressWarnings("try")
+ public void initialCorePushToZeroStore() throws InterruptedException {
+ String collectionName = getCollectionName();
+ String shardName = getShardName();
+
+ // we'll do a timed attempt to acquire the read lock, but at this point in time the sub shard
+ // should not be active and indexing so we shouldn't wait long
+ try (ZeroAccessLocks.NoThrowAutoCloseable ignore =
+ metadataCacheManager
+ .getOrCreateCoreMetadata(solrCore.getName())
+ .getZeroAccessLocks()
+ .acquireIndexingLock(LOCK_ACQUIRE_TIMEOUT)) {
+ ZeroMetadataVersion shardMetadataVersion =
+ metadataController.readMetadataValue(collectionName, shardName);
+ // TODO if split failed and retried, will it use a different core name? Or fail forever?
+ if (!metadataController.hasDefaultNodeSuffix(shardMetadataVersion)) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "New shard has zk information that is not default. "
+ + "The suffix value is "
+ + shardMetadataVersion.getMetadataSuffix()
+ + getTaskDescription());
+ }
+ // sync local cache with zk's default information i.e. equivalent of no-op pull
+ // this syncing is necessary for the zk conditional update to succeed at the end of core
+ // push
+ try (ZeroAccessLocks.NoThrowAutoCloseable ignore2 =
+ metadataCacheManager
+ .getOrCreateCoreMetadata(solrCore.getName())
+ .getZeroAccessLocks()
+ .acquirePushLock(LOCK_ACQUIRE_TIMEOUT)) {
+ // Push yet to happen so metadata cache not assumed up to date
+ metadataCacheManager.updateCoreMetadata(
+ solrCore.getName(), shardMetadataVersion, new ZeroStoreShardMetadata(), false);
+ endToEndPushCoreToZeroStore();
+ }
+ } catch (ZeroLockException e) {
+
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "Push lock: " + e.getMessage() + " " + getTaskDescription());
+ }
+ }
+
+ /**
+ * Writes to the Zero store all the files that should be written to it, including the contents of
+ * {@link ZeroStoreShardMetadata} as shard.metadata (with a suffix). This method does not update
+ * ZooKeeper with the new metadataSuffix. Until this happens, the writes to Zero store are not
+ * taken into account.
+ */
+ @VisibleForTesting
+ protected ZeroStoreShardMetadata pushFilesToZeroStore(
+ MetadataComparisonResult metadataComparisonResult) throws Exception {
+
+ String collectionName = getCollectionName();
+ String shardName = getShardName();
+
+ MetadataCacheEntry coreMetadata =
+ metadataCacheManager.getOrCreateCoreMetadata(solrCore.getName());
+ ZeroStoreShardMetadata shardMetadata = coreMetadata.getZeroShardMetadata();
+ Instant startTime = Instant.now();
+
+ FileTransferCounter counter = new FileTransferCounter();
+ boolean isSuccessful = false;
+
+ try {
+ MDCLoggingContext.setCore(solrCore);
+
+ // Creating the new ZeroStoreShardMetadata as a modified clone of the existing one
+ ZeroStoreShardMetadata newShardMetadata =
+ new ZeroStoreShardMetadata(
+ shardMetadata.getZeroFiles(),
+ shardMetadata.getZeroFilesToDelete(),
+ metadataComparisonResult.getLocalGeneration());
+
+ // Submit for delete only Zero store files previously marked for deletion, not the ones that
+ // the current update marks for deletion. That's why the enqueue for delete is done before
+ // adding the just deleted files to newShardMetadata. If the current update fails (in the
+ // ZooKeeper update phase for example), the just deleted files remain required.
+ enqueueForHardDelete(newShardMetadata);
+
+ // Files that are part of the current commit point on the Zero store but no longer needed
+ // after this push succeeds are not deleted (see above why) but moved from the set of files
+ // being part of the shard to the set of files that need to be deleted later.
+ for (ZeroFile.WithLocal bf : metadataComparisonResult.getFilesToDelete()) {
+ newShardMetadata.removeFile(bf);
+ ZeroFile.ToDelete zftd = new ZeroFile.ToDelete(bf, Instant.now());
+ newShardMetadata.addFileToDelete(zftd);
+ }
+
+ // add the old shard.metadata file to delete
+ if (!metadataController.hasDefaultNodeSuffix(coreMetadata)) {
+ ZeroFile.ToDelete zftd =
+ metadataController.newShardMetadataZeroFileToDelete(
+ collectionName, shardName, coreMetadata.getMetadataVersion().getMetadataSuffix());
+ newShardMetadata.addFileToDelete(zftd);
+ }
+
+ // When we build the LocalCoreMetadata we requested to reserve the commit point for some short
+ // duration. Assumption is it took less than this duration to get here (no blocking
+ // operations). Now we actually save the commit point for the (potentially long) time it takes
+ // to push all files to the Zero store.
+ solrCore.getDeletionPolicy().saveCommitPoint(metadataComparisonResult.getLocalGeneration());
+
+ Directory coreIndexDir = CorePuller.getDirectory(solrCore, solrCore.getIndexDir());
+ try {
+ // parallel pushes of segment files and the new shard.metadata file into the Zero store
+ CompletableFuture<?>[] futures =
+ new CompletableFuture<?>[metadataComparisonResult.getFilesToPush().size() + 1];
+
+ int i = 0;
+ // Push all segment files
+ for (ZeroFile.WithLocal zf : metadataComparisonResult.getFilesToPush()) {
+ // Sanity check that we're talking about the same file (just sanity, Solr doesn't update
+ // files so should never be different)
+ assert zf.getFileSize() == coreIndexDir.fileLength(zf.getSolrFileName());
+
+ futures[i++] = zeroStoreClient.pushFileAsync(coreIndexDir, zf);
+ newShardMetadata.addFile(zf);
+ counter.incrementActualFilesTransferred();
+ counter.incrementBytesTransferred(zf.getFileSize());
+ }
+ // push the new shard.metadata in parallel with other files
+ // As opposed to segment files, for shard.metadata not capturing file size and checksum.
+ // This is because there is nothing to compare them with, as opposed to normal segment files
+ // for which this information is captured... in the shard.metadata file :-O
+ futures[i] =
+ zeroStoreClient.pushShardMetadataAsync(
+ metadataController.newShardMetadataZeroFile(
+ getCollectionName(),
+ getShardName(),
+ metadataComparisonResult.getMetadataSuffix()),
+ newShardMetadata);
+ counter.incrementActualFilesTransferred();
+ String shardMetadataFileContent = newShardMetadata.toJson();
+ counter.incrementBytesTransferred(shardMetadataFileContent.length());
+
+ CompletableFuture.allOf(futures).get();
+ } finally {
+ solrCore.getDirectoryFactory().release(coreIndexDir);
+ solrCore
+ .getDeletionPolicy()
+ .releaseCommitPoint(metadataComparisonResult.getLocalGeneration());
+ }
+
+ isSuccessful = true;
+ return newShardMetadata;
+ } finally {
+ MDCLoggingContext.clear();
+
+ counter.setExpectedFilesTransferred(metadataComparisonResult.getFilesToPush().size());
+
+ if (log.isInfoEnabled()) {
+ log.info(
+ "PUSH runTime={} bytesTransferred={} expectedFilesAffected={} actualFilesAffected={} isSuccessful={} "
+ + "localGeneration={} zeroGeneration={}",
+ startTime.until(Instant.now(), ChronoUnit.MILLIS),
+ counter.getBytesTransferred(),
+ counter.getExpectedFilesTransferred(),
+ counter.getActualFilesTransferred(),
+ isSuccessful,
+ metadataComparisonResult.getLocalGeneration(),
+ coreMetadata.getZeroShardMetadata().getGeneration());
+ }
+ }
+ }
+
+ @VisibleForTesting
+ public void enqueueForHardDelete(ZeroStoreShardMetadata shardMetadata) {
+ Set<ZeroFile.ToDelete> filesToDelete =
+ shardMetadata.getZeroFilesToDelete().stream()
+ .filter(this::okForHardDelete)
+ .collect(Collectors.toSet());
+ if (enqueueForDelete(filesToDelete)) {
+ shardMetadata.removeFilesFromDeleted(filesToDelete);
+ }
+ }
+
+ /**
+ * @return true if the files were enqueued for deletion successfully
+ */
+ @VisibleForTesting
+ protected boolean enqueueForDelete(Set<ZeroFile.ToDelete> zeroFilesToDelete) {
+ if (zeroFilesToDelete == null || zeroFilesToDelete.isEmpty()) {
+ return false;
+ }
+ Set<ZeroFile> zeroFiles = new HashSet<>(zeroFilesToDelete);
+
+ if (zeroFiles.isEmpty()) return true;
+ try {
+ deleteProcessor.deleteFiles(zeroFiles, true);
+ return true;
+ } catch (Exception ex) {
+ return false;
+ }
+ }
+
+ private String getTaskDescription() {
+ return String.format(
+ Locale.ROOT,
+ "(col=%s,shard=%s,name=%s,thread=%s)",
+ getCollectionName(),
+ getShardName(),
+ getCoreName(),
+ Thread.currentThread().getName());
+ }
+
+ /**
+ * Returns true if a deleted Zero store file (i.e. a file marked for delete but not deleted yet)
+ * can be hard deleted now.
+ */
+ @VisibleForTesting
+ protected boolean okForHardDelete(ZeroFile.ToDelete file) {
+ // For now we only check how long ago the file was marked for delete.
+ return file.getDeletedAt().until(Instant.now(), ChronoUnit.MILLIS)
+ >= deleteProcessor.getDeleteDelayMs();
+ }
+
+ protected String getCoreName() {
+ return solrCore.getName();
+ }
+
+ protected String getShardName() {
+ return solrCore.getCoreDescriptor().getCloudDescriptor().getShardId();
+ }
+
+ protected String getCollectionName() {
+ return solrCore.getCoreDescriptor().getCollectionName();
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/zero/process/CorePusherExecutionInfo.java b/solr/core/src/java/org/apache/solr/zero/process/CorePusherExecutionInfo.java
new file mode 100644
index 00000000000..ffc3a478a5e
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/zero/process/CorePusherExecutionInfo.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.zero.process;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import org.apache.solr.zero.client.ZeroFile;
+
+public class CorePusherExecutionInfo {
+
+ private final boolean hasPushed;
+ private final long zeroGeneration;
+ private final String metadataSuffix;
+ private final Collection<ZeroFile.WithLocal> filesToDelete;
+ private final Collection<ZeroFile.WithLocal> filesToPush;
+ private final long pushLockWaitTimeMs;
+ private final long actualPushTimeMs;
+ private final long metadataUpdateTimeMs;
+ private final long totalTimeMs;
+
+ public CorePusherExecutionInfo(
+ boolean hasPushed,
+ long zeroGeneration,
+ String metadataSuffix,
+ Collection<ZeroFile.WithLocal> filesToPush,
+ Collection<ZeroFile.WithLocal> filesToDelete,
+ long pushLockWaitTimeMs,
+ long actualPushTimeMs,
+ long metadataUpdateTimeMs,
+ long totalTimeMs) {
+ this.hasPushed = hasPushed;
+ this.zeroGeneration = zeroGeneration;
+ this.metadataSuffix = metadataSuffix;
+ this.filesToPush = filesToPush;
+ this.filesToDelete = filesToDelete;
+ this.pushLockWaitTimeMs = pushLockWaitTimeMs;
+ this.actualPushTimeMs = actualPushTimeMs;
+ this.metadataUpdateTimeMs = metadataUpdateTimeMs;
+ this.totalTimeMs = totalTimeMs;
+ }
+
+ public static CorePusherExecutionInfo localAlreadyUpToDate(
+ long zeroGeneration, long pushLockWaitTimeMs, long totalTimeMs) {
+ return new CorePusherExecutionInfo(
+ false,
+ zeroGeneration,
+ null,
+ new ArrayList<>(),
+ new ArrayList<>(),
+ pushLockWaitTimeMs,
+ 0,
+ 0,
+ totalTimeMs);
+ }
+
+ public static CorePusherExecutionInfo oddEmptyDiffCommitPoint(
+ long zeroGeneration,
+ Collection<ZeroFile.WithLocal> filesToPush,
+ Collection<ZeroFile.WithLocal> filesToDelete,
+ long pushLockWaitTimeMs,
+ long totalTimeMs) {
+ return new CorePusherExecutionInfo(
+ false,
+ zeroGeneration,
+ null,
+ filesToPush,
+ filesToDelete,
+ pushLockWaitTimeMs,
+ 0,
+ 0,
+ totalTimeMs);
+ }
+
+ public boolean hasPushed() {
+ return hasPushed;
+ }
+
+ public long getZeroGeneration() {
+ return zeroGeneration;
+ }
+
+ public String getMetadataSuffix() {
+ return metadataSuffix;
+ }
+
+ public long getPushLockWaitTimeMs() {
+ return pushLockWaitTimeMs;
+ }
+
+ public long getActualPushTimeMs() {
+ return actualPushTimeMs;
+ }
+
+ public long getMetadataUpdateTimeMs() {
+ return metadataUpdateTimeMs;
+ }
+
+ public long getTotalTimeMs() {
+ return totalTimeMs;
+ }
+
+ public int getNumFilesPushed() {
+ return filesToPush.size();
+ }
+
+ public long getSizeBytesPushed() {
+ return filesToPush.stream().mapToLong(ZeroFile.WithLocal::getFileSize).sum();
+ }
+
+ public int getNumFilesToDelete() {
+ return filesToDelete.size();
+ }
+
+ @Override
+ public String toString() {
+ return "ExecutionInfo [hasPushed="
+ + hasPushed
+ + ", zeroGeneration="
+ + zeroGeneration
+ + ", metadataSuffix="
+ + metadataSuffix
+ + ", filesToPush="
+ + filesToPush
+ + ", filesToDelete="
+ + filesToDelete
+ + ", pushLockWaitTimeMs="
+ + pushLockWaitTimeMs
+ + ", actualPushTimeMs="
+ + actualPushTimeMs
+ + ", metadataUpdateTimeMs="
+ + metadataUpdateTimeMs
+ + ", totalTimeMs="
+ + totalTimeMs
+ + "]";
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/zero/process/DeleteProcessor.java b/solr/core/src/java/org/apache/solr/zero/process/DeleteProcessor.java
new file mode 100644
index 00000000000..b31000abce7
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/zero/process/DeleteProcessor.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.zero.process;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.Closeable;
+import java.lang.invoke.MethodHandles;
+import java.util.Set;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.TimeUnit;
+import org.apache.lucene.util.NamedThreadFactory;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.ExecutorUtil.MDCAwareThreadPoolExecutor;
+import org.apache.solr.zero.client.ZeroFile;
+import org.apache.solr.zero.client.ZeroStoreClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A generic deletion processor used for deleting files from Zero store. Each processor manages its
+ * own task bounded thread pool for processing {@link DeleterTask} asynchronously. Processors
+ * support retrying tasks if necessary but retry decisions are left to the individual task
+ * implementations.
+ *
+ * <p>Instances of {@link DeleteProcessor} are managed by the {@link ZeroStoreManager}.
+ */
+public class DeleteProcessor implements Closeable {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private final String name;
+ private final int almostMaxQueueSize;
+
+ /**
+ * Note we sleep() after each failed attempt, so multiply this value by {@link #fixedRetryDelay}
+ * to find out how long we'll retry (at least) if Zero store access fails for some reason ("at
+ * least" because we re-enqueue at the tail of the queue, so there might be additional processing
+ * delay if the queue is not empty and is processed before the enqueued retry is processed).
+ */
+ private final int maxDeleteAttempts;
+
+ private final long fixedRetryDelay;
+ private final ZeroStoreClient zeroStoreClient;
+ private final BlockingQueue<Runnable> deleteQueue;
+ private final MDCAwareThreadPoolExecutor deleteExecutor;
+ private final long deleteDelayMs;
+
+ /**
+ * @param name identifying the processor
+ * @param zeroStoreClient the zeroStoreClient to use in this processor
+ * @param almostMaxQueueSize the target max queue size
+ * @param numDeleterThreads the number of threads to configure in the underlying thread pool
+ * @param defaultMaxDeleteAttempts maximum number of attempts to retry any task enqueued in this
+ * processor
+ * @param fixedRetryDelay fixed time delay in ms between retry attempts
+ */
+ public DeleteProcessor(
+ String name,
+ ZeroStoreClient zeroStoreClient,
+ int almostMaxQueueSize,
+ int numDeleterThreads,
+ int defaultMaxDeleteAttempts,
+ long fixedRetryDelay,
+ long deleteDelayMs) {
+ this.name = name;
+ this.almostMaxQueueSize = almostMaxQueueSize;
+ this.maxDeleteAttempts = defaultMaxDeleteAttempts;
+ this.fixedRetryDelay = fixedRetryDelay;
+ this.deleteDelayMs = deleteDelayMs;
+ NamedThreadFactory threadFactory = new NamedThreadFactory(name);
+
+ // Note this queue MUST NOT BE BOUNDED, or we risk deadlocks given that DeleterTask's
+ // re-enqueue themselves upon failure
+ deleteQueue = new LinkedBlockingDeque<>();
+
+ deleteExecutor =
+ new MDCAwareThreadPoolExecutor(
+ numDeleterThreads, numDeleterThreads, 0L, TimeUnit.SECONDS, deleteQueue, threadFactory);
+ this.zeroStoreClient = zeroStoreClient;
+ }
+
+ /**
+ * Enqueues the given set of files for deletion from Zero store as an async task.
+ *
+ * @param zeroFiles list of files to delete from Zero store
+ * @param allowRetry flag indicating if the task should be retried if it fails
+ */
+ public CompletableFuture<DeleterTask.Result> deleteFiles(
+ Set<ZeroFile> zeroFiles, boolean allowRetry) {
+ String collectionName = zeroFiles.iterator().next().getCollectionName();
+ DeleterTask task =
+ new FilesDeletionTask(
+ zeroStoreClient, collectionName, zeroFiles, allowRetry, maxDeleteAttempts);
+ return enqueue(task, false);
+ }
+
+ /**
+ * Enqueues a task to delete all files belonging to the given collection from Zero store as an
+ * async task.
+ *
+ * @param collectionName the name of the collection to be deleted from Zero store
+ * @param allowRetry flag indicating if the task should be retried if it fails
+ */
+ public CompletableFuture<DeleterTask.Result> deleteCollection(
+ String collectionName, boolean allowRetry) {
+ DeleterTask task =
+ new CollectionDeletionTask(zeroStoreClient, collectionName, allowRetry, maxDeleteAttempts);
+ return enqueue(task, false);
+ }
+
+ /**
+ * Enqueues a task to delete all files belonging to the given collection and shard as an async
+ * task.
+ *
+ * @param collectionName targeted collection
+ * @param shardName shard to delete
+ * @param allowRetry flag indicating if the task should be retried if it fails
+ */
+ public CompletableFuture<DeleterTask.Result> deleteShard(
+ String collectionName, String shardName, boolean allowRetry) {
+ DeleterTask task =
+ new ShardDeletionTask(
+ zeroStoreClient, collectionName, shardName, allowRetry, maxDeleteAttempts);
+ return enqueue(task, false);
+ }
+
+ /**
+ * Enqueues a task to be processed by a thread in the {@link DeleteProcessor#deleteExecutor}
+ * thread pool. The callback is handled by the same execution thread and will re-enqueue a task
+ * that has failed and should be retried. Tasks that are enqueued via the retry mechanism are not
+ * bound by the same size constraints as newly minted tasks are.
+ *
+ * @return CompletableFuture to allow calling threads the capability to block on the computation
+ * results as needed, retrieved suppressed exceptions in retry, etc
+ */
+ @VisibleForTesting
+ protected CompletableFuture<DeleterTask.Result> enqueue(DeleterTask task, boolean isRetry) {
+ if (!isRetry && deleteQueue.size() > almostMaxQueueSize) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "Unable to enqueue deletion task: " + task.toString());
+ }
+
+ return CompletableFuture.supplyAsync(task::call, deleteExecutor)
+ .thenCompose(
+ result -> {
+ // the callback will execute on the same thread as the executing task
+ if (!result.isSuccess() && result.shouldRetry()) {
+ try {
+ // Some delay before retry... (could move this delay to before trying to delete a
+ // file that previously failed to be deleted, that way if the queue is busy, and
+ // it took time to retry, we don't add a delay on top of that. On the other hand,
+ // an exception here could be an issue with the Zero store itself and nothing
+ // specific to the file at hand, so slowing all delete attempts for all files
+ // might make sense.
+ Thread.sleep(fixedRetryDelay);
+ return enqueue(result.getTask(), result.shouldRetry());
+ } catch (Exception ex) {
+ log.error(
+ "Could not re-enqueue failed deleter task that should have been enqueued!",
+ ex);
+ }
+ }
+ return CompletableFuture.completedFuture(result);
+ });
+ }
+
+ @Override
+ public void close() {
+ deleteExecutor.shutdown();
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public long getDeleteDelayMs() {
+ return deleteDelayMs;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/zero/process/DeleterTask.java b/solr/core/src/java/org/apache/solr/zero/process/DeleterTask.java
new file mode 100644
index 00000000000..9929f906adb
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/zero/process/DeleterTask.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.zero.process;
+
+import java.lang.invoke.MethodHandles;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.solr.zero.client.ZeroFile;
+import org.apache.solr.zero.client.ZeroStoreClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Generic deletion task for files located on the Zero store */
+public abstract class DeleterTask implements Callable<DeleterTask.Result> {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ protected final ZeroStoreClient zeroStoreClient;
+
+ private final Set<ZeroFile> zeroFiles;
+ private final AtomicInteger attempt;
+ private final Instant queuedTime;
+ private final int maxAttempts;
+ private final boolean allowRetry;
+ private Throwable err;
+
+ public DeleterTask(
+ ZeroStoreClient zeroStoreClient,
+ Set<ZeroFile> zeroFiles,
+ boolean allowRetry,
+ int maxAttempts) {
+ this.zeroStoreClient = zeroStoreClient;
+ this.zeroFiles = zeroFiles;
+ this.attempt = new AtomicInteger(0);
+ this.queuedTime = Instant.now();
+ this.allowRetry = allowRetry;
+ this.maxAttempts = maxAttempts;
+ }
+
+ public abstract String getBasePath();
+
+ /** Return a String representing the action performed by the DeleterTask for logging purposes */
+ public abstract String getActionName();
+
+ public abstract void setMDCContext();
+
+ @Override
+ public DeleterTask.Result call() {
+ List<ZeroFile> filesDeleted = new ArrayList<>();
+ final Instant startTime = Instant.now();
+ boolean isSuccess = true;
+ boolean shouldRetry = false;
+ try {
+ filesDeleted.addAll(doDelete());
+ attempt.incrementAndGet();
+ return new DeleterTask.Result(this, filesDeleted, isSuccess, shouldRetry, err);
+ } catch (Exception ex) {
+ if (err == null) {
+ err = ex;
+ } else {
+ err.addSuppressed(ex);
+ }
+ int attempts = attempt.incrementAndGet();
+ isSuccess = false;
+ log.warn("{} DeleterTask failed on attempt={}", this, attempts, ex);
+ if (allowRetry) {
+ if (attempts < maxAttempts) {
+ shouldRetry = true;
+ } else {
+ log.warn("{} reached {} attempt limit. This task won't be retried.", this, maxAttempts);
+ }
+ }
+ } finally {
+ Instant now = Instant.now();
+ long runTime = startTime.until(now, ChronoUnit.MILLIS);
+ long startLatency = this.queuedTime.until(now, ChronoUnit.MILLIS);
+ if (log.isInfoEnabled()) {
+ log.info(
+ "{} path={} runTime={} startLatency={} isSuccess={}",
+ this,
+ getBasePath(),
+ runTime,
+ startLatency,
+ isSuccess);
+ }
+ }
+ return new DeleterTask.Result(this, filesDeleted, isSuccess, shouldRetry, err);
+ }
+
+ public Collection<ZeroFile> doDelete() throws Exception {
+ zeroStoreClient.deleteZeroFiles(zeroFiles);
+ return zeroFiles;
+ }
+
+ /** Represents the result of a deletion task */
+ public static class Result {
+ private final DeleterTask task;
+ private final Collection<ZeroFile> filesDeleted;
+ private boolean isSuccess;
+ private final boolean shouldRetry;
+ private final Throwable err;
+
+ public Result(
+ DeleterTask task,
+ Collection<ZeroFile> filesDeleted,
+ boolean isSuccess,
+ boolean shouldRetry,
+ Throwable errs) {
+ this.task = task;
+ this.filesDeleted = filesDeleted;
+ this.isSuccess = isSuccess;
+ this.shouldRetry = shouldRetry;
+ this.err = errs;
+ }
+
+ public boolean isSuccess() {
+ return isSuccess;
+ }
+
+ public void updateSuccess(boolean s) {
+ isSuccess = s;
+ }
+
+ public boolean shouldRetry() {
+ return shouldRetry;
+ }
+
+ public DeleterTask getTask() {
+ return task;
+ }
+
+ /**
+ * @return the files that are being deleted. Note if the task wasn't successful there is no
+ * guarantee all of these files were in fact deleted from the Zero store
+ */
+ public Collection<ZeroFile> getFilesDeleted() {
+ return filesDeleted;
+ }
+
+ public Throwable getError() {
+ return err;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ Locale.ROOT,
+ "action=%s totalFilesSpecified=%d allowRetry=%b queuedTime=%s attemptsTried=%d",
+ getActionName(),
+ zeroFiles.size(),
+ allowRetry,
+ queuedTime.toString(),
+ attempt.get());
+ }
+
+ public int getAttempts() {
+ return attempt.get();
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/zero/process/FilesDeletionTask.java b/solr/core/src/java/org/apache/solr/zero/process/FilesDeletionTask.java
new file mode 100644
index 00000000000..fcacd5883cb
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/zero/process/FilesDeletionTask.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.zero.process;
+
+import java.util.Locale;
+import java.util.Set;
+import org.apache.solr.logging.MDCLoggingContext;
+import org.apache.solr.zero.client.ZeroFile;
+import org.apache.solr.zero.client.ZeroStoreClient;
+
+public class FilesDeletionTask extends DeleterTask {
+
+ protected final String collectionName;
+
+ public FilesDeletionTask(
+ ZeroStoreClient zeroStoreClient,
+ String collectionName,
+ Set<ZeroFile> zeroFiles,
+ boolean allowRetry,
+ int maxRetryAttempt) {
+ super(zeroStoreClient, zeroFiles, allowRetry, maxRetryAttempt);
+ this.collectionName = collectionName;
+ }
+
+ @Override
+ public String getActionName() {
+ return "DELETE_FILES";
+ }
+
+ @Override
+ public void setMDCContext() {
+ MDCLoggingContext.setCollection(collectionName);
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ Locale.ROOT, "FilesDeletionTask %s collection=%s", super.toString(), collectionName);
+ }
+
+ @Override
+ public String getBasePath() {
+ return zeroStoreClient.getCollectionURI(collectionName).toString();
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/zero/process/ShardDeletionTask.java b/solr/core/src/java/org/apache/solr/zero/process/ShardDeletionTask.java
new file mode 100644
index 00000000000..f30068bde21
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/zero/process/ShardDeletionTask.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.zero.process;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.file.NoSuchFileException;
+import java.util.Locale;
+import org.apache.solr.logging.MDCLoggingContext;
+import org.apache.solr.zero.client.ZeroStoreClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** A file deletion task that deletes all files from Zero store under given path */
+public class ShardDeletionTask extends FilesDeletionTask {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ protected final String shardName;
+
+ public ShardDeletionTask(
+ ZeroStoreClient zeroStoreClient,
+ String collectionName,
+ String shardName,
+ boolean allowRetry,
+ int maxRetryAttempt) {
+ super(
+ zeroStoreClient,
+ collectionName,
+ zeroStoreClient.listShardZeroFiles(collectionName, shardName),
+ allowRetry,
+ maxRetryAttempt);
+ this.shardName = shardName;
+ }
+
+ @Override
+ public String getActionName() {
+ return "DELETE_SHARD_FILES";
+ }
+
+ @Override
+ public void setMDCContext() {
+ super.setMDCContext();
+ MDCLoggingContext.setShard(shardName);
+ }
+
+ @Override
+ public String toString() {
+ return String.format(Locale.ROOT, "ShardDeletionTask %s shard=%s", super.toString(), shardName);
+ }
+
+ @Override
+ public String getBasePath() {
+ return zeroStoreClient.getShardURI(collectionName, shardName).toString();
+ }
+
+ @Override
+ public DeleterTask.Result call() {
+ DeleterTask.Result result = super.call();
+ if (result.isSuccess()) {
+ try {
+ zeroStoreClient.deleteShardDirectory(collectionName, shardName);
+ } catch (NoSuchFileException ex) {
+ if (log.isWarnEnabled())
+ log.warn(
+ "Could not delete Zero store directory for collection={} shard={} as it does not exist",
+ collectionName,
+ shardName);
+ } catch (IOException ex) {
+ result.updateSuccess(false);
+ if (log.isWarnEnabled())
+ log.warn(
+ "Could not delete Zero store directory for collection={} shard={} after all files have been deleted",
+ collectionName,
+ shardName);
+ }
+ }
+ return result;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/zero/process/ZeroAccessLocks.java b/solr/core/src/java/org/apache/solr/zero/process/ZeroAccessLocks.java
new file mode 100644
index 00000000000..9472edfc0c1
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/zero/process/ZeroAccessLocks.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.zero.process;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.lang.invoke.MethodHandles;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.solr.zero.exception.ZeroLockException;
+import org.apache.solr.zero.metadata.ZeroMetadataVersion;
+import org.apache.solr.zero.metadata.ZeroStoreShardMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Locking logic (and locks) for read and write access to the Zero store */
+public class ZeroAccessLocks {
+ /**
+ * Should only be created once at initialization time, subsequent updates should reuse same lock
+ * instance. see {@link
+ * org.apache.solr.zero.metadata.MetadataCacheManager.MetadataCacheEntry#updatedOf(ZeroMetadataVersion,
+ * ZeroStoreShardMetadata, boolean)} and other overload.
+ *
+ * <p>We don't need fair ordering policy for this lock, which normally has lower throughput. We
+ * rely on cacheLikelyUpToDate and isLeader at query time so that queries may not contend on this
+ * lock and let the steady state indexing do its job without contention.
+ *
+ * <p>On indexing side we rely on read lock and we can have multiple readers just fine. Write lock
+ * is only needed in a fail over scenario(leader changed) where we need to pull from Zero store
+ * but that is only needed to be done by one thread. Therefore, we acquire write lock with a
+ * timeout and check for that condition after the timeout. Therefore, no concern of starvation
+ * there either.
+ */
+ private final ReentrantReadWriteLock indexingAndPullLock;
+
+ /**
+ * Should only be created once at initialization time, subsequent updates should reuse same lock
+ * instance. see {@link
+ * org.apache.solr.zero.metadata.MetadataCacheManager.MetadataCacheEntry#updatedOf(ZeroMetadataVersion,
+ * ZeroStoreShardMetadata, boolean)} and other overload.
+ *
+ * <p>Use a fair ordering policy for this lock, to ensure that the longest waiting thread gets to
+ * acquire the lock first. This will help prevent threads that may starve waiting to acquire the
+ * lock to push, even though another thread may have already completed the work on their behalf.
+ * See {@link CorePusher#endToEndPushCoreToZeroStore()} for details.
+ */
+ private final ReentrantLock pushLock;
+
+ private final CloseableLock closableIndexingLock;
+ private final CloseableLock closablePullLock;
+ private final CloseableLock closablePushLock;
+
+ public ZeroAccessLocks() {
+ this(new ReentrantReadWriteLock(), new ReentrantLock(true));
+ }
+
+ @VisibleForTesting
+ public ZeroAccessLocks(ReentrantReadWriteLock indexingAndPullLock, ReentrantLock pushLock) {
+ this.indexingAndPullLock = indexingAndPullLock;
+ this.pushLock = pushLock;
+
+ closableIndexingLock = new CloseableLock(indexingAndPullLock.readLock());
+ closablePullLock = new CloseableLock(indexingAndPullLock.writeLock());
+ closablePushLock = new CloseableLock(pushLock);
+ }
+
+ /**
+ * This lock must be acquired before starting indexing (or any other update) on a core. It
+ * disallows pulls from occurring during indexing, because otherwise the subsequent Zero store
+ * update could end up overwriting a previous update and leading to data loss. This lock does
+ * allow multiple indexing batches to proceed concurrently on a core. This lock must be held
+ * during the whole indexing duration, including when the core is pushed to the Zero store at the
+ * end of indexing (pushing requires first calling {@link #acquirePushLock(long)})
+ */
+ public NoThrowAutoCloseable acquireIndexingLock(long timeoutSecond)
+ throws ZeroLockException, InterruptedException {
+ return closableIndexingLock.tryLockAsResource(timeoutSecond, TimeUnit.SECONDS);
+ }
+
+ /**
+ * This lock protects pulls from each other (only one pull at any given time for efficiency
+ * reasons given only one can succeed), and prevents indexing from happening concurrently with
+ * pulls (this could lead to data loss). This lock must be held for the whole duration of a core
+ * pull from the Zero store.
+ */
+ public NoThrowAutoCloseable acquirePullLock(long timeoutSecond)
+ throws ZeroLockException, InterruptedException {
+ return closablePullLock.tryLockAsResource(timeoutSecond, TimeUnit.SECONDS);
+ }
+
+ /**
+ * This lock is required for pushing the core to Zero store. Before acquiring this lock, {@link
+ * #acquireIndexingLock(long)} must have been called (both are needed for pushing). Although
+ * multiple indexing batches can index in parallel (multiple threads can concurrently call {@link
+ * #acquireIndexingLock(long)}, only one push can be done in parallel to the Zero store, and only
+ * one thread at a time will succeed calling this method (for efficiency reasons, because if
+ * multiple pushes happen in parallel only one will succeed).
+ */
+ public NoThrowAutoCloseable acquirePushLock(long timeoutSecond)
+ throws ZeroLockException, InterruptedException {
+ return closablePushLock.tryLockAsResource(timeoutSecond, TimeUnit.SECONDS);
+ }
+
+ /**
+ * To update the cached core metadata, need either a pull write lock or push lock along with pull
+ * read lock
+ */
+ public boolean canUpdateCoreMetadata() {
+ return indexingAndPullLock.getWriteHoldCount() > 0
+ || (pushLock.getHoldCount() > 0 && indexingAndPullLock.getReadHoldCount() > 0);
+ }
+
+ public int getIndexingHoldCount() {
+ // The indexing lock is the read component. See closableIndexingLock assignment
+ return indexingAndPullLock.getReadHoldCount();
+ }
+
+ public int getPullHoldCount() {
+ // The pull lock is the write component. See closablePullLock assignment
+ return indexingAndPullLock.getWriteHoldCount();
+ }
+
+ @Override
+ public String toString() {
+ return "pullLockHoldCount="
+ + getPullHoldCount()
+ + " indexingLockHoldCount="
+ + getIndexingHoldCount()
+ + " pushLockHoldCount="
+ + pushLock.getHoldCount();
+ }
+
+ public interface NoThrowAutoCloseable extends AutoCloseable {
+ @Override
+ void close();
+ }
+
+ public static class CloseableLock {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private final ZeroAccessLocks.NoThrowAutoCloseable unlocker = CloseableLock.this::unlock;
+
+ private final Lock lock;
+
+ public CloseableLock(Lock lock) {
+ assert (lock != null);
+ this.lock = lock;
+ }
+
+ public ZeroAccessLocks.NoThrowAutoCloseable tryLockAsResource(long timeout, TimeUnit unit)
+ throws InterruptedException, ZeroLockException {
+ Instant startTime = Instant.now();
+ boolean success = lock.tryLock(timeout, unit);
+ if (!success) {
+ if (log.isWarnEnabled()) {
+ log.warn(
+ "Acquiring lock timed out after: {} ",
+ startTime.until(Instant.now(), ChronoUnit.MILLIS));
+ }
+ throw new ZeroLockException("Couldn't acquire lock within %d ms ", timeout);
+ } else {
+ if (log.isDebugEnabled()) {
+ log.debug("Lock acquired in {} ms", startTime.until(Instant.now(), ChronoUnit.MILLIS));
+ }
+ return unlocker;
+ }
+ }
+
+ public void unlock() {
+ lock.unlock();
+ log.info("Lock released successfully");
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/zero/process/ZeroCoreIndexingBatchProcessor.java b/solr/core/src/java/org/apache/solr/zero/process/ZeroCoreIndexingBatchProcessor.java
new file mode 100644
index 00000000000..427356572e8
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/zero/process/ZeroCoreIndexingBatchProcessor.java
@@ -0,0 +1,337 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.zero.process;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.Closeable;
+import java.lang.invoke.MethodHandles;
+import org.apache.solr.cloud.CloudDescriptor;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.logging.MDCLoggingContext;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.zero.exception.ZeroLockException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class implements the features required for a ZERO core (replica) to do indexing. It brings
+ * it up to date (if needed) before indexing starts and pushes the updated core after a successful
+ * commit.
+ */
+public class ZeroCoreIndexingBatchProcessor implements Closeable {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ /**
+ * Time indexing thread needs to wait to try acquiring pull lock before checking if someone else
+ * has already done the pull.
+ */
+ public static final int SECONDS_TO_WAIT_PULL_LOCK = 5;
+
+ /**
+ * If an indexing thread is waiting on the indexing lock, then it has not performed local indexing
+ * yet. If it is waiting, that means another thread is currently pulling index files for the core.
+ * Other threads can be indexing concurrently so the longest wait time is proportional to the time
+ * it takes to hydrate (load) a full core from Zero store. This can take time on saturated
+ * clusters.
+ */
+ public static final int SECONDS_TO_WAIT_INDEXING_LOCK = 120;
+
+ private final SolrCore core;
+ private final String collectionName;
+ private final String shardName;
+ private IndexingBatchState state;
+ private final ZeroStoreManager zeroStoreManager;
+ private final ZeroAccessLocks locks;
+
+ private ZeroAccessLocks.NoThrowAutoCloseable coreIndexingLock;
+ private final SolrQueryResponse response;
+
+ public ZeroCoreIndexingBatchProcessor(
+ SolrCore core, ClusterState clusterState, SolrQueryResponse response) {
+ this.core = core;
+ this.response = response;
+ CloudDescriptor cloudDescriptor = core.getCoreDescriptor().getCloudDescriptor();
+ collectionName = cloudDescriptor.getCollectionName();
+ shardName = cloudDescriptor.getShardId();
+
+ CoreContainer coreContainer = core.getCoreContainer();
+ zeroStoreManager = coreContainer.getZeroStoreManager();
+ locks = zeroStoreManager.getZeroAccessLocks(core.getName());
+
+ DocCollection collection = clusterState.getCollection(collectionName);
+ if (!collection.isZeroIndex()) {
+ throw new SolrException(
+ ErrorCode.SERVER_ERROR, collectionName + " is not a Zero collection.");
+ }
+
+ Slice shard = collection.getSlicesMap().get(shardName);
+ if (shard == null) {
+ throw new SolrException(
+ ErrorCode.SERVER_ERROR,
+ "Indexing batch received for an unknown shard,"
+ + " collection="
+ + collectionName
+ + " shard="
+ + shardName
+ + " core="
+ + core.getName());
+ }
+
+ if (!Slice.State.ACTIVE.equals(shard.getState())) {
+ // This happens when we buffer updates for a sub shard.
+ // ZERO replica should eventually stop supporting buffered updates and then this should
+ // become a real exception
+ log.warn(
+ "Processing an indexing batch for a non-active shard, collection={} shard={} core={}",
+ collectionName,
+ shardName,
+ core.getName());
+
+ // TODO it would have been better to have instead the code below, but then ZERO
+ // replica split tests fail. Something to look into.
+ // In ZeroStoreUpdateProcessor.rejectIfConstructing() an indexing batch would be rejected
+ // if a shard splits, so seeing activity here might be due to post split actions?
+
+ // // ZERO replica do not support buffered updates (not until transaction log is updated
+ // // to go to Zero store, as the Solr nodes are considered stateless)
+ // throw new SolrException(
+ // ErrorCode.SERVER_ERROR,
+ // "Processing an indexing batch for a non-active shard, collection="
+ // + collectionName
+ // + " shard="
+ // + shardName
+ // + " core="
+ // + core.getName());
+ }
+
+ state = IndexingBatchState.NOT_STARTED;
+ }
+
+ /**
+ * Should be called whenever a document is about to be added/deleted from the ZERO core. If it is
+ * the first doc of the core, this method will mark the start of an indexing batch and bring a
+ * stale ZERO core upto date by pulling from the Zero store.
+ */
+ public void addOrDeleteGoingToBeIndexedLocally() {
+ // Following logic is built on the assumption that one particular instance of this processor
+ // will solely be consumed by a single thread. And all the documents of indexing batch will be
+ // processed by this one instance.
+ try {
+ MDCLoggingContext.setCore(core);
+
+ if (IndexingBatchState.NOT_STARTED.equals(state)) {
+ startIndexingBatch();
+ } else if (IndexingBatchState.STARTED.equals(state)) {
+ // do nothing, we only use this method to start an indexing batch once
+ } else if (IndexingBatchState.COMMITTED.equals(state)) {
+ throw new SolrException(
+ ErrorCode.SERVER_ERROR,
+ "Why are we adding/deleting a doc through an already committed indexing batch?"
+ + " collection="
+ + collectionName
+ + " shard="
+ + shardName
+ + " core="
+ + core.getName());
+ } else {
+ throwUnknownStateError();
+ }
+ } finally {
+ MDCLoggingContext.clear();
+ }
+ }
+
+ @VisibleForTesting
+ protected void startIndexingBatch() {
+ // Following pull logic should only run once before the first add/delete of an indexing batch is
+ // processed by this processor
+ assert IndexingBatchState.NOT_STARTED.equals(state);
+
+ if (coreIndexingLock != null) {
+ String err = "Indexing thread already has an indexing lock but indexing has not yet started!";
+ if (locks.getIndexingHoldCount() != 0) {
+ err += " The current thread also holds a lock on it when it shouldn't!";
+ }
+ throw new SolrException(
+ ErrorCode.SERVER_ERROR,
+ err
+ + " collection="
+ + collectionName
+ + " shard="
+ + shardName
+ + " core="
+ + core.getName());
+ }
+
+ try {
+ zeroStoreManager.setIndexingBatchReceived(core);
+ state = IndexingBatchState.STARTED;
+ CorePullStatus pullStatus = zeroStoreManager.pullCoreFromZeroStore(core);
+ response.addToLog("pull.done", pullStatus == CorePullStatus.SUCCESS);
+ coreIndexingLock = locks.acquireIndexingLock(getIndexingLockTimeout());
+ zeroStoreManager.setIndexingStarted(core);
+ } catch (ZeroLockException ex) {
+ throw new SolrException(
+ ErrorCode.SERVER_ERROR,
+ "Indexing thread timed out trying to acquire the pull read lock in "
+ + getIndexingLockTimeout()
+ + " seconds"
+ + " collection="
+ + collectionName
+ + " shard="
+ + shardName
+ + " core="
+ + core.getName());
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new SolrException(
+ ErrorCode.SERVER_ERROR,
+ "Indexing thread interrupted while trying to acquire pull read lock."
+ + " collection="
+ + collectionName
+ + " shard="
+ + shardName
+ + " core="
+ + core.getName(),
+ ex);
+ }
+ }
+
+ /**
+ * Should be called after the ZERO core has successfully hard committed locally. This method will
+ * push the updated core to the Zero store. If there was no local add/delete of a document for
+ * this processor then the push will be skipped.
+ */
+ public void hardCommitCompletedLocally() {
+ finishIndexingBatch();
+ }
+
+ protected void finishIndexingBatch() {
+ String coreName = core.getName();
+ if (IndexingBatchState.NOT_STARTED.equals(state)) {
+ // This is an isolated commit since we've not seen any added/deleted doc
+ // ZERO replicas currently require (and force) hard commits after each batch, isolated commits
+ // are therefore ignored
+ state = IndexingBatchState.COMMITTED;
+ if (log.isInfoEnabled()) {
+ log.info(
+ "Isolated commit encountered for a ZERO replica, ignoring writing to Zero store."
+ + " collection={} shard={} core={}",
+ collectionName,
+ shardName,
+ coreName);
+ }
+ } else if (IndexingBatchState.STARTED.equals(state)) {
+ if (locks.getIndexingHoldCount() != 1) {
+ throw new SolrException(
+ ErrorCode.SERVER_ERROR,
+ "Indexing lock hold count is not 1 (is "
+ + locks.getIndexingHoldCount()
+ + ") collection="
+ + collectionName
+ + " shard="
+ + shardName
+ + " core="
+ + coreName);
+ }
+
+ zeroStoreManager.setIndexingFinished(core);
+ state = IndexingBatchState.COMMITTED;
+ CorePusherExecutionInfo pushExecutionInfo = zeroStoreManager.pushCoreToZeroStore(core);
+ addToLog(pushExecutionInfo);
+
+ } else if (IndexingBatchState.COMMITTED.equals(state)) {
+ throw new SolrException(
+ ErrorCode.SERVER_ERROR,
+ "Why are we committing an already committed indexing batch?"
+ + " collection="
+ + collectionName
+ + " shard="
+ + shardName
+ + " core="
+ + coreName);
+ } else {
+ throwUnknownStateError();
+ }
+ }
+
+ private void addToLog(CorePusherExecutionInfo pushExecutionInfo) {
+ response.addToLog("push.done", pushExecutionInfo.hasPushed());
+ response.addToLog("push.zeroGeneration", pushExecutionInfo.getZeroGeneration());
+ response.addToLog("push.metadataSuffix", pushExecutionInfo.getMetadataSuffix());
+ response.addToLog("push.lockWaiTimeMs", pushExecutionInfo.getPushLockWaitTimeMs());
+ response.addToLog("push.actualPushTimeMs", pushExecutionInfo.getActualPushTimeMs());
+ response.addToLog("push.metadataUpdateTimeMs", pushExecutionInfo.getMetadataUpdateTimeMs());
+ response.addToLog("push.totalTimeMs", pushExecutionInfo.getTotalTimeMs());
+ response.addToLog("push.numFiles", pushExecutionInfo.getNumFilesPushed());
+ response.addToLog("push.sizeBytes", pushExecutionInfo.getSizeBytesPushed());
+ response.addToLog("push.numFilesToDelete", pushExecutionInfo.getNumFilesToDelete());
+ }
+
+ private void throwUnknownStateError() {
+ throw new SolrException(
+ ErrorCode.SERVER_ERROR,
+ "Programmer's error, unknown IndexingBatchState"
+ + state
+ + " collection="
+ + collectionName
+ + " shard="
+ + shardName
+ + " core="
+ + core.getName());
+ }
+
+ @VisibleForTesting
+ protected int getIndexingLockTimeout() {
+ return SECONDS_TO_WAIT_INDEXING_LOCK;
+ }
+
+ @Override
+ public void close() {
+ if (!IndexingBatchState.NOT_STARTED.equals(state)) {
+ try {
+ zeroStoreManager.setIndexingFinished(core);
+ } catch (Exception ex) {
+ log.error("Error recording the finish of a ZERO core indexing batch", ex);
+ }
+ }
+ // TODO would be nice to decide in a more deterministic way if the lock has to be released
+ if (locks.getIndexingHoldCount() > 0) {
+ // release read lock
+ coreIndexingLock.close();
+ if (locks.getIndexingHoldCount() != 0) {
+ log.error(
+ "Indexing lock of a ZERO core was unlocked but this thread still holds it: {}",
+ locks.getIndexingHoldCount());
+ }
+ }
+ }
+
+ private enum IndexingBatchState {
+ NOT_STARTED,
+ STARTED,
+ COMMITTED
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/zero/process/ZeroStoreManager.java b/solr/core/src/java/org/apache/solr/zero/process/ZeroStoreManager.java
new file mode 100644
index 00000000000..264607f72f4
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/zero/process/ZeroStoreManager.java
@@ -0,0 +1,692 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.zero.process;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import org.apache.solr.cloud.CloudDescriptor;
+import org.apache.solr.cloud.ZkController;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.params.CollectionAdminParams;
+import org.apache.solr.common.util.CollectionUtil;
+import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.common.util.SolrNamedThreadFactory;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.CoreDescriptor;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.ZeroConfig;
+import org.apache.solr.zero.client.ZeroStoreClient;
+import org.apache.solr.zero.client.ZeroStoreClientFactory;
+import org.apache.solr.zero.exception.ZeroException;
+import org.apache.solr.zero.exception.ZeroLockException;
+import org.apache.solr.zero.metadata.MetadataCacheManager;
+import org.apache.solr.zero.metadata.ZeroMetadataController;
+import org.apache.solr.zero.metadata.ZeroMetadataVersion;
+import org.apache.solr.zero.util.DeduplicatingList;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** This is the main entry point to the ZERO implementation from existing SolrCloud classes. */
+public class ZeroStoreManager {
+
+ /**
+ * If more than this number of DISTINCT cores need to be enqueued for async pull, the enqueue
+ * might block. We likely do not want the enqueue to block (currently enqueue is done from {@link
+ * org.apache.solr.servlet.HttpSolrCall#init()} so it would block a Jetty thread), so keep this
+ * number high enough. They day is far though before a single SolrCloud node is able to support
+ * that number of cores. That constant is likely more than an order of magnitude too big.
+ */
+ private static final int CORE_TO_PULL_LIST_MAX_SIZE = 100000;
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private final CoreContainer coreContainer;
+ private ZeroMetadataController zeroMetadataController;
+ private MetadataCacheManager metadataCacheManager;
+ private ZeroStoreClient zeroStoreClient;
+
+ /** A generic delete processor that is used only to delete individual files */
+ private DeleteProcessor corePusherFileDeleteProcessor;
+
+ private final DeduplicatingList<String, CorePuller> corePullTaskQueue;
+
+ /**
+ * This executor feeds off the {@link #corePullTaskQueue} {@link DeduplicatingList}, disguising as
+ * a bare bones {@link BlockingQueue} by using {@link CorePullerBlockingQueue}. This executor is
+ * used in a very specific way: nothing is ever submitted to it, it only consumes the queue, with
+ * entries added directly into the queue. This imposes constraints on the worker threads of the
+ * executor.
+ *
+ * <p>The reason is that we want to maintain the queue of cores to pull as a {@link
+ * DeduplicatingList} of {@link CorePuller} in order to effectively be able to deduplicate
+ * inserts. A {@link ThreadPoolExecutor} only accepts queues of {@link Runnable}, so we can't
+ * easily do sumbits (unless we submitted {@link CorePuller} instances already wrapped within a
+ * {@link Runnable}, but then the deduplication code would become ugly).
+ */
+ private final ExecutorUtil.MDCAwareThreadPoolExecutor asyncCorePullExecutor;
+
+ private DeleteProcessor overseerDeleteProcessor;
+ private CorePullerFactory corePullerFactory;
+ private final ZeroConfig config;
+
+ public ZeroStoreManager(CoreContainer coreContainer, ZeroConfig config) {
+ this.coreContainer = coreContainer;
+
+ zeroMetadataController =
+ new ZeroMetadataController(coreContainer.getZkController().getSolrCloudManager());
+ metadataCacheManager = new MetadataCacheManager(config, coreContainer);
+
+ // The default CorePuller creation below passes null for the pullEndNotification which
+ // means no pull end notifications. A different factory is installed by tests when a
+ // notification is needed
+ corePullerFactory =
+ (solrCore,
+ zeroStoreClient,
+ metadataCacheManager,
+ metadataController,
+ maxFailedCorePullAttempts) ->
+ new CorePuller(
+ solrCore,
+ zeroStoreClient,
+ metadataCacheManager,
+ metadataController,
+ maxFailedCorePullAttempts,
+ null);
+
+ this.zeroStoreClient =
+ ZeroStoreClientFactory.newInstance(
+ coreContainer.getNodeConfig(), coreContainer.getMetricManager());
+
+ corePusherFileDeleteProcessor =
+ new DeleteProcessor(
+ "PusherFileDeleter",
+ zeroStoreClient,
+ config.getAlmostMaxDeleterQueueSize(),
+ config.getDeleterThreadPoolSize(),
+ config.getMaxDeleteAttempts(),
+ config.getDeleteSleepMsFailedAttempt(),
+ config.getDeleteDelayMs());
+
+ // Non-Overseer nodes will initiate a delete processor but the underlying pool will sit idle
+ // until the node is elected and tasks are added. The overhead should be small.
+ overseerDeleteProcessor =
+ new DeleteProcessor(
+ "OverseerDeleteProcessor",
+ zeroStoreClient,
+ config.getAlmostMaxDeleterQueueSize(),
+ config.getDeleterThreadPoolSize(),
+ config.getMaxDeleteAttempts(),
+ config.getDeleteSleepMsFailedAttempt(),
+ config.getDeleteDelayMs());
+
+ this.config = config;
+
+ // Queue containing cores to pull from the Zero store
+ this.corePullTaskQueue = CorePuller.getDeduplicatingList(CORE_TO_PULL_LIST_MAX_SIZE);
+ // The executor executing the code pulling these cores. Actual files are pulled using the
+ // file pull executor defined in ZeroStoreClient.
+ //
+ // Tasks are added directly into the queue, not via the executor. We therefore set a fixed
+ // thread pool size (i.e. core size = max size) to not let threads terminate but have them
+ // wait for tasks to arrive in the queue.
+ // TODO All pushing and pulling executors likely belong in the same place
+ asyncCorePullExecutor =
+ new ExecutorUtil.MDCAwareThreadPoolExecutor(
+ config.getNumCorePullerThreads(),
+ config.getNumCorePullerThreads(),
+ 0L,
+ TimeUnit.MILLISECONDS,
+ new CorePullerBlockingQueue(corePullTaskQueue, this),
+ new SolrNamedThreadFactory("asyncCorePullExecutor"));
+ // We also need to pre-start all threads because nothing will start them for us (there are no
+ // direct submits to this ThreadPoolExecutor).
+ asyncCorePullExecutor.prestartAllCoreThreads();
+
+ if (log.isInfoEnabled()) {
+ log.info("asyncCorePullExecutor initialized");
+ }
+ }
+
+ public ZeroStoreClient getZeroStoreClient() {
+ return zeroStoreClient;
+ }
+
+ /** For testing purposes only. Current Zero store client is closed before replacing it. */
+ @VisibleForTesting
+ public void replaceZeroStoreClient(ZeroStoreClient zeroStoreClient) {
+ if (this.zeroStoreClient != null) {
+ this.zeroStoreClient.shutdown();
+ }
+ this.zeroStoreClient = zeroStoreClient;
+ }
+
+ @VisibleForTesting
+ public void replaceDeleteProcessors(
+ DeleteProcessor deleteProcessor, DeleteProcessor overseerDeleteProcessor) {
+ if (this.corePusherFileDeleteProcessor != null) {
+ this.corePusherFileDeleteProcessor.close();
+ }
+ this.corePusherFileDeleteProcessor = deleteProcessor;
+
+ if (this.overseerDeleteProcessor != null) {
+ this.overseerDeleteProcessor.close();
+ }
+ this.overseerDeleteProcessor = overseerDeleteProcessor;
+ }
+
+ public ZeroMetadataController getZeroMetadataController() {
+ return zeroMetadataController;
+ }
+
+ public MetadataCacheManager getMetadataCacheManager() {
+ return metadataCacheManager;
+ }
+
+ public CoreContainer getCoreContainer() {
+ return coreContainer;
+ }
+
+ public ZeroConfig getConfig() {
+ return config;
+ }
+
+ /**
+ * Check and evict any existing entry for Zero store metadata in the MetadataCacheManager. Zero
+ * store metadata should be treated the way CoreDescriptors in SolrCores are, and removed from
+ * this cache when the descriptors are removed from SolrCores.
+ */
+ public void evictCoreZeroMetadata(CoreDescriptor cd) {
+ if (cd != null
+ && cd.getCloudDescriptor().getReplicaType().equals(Replica.Type.ZERO)
+ && metadataCacheManager.removeCoreMetadata(cd.getName())) {
+ String collectionName = cd.getCollectionName();
+ String shardId = cd.getCloudDescriptor().getShardId();
+ if (log.isInfoEnabled()) {
+ log.info(
+ "Evicted core {} for collection {} and shard {} from ZERO core concurrency cache",
+ cd.getName(),
+ collectionName,
+ shardId);
+ }
+ }
+ }
+
+ public CorePuller newCorePuller(SolrCore solrCore) {
+ return corePullerFactory.createPuller(
+ solrCore,
+ zeroStoreClient,
+ metadataCacheManager,
+ zeroMetadataController,
+ config.getMaxFailedCorePullAttempts());
+ }
+
+ public CorePusher newCorePusher(SolrCore solrCore) {
+ return new CorePusher(
+ solrCore,
+ zeroStoreClient,
+ corePusherFileDeleteProcessor,
+ metadataCacheManager,
+ zeroMetadataController);
+ }
+
+ /**
+ * This method is called at the end of an indexing batch. It pushes new content of the local core
+ * present in the Zero store. When called after the first indexing batch for a core, it pushes all
+ * files of the core (i.e. this could be the first write of that shard to the Zero store).
+ */
+ public CorePusherExecutionInfo pushCoreToZeroStore(SolrCore core) {
+ CorePusher corePusher = newCorePusher(core);
+ return corePusher.endToEndPushCoreToZeroStore();
+ }
+
+ public void shutdown() {
+ stopBackgroundCorePulling();
+
+ if (corePusherFileDeleteProcessor != null) {
+ corePusherFileDeleteProcessor.close();
+ if (log.isInfoEnabled()) {
+ log.info("DeleteProcessor {} has shutdown", corePusherFileDeleteProcessor.getName());
+ }
+ }
+ if (overseerDeleteProcessor != null) {
+ overseerDeleteProcessor.close();
+ if (log.isInfoEnabled()) {
+ log.info("Overseer DeleteProcessor {} has shutdown", overseerDeleteProcessor.getName());
+ }
+ }
+
+ if (zeroStoreClient != null) {
+ zeroStoreClient.shutdown();
+ zeroStoreClient = null;
+ }
+ }
+
+ @VisibleForTesting
+ void replaceMetadataCacheManager(MetadataCacheManager metadataCacheManager) {
+ this.metadataCacheManager = metadataCacheManager;
+ }
+
+ @VisibleForTesting
+ void replaceCorePullerFactory(CorePullerFactory corePullerFactory) {
+ this.corePullerFactory = corePullerFactory;
+ }
+
+ @VisibleForTesting
+ void replaceZeroShardMetadataController(ZeroMetadataController metadataController) {
+ this.zeroMetadataController = metadataController;
+ }
+
+ /**
+ * This is used for optimizations and skipping pull in some cases. Unclear if the cost of the
+ * optimization (running this method) is not higher than the potential savings.
+ */
+ public boolean isLeader(SolrCore core) {
+ try {
+ if (!coreContainer.isZooKeeperAware()) {
+ // not solr cloud
+ return false;
+ }
+
+ CoreDescriptor coreDescriptor = coreContainer.getCoreDescriptor(core.getName());
+ if (coreDescriptor == null) {
+ // core descriptor does not exist
+ return false;
+ }
+
+ CloudDescriptor cd = coreDescriptor.getCloudDescriptor();
+ if (cd == null || cd.getReplicaType() != Replica.Type.ZERO) {
+ // not a ZERO replica
+ return false;
+ }
+
+ ZkController zkController = coreContainer.getZkController();
+ Replica leaderReplica =
+ zkController.getZkStateReader().getLeaderRetry(cd.getCollectionName(), cd.getShardId());
+ // not a leader replica
+ return leaderReplica != null && cd.getCoreNodeName().equals(leaderReplica.getName());
+ } catch (Exception ex) {
+ log.warn(
+ String.format(
+ Locale.ROOT,
+ "Could not establish if current replica is leader for the given core, collection=%s shard=%s core=%s",
+ core.getCoreDescriptor().getCollectionName(),
+ core.getCoreDescriptor().getCloudDescriptor().getShardId(),
+ core.getName()),
+ ex);
+ // we will proceed further as we are not a leader
+ }
+ return false;
+ }
+
+ @VisibleForTesting
+ @FunctionalInterface
+ public interface CorePullerFactory {
+ CorePuller createPuller(
+ SolrCore solrCore,
+ ZeroStoreClient zeroStoreClient,
+ MetadataCacheManager metadataCacheManager,
+ ZeroMetadataController metadataController,
+ int maxFailedCorePullAttempts);
+ }
+
+ /**
+ * Returns the list of core properties that are needed to create a core corresponding to provided
+ * {@code replica} of the {@code collection}. These cores are ZERO cores found in ZooKeeper that
+ * do not have a local presence on the disk at startup. See {@link
+ * ZeroStoreManager#discoverAdditionalCoreDescriptorsForZeroReplicas}
+ */
+ public Map<String, String> getZeroCoreProperties(DocCollection collection, Replica replica) {
+ // "numShards" is a property that is found in core descriptors. But it is only set on the
+ // cores created at collection creation time. It is not part of cores created by addition of
+ // replicas/shards or shard splits. Once set, it is not even kept in sync with latest number
+ // of shards. Therefore, we do not put it in any of missing cores we create.
+
+ Map<String, String> params = new HashMap<>();
+ params.put(CoreDescriptor.CORE_COLLECTION, collection.getName());
+ params.put(CoreDescriptor.CORE_NODE_NAME, replica.getName());
+ params.put(
+ CoreDescriptor.CORE_SHARD,
+ collection.getShardId(replica.getNodeName(), replica.getCoreName()));
+ params.put(CloudDescriptor.REPLICA_TYPE, Replica.Type.ZERO.name());
+ params.put(CollectionAdminParams.COLL_CONF, collection.getConfigName());
+ return params;
+ }
+
+ /** True if the core has ever synced with the Zero store, otherwise, false. */
+ public boolean hasCoreEverSyncedWithZeroStore(SolrCore core) {
+ MetadataCacheManager.MetadataCacheEntry coreMetadata =
+ metadataCacheManager.getOrCreateCoreMetadata(core.getName());
+ // If the cached value has moved beyond the initialization value then we must have pushed
+ // (performed indexing as a leader) or pulled (including the no-op pull that happens when the
+ // shard is at ZeroShardMetadataController.METADATA_NODE_DEFAULT_VALUE )
+ if (!metadataCacheManager.hasInitialCacheSuffixValue(coreMetadata)) return true;
+
+ // If local cache doesn't know about any updates, check in ZK
+ ZeroMetadataVersion shardMetadataVersion =
+ zeroMetadataController.readMetadataValue(getCollectionName(core), getShardName(core));
+ return zeroMetadataController.hasDefaultNodeSuffix(shardMetadataVersion);
+ }
+
+ /** If the core has never synced with the Zero store then fail the query. */
+ public void ensureZeroCoreFreshness(SolrCore core) {
+ CloudDescriptor cloudDescriptor = core.getCoreDescriptor().getCloudDescriptor();
+ if (cloudDescriptor != null && cloudDescriptor.getReplicaType() == Replica.Type.ZERO) {
+ boolean hasCoreEverSynced = hasCoreEverSyncedWithZeroStore(core);
+ if (!hasCoreEverSynced) {
+ // The message in this exception is verified by some tests. Change here, change there...
+ throw new SolrException(
+ SolrException.ErrorCode.SERVICE_UNAVAILABLE,
+ core.getName()
+ + " is not fresh enough because it has never synced with the Zero store.");
+ }
+ }
+ }
+
+ public CorePullStatus pullCoreFromZeroStore(SolrCore core) throws SolrException {
+ try {
+ CorePuller corePuller = newCorePuller(core);
+ return corePuller.pullCoreWithRetries(
+ isLeader(core),
+ ZeroCoreIndexingBatchProcessor.SECONDS_TO_WAIT_PULL_LOCK,
+ getConfig().getCorePullRetryDelay());
+ } catch (ZeroException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e.getMessage(), e);
+ }
+ }
+
+ public void setIndexingBatchReceived(SolrCore core) {
+ getMetadataCacheManager()
+ .recordState(core, MetadataCacheManager.ZeroCoreStage.INDEXING_BATCH_RECEIVED);
+ }
+
+ public void setIndexingStarted(SolrCore core) {
+ getMetadataCacheManager()
+ .recordState(core, MetadataCacheManager.ZeroCoreStage.LOCAL_INDEXING_STARTED);
+ }
+
+ public void setIndexingFinished(SolrCore core) {
+ getMetadataCacheManager()
+ .recordState(core, MetadataCacheManager.ZeroCoreStage.INDEXING_BATCH_FINISHED);
+ }
+
+ public ZeroAccessLocks getZeroAccessLocks(String coreName) {
+ return getMetadataCacheManager().getOrCreateCoreMetadata(coreName).getZeroAccessLocks();
+ }
+
+ /**
+ * Acquires the indexing lock that must be held during the split process using the {@link
+ * ZeroCoreIndexingBatchProcessor#SECONDS_TO_WAIT_PULL_LOCK} timeout.
+ *
+ * <p>For indexing, the same lock is acquired in {@link
+ * ZeroCoreIndexingBatchProcessor#startIndexingBatch} using the actual indexing timeout.
+ */
+ public AutoCloseable acquireIndexingLockForSplit(SolrCore core) throws SolrException {
+ try {
+ // TODO Maybe use ZeroCoreIndexingBatchProcessor.SECONDS_TO_WAIT_INDEXING_LOCK instead?
+ return metadataCacheManager
+ .getOrCreateCoreMetadata(core.getName())
+ .getZeroAccessLocks()
+ .acquireIndexingLock(ZeroCoreIndexingBatchProcessor.SECONDS_TO_WAIT_PULL_LOCK);
+ } catch (ZeroLockException e) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ String.format(
+ Locale.ROOT,
+ "Failed to acquire indexing lock within %s seconds. "
+ + "collection=%s shard=%s core=%s",
+ ZeroCoreIndexingBatchProcessor.SECONDS_TO_WAIT_PULL_LOCK,
+ getCollectionName(core),
+ getShardName(core),
+ core.getName()));
+ } catch (InterruptedException e) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ String.format(
+ Locale.ROOT,
+ "Thread interrupted while trying to acquire pull read lock."
+ + " collection=%s shard=%s core=%s",
+ getCollectionName(core),
+ getShardName(core),
+ core.getName()));
+ }
+ }
+
+ public void initialCorePushToZeroStore(SolrCore core) throws InterruptedException {
+ CloudDescriptor cloudDesc = core.getCoreDescriptor().getCloudDescriptor();
+ String collectionName = cloudDesc.getCollectionName();
+ String shardName = cloudDesc.getShardId();
+ String coreName = core.getName();
+
+ if (log.isInfoEnabled()) {
+ log.info(
+ "Attempting to push to Zero store for collection={} shard={} core={}",
+ collectionName,
+ shardName,
+ coreName);
+ }
+
+ CorePusher pusher = newCorePusher(core);
+
+ pusher.initialCorePushToZeroStore();
+
+ if (log.isInfoEnabled()) {
+ log.info(
+ "Successfully pushed to Zero store for collection={} shard={} core={}",
+ collectionName,
+ shardName,
+ coreName);
+ }
+ }
+
+ /**
+ * Enqueues the core (wrapped in a {@link CorePuller}) into the deduplicating list to be
+ * eventually pulled from Zero store.
+ *
+ * <p>Eventually, the {@link #asyncCorePullExecutor} will call {@link
+ * CorePuller#lockAndPullCore(boolean, long)}. The actual execution will happen in {@link
+ * CorePullerBlockingQueue.RunnableAsyncCorePull#run()}, built in {@link
+ * CorePullerBlockingQueue#take()}.
+ *
+ * <p>In case of pull errors (even obviously transient ones) there are no retries. Given the
+ * enqueue is triggered by queries, a subsequent query will trigger a new pull request. Note that
+ * upon completion, the {@link CorePuller#pullEndNotification} is called by calling {@link
+ * CorePuller#notifyPullEnd(CorePullStatus)}.
+ *
+ * <p>TODO more work needed because we shouldn't try to pull too often
+ */
+ public void enqueueCorePullFromZeroStore(SolrCore core) throws Exception {
+ CorePuller puller = newCorePuller(core);
+ corePullTaskQueue.addDeduplicated(puller, false);
+ }
+
+ @VisibleForTesting
+ void stopBackgroundCorePulling() {
+ // Stopping the executor will stop the async pulls.
+ // Tests do not need to restart the pulls, we're ok.
+ ExecutorUtil.shutdownAndAwaitTermination(asyncCorePullExecutor);
+ }
+
+ public void deleteShard(String collectionName, String shardName) throws SolrException {
+ zeroMetadataController.cleanUpMetadataNodes(collectionName, shardName);
+
+ CompletableFuture<DeleterTask.Result> deleteFuture =
+ overseerDeleteProcessor.deleteShard(collectionName, shardName, false);
+
+ DeleterTask.Result result = null;
+ Throwable t = null;
+ try {
+ // TODO: Find a reasonable timeout value
+ result = deleteFuture.get(60, TimeUnit.SECONDS);
+ } catch (Exception ex) {
+ t = ex;
+ }
+ if (t != null || !result.isSuccess()) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "Could not complete deleting shard"
+ + shardName
+ + " from Zero store belonging to collection "
+ + collectionName
+ + ". Files belonging to this shard may be orphaned.",
+ t);
+ }
+ }
+
+ public void deleteCollection(String collectionName) throws SolrException {
+ // deletes all files belonging to this collection
+ CompletableFuture<DeleterTask.Result> deleteFuture =
+ overseerDeleteProcessor.deleteCollection(collectionName, false);
+
+ DeleterTask.Result result = null;
+ Throwable t = null;
+ try {
+ // TODO: Find a reasonable timeout value
+ result = deleteFuture.get(60, TimeUnit.SECONDS);
+ } catch (Exception ex) {
+ t = ex;
+ }
+ if (t != null || !result.isSuccess()) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "Could not complete deleting collection "
+ + collectionName
+ + " from Zero store, files belonging to this collection"
+ + " may be orphaned.",
+ t);
+ }
... 10929 lines suppressed ...