You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ho...@apache.org on 2021/11/09 17:06:30 UTC
[solr-operator] branch main updated: Add ability to schedule
recurring incremental backups (#359)
This is an automated email from the ASF dual-hosted git repository.
houston pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr-operator.git
The following commit(s) were added to refs/heads/main by this push:
new ff3bcd1 Add ability to schedule recurring incremental backups (#359)
ff3bcd1 is described below
commit ff3bcd176f1ba6bf671970e33dc57a63c499a9b4
Author: Houston Putman <ho...@apache.org>
AuthorDate: Tue Nov 9 12:06:27 2021 -0500
Add ability to schedule recurring incremental backups (#359)
---
api/v1beta1/solrbackup_types.go | 81 ++++++++++----
api/v1beta1/solrcloud_types.go | 1 -
api/v1beta1/zz_generated.deepcopy.go | 82 +++++++++++---
config/crd/bases/solr.apache.org_solrbackups.yaml | 121 +++++++++++++++++++-
controllers/common.go | 3 +
controllers/solrbackup_controller.go | 123 ++++++++++++++-------
controllers/solrcloud_controller.go | 4 +-
controllers/solrprometheusexporter_controller.go | 12 +-
controllers/util/backup_util.go | 84 +++++++-------
controllers/util/solr_api/api.go | 55 ++++++++-
.../{common.go => util/solr_api/node_command.go} | 18 ++-
controllers/util/solr_update_util.go | 2 +-
docs/solr-backup/README.md | 87 ++++++++++++++-
helm/solr-operator/Chart.yaml | 11 +-
helm/solr-operator/crds/crds.yaml | 121 +++++++++++++++++++-
main.go | 3 +-
16 files changed, 660 insertions(+), 148 deletions(-)
diff --git a/api/v1beta1/solrbackup_types.go b/api/v1beta1/solrbackup_types.go
index ec0d151..2665211 100644
--- a/api/v1beta1/solrbackup_types.go
+++ b/api/v1beta1/solrbackup_types.go
@@ -21,7 +21,6 @@ import (
"fmt"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "strings"
)
// SolrBackupSpec defines the desired state of SolrBackup
@@ -50,6 +49,13 @@ type SolrBackupSpec struct {
// +optional
Location string `json:"location,omitempty"`
+ // Set this backup to be taken recurrently, with options for scheduling and storage.
+ //
+ // NOTE: This is only supported for Solr Clouds version 8.9+, as it uses the incremental backup API.
+ //
+ // +optional
+ Recurrence *BackupRecurrence `json:"recurrence,omitempty"`
+
// Persistence is the specification on how to persist the backup data.
// This feature has been removed as of v0.5.0. Any options specified here will not be used.
//
@@ -67,6 +73,39 @@ func (spec *SolrBackupSpec) withDefaults() (changed bool) {
return changed
}
+// BackupRecurrence defines the recurrence of the incremental backup
+type BackupRecurrence struct {
+ // Perform a backup on the given schedule, in CRON format.
+ //
+ // Multiple CRON syntaxes are supported
+ // - Standard CRON (e.g. "CRON_TZ=Asia/Seoul 0 6 * * ?")
+ // - Predefined Schedules (e.g. "@yearly", "@weekly", "@daily", etc.)
+ // - Intervals (e.g. "@every 10h30m")
+ //
+ // For more information please check this reference:
+ // https://pkg.go.dev/github.com/robfig/cron/v3?utm_source=godoc#hdr-CRON_Expression_Format
+ Schedule string `json:"schedule"`
+
+ // Define the number of backup points to save for this backup at any given time.
+ // The oldest backups will be deleted if too many exist when a backup is taken.
+ // If not provided, this defaults to 5.
+ //
+ // +kubebuilder:default:=5
+ // +kubebuilder:validation:Minimum:=1
+ // +optional
+ MaxSaved int `json:"maxSaved,omitempty"`
+
+ // Disable the recurring backups. Note this will not affect any currently-running backup.
+ //
+ // +kubebuilder:default:=false
+ // +optional
+ Disabled bool `json:"disabled,omitempty"`
+}
+
+func (recurrence *BackupRecurrence) IsEnabled() bool {
+ return recurrence != nil && !recurrence.Disabled
+}
+
// PersistenceSource defines the location and method of persisting the backup data.
// Exactly one member must be specified.
//
@@ -160,27 +199,29 @@ type VolumePersistenceSource struct {
BusyBoxImage ContainerImage `json:"busyBoxImage,omitempty"`
}
-// Deprecated: Will be unused as of v0.5.0
-func (spec *VolumePersistenceSource) withDefaults(backupName string) (changed bool) {
- changed = spec.BusyBoxImage.withDefaults(DefaultBusyBoxImageRepo, DefaultBusyBoxImageVersion, DefaultPullPolicy) || changed
-
- if spec.Path != "" && strings.HasPrefix(spec.Path, "/") {
- spec.Path = strings.TrimPrefix(spec.Path, "/")
- changed = true
- }
+// SolrBackupStatus defines the observed state of SolrBackup
+type SolrBackupStatus struct {
+ // The current Backup Status, which all fields are added to this struct
+ IndividualSolrBackupStatus `json:",inline"`
- if spec.Filename == "" {
- spec.Filename = backupName + ".tgz"
- changed = true
- }
+ // The scheduled time for the next backup to occur
+ // +optional
+ NextScheduledTime *metav1.Time `json:"nextScheduledTime,omitempty"`
- return changed
+ // The status history of recurring backups
+ // +optional
+ History []IndividualSolrBackupStatus `json:"history,omitempty"`
}
-// SolrBackupStatus defines the observed state of SolrBackup
-type SolrBackupStatus struct {
+// IndividualSolrBackupStatus defines the observed state of a single issued SolrBackup
+type IndividualSolrBackupStatus struct {
// Version of the Solr being backed up
- SolrVersion string `json:"solrVersion"`
+ // +optional
+ SolrVersion string `json:"solrVersion,omitempty"`
+
+ // The time that this backup was initiated
+ // +optional
+ StartTime metav1.Time `json:"startTimestamp,omitempty"`
// The status of each collection's backup progress
// +optional
@@ -289,8 +330,10 @@ func (sb *SolrBackup) PersistenceJobName() string {
//+kubebuilder:categories=all
//+kubebuilder:subresource:status
//+kubebuilder:printcolumn:name="Cloud",type="string",JSONPath=".spec.solrCloud",description="Solr Cloud"
-//+kubebuilder:printcolumn:name="Finished",type="boolean",JSONPath=".status.finished",description="Whether the backup has finished"
-//+kubebuilder:printcolumn:name="Successful",type="boolean",JSONPath=".status.successful",description="Whether the backup was successful"
+//+kubebuilder:printcolumn:name="Started",type="date",JSONPath=".status.startTimestamp",description="Most recent time the backup started"
+//+kubebuilder:printcolumn:name="Finished",type="boolean",JSONPath=".status.finished",description="Whether the most recent backup has finished"
+//+kubebuilder:printcolumn:name="Successful",type="boolean",JSONPath=".status.successful",description="Whether the most recent backup was successful"
+//+kubebuilder:printcolumn:name="NextBackup",type="string",JSONPath=".status.nextScheduledTime",description="Next scheduled time for a recurrent backup",format="date-time"
//+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
// SolrBackup is the Schema for the solrbackups API
diff --git a/api/v1beta1/solrcloud_types.go b/api/v1beta1/solrcloud_types.go
index ff265ec..3c6b1ef 100644
--- a/api/v1beta1/solrcloud_types.go
+++ b/api/v1beta1/solrcloud_types.go
@@ -35,7 +35,6 @@ const (
DefaultSolrReplicas = int32(3)
DefaultSolrRepo = "library/solr"
DefaultSolrVersion = "8.9"
- DefaultSolrStorage = "5Gi"
DefaultSolrJavaMem = "-Xms1g -Xmx2g"
DefaultSolrOpts = ""
DefaultSolrLogLevel = "INFO"
diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go
index 31031ee..4e3450b 100644
--- a/api/v1beta1/zz_generated.deepcopy.go
+++ b/api/v1beta1/zz_generated.deepcopy.go
@@ -77,6 +77,21 @@ func (in *BackupPersistenceStatus) DeepCopy() *BackupPersistenceStatus {
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *BackupRecurrence) DeepCopyInto(out *BackupRecurrence) {
+ *out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupRecurrence.
+func (in *BackupRecurrence) DeepCopy() *BackupRecurrence {
+ if in == nil {
+ return nil
+ }
+ out := new(BackupRecurrence)
+ in.DeepCopyInto(out)
+ return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *CollectionBackupStatus) DeepCopyInto(out *CollectionBackupStatus) {
*out = *in
if in.StartTime != nil {
@@ -299,6 +314,43 @@ func (in *GcsRepository) DeepCopy() *GcsRepository {
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *IndividualSolrBackupStatus) DeepCopyInto(out *IndividualSolrBackupStatus) {
+ *out = *in
+ in.StartTime.DeepCopyInto(&out.StartTime)
+ if in.CollectionBackupStatuses != nil {
+ in, out := &in.CollectionBackupStatuses, &out.CollectionBackupStatuses
+ *out = make([]CollectionBackupStatus, len(*in))
+ for i := range *in {
+ (*in)[i].DeepCopyInto(&(*out)[i])
+ }
+ }
+ if in.PersistenceStatus != nil {
+ in, out := &in.PersistenceStatus, &out.PersistenceStatus
+ *out = new(BackupPersistenceStatus)
+ (*in).DeepCopyInto(*out)
+ }
+ if in.FinishTime != nil {
+ in, out := &in.FinishTime, &out.FinishTime
+ *out = (*in).DeepCopy()
+ }
+ if in.Successful != nil {
+ in, out := &in.Successful, &out.Successful
+ *out = new(bool)
+ **out = **in
+ }
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IndividualSolrBackupStatus.
+func (in *IndividualSolrBackupStatus) DeepCopy() *IndividualSolrBackupStatus {
+ if in == nil {
+ return nil
+ }
+ out := new(IndividualSolrBackupStatus)
+ in.DeepCopyInto(out)
+ return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *IngressOptions) DeepCopyInto(out *IngressOptions) {
*out = *in
if in.Annotations != nil {
@@ -803,6 +855,11 @@ func (in *SolrBackupSpec) DeepCopyInto(out *SolrBackupSpec) {
*out = make([]string, len(*in))
copy(*out, *in)
}
+ if in.Recurrence != nil {
+ in, out := &in.Recurrence, &out.Recurrence
+ *out = new(BackupRecurrence)
+ **out = **in
+ }
if in.Persistence != nil {
in, out := &in.Persistence, &out.Persistence
*out = new(PersistenceSource)
@@ -823,27 +880,18 @@ func (in *SolrBackupSpec) DeepCopy() *SolrBackupSpec {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *SolrBackupStatus) DeepCopyInto(out *SolrBackupStatus) {
*out = *in
- if in.CollectionBackupStatuses != nil {
- in, out := &in.CollectionBackupStatuses, &out.CollectionBackupStatuses
- *out = make([]CollectionBackupStatus, len(*in))
+ in.IndividualSolrBackupStatus.DeepCopyInto(&out.IndividualSolrBackupStatus)
+ if in.NextScheduledTime != nil {
+ in, out := &in.NextScheduledTime, &out.NextScheduledTime
+ *out = (*in).DeepCopy()
+ }
+ if in.History != nil {
+ in, out := &in.History, &out.History
+ *out = make([]IndividualSolrBackupStatus, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
- if in.PersistenceStatus != nil {
- in, out := &in.PersistenceStatus, &out.PersistenceStatus
- *out = new(BackupPersistenceStatus)
- (*in).DeepCopyInto(*out)
- }
- if in.FinishTime != nil {
- in, out := &in.FinishTime, &out.FinishTime
- *out = (*in).DeepCopy()
- }
- if in.Successful != nil {
- in, out := &in.Successful, &out.Successful
- *out = new(bool)
- **out = **in
- }
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SolrBackupStatus.
diff --git a/config/crd/bases/solr.apache.org_solrbackups.yaml b/config/crd/bases/solr.apache.org_solrbackups.yaml
index 248515a..aa640fc 100644
--- a/config/crd/bases/solr.apache.org_solrbackups.yaml
+++ b/config/crd/bases/solr.apache.org_solrbackups.yaml
@@ -35,14 +35,23 @@ spec:
jsonPath: .spec.solrCloud
name: Cloud
type: string
- - description: Whether the backup has finished
+ - description: Most recent time the backup started
+ jsonPath: .status.startTimestamp
+ name: Started
+ type: date
+ - description: Whether the most recent backup has finished
jsonPath: .status.finished
name: Finished
type: boolean
- - description: Whether the backup was successful
+ - description: Whether the most recent backup was successful
jsonPath: .status.successful
name: Successful
type: boolean
+ - description: Next scheduled time for a recurrent backup
+ format: date-time
+ jsonPath: .status.nextScheduledTime
+ name: NextBackup
+ type: string
- jsonPath: .metadata.creationTimestamp
name: Age
type: date
@@ -1051,6 +1060,24 @@ spec:
- source
type: object
type: object
+ recurrence:
+ description: "Set this backup to be taken recurrently, with options for scheduling and storage. \n NOTE: This is only supported for Solr Clouds version 8.9+, as it uses the incremental backup API."
+ properties:
+ disabled:
+ default: false
+ description: Disable the recurring backups. Note this will not affect any currently-running backup.
+ type: boolean
+ maxSaved:
+ default: 5
+ description: Define the number of backup points to save for this backup at any given time. The oldest backups will be deleted if too many exist when a backup is taken. If not provided, this defaults to 5.
+ minimum: 1
+ type: integer
+ schedule:
+ description: "Perform a backup on the given schedule, in CRON format. \n Multiple CRON syntaxes are supported - Standard CRON (e.g. \"CRON_TZ=Asia/Seoul 0 6 * * ?\") - Predefined Schedules (e.g. \"@yearly\", \"@weekly\", \"@daily\", etc.) - Intervals (e.g. \"@every 10h30m\") \n For more information please check this reference: https://pkg.go.dev/github.com/robfig/cron/v3?utm_source=godoc#hdr-CRON_Expression_Format"
+ type: string
+ required:
+ - schedule
+ type: object
repositoryName:
description: The name of the repository to use for the backup. Defaults to "legacy_local_repository" if not specified (the auto-configured repository for legacy singleton volumes).
maxLength: 100
@@ -1111,6 +1138,90 @@ spec:
finished:
description: Whether the backup has finished
type: boolean
+ history:
+ description: The status history of recurring backups
+ items:
+ description: IndividualSolrBackupStatus defines the observed state of a single issued SolrBackup
+ properties:
+ collectionBackupStatuses:
+ description: The status of each collection's backup progress
+ items:
+ description: CollectionBackupStatus defines the progress of a Solr Collection's backup
+ properties:
+ asyncBackupStatus:
+ description: The status of the asynchronous backup call to solr
+ type: string
+ backupName:
+ description: BackupName of this collection's backup in Solr
+ type: string
+ collection:
+ description: Solr Collection name
+ type: string
+ finishTimestamp:
+ description: Time that the collection backup finished at
+ format: date-time
+ type: string
+ finished:
+ description: Whether the backup has finished
+ type: boolean
+ inProgress:
+ description: Whether the collection is being backed up
+ type: boolean
+ startTimestamp:
+ description: Time that the collection backup started at
+ format: date-time
+ type: string
+ successful:
+ description: Whether the backup was successful
+ type: boolean
+ required:
+ - collection
+ type: object
+ type: array
+ finishTimestamp:
+ description: Version of the Solr being backed up
+ format: date-time
+ type: string
+ finished:
+ description: Whether the backup has finished
+ type: boolean
+ persistenceStatus:
+ description: Whether the backups are in progress of being persisted. This feature has been removed as of v0.5.0.
+ properties:
+ finishTimestamp:
+ description: Time that the collection backup finished at
+ format: date-time
+ type: string
+ finished:
+ description: Whether the persistence has finished
+ type: boolean
+ inProgress:
+ description: Whether the collection is being backed up
+ type: boolean
+ startTimestamp:
+ description: Time that the collection backup started at
+ format: date-time
+ type: string
+ successful:
+ description: Whether the backup was successful
+ type: boolean
+ type: object
+ solrVersion:
+ description: Version of the Solr being backed up
+ type: string
+ startTimestamp:
+ description: The time that this backup was initiated
+ format: date-time
+ type: string
+ successful:
+ description: Whether the backup was successful
+ type: boolean
+ type: object
+ type: array
+ nextScheduledTime:
+ description: The scheduled time for the next backup to occur
+ format: date-time
+ type: string
persistenceStatus:
description: Whether the backups are in progress of being persisted. This feature has been removed as of v0.5.0.
properties:
@@ -1135,11 +1246,13 @@ spec:
solrVersion:
description: Version of the Solr being backed up
type: string
+ startTimestamp:
+ description: The time that this backup was initiated
+ format: date-time
+ type: string
successful:
description: Whether the backup was successful
type: boolean
- required:
- - solrVersion
type: object
type: object
served: true
diff --git a/controllers/common.go b/controllers/common.go
index 38a056e..b86fb00 100644
--- a/controllers/common.go
+++ b/controllers/common.go
@@ -24,6 +24,9 @@ import (
// Set the requeueAfter if it has not been set, or is greater than the new time to requeue at
func updateRequeueAfter(requeueOrNot *reconcile.Result, newWait time.Duration) {
+ if newWait <= 0 {
+ requeueOrNot.RequeueAfter = 0
+ }
if requeueOrNot.RequeueAfter <= 0 || requeueOrNot.RequeueAfter > newWait {
requeueOrNot.RequeueAfter = newWait
}
diff --git a/controllers/solrbackup_controller.go b/controllers/solrbackup_controller.go
index 4e8d41b..c5905c1 100644
--- a/controllers/solrbackup_controller.go
+++ b/controllers/solrbackup_controller.go
@@ -30,7 +30,6 @@ import (
"github.com/apache/solr-operator/controllers/util"
"github.com/go-logr/logr"
- batchv1 "k8s.io/api/batch/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
@@ -48,7 +47,7 @@ import (
type SolrBackupReconciler struct {
client.Client
Scheme *runtime.Scheme
- config *rest.Config
+ Config *rest.Config
}
//+kubebuilder:rbac:groups="",resources=pods/exec,verbs=create
@@ -80,45 +79,93 @@ func (r *SolrBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request)
return reconcile.Result{}, err
}
- oldStatus := backup.Status.DeepCopy()
-
changed := backup.WithDefaults()
if changed {
logger.Info("Setting default settings for solr-backup")
- if err := r.Update(ctx, backup); err != nil {
+ if err = r.Update(ctx, backup); err != nil {
return reconcile.Result{}, err
}
return reconcile.Result{Requeue: true}, nil
}
- // When working with the collection backups, auto-requeue after 5 seconds
- // to check on the status of the async solr backup calls
+ oldStatus := backup.Status.DeepCopy()
+
requeueOrNot := reconcile.Result{}
- solrCloud, _, err := r.reconcileSolrCloudBackup(ctx, backup, logger)
- if err != nil {
- // TODO Should we be failing the backup for some sub-set of errors here?
- logger.Error(err, "Error while taking SolrCloud backup")
-
- // Requeue after 10 seconds for errors.
- requeueOrNot = reconcile.Result{Requeue: true, RequeueAfter: time.Second * 10}
- } else if solrCloud != nil && !backup.Status.Finished {
- // Only requeue if the SolrCloud we are backing up exists and we are not finished with the backups.
- requeueOrNot = reconcile.Result{Requeue: true, RequeueAfter: time.Second * 5}
- } else if backup.Status.Finished && backup.Status.FinishTime == nil {
- now := metav1.Now()
- backup.Status.FinishTime = &now
+ var backupNeedsToWait bool
+
+ // Check if we should start the next backup
+ if backup.Status.NextScheduledTime != nil {
+ // If the backup no longer enabled, remove the next scheduled time
+ if !backup.Spec.Recurrence.IsEnabled() {
+ backup.Status.NextScheduledTime = nil
+ backupNeedsToWait = false
+ } else if backup.Status.NextScheduledTime.UTC().Before(time.Now().UTC()) {
+ // We have hit the next scheduled restart time.
+ backupNeedsToWait = false
+ backup.Status.NextScheduledTime = nil
+
+ // Add the current backup to the front of the history.
+ // If there is no max
+ backup.Status.History = append([]solrv1beta1.IndividualSolrBackupStatus{backup.Status.IndividualSolrBackupStatus}, backup.Status.History...)
+
+ // Remove history if we have too much saved
+ if len(backup.Status.History) > backup.Spec.Recurrence.MaxSaved {
+ backup.Status.History = backup.Status.History[:backup.Spec.Recurrence.MaxSaved]
+ }
+
+ // Reset Current, which is fine since it is now in the history.
+ backup.Status.IndividualSolrBackupStatus = solrv1beta1.IndividualSolrBackupStatus{}
+ } else {
+ // If we have not hit the next scheduled restart, wait to requeue until that is true.
+ updateRequeueAfter(&requeueOrNot, backup.Status.NextScheduledTime.UTC().Sub(time.Now().UTC()))
+ backupNeedsToWait = true
+ }
+ } else {
+ backupNeedsToWait = false
+ }
+
+ // Do backup work if we are not waiting and the current backup is not finished
+ if !backupNeedsToWait && !backup.Status.IndividualSolrBackupStatus.Finished {
+ solrCloud, _, err1 := r.reconcileSolrCloudBackup(ctx, backup, &backup.Status.IndividualSolrBackupStatus, logger)
+ if err1 != nil {
+ // TODO Should we be failing the backup for some sub-set of errors here?
+ logger.Error(err1, "Error while taking SolrCloud backup")
+
+ // Requeue after 10 seconds for errors.
+ updateRequeueAfter(&requeueOrNot, time.Second*10)
+ } else if backup.Status.IndividualSolrBackupStatus.Finished {
+ // Set finish time
+ now := metav1.Now()
+ backup.Status.IndividualSolrBackupStatus.FinishTime = &now
+ } else if solrCloud != nil {
+ // When working with the collection backups, auto-requeue after 5 seconds
+ // to check on the status of the async solr backup calls
+ updateRequeueAfter(&requeueOrNot, time.Second*5)
+ }
+ }
+
+ // Schedule the next backupTime, if it doesn't have a next scheduled time, it has recurrence and the current backup is finished
+ if backup.Status.NextScheduledTime == nil && backup.Spec.Recurrence.IsEnabled() && backup.Status.IndividualSolrBackupStatus.Finished {
+ if nextBackupTime, err1 := util.ScheduleNextBackup(backup.Spec.Recurrence.Schedule, backup.Status.IndividualSolrBackupStatus.StartTime.Time); err1 != nil {
+ logger.Error(err1, "Could not schedule new backup due to bad cron schedule", "cron", backup.Spec.Recurrence.Schedule)
+ } else {
+ logger.Info("Scheduling Next Backup", "time", nextBackupTime)
+ convTime := metav1.NewTime(nextBackupTime)
+ backup.Status.NextScheduledTime = &convTime
+ updateRequeueAfter(&requeueOrNot, backup.Status.NextScheduledTime.Sub(time.Now()))
+ }
}
- if !reflect.DeepEqual(oldStatus, &backup.Status) {
- logger.Info("Updating status for solr-backup")
+ if !reflect.DeepEqual(*oldStatus, backup.Status) {
+ logger.Info("Updating status for solr-backup", "newStatus", backup.Status, "oldStatus", oldStatus)
err = r.Status().Update(ctx, backup)
}
return requeueOrNot, err
}
-func (r *SolrBackupReconciler) reconcileSolrCloudBackup(ctx context.Context, backup *solrv1beta1.SolrBackup, logger logr.Logger) (solrCloud *solrv1beta1.SolrCloud, actionTaken bool, err error) {
+func (r *SolrBackupReconciler) reconcileSolrCloudBackup(ctx context.Context, backup *solrv1beta1.SolrBackup, currentBackupStatus *solrv1beta1.IndividualSolrBackupStatus, logger logr.Logger) (solrCloud *solrv1beta1.SolrCloud, actionTaken bool, err error) {
// Get the solrCloud that this backup is for.
solrCloud = &solrv1beta1.SolrCloud{}
@@ -139,7 +186,7 @@ func (r *SolrBackupReconciler) reconcileSolrCloudBackup(ctx context.Context, bac
}
// First check if the collection backups have been completed
- collectionBackupsFinished := util.UpdateStatusOfCollectionBackups(backup)
+ collectionBackupsFinished := util.UpdateStatusOfCollectionBackups(currentBackupStatus)
// If the collectionBackups are complete, then nothing else has to be done here
if collectionBackupsFinished {
@@ -156,9 +203,9 @@ func (r *SolrBackupReconciler) reconcileSolrCloudBackup(ctx context.Context, bac
}
// This should only occur before the backup processes have been started
- if backup.Status.SolrVersion == "" {
+ if currentBackupStatus.StartTime.IsZero() {
// Prep the backup directory in the persistentVolume
- err = util.EnsureDirectoryForBackup(solrCloud, backupRepository, backup, r.config)
+ err = util.EnsureDirectoryForBackup(solrCloud, backupRepository, backup, r.Config)
if err != nil {
return solrCloud, actionTaken, err
}
@@ -170,30 +217,31 @@ func (r *SolrBackupReconciler) reconcileSolrCloudBackup(ctx context.Context, bac
}
// Only set the solr version at the start of the backup. This shouldn't change throughout the backup.
- backup.Status.SolrVersion = solrCloud.Status.Version
+ currentBackupStatus.SolrVersion = solrCloud.Status.Version
+ currentBackupStatus.StartTime = metav1.Now()
}
// Go through each collection specified and reconcile the backup.
for _, collection := range backup.Spec.Collections {
// This will in-place update the CollectionBackupStatus in the backup object
- if _, err = reconcileSolrCollectionBackup(ctx, backup, solrCloud, backupRepository, collection, logger); err != nil {
+ if _, err = reconcileSolrCollectionBackup(ctx, backup, currentBackupStatus, solrCloud, backupRepository, collection, logger); err != nil {
break
}
}
// First check if the collection backups have been completed
- util.UpdateStatusOfCollectionBackups(backup)
+ util.UpdateStatusOfCollectionBackups(currentBackupStatus)
return solrCloud, actionTaken, err
}
-func reconcileSolrCollectionBackup(ctx context.Context, backup *solrv1beta1.SolrBackup, solrCloud *solrv1beta1.SolrCloud, backupRepository *solrv1beta1.SolrBackupRepository, collection string, logger logr.Logger) (finished bool, err error) {
+func reconcileSolrCollectionBackup(ctx context.Context, backup *solrv1beta1.SolrBackup, currentBackupStatus *solrv1beta1.IndividualSolrBackupStatus, solrCloud *solrv1beta1.SolrCloud, backupRepository *solrv1beta1.SolrBackupRepository, collection string, logger logr.Logger) (finished bool, err error) {
now := metav1.Now()
collectionBackupStatus := solrv1beta1.CollectionBackupStatus{}
collectionBackupStatus.Collection = collection
backupIndex := -1
// Get the backup status for this collection, if one exists
- for i, status := range backup.Status.CollectionBackupStatuses {
+ for i, status := range currentBackupStatus.CollectionBackupStatuses {
if status.Collection == collection {
collectionBackupStatus = status
backupIndex = i
@@ -201,7 +249,9 @@ func reconcileSolrCollectionBackup(ctx context.Context, backup *solrv1beta1.Solr
}
// If the collection backup hasn't started, start it
- if !collectionBackupStatus.InProgress && !collectionBackupStatus.Finished {
+ if collectionBackupStatus.Finished {
+ return true, nil
+ } else if !collectionBackupStatus.InProgress {
// Start the backup by calling solr
var started bool
started, err = util.StartBackupForCollection(ctx, solrCloud, backupRepository, backup, collection, logger)
@@ -239,9 +289,9 @@ func reconcileSolrCollectionBackup(ctx context.Context, backup *solrv1beta1.Solr
}
if backupIndex < 0 {
- backup.Status.CollectionBackupStatuses = append(backup.Status.CollectionBackupStatuses, collectionBackupStatus)
+ currentBackupStatus.CollectionBackupStatuses = append(currentBackupStatus.CollectionBackupStatuses, collectionBackupStatus)
} else {
- backup.Status.CollectionBackupStatuses[backupIndex] = collectionBackupStatus
+ currentBackupStatus.CollectionBackupStatuses[backupIndex] = collectionBackupStatus
}
return collectionBackupStatus.Finished, err
@@ -249,11 +299,10 @@ func reconcileSolrCollectionBackup(ctx context.Context, backup *solrv1beta1.Solr
// SetupWithManager sets up the controller with the Manager.
func (r *SolrBackupReconciler) SetupWithManager(mgr ctrl.Manager) (err error) {
- r.config = mgr.GetConfig()
+ r.Config = mgr.GetConfig()
ctrlBuilder := ctrl.NewControllerManagedBy(mgr).
- For(&solrv1beta1.SolrBackup{}).
- Owns(&batchv1.Job{})
+ For(&solrv1beta1.SolrBackup{})
ctrlBuilder, err = r.indexAndWatchForSolrClouds(mgr, ctrlBuilder)
if err != nil {
diff --git a/controllers/solrcloud_controller.go b/controllers/solrcloud_controller.go
index 8d0153c..0674825 100644
--- a/controllers/solrcloud_controller.go
+++ b/controllers/solrcloud_controller.go
@@ -103,7 +103,7 @@ func (r *SolrCloudReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
changed := instance.WithDefaults()
if changed {
logger.Info("Setting default settings for SolrCloud")
- if err := r.Update(ctx, instance); err != nil {
+ if err = r.Update(ctx, instance); err != nil {
return reconcile.Result{}, err
}
return reconcile.Result{Requeue: true}, nil
@@ -323,7 +323,7 @@ func (r *SolrCloudReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
// Set the annotation for a scheduled restart, if necessary.
if nextRestartAnnotation, reconcileWaitDuration, err := util.ScheduleNextRestart(instance.Spec.UpdateStrategy.RestartSchedule, foundStatefulSet.Spec.Template.Annotations); err != nil {
- logger.Error(err, "Cannot parse restartSchedule cron: %s", instance.Spec.UpdateStrategy.RestartSchedule)
+ logger.Error(err, "Cannot parse restartSchedule cron", "cron", instance.Spec.UpdateStrategy.RestartSchedule)
} else {
if nextRestartAnnotation != "" {
// Set the new restart time annotation
diff --git a/controllers/solrprometheusexporter_controller.go b/controllers/solrprometheusexporter_controller.go
index b6dd129..ae8f53a 100644
--- a/controllers/solrprometheusexporter_controller.go
+++ b/controllers/solrprometheusexporter_controller.go
@@ -217,7 +217,7 @@ func (r *SolrPrometheusExporterReconciler) Reconcile(ctx context.Context, req ct
// Set the annotation for a scheduled restart, if necessary.
if nextRestartAnnotation, reconcileWaitDuration, err := util.ScheduleNextRestart(prometheusExporter.Spec.RestartSchedule, foundDeploy.Spec.Template.Annotations); err != nil {
- logger.Error(err, "Cannot parse restartSchedule cron: %s", prometheusExporter.Spec.RestartSchedule)
+ logger.Error(err, "Cannot parse restartSchedule cron", "cron", prometheusExporter.Spec.RestartSchedule)
} else {
if nextRestartAnnotation != "" {
if deploy.Spec.Template.Annotations == nil {
@@ -383,7 +383,7 @@ func (r *SolrPrometheusExporterReconciler) indexAndWatchForSolrClouds(mgr ctrl.M
solrCloudField := ".spec.solrReference.cloud.name"
if err := mgr.GetFieldIndexer().IndexField(context.Background(), &solrv1beta1.SolrPrometheusExporter{}, solrCloudField, func(rawObj client.Object) []string {
- // grab the SolrCloud object, extract the used configMap...
+ // grab the SolrPrometheusExporter object, extract the used SolrCloud...
exporter := rawObj.(*solrv1beta1.SolrPrometheusExporter)
if exporter.Spec.SolrReference.Cloud == nil {
return nil
@@ -429,7 +429,7 @@ func (r *SolrPrometheusExporterReconciler) indexAndWatchForProvidedConfigMaps(mg
providedConfigMapField := ".spec.customKubeOptions.configMapOptions.providedConfigMap"
if err := mgr.GetFieldIndexer().IndexField(context.Background(), &solrv1beta1.SolrPrometheusExporter{}, providedConfigMapField, func(rawObj client.Object) []string {
- // grab the SolrCloud object, extract the used configMap...
+ // grab the SolrPrometheusExporter object, extract the used configMap...
exporter := rawObj.(*solrv1beta1.SolrPrometheusExporter)
if exporter.Spec.CustomKubeOptions.ConfigMapOptions == nil {
return nil
@@ -475,7 +475,7 @@ func (r *SolrPrometheusExporterReconciler) indexAndWatchForKeystoreSecret(mgr ct
tlsSecretField := ".spec.solrReference.solrTLS.pkcs12Secret"
if err := mgr.GetFieldIndexer().IndexField(context.Background(), &solrv1beta1.SolrPrometheusExporter{}, tlsSecretField, func(rawObj client.Object) []string {
- // grab the SolrCloud object, extract the referenced TLS secret...
+ // grab the SolrPrometheusExporter object, extract the referenced TLS secret...
exporter := rawObj.(*solrv1beta1.SolrPrometheusExporter)
if exporter.Spec.SolrReference.SolrTLS == nil || exporter.Spec.SolrReference.SolrTLS.PKCS12Secret == nil {
return nil
@@ -493,7 +493,7 @@ func (r *SolrPrometheusExporterReconciler) indexAndWatchForTruststoreSecret(mgr
tlsSecretField := ".spec.solrReference.solrTLS.trustStoreSecret"
if err := mgr.GetFieldIndexer().IndexField(context.Background(), &solrv1beta1.SolrPrometheusExporter{}, tlsSecretField, func(rawObj client.Object) []string {
- // grab the SolrCloud object, extract the referenced truststore secret...
+ // grab the SolrPrometheusExporter object, extract the referenced truststore secret...
exporter := rawObj.(*solrv1beta1.SolrPrometheusExporter)
if exporter.Spec.SolrReference.SolrTLS == nil || exporter.Spec.SolrReference.SolrTLS.TrustStoreSecret == nil {
return nil
@@ -511,7 +511,7 @@ func (r *SolrPrometheusExporterReconciler) indexAndWatchForBasicAuthSecret(mgr c
secretField := ".spec.solrReference.basicAuthSecret"
if err := mgr.GetFieldIndexer().IndexField(context.Background(), &solrv1beta1.SolrPrometheusExporter{}, secretField, func(rawObj client.Object) []string {
- // grab the SolrCloud object, extract the referenced TLS secret...
+ // grab the SolrPrometheusExporter object, extract the referenced BasicAuth secret...
exporter := rawObj.(*solrv1beta1.SolrPrometheusExporter)
if exporter.Spec.SolrReference.BasicAuthSecret == "" {
return nil
diff --git a/controllers/util/backup_util.go b/controllers/util/backup_util.go
index 0119665..ac89129 100644
--- a/controllers/util/backup_util.go
+++ b/controllers/util/backup_util.go
@@ -24,12 +24,15 @@ import (
solr "github.com/apache/solr-operator/api/v1beta1"
"github.com/apache/solr-operator/controllers/util/solr_api"
"github.com/go-logr/logr"
+ "github.com/robfig/cron/v3"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/remotecommand"
"net/url"
+ "strconv"
+ "time"
)
func GetBackupRepositoryByName(backupRepos []solr.SolrBackupRepository, repositoryName string) *solr.SolrBackupRepository {
@@ -54,19 +57,20 @@ func AsyncIdForCollectionBackup(collection string, backupName string) string {
return fmt.Sprintf("%s-%s", backupName, collection)
}
-func UpdateStatusOfCollectionBackups(backup *solr.SolrBackup) (allFinished bool) {
+func UpdateStatusOfCollectionBackups(backupStatus *solr.IndividualSolrBackupStatus) (allFinished bool) {
// Check if all collection backups have been completed, this is updated in the loop
- allFinished = len(backup.Status.CollectionBackupStatuses) > 0
+ allFinished = len(backupStatus.CollectionBackupStatuses) > 0
- allSuccessful := len(backup.Status.CollectionBackupStatuses) > 0
+ allSuccessful := len(backupStatus.CollectionBackupStatuses) > 0
- for _, collectionStatus := range backup.Status.CollectionBackupStatuses {
+ for _, collectionStatus := range backupStatus.CollectionBackupStatuses {
allFinished = allFinished && collectionStatus.Finished
allSuccessful = allSuccessful && (collectionStatus.Successful != nil && *collectionStatus.Successful)
}
- backup.Status.Finished = allFinished
- if allFinished && backup.Status.Successful == nil {
- backup.Status.Successful = &allSuccessful
+
+ backupStatus.Finished = allFinished
+ if allFinished && backupStatus.Successful == nil {
+ backupStatus.Successful = &allSuccessful
}
return
}
@@ -79,6 +83,11 @@ func GenerateQueryParamsForBackup(backupRepository *solr.SolrBackupRepository, b
queryParams.Add("async", AsyncIdForCollectionBackup(collection, backup.Name))
queryParams.Add("location", BackupLocationPath(backupRepository, backup.Spec.Location))
queryParams.Add("repository", backup.Spec.RepositoryName)
+
+ if backup.Spec.Recurrence.IsEnabled() {
+ queryParams.Add("maxNumBackupPoints", strconv.Itoa(backup.Spec.Recurrence.MaxSaved))
+ }
+
return queryParams
}
@@ -101,45 +110,34 @@ func StartBackupForCollection(ctx context.Context, cloud *solr.SolrCloud, backup
}
func CheckBackupForCollection(ctx context.Context, cloud *solr.SolrCloud, collection string, backupName string, logger logr.Logger) (finished bool, success bool, asyncStatus string, err error) {
- queryParams := url.Values{}
- queryParams.Add("action", "REQUESTSTATUS")
- queryParams.Add("requestid", AsyncIdForCollectionBackup(collection, backupName))
-
- resp := &solr_api.SolrAsyncResponse{}
-
logger.Info("Calling to check on collection backup", "solrCloud", cloud.Name, "collection", collection)
- err = solr_api.CallCollectionsApi(ctx, cloud, queryParams, resp)
+
+ var message string
+ asyncStatus, message, err = solr_api.CheckAsyncRequest(ctx, cloud, AsyncIdForCollectionBackup(collection, backupName))
if err == nil {
- if resp.ResponseHeader.Status == 0 {
- asyncStatus = resp.Status.AsyncState
- if resp.Status.AsyncState == "completed" {
- finished = true
- success = true
- }
- if resp.Status.AsyncState == "failed" {
- finished = true
- success = false
- }
+ if asyncStatus == "completed" {
+ finished = true
+ success = true
+ }
+ if asyncStatus == "failed" {
+ finished = true
+ success = false
}
} else {
- logger.Error(err, "Error checking on collection backup", "solrCloud", cloud.Name, "collection", collection)
+ logger.Error(err, "Error checking on collection backup", "solrCloud", cloud.Name, "collection", collection, "message", message)
}
return finished, success, asyncStatus, err
}
func DeleteAsyncInfoForBackup(ctx context.Context, cloud *solr.SolrCloud, collection string, backupName string, logger logr.Logger) (err error) {
- queryParams := url.Values{}
- queryParams.Add("action", "DELETESTATUS")
- queryParams.Add("requestid", AsyncIdForCollectionBackup(collection, backupName))
-
- resp := &solr_api.SolrAsyncResponse{}
-
logger.Info("Calling to delete async info for backup command.", "solrCloud", cloud.Name, "collection", collection)
- err = solr_api.CallCollectionsApi(ctx, cloud, queryParams, resp)
+ var message string
+ message, err = solr_api.DeleteAsyncRequest(ctx, cloud, AsyncIdForCollectionBackup(collection, backupName))
+
if err != nil {
- logger.Error(err, "Error deleting async data for collection backup", "solrCloud", cloud.Name, "collection", collection)
+ logger.Error(err, "Error deleting async data for collection backup", "solrCloud", cloud.Name, "collection", collection, "message", message)
}
return err
@@ -153,15 +151,15 @@ func EnsureDirectoryForBackup(solrCloud *solr.SolrCloud, backupRepository *solr.
solrCloud.GetAllSolrPodNames()[0],
solrCloud.Namespace,
[]string{"/bin/bash", "-c", "rm -rf " + backupPath + " && mkdir -p " + backupPath},
- *config,
+ config,
)
}
return nil
}
-func RunExecForPod(podName string, namespace string, command []string, config rest.Config) (err error) {
+func RunExecForPod(podName string, namespace string, command []string, config *rest.Config) (err error) {
client := &kubernetes.Clientset{}
- if client, err = kubernetes.NewForConfig(&config); err != nil {
+ if client, err = kubernetes.NewForConfig(config); err != nil {
return err
}
req := client.CoreV1().RESTClient().Post().
@@ -170,7 +168,7 @@ func RunExecForPod(podName string, namespace string, command []string, config re
Namespace(namespace).
SubResource("exec")
scheme := runtime.NewScheme()
- if err := corev1.AddToScheme(scheme); err != nil {
+ if err = corev1.AddToScheme(scheme); err != nil {
return fmt.Errorf("error adding to scheme: %v", err)
}
@@ -184,7 +182,8 @@ func RunExecForPod(podName string, namespace string, command []string, config re
TTY: false,
}, parameterCodec)
- exec, err := remotecommand.NewSPDYExecutor(&config, "POST", req.URL())
+ var exec remotecommand.Executor
+ exec, err = remotecommand.NewSPDYExecutor(config, "POST", req.URL())
if err != nil {
return fmt.Errorf("error while creating Executor: %v", err)
}
@@ -202,3 +201,12 @@ func RunExecForPod(podName string, namespace string, command []string, config re
return nil
}
+
+func ScheduleNextBackup(restartSchedule string, lastBackupTime time.Time) (nextBackup time.Time, err error) {
+ if parsedSchedule, parseErr := cron.ParseStandard(restartSchedule); parseErr != nil {
+ err = parseErr
+ } else {
+ nextBackup = parsedSchedule.Next(lastBackupTime)
+ }
+ return
+}
diff --git a/controllers/util/solr_api/api.go b/controllers/util/solr_api/api.go
index 5e4baa1..4ab0a07 100644
--- a/controllers/util/solr_api/api.go
+++ b/controllers/util/solr_api/api.go
@@ -51,10 +51,10 @@ type SolrAsyncResponse struct {
ResponseHeader SolrResponseHeader `json:"responseHeader"`
// +optional
- RequestId string `json:"requestId"`
+ RequestId string `json:"requestId,omitempty"`
// +optional
- Status SolrAsyncStatus `json:"status"`
+ Status SolrAsyncStatus `json:"status,omitempty"`
}
type SolrResponseHeader struct {
@@ -65,9 +65,56 @@ type SolrResponseHeader struct {
type SolrAsyncStatus struct {
// Possible states can be found here: https://github.com/apache/solr/blob/releases/lucene-solr%2F8.8.1/solr/solrj/src/java/org/apache/solr/client/solrj/response/RequestStatusState.java
- AsyncState string `json:"state"`
+ // +optional
+ AsyncState string `json:"state,omitempty"`
+
+ // +optional
+ Message string `json:"msg,omitempty"`
+}
+
+type SolrAsyncStatusResponse struct {
+ ResponseHeader SolrResponseHeader `json:"responseHeader"`
+
+ // +optional
+ Status SolrAsyncStatus `json:"status,omitempty"`
+}
+
+type SolrDeleteRequestStatus struct {
+ ResponseHeader SolrResponseHeader `json:"responseHeader"`
+
+ // Status of the delete request
+ // +optional
+ Status string `json:"status,omitempty"`
+}
+
+func CheckAsyncRequest(ctx context.Context, cloud *solr.SolrCloud, asyncId string) (asyncState string, message string, err error) {
+ asyncStatus := &SolrAsyncStatusResponse{}
+
+ queryParams := url.Values{}
+ queryParams.Set("action", "REQUESTSTATUS")
+ queryParams.Set("requestid", asyncId)
+ if err = CallCollectionsApi(ctx, cloud, queryParams, asyncStatus); err == nil {
+ if _, err = CheckForCollectionsApiError("REQUESTSTATUS", asyncStatus.ResponseHeader); err == nil {
+ asyncState = asyncStatus.Status.AsyncState
+ message = asyncStatus.Status.Message
+ }
+ }
+
+ return
+}
+
+func DeleteAsyncRequest(ctx context.Context, cloud *solr.SolrCloud, asyncId string) (message string, err error) {
+ deleteStatus := &SolrDeleteRequestStatus{}
+
+ queryParams := url.Values{}
+ queryParams.Set("action", "DELETESTATUS")
+ queryParams.Set("requestid", asyncId)
+ if err = CallCollectionsApi(ctx, cloud, queryParams, deleteStatus); err == nil {
+ _, err = CheckForCollectionsApiError("DELETESTATUS", deleteStatus.ResponseHeader)
+ message = deleteStatus.Status
+ }
- Message string `json:"msg"`
+ return
}
func CallCollectionsApi(ctx context.Context, cloud *solr.SolrCloud, urlParams url.Values, response interface{}) (err error) {
diff --git a/controllers/common.go b/controllers/util/solr_api/node_command.go
similarity index 68%
copy from controllers/common.go
copy to controllers/util/solr_api/node_command.go
index 38a056e..d98336b 100644
--- a/controllers/common.go
+++ b/controllers/util/solr_api/node_command.go
@@ -15,16 +15,14 @@
* limitations under the License.
*/
-package controllers
+package solr_api
-import (
- "sigs.k8s.io/controller-runtime/pkg/reconcile"
- "time"
-)
+type SolrReplaceNodeResponse struct {
+ ResponseHeader SolrResponseHeader `json:"responseHeader"`
-// Set the requeueAfter if it has not been set, or is greater than the new time to requeue at
-func updateRequeueAfter(requeueOrNot *reconcile.Result, newWait time.Duration) {
- if requeueOrNot.RequeueAfter <= 0 || requeueOrNot.RequeueAfter > newWait {
- requeueOrNot.RequeueAfter = newWait
- }
+ // +optional
+ Success string `json:"success,omitempty"`
+
+ // +optional
+ Failure string `json:"failure,omitempty"`
}
diff --git a/controllers/util/solr_update_util.go b/controllers/util/solr_update_util.go
index 5ce6038..199137f 100644
--- a/controllers/util/solr_update_util.go
+++ b/controllers/util/solr_update_util.go
@@ -77,7 +77,7 @@ func scheduleNextRestartWithTime(restartSchedule string, podTemplateAnnotations
err = parseErr
} else {
nextRestartTime := parsedSchedule.Next(lastScheduledTime)
- nextRestart = parsedSchedule.Next(lastScheduledTime).Format(time.RFC3339)
+ nextRestart = nextRestartTime.Format(time.RFC3339)
reconcileWaitDurationTmp := nextRestartTime.Sub(currentTime)
reconcileWaitDuration = &reconcileWaitDurationTmp
}
diff --git a/docs/solr-backup/README.md b/docs/solr-backup/README.md
index d5fa604..23c9d3d 100644
--- a/docs/solr-backup/README.md
+++ b/docs/solr-backup/README.md
@@ -28,6 +28,7 @@ For detailed information on how to best configure backups for your use case, ple
This page outlines how to create and delete a Kubernetes SolrBackup
- [Creation](#creating-an-example-solrbackup)
+- [Recurring/Scheduled Backups](#recurring-backups)
- [Deletion](#deleting-an-example-solrbackup)
- [Repository Types](#supported-repository-types)
- [GCS](#gcs-backup-repositories)
@@ -99,17 +100,97 @@ The status of our triggered backup can be checked with the command below.
```bash
$ kubectl get solrbackups
-NAME CLOUD FINISHED SUCCESSFUL AGE
-local-backup example true true 72s
+NAME CLOUD STARTED FINISHED SUCCESSFUL NEXTBACKUP AGE
+test example 123m true false 161m
```
+## Recurring Backups
+_Since v0.5.0_
+
+The Solr Operator enables taking recurring updates, at a set interval.
+Note that this feature requires a SolrCloud running Solr `8.9.0` or older, because it relies on `Incremental` backups.
+
+By default the Solr Operator will save a maximum of **5** backups at a time, however users can override this using `SolrBackup.spec.recurrence.maxSaved`.
+When using `recurrence`, users must provide a Cron-style `schedule` for the interval at which backups should be taken.
+Please refer to the [GoLang cron-spec](https://pkg.go.dev/github.com/robfig/cron/v3?utm_source=godoc#hdr-CRON_Expression_Format) for more information on allowed syntax.
+
+```yaml
+apiVersion: solr.apache.org/v1beta1
+kind: SolrBackup
+metadata:
+ name: local-backup
+ namespace: default
+spec:
+ repositoryName: "local-collection-backups-1"
+ solrCloud: example
+ collections:
+ - techproducts
+ - books
+ recurrence: # Store one backup daily, and keep a week at a time.
+ schedule: "@daily"
+ maxSaved: 7
+```
+
+If using `kubectl`, the standard `get` command will return the time the backup was last started and when the next backup will occur.
+
+```bash
+$ kubectl get solrbackups
+NAME CLOUD STARTED FINISHED SUCCESSFUL NEXTBACKUP AGE
+test example 123m true true 2021-11-09T00:00:00Z 161m
+```
+
+Much like when not taking a recurring backup, `SolrBackup.status` will contain the information from the latest, or currently running, backup.
+The results of previous backup attempts are stored under `SolrBackup.status.history` (sorted from most recent to oldest).
+
+You are able to **add or remove** `recurrence` to/from an existing `SolrBackup` object, no matter what stage that `SolrBackup` object is in.
+If you add recurrence, then a new backup will be scheduled based on the `startTimestamp` of the last backup.
+If you remove recurrence, then the `nextBackupTime` will be removed.
+However, if the recurrent backup is already underway, it will not be stopped.
+
+### Backup Scheduling
+
+Backups are scheduled based on the `startTimestamp` of the last backup.
+Therefore if a interval schedule such as `@every 1h` is used, and a backup starts on `2021-11-09T03:10:00Z` and ends on `2021-11-09T05:30:00Z`, then the next backup will be started at `2021-11-09T04:10:00Z`.
+If the interval is shorter than the time it takes to complete a backup, then the next backup will started directly after the previous backup completes (even though it is delayed from its given schedule).
+And the next backup will be scheduled based on the `startTimestamp` of the delayed backup.
+So there is a possibility of skew overtime if backups take longer than the allotted schedule.
+
+If a guaranteed schedule is important, it is recommended to use intervals that are guaranteed to be longer than the time it takes to complete a backup.
+
+### Temporarily Disabling Recurring Backups
+
+It is also easy to temporarily disable backups for a time.
+Merely add `disabled: true` under the `recurrence` section of the `SolrBackup` resource.
+And set `disabled: false`, or just remove the property to re-enable backups.
+
+Since backups are scheduled based on the `startTimestamp` of the last backup, a new backup may start immediately after you re-enable the recurrence.
+
+```yaml
+apiVersion: solr.apache.org/v1beta1
+kind: SolrBackup
+metadata:
+ name: local-backup
+ namespace: default
+spec:
+ repositoryName: "local-collection-backups-1"
+ solrCloud: example
+ collections:
+ - techproducts
+ - books
+ recurrence: # Store one backup daily, and keep a week at a time.
+ schedule: "@daily"
+ maxSaved: 7
+ disabled: true
+```
+
+**Note: this will not stop any backups running at the time that `disabled: true` is set, it will only affect scheduling future backups.**
+
## Deleting an example SolrBackup
Once the operator completes a backup, the SolrBackup instance can be safely deleted.
```bash
$ kubectl delete solrbackup local-backup
-TODO command output
```
Note that deleting SolrBackup instances doesn't delete the backed up data, which the operator views as already persisted and outside its control.
diff --git a/helm/solr-operator/Chart.yaml b/helm/solr-operator/Chart.yaml
index 257cb62..71f9689 100644
--- a/helm/solr-operator/Chart.yaml
+++ b/helm/solr-operator/Chart.yaml
@@ -41,7 +41,7 @@ dependencies:
condition: zookeeper-operator.install
annotations:
artifacthub.io/operator: "true"
- artifacthub.io/operatorCapabilities: Seamless Upgrades
+ artifacthub.io/operatorCapabilities: Full Lifecycle
artifacthub.io/prerelease: "true"
artifacthub.io/recommendations: |
- url: https://artifacthub.io/packages/helm/apache-solr/solr
@@ -183,6 +183,15 @@ annotations:
url: https://github.com/apache/solr-operator/issues/326
- name: Github PR
url: https://github.com/apache/solr-operator/pull/358
+ - kind: added
+ description: Scheduled/Recurring SolrBackup support
+ links:
+ - name: Github Issue
+ url: https://github.com/apache/solr-operator/issues/303
+ - name: Github PR
+ url: https://github.com/apache/solr-operator/pull/359
+ - name: SolrBackup Documentation
+ url: https://apache.github.io/solr-operator/docs/solr-backup#recurring-backups
artifacthub.io/images: |
- name: solr-operator
image: apache/solr-operator:v0.5.0-prerelease
diff --git a/helm/solr-operator/crds/crds.yaml b/helm/solr-operator/crds/crds.yaml
index 6ff1bd1..7cd6c99 100644
--- a/helm/solr-operator/crds/crds.yaml
+++ b/helm/solr-operator/crds/crds.yaml
@@ -35,14 +35,23 @@ spec:
jsonPath: .spec.solrCloud
name: Cloud
type: string
- - description: Whether the backup has finished
+ - description: Most recent time the backup started
+ jsonPath: .status.startTimestamp
+ name: Started
+ type: date
+ - description: Whether the most recent backup has finished
jsonPath: .status.finished
name: Finished
type: boolean
- - description: Whether the backup was successful
+ - description: Whether the most recent backup was successful
jsonPath: .status.successful
name: Successful
type: boolean
+ - description: Next scheduled time for a recurrent backup
+ format: date-time
+ jsonPath: .status.nextScheduledTime
+ name: NextBackup
+ type: string
- jsonPath: .metadata.creationTimestamp
name: Age
type: date
@@ -1051,6 +1060,24 @@ spec:
- source
type: object
type: object
+ recurrence:
+ description: "Set this backup to be taken recurrently, with options for scheduling and storage. \n NOTE: This is only supported for Solr Clouds version 8.9+, as it uses the incremental backup API."
+ properties:
+ disabled:
+ default: false
+ description: Disable the recurring backups. Note this will not affect any currently-running backup.
+ type: boolean
+ maxSaved:
+ default: 5
+ description: Define the number of backup points to save for this backup at any given time. The oldest backups will be deleted if too many exist when a backup is taken. If not provided, this defaults to 5.
+ minimum: 1
+ type: integer
+ schedule:
+ description: "Perform a backup on the given schedule, in CRON format. \n Multiple CRON syntaxes are supported - Standard CRON (e.g. \"CRON_TZ=Asia/Seoul 0 6 * * ?\") - Predefined Schedules (e.g. \"@yearly\", \"@weekly\", \"@daily\", etc.) - Intervals (e.g. \"@every 10h30m\") \n For more information please check this reference: https://pkg.go.dev/github.com/robfig/cron/v3?utm_source=godoc#hdr-CRON_Expression_Format"
+ type: string
+ required:
+ - schedule
+ type: object
repositoryName:
description: The name of the repository to use for the backup. Defaults to "legacy_local_repository" if not specified (the auto-configured repository for legacy singleton volumes).
maxLength: 100
@@ -1111,6 +1138,90 @@ spec:
finished:
description: Whether the backup has finished
type: boolean
+ history:
+ description: The status history of recurring backups
+ items:
+ description: IndividualSolrBackupStatus defines the observed state of a single issued SolrBackup
+ properties:
+ collectionBackupStatuses:
+ description: The status of each collection's backup progress
+ items:
+ description: CollectionBackupStatus defines the progress of a Solr Collection's backup
+ properties:
+ asyncBackupStatus:
+ description: The status of the asynchronous backup call to solr
+ type: string
+ backupName:
+ description: BackupName of this collection's backup in Solr
+ type: string
+ collection:
+ description: Solr Collection name
+ type: string
+ finishTimestamp:
+ description: Time that the collection backup finished at
+ format: date-time
+ type: string
+ finished:
+ description: Whether the backup has finished
+ type: boolean
+ inProgress:
+ description: Whether the collection is being backed up
+ type: boolean
+ startTimestamp:
+ description: Time that the collection backup started at
+ format: date-time
+ type: string
+ successful:
+ description: Whether the backup was successful
+ type: boolean
+ required:
+ - collection
+ type: object
+ type: array
+ finishTimestamp:
+ description: Version of the Solr being backed up
+ format: date-time
+ type: string
+ finished:
+ description: Whether the backup has finished
+ type: boolean
+ persistenceStatus:
+ description: Whether the backups are in progress of being persisted. This feature has been removed as of v0.5.0.
+ properties:
+ finishTimestamp:
+ description: Time that the collection backup finished at
+ format: date-time
+ type: string
+ finished:
+ description: Whether the persistence has finished
+ type: boolean
+ inProgress:
+ description: Whether the collection is being backed up
+ type: boolean
+ startTimestamp:
+ description: Time that the collection backup started at
+ format: date-time
+ type: string
+ successful:
+ description: Whether the backup was successful
+ type: boolean
+ type: object
+ solrVersion:
+ description: Version of the Solr being backed up
+ type: string
+ startTimestamp:
+ description: The time that this backup was initiated
+ format: date-time
+ type: string
+ successful:
+ description: Whether the backup was successful
+ type: boolean
+ type: object
+ type: array
+ nextScheduledTime:
+ description: The scheduled time for the next backup to occur
+ format: date-time
+ type: string
persistenceStatus:
description: Whether the backups are in progress of being persisted. This feature has been removed as of v0.5.0.
properties:
@@ -1135,11 +1246,13 @@ spec:
solrVersion:
description: Version of the Solr being backed up
type: string
+ startTimestamp:
+ description: The time that this backup was initiated
+ format: date-time
+ type: string
successful:
description: Whether the backup was successful
type: boolean
- required:
- - solrVersion
type: object
type: object
served: true
diff --git a/main.go b/main.go
index ae118d0..11d8606 100644
--- a/main.go
+++ b/main.go
@@ -23,7 +23,7 @@ import (
"flag"
"fmt"
"github.com/apache/solr-operator/controllers/util/solr_api"
- zk_api "github.com/apache/solr-operator/controllers/zk_api"
+ "github.com/apache/solr-operator/controllers/zk_api"
"github.com/apache/solr-operator/version"
"github.com/fsnotify/fsnotify"
"io/ioutil"
@@ -203,6 +203,7 @@ func main() {
if err = (&controllers.SolrBackupReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
+ Config: mgr.GetConfig(),
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "SolrBackup")
os.Exit(1)