You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ho...@apache.org on 2024/04/08 13:36:31 UTC
(solr-operator) branch release-0.8 updated (8cbb5b0 -> dbc8db9)
This is an automated email from the ASF dual-hosted git repository.
houston pushed a change to branch release-0.8
in repository https://gitbox.apache.org/repos/asf/solr-operator.git
from 8cbb5b0 Avoid reset of security.json if get fails (#659) (#660)
new e5c1271 Fix scaling when using ingress-addressed nodes (#692)
new dbc8db9 Fix a few issues with testing, add better cluster information
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
api/v1beta1/solrcloud_types.go | 7 +
controllers/solr_cluster_ops_util.go | 25 +++-
controllers/solrcloud_controller.go | 72 +++++-----
controllers/util/common.go | 24 +++-
controllers/util/solr_update_util.go | 14 +-
controllers/util/solr_util.go | 3 +
helm/solr-operator/Chart.yaml | 8 +-
tests/e2e/solrcloud_rolling_upgrade_test.go | 2 +-
tests/e2e/solrcloud_scaling_test.go | 199 +++++++++++++++++++++++++++-
tests/e2e/suite_test.go | 64 ++++++++-
tests/e2e/test_utils_test.go | 12 +-
11 files changed, 379 insertions(+), 51 deletions(-)
(solr-operator) 01/02: Fix scaling when using ingress-addressed nodes (#692)
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
houston pushed a commit to branch release-0.8
in repository https://gitbox.apache.org/repos/asf/solr-operator.git
commit e5c1271a24f6394fafe300c44259ad311c490162
Author: Houston Putman <ho...@apache.org>
AuthorDate: Wed Apr 3 16:49:59 2024 -0400
Fix scaling when using ingress-addressed nodes (#692)
---
api/v1beta1/solrcloud_types.go | 7 +
controllers/solr_cluster_ops_util.go | 22 ++-
controllers/solrcloud_controller.go | 71 +++++-----
controllers/util/common.go | 24 +++-
controllers/util/solr_update_util.go | 14 +-
controllers/util/solr_util.go | 3 +
helm/solr-operator/Chart.yaml | 8 +-
tests/e2e/solrcloud_rolling_upgrade_test.go | 2 +-
tests/e2e/solrcloud_scaling_test.go | 199 +++++++++++++++++++++++++++-
tests/e2e/suite_test.go | 33 +++++
tests/e2e/test_utils_test.go | 4 +-
11 files changed, 341 insertions(+), 46 deletions(-)
diff --git a/api/v1beta1/solrcloud_types.go b/api/v1beta1/solrcloud_types.go
index c028337..995a9ac 100644
--- a/api/v1beta1/solrcloud_types.go
+++ b/api/v1beta1/solrcloud_types.go
@@ -1227,6 +1227,13 @@ func (sc *SolrCloud) GetAllSolrPodNames() []string {
if sc.Spec.Replicas != nil {
replicas = int(*sc.Spec.Replicas)
}
+ if int(sc.Status.Replicas) > replicas {
+ replicas = int(sc.Status.Replicas)
+ }
+ return sc.GetSolrPodNames(replicas)
+}
+
+func (sc *SolrCloud) GetSolrPodNames(replicas int) []string {
podNames := make([]string, replicas)
statefulSetName := sc.StatefulSetName()
for i := range podNames {
diff --git a/controllers/solr_cluster_ops_util.go b/controllers/solr_cluster_ops_util.go
index c9f352a..07452eb 100644
--- a/controllers/solr_cluster_ops_util.go
+++ b/controllers/solr_cluster_ops_util.go
@@ -151,7 +151,7 @@ func retryNextQueuedClusterOpWithQueue(statefulSet *appsv1.StatefulSet, clusterO
return hasOp, err
}
-func determineScaleClusterOpLockIfNecessary(ctx context.Context, r *SolrCloudReconciler, instance *solrv1beta1.SolrCloud, statefulSet *appsv1.StatefulSet, scaleDownOpIsQueued bool, podList []corev1.Pod, logger logr.Logger) (clusterOp *SolrClusterOp, retryLaterDuration time.Duration, err error) {
+func determineScaleClusterOpLockIfNecessary(ctx context.Context, r *SolrCloudReconciler, instance *solrv1beta1.SolrCloud, statefulSet *appsv1.StatefulSet, scaleDownOpIsQueued bool, podList []corev1.Pod, blockReconciliationOfStatefulSet bool, logger logr.Logger) (clusterOp *SolrClusterOp, retryLaterDuration time.Duration, err error) {
desiredPods := int(*instance.Spec.Replicas)
configuredPods := int(*statefulSet.Spec.Replicas)
if desiredPods != configuredPods {
@@ -170,11 +170,26 @@ func determineScaleClusterOpLockIfNecessary(ctx context.Context, r *SolrCloudRec
Metadata: strconv.Itoa(configuredPods - 1),
}
} else if desiredPods > configuredPods && (instance.Spec.Scaling.PopulatePodsOnScaleUp == nil || *instance.Spec.Scaling.PopulatePodsOnScaleUp) {
+ // We need to wait for all pods to become healthy to scale up in a managed fashion, otherwise
+ // the balancing will skip some pods
if len(podList) < configuredPods {
// There are not enough pods, the statefulSet controller has yet to create the previously desired pods.
// Do not start the scale up until these missing pods are created.
return nil, time.Second * 5, nil
}
+ // If Solr nodes are advertised by their individual node services (through an ingress)
+ // then make sure that the host aliases are set for all desired pods before starting a scale-up.
+ // If the host aliases do not already include the soon-to-be created pods, then Solr might not be able to balance
+ // replicas onto the new hosts.
+ // We need to make sure that the StatefulSet is updated with these new hostAliases before the scale up occurs.
+ if instance.UsesIndividualNodeServices() && instance.Spec.SolrAddressability.External.UseExternalAddress {
+ for _, pod := range podList {
+ if len(pod.Spec.HostAliases) < desiredPods {
+ return nil, time.Second * 5, nil
+ }
+ }
+ }
+
clusterOp = &SolrClusterOp{
Operation: ScaleUpLock,
Metadata: strconv.Itoa(desiredPods),
@@ -349,7 +364,8 @@ func handleManagedCloudRollingUpdate(ctx context.Context, r *SolrCloudReconciler
}
operationComplete = true
// Only do a re-balancing for rolling restarts that migrated replicas
- if updateMetadata.RequiresReplicaMigration {
+ // If a scale-up will occur afterwards, skip the re-balancing, because it will occur after the scale-up anyway
+ if updateMetadata.RequiresReplicaMigration && *instance.Spec.Replicas <= *statefulSet.Spec.Replicas {
nextClusterOp = &SolrClusterOp{
Operation: BalanceReplicasLock,
Metadata: "RollingUpdateComplete",
@@ -371,7 +387,7 @@ func handleManagedCloudRollingUpdate(ctx context.Context, r *SolrCloudReconciler
// We won't kill pods that we need the cluster state for, but we can kill the pods that are already not running.
// This is important for scenarios where there is a bad pod config and nothing is running, but we need to do
// a restart to get a working pod config.
- state, retryLater, apiError := util.GetNodeReplicaState(ctx, instance, hasReadyPod, logger)
+ state, retryLater, apiError := util.GetNodeReplicaState(ctx, instance, statefulSet, hasReadyPod, logger)
if apiError != nil {
return false, true, 0, nil, apiError
} else if !retryLater {
diff --git a/controllers/solrcloud_controller.go b/controllers/solrcloud_controller.go
index 5ad2213..4d832cf 100644
--- a/controllers/solrcloud_controller.go
+++ b/controllers/solrcloud_controller.go
@@ -152,6 +152,11 @@ func (r *SolrCloudReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
hostNameIpMap := make(map[string]string)
// Generate a service for every Node
if instance.UsesIndividualNodeServices() {
+ // When updating the statefulSet below, the hostNameIpMap is just used to add new IPs or modify existing ones.
+ // When scaling down, the hostAliases that are no longer found here will not be removed from the hostAliases in the statefulSet pod spec.
+ // Therefore, it should be ok that we are not reconciling the node services that will be scaled down in the future.
+ // This is unfortunately the reality since we don't have the statefulSet yet to determine how many Solr pods are still running,
+ // we just have Spec.replicas which is the requested pod count.
for _, nodeName := range solrNodeNames {
err, ip := r.reconcileNodeService(ctx, logger, instance, nodeName)
if err != nil {
@@ -161,6 +166,7 @@ func (r *SolrCloudReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
if instance.Spec.SolrAddressability.External.UseExternalAddress {
if ip == "" {
// If we are using this IP in the hostAliases of the statefulSet, it needs to be set for every service before trying to update the statefulSet
+ // TODO: Make an event here
blockReconciliationOfStatefulSet = true
} else {
hostNameIpMap[instance.AdvertisedNodeHost(nodeName)] = ip
@@ -319,36 +325,6 @@ func (r *SolrCloudReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
}
}
- extAddressabilityOpts := instance.Spec.SolrAddressability.External
- if extAddressabilityOpts != nil && extAddressabilityOpts.Method == solrv1beta1.Ingress {
- // Generate Ingress
- ingress := util.GenerateIngress(instance, solrNodeNames)
-
- // Check if the Ingress already exists
- ingressLogger := logger.WithValues("ingress", ingress.Name)
- foundIngress := &netv1.Ingress{}
- err = r.Get(ctx, types.NamespacedName{Name: ingress.Name, Namespace: ingress.Namespace}, foundIngress)
- if err != nil && errors.IsNotFound(err) {
- ingressLogger.Info("Creating Ingress")
- if err = controllerutil.SetControllerReference(instance, ingress, r.Scheme); err == nil {
- err = r.Create(ctx, ingress)
- }
- } else if err == nil {
- var needsUpdate bool
- needsUpdate, err = util.OvertakeControllerRef(instance, foundIngress, r.Scheme)
- needsUpdate = util.CopyIngressFields(ingress, foundIngress, ingressLogger) || needsUpdate
-
- // Update the found Ingress and write the result back if there are any changes
- if needsUpdate && err == nil {
- ingressLogger.Info("Updating Ingress")
- err = r.Update(ctx, foundIngress)
- }
- }
- if err != nil {
- return requeueOrNot, err
- }
- }
-
var statefulSet *appsv1.StatefulSet
if !blockReconciliationOfStatefulSet {
@@ -416,6 +392,39 @@ func (r *SolrCloudReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
if err != nil {
return requeueOrNot, err
}
+ if statefulSet != nil && statefulSet.Spec.Replicas != nil {
+ solrNodeNames = instance.GetSolrPodNames(int(*statefulSet.Spec.Replicas))
+ }
+
+ extAddressabilityOpts := instance.Spec.SolrAddressability.External
+ if extAddressabilityOpts != nil && extAddressabilityOpts.Method == solrv1beta1.Ingress {
+ // Generate Ingress
+ ingress := util.GenerateIngress(instance, solrNodeNames)
+
+ // Check if the Ingress already exists
+ ingressLogger := logger.WithValues("ingress", ingress.Name)
+ foundIngress := &netv1.Ingress{}
+ err = r.Get(ctx, types.NamespacedName{Name: ingress.Name, Namespace: ingress.Namespace}, foundIngress)
+ if err != nil && errors.IsNotFound(err) {
+ ingressLogger.Info("Creating Ingress")
+ if err = controllerutil.SetControllerReference(instance, ingress, r.Scheme); err == nil {
+ err = r.Create(ctx, ingress)
+ }
+ } else if err == nil {
+ var needsUpdate bool
+ needsUpdate, err = util.OvertakeControllerRef(instance, foundIngress, r.Scheme)
+ needsUpdate = util.CopyIngressFields(ingress, foundIngress, ingressLogger) || needsUpdate
+
+ // Update the found Ingress and write the result back if there are any changes
+ if needsUpdate && err == nil {
+ ingressLogger.Info("Updating Ingress")
+ err = r.Update(ctx, foundIngress)
+ }
+ }
+ if err != nil {
+ return requeueOrNot, err
+ }
+ }
// *********************************************************
// The operations after this require a statefulSet to exist,
@@ -546,7 +555,7 @@ func (r *SolrCloudReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
// a "locked" cluster operation
if clusterOp == nil {
_, scaleDownOpIsQueued := queuedRetryOps[ScaleDownLock]
- clusterOp, retryLaterDuration, err = determineScaleClusterOpLockIfNecessary(ctx, r, instance, statefulSet, scaleDownOpIsQueued, podList, logger)
+ clusterOp, retryLaterDuration, err = determineScaleClusterOpLockIfNecessary(ctx, r, instance, statefulSet, scaleDownOpIsQueued, podList, blockReconciliationOfStatefulSet, logger)
// If the new clusterOperation is an update to a queued clusterOp, just change the operation that is already queued
if clusterOp != nil {
diff --git a/controllers/util/common.go b/controllers/util/common.go
index b887f6d..3eca00b 100644
--- a/controllers/util/common.go
+++ b/controllers/util/common.go
@@ -432,7 +432,29 @@ func CopyPodTemplates(from, to *corev1.PodTemplateSpec, basePath string, logger
if !DeepEqualWithNils(to.Spec.HostAliases, from.Spec.HostAliases) {
requireUpdate = true
- to.Spec.HostAliases = from.Spec.HostAliases
+ if to.Spec.HostAliases == nil {
+ to.Spec.HostAliases = from.Spec.HostAliases
+ } else {
+ // Do not remove aliases that are no longer used.
+ // This is in case Solr is scaling down and we want to keep the old addresses for future use.
+ for _, fromAlias := range from.Spec.HostAliases {
+ found := false
+ for i, toAlias := range to.Spec.HostAliases {
+ if fromAlias.Hostnames[0] == toAlias.Hostnames[0] {
+ found = true
+ if !DeepEqualWithNils(toAlias, fromAlias) {
+ requireUpdate = true
+ to.Spec.HostAliases[i] = fromAlias
+ break
+ }
+ }
+ }
+ if !found {
+ requireUpdate = true
+ to.Spec.HostAliases = append(to.Spec.HostAliases, fromAlias)
+ }
+ }
+ }
logger.Info("Update required because field changed", "field", basePath+"Spec.HostAliases", "from", to.Spec.HostAliases, "to", from.Spec.HostAliases)
}
diff --git a/controllers/util/solr_update_util.go b/controllers/util/solr_update_util.go
index 4f9eebe..3123544 100644
--- a/controllers/util/solr_update_util.go
+++ b/controllers/util/solr_update_util.go
@@ -24,6 +24,7 @@ import (
"github.com/apache/solr-operator/controllers/util/solr_api"
"github.com/go-logr/logr"
"github.com/robfig/cron/v3"
+ appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"net/url"
@@ -119,7 +120,7 @@ func (state NodeReplicaState) PodHasReplicas(cloud *solr.SolrCloud, podName stri
return isInClusterState && contents.replicas > 0
}
-func GetNodeReplicaState(ctx context.Context, cloud *solr.SolrCloud, hasReadyPod bool, logger logr.Logger) (state NodeReplicaState, retryLater bool, err error) {
+func GetNodeReplicaState(ctx context.Context, cloud *solr.SolrCloud, statefulSet *appsv1.StatefulSet, hasReadyPod bool, logger logr.Logger) (state NodeReplicaState, retryLater bool, err error) {
clusterResp := &solr_api.SolrClusterStatusResponse{}
overseerResp := &solr_api.SolrOverseerStatusResponse{}
@@ -138,7 +139,7 @@ func GetNodeReplicaState(ctx context.Context, cloud *solr.SolrCloud, hasReadyPod
}
}
if err == nil {
- state = findSolrNodeContents(clusterResp.ClusterStatus, overseerResp.Leader, GetAllManagedSolrNodeNames(cloud))
+ state = findSolrNodeContents(clusterResp.ClusterStatus, overseerResp.Leader, GetManagedSolrNodeNames(cloud, int(*statefulSet.Spec.Replicas)))
} else {
logger.Error(err, "Could not fetch cluster state information for cloud")
}
@@ -535,6 +536,15 @@ func GetAllManagedSolrNodeNames(solrCloud *solr.SolrCloud) map[string]bool {
return allNodeNames
}
+func GetManagedSolrNodeNames(solrCloud *solr.SolrCloud, currentlyConfiguredPodCount int) map[string]bool {
+ podNames := solrCloud.GetSolrPodNames(currentlyConfiguredPodCount)
+ allNodeNames := make(map[string]bool, len(podNames))
+ for _, podName := range podNames {
+ allNodeNames[SolrNodeName(solrCloud, podName)] = true
+ }
+ return allNodeNames
+}
+
// EvictReplicasForPodIfNecessary takes a solr Pod and migrates all replicas off of that Pod.
// For updates this will only be called for pods using ephemeral data.
// For scale-down operations, this can be called for pods using ephemeral or persistent data.
diff --git a/controllers/util/solr_util.go b/controllers/util/solr_util.go
index 47f8c07..de44d7c 100644
--- a/controllers/util/solr_util.go
+++ b/controllers/util/solr_util.go
@@ -556,6 +556,9 @@ func GenerateStatefulSet(solrCloud *solr.SolrCloud, solrCloudStatus *solr.SolrCl
VolumeClaimTemplates: pvcs,
},
}
+ if solrCloud.UsesHeadlessService() {
+ stateful.Spec.Template.Spec.Subdomain = solrCloud.HeadlessServiceName()
+ }
var imagePullSecrets []corev1.LocalObjectReference
diff --git a/helm/solr-operator/Chart.yaml b/helm/solr-operator/Chart.yaml
index b36bb87..dc6665c 100644
--- a/helm/solr-operator/Chart.yaml
+++ b/helm/solr-operator/Chart.yaml
@@ -68,7 +68,6 @@ annotations:
url: https://github.com/apache/solr-operator/issues/624
- name: Github PR
url: https://github.com/apache/solr-operator/pull/648
- - kind: fixed
description: Avoid reset of security.json if get request fails
links:
- name: Github Issue
@@ -89,6 +88,13 @@ annotations:
url: https://github.com/apache/solr-operator/issues/693
- name: Github PR
url: https://github.com/apache/solr-operator/pull/694
+ - kind: fixed
+ description: SolrClouds addressed via an Ingress now scale up and down safely.
+ links:
+ - name: Github Issue
+ url: https://github.com/apache/solr-operator/issues/682
+ - name: Github PR
+ url: https://github.com/apache/solr-operator/pull/692
artifacthub.io/images: |
- name: solr-operator
image: apache/solr-operator:v0.8.1-prerelease
diff --git a/tests/e2e/solrcloud_rolling_upgrade_test.go b/tests/e2e/solrcloud_rolling_upgrade_test.go
index 6143342..faf2175 100644
--- a/tests/e2e/solrcloud_rolling_upgrade_test.go
+++ b/tests/e2e/solrcloud_rolling_upgrade_test.go
@@ -163,7 +163,7 @@ var _ = FDescribe("E2E - SolrCloud - Rolling Upgrades", func() {
}
By("waiting for the balanceReplicas to finish")
- expectStatefulSetWithChecks(ctx, solrCloud, solrCloud.StatefulSetName(), func(g Gomega, found *appsv1.StatefulSet) {
+ expectStatefulSetWithChecksAndTimeout(ctx, solrCloud, solrCloud.StatefulSetName(), time.Second*30, time.Second, func(g Gomega, found *appsv1.StatefulSet) {
clusterOp, err := controllers.GetCurrentClusterOp(found)
g.Expect(err).ToNot(HaveOccurred(), "Error occurred while finding clusterLock for SolrCloud")
g.Expect(clusterOp).To(BeNil(), "StatefulSet should not have a balanceReplicas lock after balancing is complete.")
diff --git a/tests/e2e/solrcloud_scaling_test.go b/tests/e2e/solrcloud_scaling_test.go
index 53d3630..106dd4f 100644
--- a/tests/e2e/solrcloud_scaling_test.go
+++ b/tests/e2e/solrcloud_scaling_test.go
@@ -140,6 +140,110 @@ var _ = FDescribe("E2E - SolrCloud - Scale Down", func() {
})
})
+ FContext("with replica migration using an ingress for node addresses", func() {
+
+ BeforeEach(func() {
+ solrCloud.Spec.SolrAddressability = solrv1beta1.SolrAddressabilityOptions{
+ External: &solrv1beta1.ExternalAddressability{
+ Method: solrv1beta1.Ingress,
+ UseExternalAddress: true,
+ DomainName: "test.solr.org",
+ },
+ }
+ })
+
+ FIt("Scales Down", func(ctx context.Context) {
+ originalSolrCloud := solrCloud.DeepCopy()
+ solrCloud.Spec.Replicas = pointer.Int32(1)
+ By("triggering a scale down via solrCloud replicas")
+ Expect(k8sClient.Patch(ctx, solrCloud, client.MergeFrom(originalSolrCloud))).To(Succeed(), "Could not patch SolrCloud replicas to initiate scale down")
+
+ By("waiting for the scaleDown of first pod to begin")
+ expectStatefulSetWithChecks(ctx, solrCloud, solrCloud.StatefulSetName(), func(g Gomega, found *appsv1.StatefulSet) {
+ g.Expect(found.Spec.Replicas).To(HaveValue(BeEquivalentTo(3)), "StatefulSet should still have 3 pods, because the scale down should first move Solr replicas")
+ clusterOp, err := controllers.GetCurrentClusterOp(found)
+ g.Expect(err).ToNot(HaveOccurred(), "Error occurred while finding clusterLock for SolrCloud")
+ g.Expect(clusterOp).ToNot(BeNil(), "StatefulSet does not have a scaleDown lock.")
+ g.Expect(clusterOp.Operation).To(Equal(controllers.ScaleDownLock), "StatefulSet does not have a scaleDown lock.")
+ g.Expect(clusterOp.Metadata).To(Equal("2"), "StatefulSet scaling lock operation has the wrong metadata.")
+ })
+ queryCollection(ctx, solrCloud, solrCollection2, 0)
+
+ // Make sure that ingress still has 3 nodes at this point
+ ingress := expectIngress(ctx, solrCloud, solrCloud.CommonIngressName())
+ Expect(ingress.Spec.Rules).To(HaveLen(4), "Wrong number of ingress rules. 3 nodes + 1 common endpoint. The third node rule should not be deleted until the node itself is deleted")
+
+ By("waiting for the scaleDown of the first pod to finish")
+ expectStatefulSetWithChecks(ctx, solrCloud, solrCloud.StatefulSetName(), func(g Gomega, found *appsv1.StatefulSet) {
+ g.Expect(found.Spec.Replicas).To(HaveValue(BeEquivalentTo(2)), "StatefulSet should now have 2 pods, after the replicas have been moved off the first pod.")
+ clusterOp, err := controllers.GetCurrentClusterOp(found)
+ g.Expect(err).ToNot(HaveOccurred(), "Error occurred while finding clusterLock for SolrCloud")
+ g.Expect(clusterOp).ToNot(BeNil(), "StatefulSet does not have a scaleDown lock.")
+ g.Expect(clusterOp.Operation).To(Equal(controllers.ScaleDownLock), "StatefulSet does not have a scaleDown lock.")
+ g.Expect(clusterOp.Metadata).To(Equal("2"), "StatefulSet scaling lock operation has the wrong metadata.")
+ })
+ queryCollection(ctx, solrCloud, solrCollection2, 0)
+
+ // Wait till the pod has actually been deleted
+ expectStatefulSetWithChecks(ctx, solrCloud, solrCloud.StatefulSetName(), func(g Gomega, found *appsv1.StatefulSet) {
+ g.Expect(found.Status.Replicas).To(HaveValue(BeEquivalentTo(2)), "StatefulSet should now have 2 pods, after the replicas have been moved off the first pod.")
+ })
+
+ By("waiting for the scaleDown of second pod to begin")
+ statefulSet := expectStatefulSetWithChecks(ctx, solrCloud, solrCloud.StatefulSetName(), func(g Gomega, found *appsv1.StatefulSet) {
+ clusterOp, err := controllers.GetCurrentClusterOp(found)
+ g.Expect(err).ToNot(HaveOccurred(), "Error occurred while finding clusterLock for SolrCloud")
+ g.Expect(clusterOp).ToNot(BeNil(), "StatefulSet does not have a scaleDown lock.")
+ g.Expect(clusterOp.Operation).To(Equal(controllers.ScaleDownLock), "StatefulSet does not have a scaleDown lock.")
+ g.Expect(clusterOp.Metadata).To(Equal("1"), "StatefulSet scaling lock operation has the wrong metadata.")
+ })
+
+ // Make sure that ingress still has 2 nodes at this point
+ ingress = expectIngress(ctx, solrCloud, solrCloud.CommonIngressName())
+ Expect(ingress.Spec.Rules).To(HaveLen(3), "Wrong number of ingress rules. 2 nodes + 1 common endpoint. The second node rule should not be deleted until the node itself is deleted")
+
+ // When the next scale down happens, the 3rd solr pod (ordinal 2) should be gone, and the statefulSet replicas should be 2 across the board.
+ // The first scale down should not be complete until this is done.
+ Expect(statefulSet.Spec.Replicas).To(HaveValue(BeEquivalentTo(2)), "StatefulSet should still have 2 pods configured, because the scale down should first move Solr replicas")
+ Expect(statefulSet.Status.Replicas).To(HaveValue(BeEquivalentTo(2)), "StatefulSet should only have 2 pods running, because previous pod scale down should have completely finished")
+ // This pod check must happen after the above clusterLock and replicas check.
+ // The StatefulSet controller might take a good amount of time to actually delete the pod,
+ // and the replica migration/cluster op might already be done by the time the first pod is deleted.
+ expectNoPodNow(ctx, solrCloud, solrCloud.GetSolrPodName(2))
+ queryCollection(ctx, solrCloud, solrCollection1, 0)
+
+ By("waiting for the scaleDown to finish")
+ statefulSet = expectStatefulSetWithChecks(ctx, solrCloud, solrCloud.StatefulSetName(), func(g Gomega, found *appsv1.StatefulSet) {
+ g.Expect(found.Spec.Replicas).To(HaveValue(BeEquivalentTo(1)), "StatefulSet should now have 1 pods, after the replicas have been moved.")
+ })
+ // Once the scale down actually occurs, the clusterOp is not complete. We need to wait till the last pod is deleted
+ clusterOp, err := controllers.GetCurrentClusterOp(statefulSet)
+ Expect(clusterOp).ToNot(BeNil(), "StatefulSet does not have a scaleDown lock.")
+ Expect(clusterOp.Operation).To(Equal(controllers.ScaleDownLock), "StatefulSet does not have a scaleDown lock.")
+ Expect(clusterOp.Metadata).To(Equal("1"), "StatefulSet scaling lock operation has the wrong metadata.")
+
+ // Wait for the last pod to be deleted
+ expectStatefulSetWithChecks(ctx, solrCloud, solrCloud.StatefulSetName(), func(g Gomega, found *appsv1.StatefulSet) {
+ g.Expect(found.Status.Replicas).To(HaveValue(BeEquivalentTo(1)), "StatefulSet should now have 1 pods, after the replicas have been moved.")
+ })
+ // Once the scale down actually occurs, the statefulSet annotations should be removed very soon
+ expectStatefulSetWithChecksAndTimeout(ctx, solrCloud, solrCloud.StatefulSetName(), time.Second*2, time.Millisecond*500, func(g Gomega, found *appsv1.StatefulSet) {
+ clusterOp, err = controllers.GetCurrentClusterOp(found)
+ g.Expect(err).ToNot(HaveOccurred(), "Error occurred while finding clusterLock for SolrCloud")
+ g.Expect(clusterOp).To(BeNil(), "StatefulSet should not have a ScaleDown lock after scaling is complete.")
+ })
+
+ // Make sure that ingress has 1 node at this point
+ ingress = expectIngress(ctx, solrCloud, solrCloud.CommonIngressName())
+ Expect(ingress.Spec.Rules).To(HaveLen(2), "Wrong number of ingress rules. 1 nodes + 1 common endpoint.")
+
+ expectNoPod(ctx, solrCloud, solrCloud.GetSolrPodName(1))
+
+ queryCollection(ctx, solrCloud, solrCollection1, 0)
+ queryCollection(ctx, solrCloud, solrCollection2, 0)
+ })
+ })
+
FContext("with replica migration and deleted persistent data", func() {
BeforeEach(func() {
@@ -240,7 +344,7 @@ var _ = FDescribe("E2E - SolrCloud - Scale Up", func() {
FIt("Scales Up", func(ctx context.Context) {
originalSolrCloud := solrCloud.DeepCopy()
solrCloud.Spec.Replicas = pointer.Int32(int32(3))
- By("triggering a scale down via solrCloud replicas")
+ By("triggering a scale up via solrCloud replicas")
Expect(k8sClient.Patch(ctx, solrCloud, client.MergeFrom(originalSolrCloud))).To(Succeed(), "Could not patch SolrCloud replicas to initiate scale up")
By("waiting for the scaleUp to begin")
@@ -254,9 +358,11 @@ var _ = FDescribe("E2E - SolrCloud - Scale Up", func() {
// The first step is to increase the number of pods
// Check very often, as the new pods will be created quickly, which will cause the cluster op to change.
- statefulSet = expectStatefulSetWithChecksAndTimeout(ctx, solrCloud, solrCloud.StatefulSetName(), time.Second*5, time.Millisecond*5, func(g Gomega, found *appsv1.StatefulSet) {
- g.Expect(found.Spec.Replicas).To(HaveValue(BeEquivalentTo(3)), "StatefulSet should still have 3 pods, because the scale down should first move Solr replicas")
- })
+ if int(*statefulSet.Spec.Replicas) != 3 {
+ statefulSet = expectStatefulSetWithChecksAndTimeout(ctx, solrCloud, solrCloud.StatefulSetName(), time.Second*5, time.Millisecond*5, func(g Gomega, found *appsv1.StatefulSet) {
+ g.Expect(found.Spec.Replicas).To(HaveValue(BeEquivalentTo(3)), "StatefulSet should be configured for 3 pods when scaling up")
+ })
+ }
clusterOp, err := controllers.GetCurrentClusterOp(statefulSet)
Expect(err).ToNot(HaveOccurred(), "Error occurred while finding clusterLock for SolrCloud")
Expect(clusterOp).ToNot(BeNil(), "StatefulSet does not have a scaleUp lock.")
@@ -265,7 +371,7 @@ var _ = FDescribe("E2E - SolrCloud - Scale Up", func() {
// Wait for new pods to come up, and when they do we should be doing a balanceReplicas clusterOp
statefulSet = expectStatefulSetWithChecks(ctx, solrCloud, solrCloud.StatefulSetName(), func(g Gomega, found *appsv1.StatefulSet) {
- g.Expect(found.Status.Replicas).To(HaveValue(BeEquivalentTo(3)), "StatefulSet should still have 3 pods, because the scale down should first move Solr replicas")
+ g.Expect(found.Status.Replicas).To(HaveValue(BeEquivalentTo(3)), "StatefulSet should eventually have 3 pods created for it")
})
clusterOp, err = controllers.GetCurrentClusterOp(statefulSet)
Expect(err).ToNot(HaveOccurred(), "Error occurred while finding clusterLock for SolrCloud")
@@ -285,6 +391,89 @@ var _ = FDescribe("E2E - SolrCloud - Scale Up", func() {
})
})
+ FContext("with replica migration using an ingress for node addresses", func() {
+
+ BeforeEach(func() {
+ solrCloud.Spec.Replicas = pointer.Int32(2)
+ solrCloud.Spec.SolrAddressability = solrv1beta1.SolrAddressabilityOptions{
+ External: &solrv1beta1.ExternalAddressability{
+ Method: solrv1beta1.Ingress,
+ UseExternalAddress: true,
+ DomainName: "test.solr.org",
+ },
+ }
+ })
+
+ FIt("Scales Up", func(ctx context.Context) {
+ originalSolrCloud := solrCloud.DeepCopy()
+ solrCloud.Spec.Replicas = pointer.Int32(int32(3))
+ By("triggering a scale up via solrCloud replicas")
+ Expect(k8sClient.Patch(ctx, solrCloud, client.MergeFrom(originalSolrCloud))).To(Succeed(), "Could not patch SolrCloud replicas to initiate scale up")
+
+ By("waiting for the rolling restart to begin with hostAliases")
+ statefulSet := expectStatefulSetWithChecksAndTimeout(ctx, solrCloud, solrCloud.StatefulSetName(), time.Second*5, time.Millisecond*5, func(g Gomega, found *appsv1.StatefulSet) {
+ clusterOp, err := controllers.GetCurrentClusterOp(found)
+ g.Expect(err).ToNot(HaveOccurred(), "Error occurred while finding clusterLock for SolrCloud")
+ g.Expect(clusterOp).ToNot(BeNil(), "StatefulSet does not have a update lock to add hostAliases.")
+ g.Expect(clusterOp.Operation).To(Equal(controllers.UpdateLock), "StatefulSet does not have a update lock to add hostAliases.")
+ g.Expect(found.Spec.Template.Spec.HostAliases).To(HaveLen(3), "Host aliases in the pod template do not have length 3, which is the number of pods to scale up to")
+ })
+ expectSolrCloudWithChecksAndTimeout(ctx, solrCloud, time.Second*5, time.Millisecond*5, func(g Gomega, cloud *solrv1beta1.SolrCloud) {
+ g.Expect(cloud.Status.UpToDateNodes).To(BeEquivalentTo(0), "The Rolling Update never started, upToDateNodes should eventually be 0 when starting a restart")
+ })
+
+ By("Wait for the rolling update to be done")
+ expectSolrCloudWithChecksAndTimeout(ctx, solrCloud, time.Second*90, time.Millisecond*5, func(g Gomega, cloud *solrv1beta1.SolrCloud) {
+ g.Expect(cloud.Status.UpToDateNodes).To(BeEquivalentTo(*statefulSet.Spec.Replicas), "The Rolling Update never completed, not all replicas up to date")
+ g.Expect(cloud.Status.ReadyReplicas).To(BeEquivalentTo(*statefulSet.Spec.Replicas), "The Rolling Update never completed, not all replicas ready")
+ })
+
+ By("waiting for the scaleUp to begin")
+ statefulSet = expectStatefulSetWithChecksAndTimeout(ctx, solrCloud, solrCloud.StatefulSetName(), time.Second*5, time.Millisecond, func(g Gomega, found *appsv1.StatefulSet) {
+ clusterOp, err := controllers.GetCurrentClusterOp(found)
+ g.Expect(err).ToNot(HaveOccurred(), "Error occurred while finding clusterLock for SolrCloud")
+ g.Expect(clusterOp).ToNot(BeNil(), "StatefulSet does not have a scaleUp lock.")
+ g.Expect(clusterOp.Operation).To(Equal(controllers.ScaleUpLock), "StatefulSet does not have a scaleUp lock.")
+ g.Expect(clusterOp.Metadata).To(Equal("3"), "StatefulSet scaling lock operation has the wrong metadata.")
+ })
+
+ // The first step is to increase the number of pods
+ // Check very often, as the new pods will be created quickly, which will cause the cluster op to change.
+ if int(*statefulSet.Spec.Replicas) != 3 {
+ statefulSet = expectStatefulSetWithChecksAndTimeout(ctx, solrCloud, solrCloud.StatefulSetName(), time.Second*5, time.Millisecond*5, func(g Gomega, found *appsv1.StatefulSet) {
+ g.Expect(found.Spec.Replicas).To(HaveValue(BeEquivalentTo(3)), "StatefulSet should be configured for 3 pods when scaling up")
+ })
+ }
+ clusterOp, err := controllers.GetCurrentClusterOp(statefulSet)
+ Expect(err).ToNot(HaveOccurred(), "Error occurred while finding clusterLock for SolrCloud")
+ Expect(clusterOp).ToNot(BeNil(), "StatefulSet does not have a scaleUp lock.")
+ Expect(clusterOp.Operation).To(Equal(controllers.ScaleUpLock), "StatefulSet does not have a scaleUp lock.")
+ Expect(clusterOp.Metadata).To(Equal("3"), "StatefulSet scaling lock operation has the wrong metadata.")
+
+ // Wait for new pods to come up, and when they do we should be doing a balanceReplicas clusterOp
+ statefulSet = expectStatefulSetWithChecks(ctx, solrCloud, solrCloud.StatefulSetName(), func(g Gomega, found *appsv1.StatefulSet) {
+ g.Expect(found.Status.Replicas).To(HaveValue(BeEquivalentTo(3)), "StatefulSet should eventually have 3 pods created for it")
+ })
+ statefulSet = expectStatefulSetWithChecksAndTimeout(ctx, solrCloud, solrCloud.StatefulSetName(), time.Millisecond*50, time.Millisecond*10, func(g Gomega, found *appsv1.StatefulSet) {
+ clusterOp, err = controllers.GetCurrentClusterOp(found)
+ g.Expect(err).ToNot(HaveOccurred(), "Error occurred while finding clusterLock for SolrCloud")
+ g.Expect(clusterOp).ToNot(BeNil(), "StatefulSet does not have a balanceReplicas lock after new pods are created.")
+ g.Expect(clusterOp.Operation).To(Equal(controllers.BalanceReplicasLock), "StatefulSet does not have a balanceReplicas lock after new pods are created.")
+ g.Expect(clusterOp.Metadata).To(Equal("ScaleUp"), "StatefulSet balanceReplicas lock operation has the wrong metadata.")
+ })
+
+ By("waiting for the scaleUp to finish")
+ statefulSet = expectStatefulSetWithChecks(ctx, solrCloud, solrCloud.StatefulSetName(), func(g Gomega, found *appsv1.StatefulSet) {
+ clusterOp, err := controllers.GetCurrentClusterOp(found)
+ g.Expect(err).ToNot(HaveOccurred(), "Error occurred while finding clusterLock for SolrCloud")
+ g.Expect(clusterOp).To(BeNil(), "StatefulSet should not have a balanceReplicas lock after balancing is complete.")
+ })
+
+ queryCollection(ctx, solrCloud, solrCollection1, 0)
+ queryCollection(ctx, solrCloud, solrCollection2, 0)
+ })
+ })
+
FContext("without replica migration", func() {
BeforeEach(func() {
diff --git a/tests/e2e/suite_test.go b/tests/e2e/suite_test.go
index f27a90f..8b38a98 100644
--- a/tests/e2e/suite_test.go
+++ b/tests/e2e/suite_test.go
@@ -311,6 +311,26 @@ func writeAllSolrInfoToFiles(ctx context.Context, directory string, namespace st
&statefulSet,
)
}
+
+ // Unfortunately the services don't have the technology label
+ req, err = labels.NewRequirement("solr-cloud", selection.Exists, make([]string, 0))
+ Expect(err).ToNot(HaveOccurred())
+
+ labelSelector = labels.Everything().Add(*req)
+ listOps = &client.ListOptions{
+ Namespace: namespace,
+ LabelSelector: labelSelector,
+ }
+
+ foundServices := &corev1.ServiceList{}
+ Expect(k8sClient.List(ctx, foundServices, listOps)).To(Succeed(), "Could not fetch Solr pods")
+ Expect(foundServices).ToNot(BeNil(), "No Solr services could be found")
+ for _, service := range foundServices.Items {
+ writeAllServiceInfoToFiles(
+ directory+service.Name+".service",
+ &service,
+ )
+ }
}
// writeAllStatefulSetInfoToFiles writes the following each to a separate file with the given base name & directory.
@@ -339,6 +359,19 @@ func writeAllStatefulSetInfoToFiles(baseFilename string, statefulSet *appsv1.Sta
Expect(writeErr).ToNot(HaveOccurred(), "Could not write statefulSet events json to file")
}
+// writeAllServiceInfoToFiles writes the following each to a separate file with the given base name & directory.
+// - Service
+func writeAllServiceInfoToFiles(baseFilename string, service *corev1.Service) {
+ // Write service to a file
+ statusFile, err := os.Create(baseFilename + ".json")
+ defer statusFile.Close()
+ Expect(err).ToNot(HaveOccurred(), "Could not open file to save service status: %s", baseFilename+".json")
+ jsonBytes, marshErr := json.MarshalIndent(service, "", "\t")
+ Expect(marshErr).ToNot(HaveOccurred(), "Could not serialize service json")
+ _, writeErr := statusFile.Write(jsonBytes)
+ Expect(writeErr).ToNot(HaveOccurred(), "Could not write service json to file")
+}
+
// writeAllPodInfoToFile writes the following each to a separate file with the given base name & directory.
// - Pod Spec/Status
// - Pod Events
diff --git a/tests/e2e/test_utils_test.go b/tests/e2e/test_utils_test.go
index b28df88..c3e693c 100644
--- a/tests/e2e/test_utils_test.go
+++ b/tests/e2e/test_utils_test.go
@@ -475,10 +475,10 @@ func callSolrApiInPod(ctx context.Context, solrCloud *solrv1beta1.SolrCloud, htt
"-verbose",
"-" + strings.ToLower(httpMethod),
fmt.Sprintf(
- "\"%s://%s:%d%s%s\"",
+ "\"%s://%s%s%s%s\"",
solrCloud.UrlScheme(false),
hostname,
- solrCloud.Spec.SolrAddressability.PodPort,
+ solrCloud.NodePortSuffix(false),
apiPath,
queryParamsString),
}
(solr-operator) 02/02: Fix a few issues with testing, add better cluster information
Posted by ho...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
houston pushed a commit to branch release-0.8
in repository https://gitbox.apache.org/repos/asf/solr-operator.git
commit dbc8db914db9184112a1bbc3a4a90be3bc317491
Author: Houston Putman <ho...@apache.org>
AuthorDate: Mon Apr 8 08:35:39 2024 -0500
Fix a few issues with testing, add better cluster information
---
controllers/solr_cluster_ops_util.go | 3 +--
controllers/solrcloud_controller.go | 1 -
tests/e2e/solrcloud_rolling_upgrade_test.go | 2 +-
tests/e2e/suite_test.go | 31 ++++++++++++++++++++++++++++-
tests/e2e/test_utils_test.go | 8 +++++++-
5 files changed, 39 insertions(+), 6 deletions(-)
diff --git a/controllers/solr_cluster_ops_util.go b/controllers/solr_cluster_ops_util.go
index 07452eb..916446b 100644
--- a/controllers/solr_cluster_ops_util.go
+++ b/controllers/solr_cluster_ops_util.go
@@ -66,6 +66,7 @@ func clearClusterOpLock(statefulSet *appsv1.StatefulSet) {
}
func setClusterOpLock(statefulSet *appsv1.StatefulSet, op SolrClusterOp) error {
+ op.LastStartTime = metav1.Now()
bytes, err := json.Marshal(op)
if err != nil {
return err
@@ -124,7 +125,6 @@ func retryNextQueuedClusterOp(statefulSet *appsv1.StatefulSet) (hasOp bool, err
hasOp = len(clusterOpRetryQueue) > 0
if len(clusterOpRetryQueue) > 0 {
nextOp := clusterOpRetryQueue[0]
- nextOp.LastStartTime = metav1.Now()
err = setClusterOpLock(statefulSet, nextOp)
if err != nil {
return hasOp, err
@@ -141,7 +141,6 @@ func retryNextQueuedClusterOpWithQueue(statefulSet *appsv1.StatefulSet, clusterO
hasOp = len(clusterOpQueue) > 0
if len(clusterOpQueue) > 0 {
nextOp := clusterOpQueue[0]
- nextOp.LastStartTime = metav1.Now()
err = setClusterOpLock(statefulSet, nextOp)
if err != nil {
return hasOp, err
diff --git a/controllers/solrcloud_controller.go b/controllers/solrcloud_controller.go
index 4d832cf..9940ff9 100644
--- a/controllers/solrcloud_controller.go
+++ b/controllers/solrcloud_controller.go
@@ -574,7 +574,6 @@ func (r *SolrCloudReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
if clusterOp != nil {
// Starting a locked cluster operation!
originalStatefulSet := statefulSet.DeepCopy()
- clusterOp.LastStartTime = metav1.Now()
err = setClusterOpLock(statefulSet, *clusterOp)
if err == nil {
err = r.Patch(ctx, statefulSet, client.StrategicMergeFrom(originalStatefulSet))
diff --git a/tests/e2e/solrcloud_rolling_upgrade_test.go b/tests/e2e/solrcloud_rolling_upgrade_test.go
index faf2175..c56951d 100644
--- a/tests/e2e/solrcloud_rolling_upgrade_test.go
+++ b/tests/e2e/solrcloud_rolling_upgrade_test.go
@@ -163,7 +163,7 @@ var _ = FDescribe("E2E - SolrCloud - Rolling Upgrades", func() {
}
By("waiting for the balanceReplicas to finish")
- expectStatefulSetWithChecksAndTimeout(ctx, solrCloud, solrCloud.StatefulSetName(), time.Second*30, time.Second, func(g Gomega, found *appsv1.StatefulSet) {
+ expectStatefulSetWithChecksAndTimeout(ctx, solrCloud, solrCloud.StatefulSetName(), time.Second*70, time.Second, func(g Gomega, found *appsv1.StatefulSet) {
clusterOp, err := controllers.GetCurrentClusterOp(found)
g.Expect(err).ToNot(HaveOccurred(), "Error occurred while finding clusterLock for SolrCloud")
g.Expect(clusterOp).To(BeNil(), "StatefulSet should not have a balanceReplicas lock after balancing is complete.")
diff --git a/tests/e2e/suite_test.go b/tests/e2e/suite_test.go
index 8b38a98..1c2aec5 100644
--- a/tests/e2e/suite_test.go
+++ b/tests/e2e/suite_test.go
@@ -285,8 +285,23 @@ func writeAllSolrInfoToFiles(ctx context.Context, directory string, namespace st
req, err := labels.NewRequirement("technology", selection.In, []string{solrv1beta1.SolrTechnologyLabel, solrv1beta1.SolrPrometheusExporterTechnologyLabel})
Expect(err).ToNot(HaveOccurred())
- labelSelector := labels.Everything().Add(*req)
listOps := &client.ListOptions{
+ Namespace: namespace,
+ }
+
+ foundSolrs := &solrv1beta1.SolrCloudList{}
+ Expect(k8sClient.List(ctx, foundSolrs, listOps)).To(Succeed(), "Could not fetch SolrClouds")
+ Expect(foundSolrs).ToNot(BeNil(), "No SolrClouds could be found")
+ for _, solrCloud := range foundSolrs.Items {
+ writeSolrClusterStatusInfoToFile(
+ ctx,
+ directory+solrCloud.Name,
+ &solrCloud,
+ )
+ }
+
+ labelSelector := labels.Everything().Add(*req)
+ listOps = &client.ListOptions{
Namespace: namespace,
LabelSelector: labelSelector,
}
@@ -333,6 +348,20 @@ func writeAllSolrInfoToFiles(ctx context.Context, directory string, namespace st
}
}
+// writeSolrClusterStatusInfoToFile writes the following each to a separate file with the given base name & directory.
+// - SolrCloud's Cluster Status from the Collections API
+func writeSolrClusterStatusInfoToFile(ctx context.Context, baseFilename string, solrCloud *solrv1beta1.SolrCloud) {
+ clusterStatus := fetchClusterStatusWithErrorHandling(ctx, solrCloud, false)
+ if clusterStatus != "" {
+ // Write cluster status to a file
+ statusFile, err := os.Create(baseFilename + ".cluster-state.json")
+ defer statusFile.Close()
+ Expect(err).ToNot(HaveOccurred(), "Could not open file to save cluster status: %s", baseFilename+".cluster-state.json")
+ _, writeErr := statusFile.Write([]byte(clusterStatus))
+ Expect(writeErr).ToNot(HaveOccurred(), "Could not write cluster status json to file")
+ }
+}
+
// writeAllStatefulSetInfoToFiles writes the following each to a separate file with the given base name & directory.
// - StatefulSet Spec/Status
// - StatefulSet Events
diff --git a/tests/e2e/test_utils_test.go b/tests/e2e/test_utils_test.go
index c3e693c..486499d 100644
--- a/tests/e2e/test_utils_test.go
+++ b/tests/e2e/test_utils_test.go
@@ -313,6 +313,10 @@ func queryCollectionWithGomega(ctx context.Context, solrCloud *solrv1beta1.SolrC
}
func fetchClusterStatus(ctx context.Context, solrCloud *solrv1beta1.SolrCloud) string {
+ return fetchClusterStatusWithErrorHandling(ctx, solrCloud, true)
+}
+
+func fetchClusterStatusWithErrorHandling(ctx context.Context, solrCloud *solrv1beta1.SolrCloud, expectNoError bool) string {
response, err := callSolrApiInPod(
ctx,
solrCloud,
@@ -323,7 +327,9 @@ func fetchClusterStatus(ctx context.Context, solrCloud *solrv1beta1.SolrCloud) s
"wt": "json",
},
)
- Expect(err).ToNot(HaveOccurred(), "Could not fetch clusterStatus for cloud")
+ if expectNoError {
+ Expect(err).ToNot(HaveOccurred(), "Could not fetch clusterStatus for cloud")
+ }
return response
}