You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ho...@apache.org on 2023/05/11 20:28:17 UTC
[solr-operator] branch main updated: Improvments to e2e tests
This is an automated email from the ASF dual-hosted git repository.
houston pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr-operator.git
The following commit(s) were added to refs/heads/main by this push:
new 89422c9 Improvments to e2e tests
89422c9 is described below
commit 89422c98d4400cdafa84b8ac8b32399fb3f72103
Author: Houston Putman <ho...@apache.org>
AuthorDate: Thu May 11 15:50:06 2023 -0400
Improvments to e2e tests
- context.Context used for Eventually() pod execs
- More pod execs use Eventually()'s to remove false negatives
- Random pods are used for pod execs, instead of the first pod,
therefore retries will succeed if one pod is having an issue
---
api/v1beta1/solrcloud_types.go | 9 ++
tests/e2e/backups_test.go | 24 +--
tests/e2e/prometheus_exporter_test.go | 2 +-
tests/e2e/resource_utils_test.go | 22 +--
tests/e2e/solrcloud_basic_test.go | 9 +-
tests/e2e/solrcloud_rolling_upgrade_test.go | 8 +-
tests/e2e/suite_test.go | 21 ++-
tests/e2e/test_utils_test.go | 230 ++++++++++++++++------------
8 files changed, 182 insertions(+), 143 deletions(-)
diff --git a/api/v1beta1/solrcloud_types.go b/api/v1beta1/solrcloud_types.go
index 8c9145c..b45a027 100644
--- a/api/v1beta1/solrcloud_types.go
+++ b/api/v1beta1/solrcloud_types.go
@@ -21,6 +21,7 @@ import (
"fmt"
"github.com/go-logr/logr"
zkApi "github.com/pravega/zookeeper-operator/api/v1beta1"
+ "math/rand"
"strconv"
"strings"
@@ -1211,6 +1212,14 @@ func (sc *SolrCloud) GetAllSolrPodNames() []string {
return podNames
}
+func (sc *SolrCloud) GetRandomSolrPodName() string {
+ replicas := 1
+ if sc.Spec.Replicas != nil {
+ replicas = int(*sc.Spec.Replicas)
+ }
+ return sc.GetSolrPodName(rand.Intn(replicas))
+}
+
func (sc *SolrCloud) GetSolrPodName(podNumber int) string {
return fmt.Sprintf("%s-%d", sc.StatefulSetName(), podNumber)
}
diff --git a/tests/e2e/backups_test.go b/tests/e2e/backups_test.go
index 3188bf7..1057522 100644
--- a/tests/e2e/backups_test.go
+++ b/tests/e2e/backups_test.go
@@ -72,7 +72,7 @@ var _ = FDescribe("E2E - Backups", Ordered, func() {
})
By("creating a Solr Collection to backup")
- createAndQueryCollection(solrCloud, solrCollection, 1, 2)
+ createAndQueryCollection(ctx, solrCloud, solrCollection, 1, 2)
})
BeforeEach(func() {
@@ -122,10 +122,10 @@ var _ = FDescribe("E2E - Backups", Ordered, func() {
Expect(foundSolrBackup.Status.History[len(foundSolrBackup.Status.History)-1].Successful).To(PointTo(BeTrue()), "The latest backup was not successful")
lastBackupId := 0
- checkBackup(solrCloud, solrBackup, func(collection string, backupListResponse *solr_api.SolrBackupListResponse) {
- Expect(backupListResponse.Backups).To(HaveLen(3), "The wrong number of recurring backups have been saved")
+ checkBackup(ctx, solrCloud, solrBackup, func(g Gomega, collection string, backupListResponse *solr_api.SolrBackupListResponse) {
+ g.Expect(backupListResponse.Backups).To(HaveLen(3), "The wrong number of recurring backups have been saved")
lastBackupId = backupListResponse.Backups[len(backupListResponse.Backups)-1].BackupId
- Expect(lastBackupId).To(BeNumerically(">", 3), "The last backup ID is too low")
+ g.Expect(lastBackupId).To(BeNumerically(">", 3), "The last backup ID is too low")
})
By("disabling further backup recurrence")
@@ -138,10 +138,10 @@ var _ = FDescribe("E2E - Backups", Ordered, func() {
// Use start time because we might have disabled the recurrence mid-backup, and the finish time might not have been set
Expect(nextFoundSolrBackup.Status.StartTime).To(Equal(foundSolrBackup.Status.StartTime), "The last backup start time should be unchanged after recurrence is disabled")
- checkBackup(solrCloud, solrBackup, func(collection string, backupListResponse *solr_api.SolrBackupListResponse) {
- Expect(backupListResponse.Backups).To(HaveLen(3), "The wrong number of recurring backups have been saved")
+ checkBackup(ctx, solrCloud, solrBackup, func(g Gomega, collection string, backupListResponse *solr_api.SolrBackupListResponse) {
+ g.Expect(backupListResponse.Backups).To(HaveLen(3), "The wrong number of recurring backups have been saved")
newLastBackupId := backupListResponse.Backups[len(backupListResponse.Backups)-1].BackupId
- Expect(newLastBackupId).To(Equal(lastBackupId), "The last backup ID should not have been changed since the backup recurrence was disabled")
+ g.Expect(newLastBackupId).To(Equal(lastBackupId), "The last backup ID should not have been changed since the backup recurrence was disabled")
})
})
})
@@ -158,8 +158,8 @@ var _ = FDescribe("E2E - Backups", Ordered, func() {
g.Expect(backup.Status.Successful).To(PointTo(BeTrue()), "Backup did not successfully complete")
})
- checkBackup(solrCloud, solrBackup, func(collection string, backupListResponse *solr_api.SolrBackupListResponse) {
- Expect(backupListResponse.Backups).To(HaveLen(1), "A non-recurring backupList should have a length of 1")
+ checkBackup(ctx, solrCloud, solrBackup, func(g Gomega, collection string, backupListResponse *solr_api.SolrBackupListResponse) {
+ g.Expect(backupListResponse.Backups).To(HaveLen(1), "A non-recurring backupList should have a length of 1")
})
// Make sure nothing else happens after the backup is complete
@@ -169,9 +169,9 @@ var _ = FDescribe("E2E - Backups", Ordered, func() {
g.Expect(backup.Status.NextScheduledTime).To(BeNil(), "There should be no nextScheduledTime for a non-recurring backup")
})
- checkBackup(solrCloud, solrBackup, func(collection string, backupListResponse *solr_api.SolrBackupListResponse) {
- Expect(backupListResponse.Backups).To(HaveLen(1), "A non-recurring backupList should have a length of 1")
- Expect(backupListResponse.Backups[0].BackupId).To(Equal(0), "A non-recurring backup should have an ID of 1")
+ checkBackup(ctx, solrCloud, solrBackup, func(g Gomega, collection string, backupListResponse *solr_api.SolrBackupListResponse) {
+ g.Expect(backupListResponse.Backups).To(HaveLen(1), "A non-recurring backupList should have a length of 1")
+ g.Expect(backupListResponse.Backups[0].BackupId).To(Equal(0), "A non-recurring backup should have an ID of 1")
})
})
})
diff --git a/tests/e2e/prometheus_exporter_test.go b/tests/e2e/prometheus_exporter_test.go
index e5d840c..23ac6a4 100644
--- a/tests/e2e/prometheus_exporter_test.go
+++ b/tests/e2e/prometheus_exporter_test.go
@@ -58,7 +58,7 @@ var _ = FDescribe("E2E - Prometheus Exporter", Ordered, func() {
})
By("creating a Solr Collection to query metrics for")
- createAndQueryCollection(solrCloud, solrCollection, 1, 2)
+ createAndQueryCollection(ctx, solrCloud, solrCollection, 1, 2)
})
BeforeEach(func() {
diff --git a/tests/e2e/resource_utils_test.go b/tests/e2e/resource_utils_test.go
index 71aa698..3a9853f 100644
--- a/tests/e2e/resource_utils_test.go
+++ b/tests/e2e/resource_utils_test.go
@@ -76,7 +76,7 @@ func expectSolrCloudWithChecks(ctx context.Context, solrCloud *solrv1beta1.SolrC
if additionalChecks != nil {
additionalChecks(g, foundSolrCloud)
}
- }).Should(Succeed())
+ }).WithContext(ctx).Should(Succeed())
return foundSolrCloud
}
@@ -238,20 +238,6 @@ func expectStatefulSetWithChecks(ctx context.Context, parentResource client.Obje
additionalChecks(g, statefulSet)
}
}).Should(Succeed())
-
- By("recreating the StatefulSet after it is deleted")
- ExpectWithOffset(resolveOffset(additionalOffset), k8sClient.Delete(ctx, statefulSet)).To(Succeed())
- EventuallyWithOffset(
- resolveOffset(additionalOffset),
- func() (types.UID, error) {
- newResource := &appsv1.StatefulSet{}
- err := k8sClient.Get(ctx, resourceKey(parentResource, statefulSetName), newResource)
- if err != nil {
- return "", err
- }
- return newResource.UID, nil
- }).Should(And(Not(BeEmpty()), Not(Equal(statefulSet.UID))), "New StatefulSet, with new UID, not created.")
-
return statefulSet
}
@@ -277,6 +263,12 @@ func expectNoStatefulSet(ctx context.Context, parentResource client.Object, stat
}).Should(MatchError("statefulsets.apps \""+statefulSetName+"\" not found"), "StatefulSet exists when it should not")
}
+func expectNoPod(ctx context.Context, parentResource client.Object, podName string, additionalOffset ...int) {
+ EventuallyWithOffset(resolveOffset(additionalOffset), func() error {
+ return k8sClient.Get(ctx, resourceKey(parentResource, podName), &corev1.Pod{})
+ }).Should(MatchError("pods \""+podName+"\" not found"), "Pod exists when it should not")
+}
+
func expectService(ctx context.Context, parentResource client.Object, serviceName string, selectorLables map[string]string, isHeadless bool, additionalOffset ...int) *corev1.Service {
return expectServiceWithChecks(ctx, parentResource, serviceName, selectorLables, isHeadless, nil, resolveOffset(additionalOffset))
}
diff --git a/tests/e2e/solrcloud_basic_test.go b/tests/e2e/solrcloud_basic_test.go
index 673377d..c600d72 100644
--- a/tests/e2e/solrcloud_basic_test.go
+++ b/tests/e2e/solrcloud_basic_test.go
@@ -28,10 +28,6 @@ import (
var _ = FDescribe("E2E - SolrCloud - Basic", func() {
var (
solrCloud *solrv1beta1.SolrCloud
-
- solrCollection1 = "e2e-1"
-
- solrCollection2 = "e2e-2"
)
BeforeEach(func() {
@@ -52,10 +48,7 @@ var _ = FDescribe("E2E - SolrCloud - Basic", func() {
})
By("creating a first Solr Collection")
- createAndQueryCollection(solrCloud, solrCollection1, 1, 2)
-
- By("creating a second Solr Collection")
- createAndQueryCollection(solrCloud, solrCollection2, 2, 1)
+ createAndQueryCollection(ctx, solrCloud, "basic", 1, 1)
})
FContext("Provided Zookeeper", func() {
diff --git a/tests/e2e/solrcloud_rolling_upgrade_test.go b/tests/e2e/solrcloud_rolling_upgrade_test.go
index c9256ff..81b730c 100644
--- a/tests/e2e/solrcloud_rolling_upgrade_test.go
+++ b/tests/e2e/solrcloud_rolling_upgrade_test.go
@@ -53,10 +53,10 @@ var _ = FDescribe("E2E - SolrCloud - Rolling Upgrades", func() {
})
By("creating a first Solr Collection")
- createAndQueryCollection(solrCloud, solrCollection1, 1, 2)
+ createAndQueryCollection(ctx, solrCloud, solrCollection1, 1, 2)
By("creating a second Solr Collection")
- createAndQueryCollection(solrCloud, solrCollection2, 2, 1)
+ createAndQueryCollection(ctx, solrCloud, solrCollection2, 2, 1)
})
FContext("Managed Update - Ephemeral Data - Slow", func() {
@@ -143,8 +143,8 @@ var _ = FDescribe("E2E - SolrCloud - Rolling Upgrades", func() {
}
By("checking that the collections can be queried after the restart")
- queryCollection(solrCloud, solrCollection1, 0)
- queryCollection(solrCloud, solrCollection2, 0)
+ queryCollection(ctx, solrCloud, solrCollection1, 0)
+ queryCollection(ctx, solrCloud, solrCollection2, 0)
})
})
})
diff --git a/tests/e2e/suite_test.go b/tests/e2e/suite_test.go
index 14eeb3d..c53026c 100644
--- a/tests/e2e/suite_test.go
+++ b/tests/e2e/suite_test.go
@@ -81,11 +81,6 @@ var _ = SynchronizedBeforeSuite(func(ctx context.Context) {
logger = zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))
logf.SetLogger(logger)
- // Run this once before all tests, not per-test-process
- By("starting the test solr operator")
- solrOperatorRelease := runSolrOperator(ctx)
- Expect(solrOperatorRelease).ToNot(BeNil())
-
var err error
k8sConfig, err = config.GetConfig()
Expect(err).NotTo(HaveOccurred(), "Could not load in default kubernetes config")
@@ -94,9 +89,19 @@ var _ = SynchronizedBeforeSuite(func(ctx context.Context) {
Expect(err).NotTo(HaveOccurred(), "Could not create controllerRuntime Kubernetes client")
// Set up a shared Zookeeper Cluster to be used for most SolrClouds
- // This will significantly speed up tests
- By("starting a shared zookeeper cluster")
- runSharedZookeeperCluster(ctx)
+ // This will significantly speed up tests.
+ // The zookeeper will not be healthy until after the zookeeper operator is released with the solr operator
+ By("creating a shared zookeeper cluster")
+ zookeeper := runSharedZookeeperCluster(ctx)
+
+ // Run this once before all tests, not per-test-process
+ By("starting the test solr operator")
+ solrOperatorRelease := runSolrOperator(ctx)
+ Expect(solrOperatorRelease).ToNot(BeNil())
+
+ By("Waiting for the Zookeeper to come up healthy")
+ // This check must be done after the solr operator is installed
+ waitForSharedZookeeperCluster(ctx, zookeeper)
}, func(ctx context.Context) {
// Run these in each parallel test process before the tests
rand.Seed(GinkgoRandomSeed() + int64(GinkgoParallelProcess()))
diff --git a/tests/e2e/test_utils_test.go b/tests/e2e/test_utils_test.go
index c3a262e..a1adff8 100644
--- a/tests/e2e/test_utils_test.go
+++ b/tests/e2e/test_utils_test.go
@@ -143,26 +143,35 @@ func runSharedZookeeperCluster(ctx context.Context) *zkApi.ZookeeperCluster {
},
}
- By("creating the shared ZK cluster")
Expect(k8sClient.Create(ctx, zookeeper)).To(Succeed(), "Failed to create shared Zookeeper Cluster")
- By("Waiting for the Zookeeper to come up healthy")
- zookeeper = expectZookeeperClusterWithChecks(ctx, zookeeper, zookeeper.Name, func(g Gomega, found *zkApi.ZookeeperCluster) {
- g.Expect(found.Status.ReadyReplicas).To(Equal(found.Spec.Replicas), "The ZookeeperCluster should have all nodes come up healthy")
- })
-
DeferCleanup(func(ctx context.Context) {
- By("tearing down the shared Zookeeper Cluster")
stopSharedZookeeperCluster(ctx, zookeeper)
})
return zookeeper
}
-// Run Solr Operator for e2e testing of resources
+// Check if the sharedZookeeper Cluster is running
+func waitForSharedZookeeperCluster(ctx context.Context, sharedZk *zkApi.ZookeeperCluster) *zkApi.ZookeeperCluster {
+ sharedZk = expectZookeeperClusterWithChecks(ctx, sharedZk, sharedZk.Name, func(g Gomega, found *zkApi.ZookeeperCluster) {
+ g.Expect(found.Status.ReadyReplicas).To(Equal(found.Spec.Replicas), "The ZookeeperCluster should have all nodes come up healthy")
+ }, 1)
+
+ // Cleanup here, because we want to delete the zk before deleting the Zookeeper Operator.
+ // DeferCleanup() is a stack, so this cleanup needs to be called after the solr operator DeferCleanup().
+ DeferCleanup(func(ctx context.Context) {
+ stopSharedZookeeperCluster(ctx, sharedZk)
+ })
+
+ return sharedZk
+}
+
+// Stop the sharedZookeeperCluster
func stopSharedZookeeperCluster(ctx context.Context, sharedZk *zkApi.ZookeeperCluster) {
err := k8sClient.Get(ctx, client.ObjectKey{Namespace: sharedZk.Namespace, Name: sharedZk.Name}, sharedZk)
if err == nil {
+ By("tearing down the shared Zookeeper Cluster")
Expect(k8sClient.Delete(ctx, sharedZk)).To(Succeed(), "Failed to delete shared Zookeeper Cluster")
}
}
@@ -194,12 +203,11 @@ func createOrRecreateNamespace(ctx context.Context, namespaceName string) {
})
}
-func createAndQueryCollection(solrCloud *solrv1beta1.SolrCloud, collection string, shards int, replicasPerShard int, nodes ...int) {
- createAndQueryCollectionWithGomega(solrCloud, collection, shards, replicasPerShard, Default, nodes...)
+func createAndQueryCollection(ctx context.Context, solrCloud *solrv1beta1.SolrCloud, collection string, shards int, replicasPerShard int, nodes ...int) {
+ createAndQueryCollectionWithGomega(ctx, solrCloud, collection, shards, replicasPerShard, Default, 1, nodes...)
}
-func createAndQueryCollectionWithGomega(solrCloud *solrv1beta1.SolrCloud, collection string, shards int, replicasPerShard int, g Gomega, nodes ...int) {
- pod := solrCloud.GetAllSolrPodNames()[0]
+func createAndQueryCollectionWithGomega(ctx context.Context, solrCloud *solrv1beta1.SolrCloud, collection string, shards int, replicasPerShard int, g Gomega, additionalOffset int, nodes ...int) {
asyncId := fmt.Sprintf("create-collection-%s-%d-%d", collection, shards, replicasPerShard)
var nodeSet []string
@@ -211,29 +219,35 @@ func createAndQueryCollectionWithGomega(solrCloud *solrv1beta1.SolrCloud, collec
createNodeSet = "&createNodeSet=" + strings.Join(nodeSet, ",")
}
- response, err := runExecForContainer(
- util.SolrNodeContainer,
- pod,
- solrCloud.Namespace,
- []string{
- "curl",
- fmt.Sprintf(
- "http://localhost:%d/solr/admin/collections?action=CREATE&name=%s&replicationFactor=%d&numShards=%d%s&async=%s",
- solrCloud.Spec.SolrAddressability.PodPort,
- collection,
- replicasPerShard,
- shards,
- createNodeSet,
- asyncId),
- },
- )
- g.Expect(err).ToNot(HaveOccurred(), "Error occurred while creating Solr Collection")
- g.Expect(response).To(ContainSubstring("\"status\":0"), "Error occurred while creating Solr Collection")
+ additionalOffset += 1
+ g.EventuallyWithOffset(additionalOffset, func(innerG Gomega) {
+ response, err := runExecForContainer(
+ ctx,
+ util.SolrNodeContainer,
+ solrCloud.GetRandomSolrPodName(),
+ solrCloud.Namespace,
+ []string{
+ "curl",
+ fmt.Sprintf(
+ "http://localhost:%d/solr/admin/collections?action=CREATE&name=%s&replicationFactor=%d&numShards=%d%s&async=%s",
+ solrCloud.Spec.SolrAddressability.PodPort,
+ collection,
+ replicasPerShard,
+ shards,
+ createNodeSet,
+ asyncId),
+ },
+ )
+ innerG.Expect(err).ToNot(HaveOccurred(), "Error occurred while starting async command to create Solr Collection")
+ innerG.Expect(response).To(ContainSubstring("\"status\":0"), "Error occurred while starting async command to create Solr Collection")
+ }, time.Second*5).WithContext(ctx).Should(Succeed(), "Collection creation command start was not successful")
+ // Only wait 5 seconds when trying to create the asyncCommand
- g.Eventually(func(innerG Gomega) {
- response, err = runExecForContainer(
+ g.EventuallyWithOffset(additionalOffset, func(innerG Gomega) {
+ response, err := runExecForContainer(
+ ctx,
util.SolrNodeContainer,
- pod,
+ solrCloud.GetRandomSolrPodName(),
solrCloud.Namespace,
[]string{
"curl",
@@ -243,10 +257,8 @@ func createAndQueryCollectionWithGomega(solrCloud *solrv1beta1.SolrCloud, collec
asyncId),
},
)
- innerG.Expect(err).ToNot(HaveOccurred(), "Error occurred while checking if Solr Collection has been created")
- innerG.Expect(response).To(ContainSubstring("\"status\":0"), "Error occurred while creating Solr Collection")
- innerG.Expect(response).To(ContainSubstring("\"state\":\"completed\""), "Did not finish creating Solr Collection in time")
- if strings.Contains(response, "\"state\":\"failed\"") {
+ innerG.Expect(err).ToNot(HaveOccurred(), "Error occurred while checking if Solr Collection creation command was successful")
+ if strings.Contains(response, "\"state\":\"failed\"") || strings.Contains(response, "\"state\":\"notfound\"") {
StopTrying("A failure occurred while creating the Solr Collection").
Attach("Collection", collection).
Attach("Shards", shards).
@@ -254,43 +266,69 @@ func createAndQueryCollectionWithGomega(solrCloud *solrv1beta1.SolrCloud, collec
Attach("Response", response).
Now()
}
- }).Should(Succeed(), "Collection creation was not successful")
-
- response, err = runExecForContainer(
- util.SolrNodeContainer,
- pod,
- solrCloud.Namespace,
- []string{
- "curl",
- fmt.Sprintf(
- "http://localhost:%d/solr/admin/collections?action=DELETESTATUS&requestid=%s",
- solrCloud.Spec.SolrAddressability.PodPort,
- asyncId),
- },
- )
- g.Expect(err).ToNot(HaveOccurred(), "Error occurred while deleting Solr CollectionsAPI AsyncID")
- g.Expect(response).To(ContainSubstring("\"status\":0"), "Error occurred while deleting Solr CollectionsAPI AsyncID")
+ innerG.Expect(response).To(ContainSubstring("\"status\":0"), "A failure occurred while creating the Solr Collection")
+ innerG.Expect(response).To(ContainSubstring("\"state\":\"completed\""), "Did not finish creating Solr Collection in time")
+ }).WithContext(ctx).Should(Succeed(), "Collection creation was not successful")
- queryCollectionWithGomega(solrCloud, collection, 0, g)
+ g.EventuallyWithOffset(additionalOffset, func(innerG Gomega) {
+ response, err := runExecForContainer(
+ ctx,
+ util.SolrNodeContainer,
+ solrCloud.GetRandomSolrPodName(),
+ solrCloud.Namespace,
+ []string{
+ "curl",
+ fmt.Sprintf(
+ "http://localhost:%d/solr/admin/collections?action=DELETESTATUS&requestid=%s",
+ solrCloud.Spec.SolrAddressability.PodPort,
+ asyncId),
+ },
+ )
+ innerG.Expect(err).ToNot(HaveOccurred(), "Error occurred while deleting Solr CollectionsAPI AsyncID")
+ innerG.Expect(response).To(ContainSubstring("\"status\":0"), "Error occurred while deleting Solr CollectionsAPI AsyncID")
+ }, time.Second*5).WithContext(ctx).Should(Succeed(), "Could not delete aysncId after collection creation")
+ // Only wait 5 seconds when trying to delete the async requestId
+
+ queryCollectionWithGomega(ctx, solrCloud, collection, 0, g, additionalOffset)
}
-func queryCollection(solrCloud *solrv1beta1.SolrCloud, collection string, docCount int) {
- queryCollectionWithGomega(solrCloud, collection, docCount, Default)
+func queryCollection(ctx context.Context, solrCloud *solrv1beta1.SolrCloud, collection string, docCount int, additionalOffset ...int) {
+ queryCollectionWithGomega(ctx, solrCloud, collection, docCount, Default, resolveOffset(additionalOffset))
}
-func queryCollectionWithGomega(solrCloud *solrv1beta1.SolrCloud, collection string, docCount int, g Gomega) {
- pod := solrCloud.GetAllSolrPodNames()[0]
- response, err := runExecForContainer(
- util.SolrNodeContainer,
- pod,
- solrCloud.Namespace,
- []string{
- "curl",
- fmt.Sprintf("http://localhost:%d/solr/%s/select", solrCloud.Spec.SolrAddressability.PodPort, collection),
- },
- )
- g.Expect(err).ToNot(HaveOccurred(), "Error occurred while querying empty Solr Collection")
- g.Expect(response).To(ContainSubstring("\"numFound\":%d", docCount), "Error occurred while querying Solr Collection '%s'", collection)
+func queryCollectionWithGomega(ctx context.Context, solrCloud *solrv1beta1.SolrCloud, collection string, docCount int, g Gomega, additionalOffset ...int) {
+ g.EventuallyWithOffset(resolveOffset(additionalOffset), func(innerG Gomega) {
+ response, err := runExecForContainer(
+ ctx,
+ util.SolrNodeContainer,
+ solrCloud.GetRandomSolrPodName(),
+ solrCloud.Namespace,
+ []string{
+ "curl",
+ fmt.Sprintf("http://localhost:%d/solr/%s/select?rows=0", solrCloud.Spec.SolrAddressability.PodPort, collection),
+ },
+ )
+ g.ExpectWithOffset(resolveOffset(additionalOffset), err).ToNot(HaveOccurred(), "Error occurred while querying empty Solr Collection")
+ g.ExpectWithOffset(resolveOffset(additionalOffset), response).To(ContainSubstring("\"numFound\":%d", docCount), "Error occurred while querying Solr Collection '%s'", collection)
+ }, time.Second*5).WithContext(ctx).Should(Succeed(), "Could not successfully query collection")
+ // Only wait 5 seconds for the collection to be query-able
+}
+
+func queryCollectionWithNoReplicaAvailable(ctx context.Context, solrCloud *solrv1beta1.SolrCloud, collection string, additionalOffset ...int) {
+ EventuallyWithOffset(resolveOffset(additionalOffset), func(g Gomega) {
+ response, err := runExecForContainer(
+ ctx,
+ util.SolrNodeContainer,
+ solrCloud.GetRandomSolrPodName(),
+ solrCloud.Namespace,
+ []string{
+ "curl",
+ fmt.Sprintf("http://localhost:%d/solr/%s/select", solrCloud.Spec.SolrAddressability.PodPort, collection),
+ },
+ )
+ g.Expect(err).ToNot(HaveOccurred(), "Error occurred while querying empty Solr Collection")
+ g.Expect(response).To(ContainSubstring("Error trying to proxy request for url"), "Wrong occurred while querying Solr Collection '%s', expected a proxy forwarding error", collection)
+ }, time.Second*5).WithContext(ctx).Should(Succeed(), "Could not successfully query collection")
}
func getPrometheusExporterPod(ctx context.Context, solrPrometheusExporter *solrv1beta1.SolrPrometheusExporter) (podName string) {
@@ -316,14 +354,15 @@ func getPrometheusExporterPod(ctx context.Context, solrPrometheusExporter *solrv
return podName
}
-func checkMetrics(ctx context.Context, solrPrometheusExporter *solrv1beta1.SolrPrometheusExporter, solrCloud *solrv1beta1.SolrCloud, collection string) string {
- return checkMetricsWithGomega(ctx, solrPrometheusExporter, solrCloud, collection, Default)
+func checkMetrics(ctx context.Context, solrPrometheusExporter *solrv1beta1.SolrPrometheusExporter, solrCloud *solrv1beta1.SolrCloud, collection string, additionalOffset ...int) string {
+ return checkMetricsWithGomega(ctx, solrPrometheusExporter, solrCloud, collection, Default, resolveOffset(additionalOffset))
}
-func checkMetricsWithGomega(ctx context.Context, solrPrometheusExporter *solrv1beta1.SolrPrometheusExporter, solrCloud *solrv1beta1.SolrCloud, collection string, g Gomega) (response string) {
+func checkMetricsWithGomega(ctx context.Context, solrPrometheusExporter *solrv1beta1.SolrPrometheusExporter, solrCloud *solrv1beta1.SolrCloud, collection string, g Gomega, additionalOffset ...int) (response string) {
g.Eventually(func(innerG Gomega) {
var err error
response, err = runExecForContainer(
+ ctx,
util.SolrPrometheusExporterContainer,
getPrometheusExporterPod(ctx, solrPrometheusExporter),
solrCloud.Namespace,
@@ -342,18 +381,16 @@ func checkMetricsWithGomega(ctx context.Context, solrPrometheusExporter *solrv1b
MatchRegexp("solr_metrics_core_query_[^{]+\\{category=\"QUERY\",searchHandler=\"/select\",[^}]*collection=\"%s\",[^}]*shard=\"shard1\",[^}]*\\} [0-9]+.0", collection),
"Could not find query metrics in the PrometheusExporter response",
)
- }).WithContext(ctx).Within(time.Second * 5).ProbeEvery(time.Millisecond * 200).Should(Succeed())
+ }).WithContext(ctx).WithOffset(resolveOffset(additionalOffset)).Within(time.Second * 5).ProbeEvery(time.Millisecond * 200).Should(Succeed())
return response
}
-func checkBackup(solrCloud *solrv1beta1.SolrCloud, solrBackup *solrv1beta1.SolrBackup, checks func(collection string, backupListResponse *solr_api.SolrBackupListResponse)) {
- checkBackupWithGomega(solrCloud, solrBackup, checks, Default)
+func checkBackup(ctx context.Context, solrCloud *solrv1beta1.SolrCloud, solrBackup *solrv1beta1.SolrBackup, checks func(g Gomega, collection string, backupListResponse *solr_api.SolrBackupListResponse)) {
+ checkBackupWithGomega(ctx, solrCloud, solrBackup, checks, Default)
}
-func checkBackupWithGomega(solrCloud *solrv1beta1.SolrCloud, solrBackup *solrv1beta1.SolrBackup, checks func(collection string, backupListResponse *solr_api.SolrBackupListResponse), g Gomega) {
- solrCloudPod := solrCloud.GetAllSolrPodNames()[0]
-
+func checkBackupWithGomega(ctx context.Context, solrCloud *solrv1beta1.SolrCloud, solrBackup *solrv1beta1.SolrBackup, checks func(g Gomega, collection string, backupListResponse *solr_api.SolrBackupListResponse), g Gomega) {
repository := util.GetBackupRepositoryByName(solrCloud.Spec.BackupRepositories, solrBackup.Spec.RepositoryName)
repositoryName := repository.Name
if repositoryName == "" {
@@ -367,26 +404,29 @@ func checkBackupWithGomega(solrCloud *solrv1beta1.SolrCloud, solrBackup *solrv1b
repositoryName,
collection,
util.BackupLocationPath(repository, solrBackup.Spec.Location))
- response, err := runExecForContainer(
- util.SolrNodeContainer,
- solrCloudPod,
- solrCloud.Namespace,
- []string{
- "curl",
- curlCommand,
- },
- )
- g.Expect(err).ToNot(HaveOccurred(), "Error occurred while fetching backup '%s' for collection '%s': %s", solrBackup.Name, collection, curlCommand)
- backupListResponse := &solr_api.SolrBackupListResponse{}
+ g.Eventually(func(innerG Gomega) {
+ response, err := runExecForContainer(
+ ctx,
+ util.SolrNodeContainer,
+ solrCloud.GetRandomSolrPodName(),
+ solrCloud.Namespace,
+ []string{
+ "curl",
+ curlCommand,
+ },
+ )
+ innerG.Expect(err).ToNot(HaveOccurred(), "Error occurred while fetching backup '%s' for collection '%s': %s", solrBackup.Name, collection, curlCommand)
+ backupListResponse := &solr_api.SolrBackupListResponse{}
- g.Expect(json.Unmarshal([]byte(response), &backupListResponse)).To(Succeed(), "Could not parse json from Solr BackupList API")
+ innerG.Expect(json.Unmarshal([]byte(response), &backupListResponse)).To(Succeed(), "Could not parse json from Solr BackupList API")
- g.Expect(backupListResponse.ResponseHeader.Status).To(BeZero(), "SolrBackupList API returned exception code: %d", backupListResponse.ResponseHeader.Status)
- checks(collection, backupListResponse)
+ innerG.Expect(backupListResponse.ResponseHeader.Status).To(BeZero(), "SolrBackupList API returned exception code: %d", backupListResponse.ResponseHeader.Status)
+ checks(innerG, collection, backupListResponse)
+ }).WithContext(ctx).Within(time.Second * 5).ProbeEvery(time.Millisecond * 200).Should(Succeed())
}
}
-func runExecForContainer(container string, podName string, namespace string, command []string) (response string, err error) {
+func runExecForContainer(ctx context.Context, container string, podName string, namespace string, command []string) (response string, err error) {
req := rawK8sClient.CoreV1().RESTClient().Post().
Resource("pods").
Name(podName).
@@ -414,7 +454,7 @@ func runExecForContainer(container string, podName string, namespace string, com
}
var stdout, stderr bytes.Buffer
- err = exec.Stream(remotecommand.StreamOptions{
+ err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{
Stdout: &stdout,
Stderr: &stderr,
Tty: false,
@@ -445,7 +485,7 @@ func generateBaseSolrCloud(replicas int) *solrv1beta1.SolrCloud {
ZookeeperRef: &solrv1beta1.ZookeeperRef{
ConnectionInfo: &solrv1beta1.ZookeeperConnectionInfo{
InternalConnectionString: sharedZookeeperConnectionString,
- ChRoot: "/" + rand.String(5),
+ ChRoot: "/" + testNamespace() + "/" + rand.String(5),
},
},
// This seems to be the lowest memory & CPU that allow the tests to pass