You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by vj...@apache.org on 2025/11/20 16:25:42 UTC
(phoenix-adapters) branch main updated: Additional tests for scan and query with indexes
This is an automated email from the ASF dual-hosted git repository.
vjasani pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/phoenix-adapters.git
The following commit(s) were added to refs/heads/main by this push:
new 64fb2db Additional tests for scan and query with indexes
64fb2db is described below
commit 64fb2dbdc428c1e43d61e6e6e69c6482b0c48b7a
Author: Viraj Jasani <vj...@apache.org>
AuthorDate: Sat Nov 15 04:23:59 2025 -0800
Additional tests for scan and query with indexes
---
.../java/org/apache/phoenix/ddb/Misc1Util.java | 64 ++++++++++++++++++++++
.../java/org/apache/phoenix/ddb/ScanIndex3IT.java | 13 +++++
.../java/org/apache/phoenix/ddb/TestUtils.java | 2 -
3 files changed, 77 insertions(+), 2 deletions(-)
diff --git a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/Misc1Util.java b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/Misc1Util.java
index cbb1e0f..398b6ac 100644
--- a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/Misc1Util.java
+++ b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/Misc1Util.java
@@ -860,6 +860,14 @@ public class Misc1Util {
batchWriteItems(dynamoDbClient, phoenixDBClientV2, table1Name, table1Items);
batchWriteItems(dynamoDbClient, phoenixDBClientV2, table2Name, table2Items);
+ List<Map<String, AttributeValue>> table1ItemsToDelete =
+ selectItemsToDelete(table1Items, numPartitions, 0.22);
+ List<Map<String, AttributeValue>> table2ItemsToDelete =
+ selectItemsToDelete(table2Items, numPartitions, 0.27);
+
+ batchDeleteItems(dynamoDbClient, phoenixDBClientV2, table1Name, table1ItemsToDelete);
+ batchDeleteItems(dynamoDbClient, phoenixDBClientV2, table2Name, table2ItemsToDelete);
+
for (int partitionId = 0; partitionId < numPartitions; partitionId++) {
String pkValueTable1 = "pk_" + partitionId + "_t1";
String pkValueTable2 = "pk_" + partitionId + "_t2";
@@ -1104,6 +1112,62 @@ public class Misc1Util {
}
}
+ private static void batchDeleteItems(DynamoDbClient dynamoDbClient,
+ DynamoDbClient phoenixDBClientV2, String tableName,
+ List<Map<String, AttributeValue>> itemKeys) {
+ int batchSize = 25;
+ int totalBatches = (int) Math.ceil((double) itemKeys.size() / batchSize);
+
+ for (int batchNum = 0; batchNum < totalBatches; batchNum++) {
+ int startIdx = batchNum * batchSize;
+ int endIdx = Math.min(startIdx + batchSize, itemKeys.size());
+ List<Map<String, AttributeValue>> batchKeys = itemKeys.subList(startIdx, endIdx);
+
+ List<WriteRequest> deleteRequests = batchKeys.stream().map(key -> WriteRequest.builder()
+ .deleteRequest(DeleteRequest.builder()
+ .key(key).build()).build()).collect(Collectors.toList());
+
+ Map<String, List<WriteRequest>> requestItems =
+ Collections.singletonMap(tableName, deleteRequests);
+
+ BatchWriteItemRequest batchDeleteRequest =
+ BatchWriteItemRequest.builder().requestItems(requestItems).build();
+
+ dynamoDbClient.batchWriteItem(batchDeleteRequest);
+ phoenixDBClientV2.batchWriteItem(batchDeleteRequest);
+ }
+ }
+
+ private static List<Map<String, AttributeValue>> selectItemsToDelete(
+ List<Map<String, AttributeValue>> items, int numPartitions, double deletePercentage) {
+ List<Map<String, AttributeValue>> itemsToDelete = new ArrayList<>();
+ int itemsPerPartition = (int) Math.ceil((double) items.size() / numPartitions);
+
+ for (int partitionId = 0; partitionId < numPartitions; partitionId++) {
+ int startIdx = partitionId * itemsPerPartition;
+ int endIdx = Math.min(startIdx + itemsPerPartition, items.size());
+ int itemsInThisPartition = endIdx - startIdx;
+ int itemsToDeleteFromPartition =
+ (int) Math.ceil(itemsInThisPartition * deletePercentage);
+
+ for (int i = 0; i < itemsToDeleteFromPartition && (startIdx + i) < endIdx; i++) {
+ int itemIdx = startIdx + (i * itemsInThisPartition / itemsToDeleteFromPartition);
+ if (itemIdx < endIdx) {
+ Map<String, AttributeValue> item = items.get(itemIdx);
+ Map<String, AttributeValue> key = new HashMap<>();
+ key.put("pk", item.get("pk"));
+ key.put("sk", item.get("sk"));
+ itemsToDelete.add(key);
+ }
+ }
+ }
+
+ itemsToDelete.sort(
+ Comparator.comparing((Map<String, AttributeValue> item) -> item.get("pk").s())
+ .thenComparing(item -> Long.parseLong(item.get("sk").n())));
+ return itemsToDelete;
+ }
+
private static void queryGSIsWithoutConditions(DynamoDbClient dynamoDbClient,
DynamoDbClient phoenixDBClientV2, String tableName, String tableSuffix) {
diff --git a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/ScanIndex3IT.java b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/ScanIndex3IT.java
index a57200a..ad584a4 100644
--- a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/ScanIndex3IT.java
+++ b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/ScanIndex3IT.java
@@ -41,6 +41,7 @@ import org.junit.rules.TestName;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest;
+import software.amazon.awssdk.services.dynamodb.model.DeleteItemRequest;
import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;
import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
import software.amazon.awssdk.services.dynamodb.model.ScanResponse;
@@ -129,6 +130,18 @@ public class ScanIndex3IT {
phoenixDBClientV2.putItem(putRequest);
dynamoDbClient.putItem(putRequest);
}
+
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ if (i % 4 == 0 || i % 11 == 0) {
+ Map<String, AttributeValue> key = new HashMap<>();
+ key.put("pk", AttributeValue.builder().s("pk_" + (i % 100)).build());
+ key.put("sk", AttributeValue.builder().n(String.valueOf(i)).build());
+ DeleteItemRequest deleteRequest =
+ DeleteItemRequest.builder().tableName(TABLE_NAME).key(key).build();
+ phoenixDBClientV2.deleteItem(deleteRequest);
+ dynamoDbClient.deleteItem(deleteRequest);
+ }
+ }
}
@AfterClass
diff --git a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/TestUtils.java b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/TestUtils.java
index 6b1c9d2..7afecc9 100644
--- a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/TestUtils.java
+++ b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/TestUtils.java
@@ -341,7 +341,6 @@ public class TestUtils {
phoenixRecordsResponse = phoenixDBStreamsClientV2.getRecords(phoenixRecordsRequest);
allPhoenixRecords.addAll(phoenixRecordsResponse.records());
phoenixShardIterator = phoenixRecordsResponse.nextShardIterator();
- LOGGER.info("Phoenix shard iterator: {}", phoenixShardIterator);
} while (phoenixShardIterator != null && !phoenixRecordsResponse.records().isEmpty());
do {
@@ -350,7 +349,6 @@ public class TestUtils {
dynamoRecordsResponse = dynamoDbStreamsClient.getRecords(dynamoRecordsRequest);
allDynamoRecords.addAll(dynamoRecordsResponse.records());
dynamoShardIterator = dynamoRecordsResponse.nextShardIterator();
- LOGGER.info("Dynamo shard iterator: {}", dynamoShardIterator);
} while (dynamoShardIterator != null && !dynamoRecordsResponse.records().isEmpty());
LOGGER.info("Phoenix stream records count: {}", allPhoenixRecords.size());