You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by vj...@apache.org on 2026/05/26 23:00:26 UTC
(phoenix-adapters) branch main updated: Support list_append in UpdateExpression SET
This is an automated email from the ASF dual-hosted git repository.
virajjasani pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/phoenix-adapters.git
The following commit(s) were added to refs/heads/main by this push:
new a8674ee Support list_append in UpdateExpression SET
a8674ee is described below
commit a8674ee27e3763bee54f909d458649da05e724d8
Author: Palash Chauhan <pa...@gmail.com>
AuthorDate: Wed May 13 10:52:17 2026 -0700
Support list_append in UpdateExpression SET
---
DDB_API_REFERENCE.md | 9 +
.../ddb/UpdateExpressionConversionTest.java | 417 +++++++++++++++++++++
.../phoenix/ddb/UpdateExpressionValidationIT.java | 147 ++++++++
.../apache/phoenix/ddb/UpdateItemBaseTests.java | 233 ++++++++++++
.../ddb/bson/UpdateExpressionDdbToBson.java | 95 ++++-
5 files changed, 897 insertions(+), 4 deletions(-)
diff --git a/DDB_API_REFERENCE.md b/DDB_API_REFERENCE.md
index 1916fb1..375ef15 100644
--- a/DDB_API_REFERENCE.md
+++ b/DDB_API_REFERENCE.md
@@ -899,6 +899,10 @@ Modifies specific attributes of an existing item (or creates it if using `SET` o
**UpdateExpression syntax:**
```
SET #name = :newName, age = :newAge
+SET counter = counter + :increment, score = score - :penalty
+SET title = if_not_exists(title, :defaultTitle)
+SET events = list_append(events, :newEvents)
+SET queue = list_append(if_not_exists(queue, :empty), :newItems)
REMOVE obsolete_field
ADD view_count :increment
DELETE tags :tagsToRemove
@@ -910,6 +914,11 @@ Supported clauses:
- `ADD` -- Add to number or add elements to a set
- `DELETE` -- Remove elements from a set
+Supported `SET` functions and operators:
+- `+` / `-` -- arithmetic on numeric attributes (e.g. `counter = counter + :n`)
+- `if_not_exists(path, :fallback)` -- use existing value if present, otherwise fall back
+- `list_append(operand1, operand2)` -- concatenate two lists. Each operand may be a literal list placeholder (e.g. `:newItems`), an attribute path (e.g. `events`, `nested.queue`), or `if_not_exists(path, :emptyList)`. Both operands must resolve to a list; exactly two operands are required; nested `list_append(list_append(...), ...)` is not supported.
+
**Legacy AttributeUpdates format:**
```json
{
diff --git a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/UpdateExpressionConversionTest.java b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/UpdateExpressionConversionTest.java
index 87142c3..a5c73af 100644
--- a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/UpdateExpressionConversionTest.java
+++ b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/UpdateExpressionConversionTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.phoenix.ddb;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -246,6 +247,422 @@ public class UpdateExpressionConversionTest {
getComparisonValuesMap()));
}
+ @Test
+ public void testListAppendPathAndLiteral() {
+ String ddbUpdateExp = "SET myList = list_append(myList, :listVal)";
+ String expected = "{\n" +
+ " \"$SET\": {\n" +
+ " \"myList\": {\n" +
+ " \"$LIST_APPEND\": [\n" +
+ " \"myList\",\n" +
+ " [\"c\", \"d\"]\n" +
+ " ]\n" +
+ " }\n" +
+ " }\n" +
+ "}";
+ Assert.assertEquals(RawBsonDocument.parse(expected),
+ UpdateExpressionDdbToBson.getBsonDocumentForUpdateExpression(ddbUpdateExp,
+ getListAppendComparisonValuesMap()));
+ }
+
+ @Test
+ public void testListAppendLiteralAndPath() {
+ String ddbUpdateExp = "SET myList = list_append(:listVal, myList)";
+ String expected = "{\n" +
+ " \"$SET\": {\n" +
+ " \"myList\": {\n" +
+ " \"$LIST_APPEND\": [\n" +
+ " [\"c\", \"d\"],\n" +
+ " \"myList\"\n" +
+ " ]\n" +
+ " }\n" +
+ " }\n" +
+ "}";
+ Assert.assertEquals(RawBsonDocument.parse(expected),
+ UpdateExpressionDdbToBson.getBsonDocumentForUpdateExpression(ddbUpdateExp,
+ getListAppendComparisonValuesMap()));
+ }
+
+ @Test
+ public void testListAppendBothLiterals() {
+ String ddbUpdateExp = "SET myList = list_append(:listVal, :listVal2)";
+ String expected = "{\n" +
+ " \"$SET\": {\n" +
+ " \"myList\": {\n" +
+ " \"$LIST_APPEND\": [\n" +
+ " [\"c\", \"d\"],\n" +
+ " [\"e\", \"f\"]\n" +
+ " ]\n" +
+ " }\n" +
+ " }\n" +
+ "}";
+ Assert.assertEquals(RawBsonDocument.parse(expected),
+ UpdateExpressionDdbToBson.getBsonDocumentForUpdateExpression(ddbUpdateExp,
+ getListAppendComparisonValuesMap()));
+ }
+
+ /**
+ * Canonical create-or-append: if the target list does not exist, fall back to an empty
+ * list as the first operand and then append the new literal items.
+ */
+ @Test
+ public void testListAppendWithIfNotExists() {
+ String ddbUpdateExp = "SET newQueue = list_append(if_not_exists(newQueue, :emptyList), :listVal)";
+ String expected = "{\n" +
+ " \"$SET\": {\n" +
+ " \"newQueue\": {\n" +
+ " \"$LIST_APPEND\": [\n" +
+ " {\n" +
+ " \"$IF_NOT_EXISTS\": {\n" +
+ " \"newQueue\": []\n" +
+ " }\n" +
+ " },\n" +
+ " [\"c\", \"d\"]\n" +
+ " ]\n" +
+ " }\n" +
+ " }\n" +
+ "}";
+ Assert.assertEquals(RawBsonDocument.parse(expected),
+ UpdateExpressionDdbToBson.getBsonDocumentForUpdateExpression(ddbUpdateExp,
+ getListAppendComparisonValuesMap()));
+ }
+
+ /**
+ * list_append combined with arithmetic and a remove in the same expression.
+ */
+ @Test
+ public void testListAppendMixedWithArithmeticAndRemove() {
+ String ddbUpdateExp =
+ "SET myList = list_append(myList, :listVal), counter = counter + :one "
+ + "REMOVE oldField";
+ String expected = "{\n" +
+ " \"$SET\": {\n" +
+ " \"myList\": {\n" +
+ " \"$LIST_APPEND\": [\n" +
+ " \"myList\",\n" +
+ " [\"c\", \"d\"]\n" +
+ " ]\n" +
+ " },\n" +
+ " \"counter\": {\n" +
+ " \"$ADD\": [\n" +
+ " \"counter\",\n" +
+ " 1\n" +
+ " ]\n" +
+ " }\n" +
+ " },\n" +
+ " \"$UNSET\": {\n" +
+ " \"oldField\": null\n" +
+ " }\n" +
+ "}";
+ Assert.assertEquals(RawBsonDocument.parse(expected),
+ UpdateExpressionDdbToBson.getBsonDocumentForUpdateExpression(ddbUpdateExp,
+ getListAppendComparisonValuesMap()));
+ }
+
+ /**
+ * Nested document path on the LHS and as the path operand.
+ */
+ @Test
+ public void testListAppendNestedPath() {
+ String ddbUpdateExp = "SET nested.queue = list_append(nested.queue, :listVal)";
+ String expected = "{\n" +
+ " \"$SET\": {\n" +
+ " \"nested.queue\": {\n" +
+ " \"$LIST_APPEND\": [\n" +
+ " \"nested.queue\",\n" +
+ " [\"c\", \"d\"]\n" +
+ " ]\n" +
+ " }\n" +
+ " }\n" +
+ "}";
+ Assert.assertEquals(RawBsonDocument.parse(expected),
+ UpdateExpressionDdbToBson.getBsonDocumentForUpdateExpression(ddbUpdateExp,
+ getListAppendComparisonValuesMap()));
+ }
+
+ /**
+ * if_not_exists as the SECOND operand (literal first). This prepends the new items
+ * before the existing-or-fallback list.
+ */
+ @Test
+ public void testListAppendLiteralAndIfNotExists() {
+ String ddbUpdateExp = "SET newQueue = list_append(:listVal, if_not_exists(newQueue, :emptyList))";
+ String expected = "{\n" +
+ " \"$SET\": {\n" +
+ " \"newQueue\": {\n" +
+ " \"$LIST_APPEND\": [\n" +
+ " [\"c\", \"d\"],\n" +
+ " {\n" +
+ " \"$IF_NOT_EXISTS\": {\n" +
+ " \"newQueue\": []\n" +
+ " }\n" +
+ " }\n" +
+ " ]\n" +
+ " }\n" +
+ " }\n" +
+ "}";
+ Assert.assertEquals(RawBsonDocument.parse(expected),
+ UpdateExpressionDdbToBson.getBsonDocumentForUpdateExpression(ddbUpdateExp,
+ getListAppendComparisonValuesMap()));
+ }
+
+ /**
+ * Create-or-append a list while atomically incrementing a counter in the same SET clause.
+ */
+ @Test
+ public void testListAppendWithIfNotExistsAndCounterIncrement() {
+ String ddbUpdateExp =
+ "SET events = list_append(if_not_exists(events, :empty_list), :new_evts), "
+ + "updateCounter = updateCounter + :one";
+ String expected = "{\n" +
+ " \"$SET\": {\n" +
+ " \"events\": {\n" +
+ " \"$LIST_APPEND\": [\n" +
+ " {\n" +
+ " \"$IF_NOT_EXISTS\": {\n" +
+ " \"events\": []\n" +
+ " }\n" +
+ " },\n" +
+ " [\"ev1\", \"ev2\"]\n" +
+ " ]\n" +
+ " },\n" +
+ " \"updateCounter\": {\n" +
+ " \"$ADD\": [\n" +
+ " \"updateCounter\",\n" +
+ " 1\n" +
+ " ]\n" +
+ " }\n" +
+ " }\n" +
+ "}";
+ Map<String, AttributeValue> attributeMap = new HashMap<>();
+ attributeMap.put(":empty_list",
+ AttributeValue.builder().l(Collections.emptyList()).build());
+ attributeMap.put(":new_evts", AttributeValue.builder().l(
+ AttributeValue.builder().s("ev1").build(),
+ AttributeValue.builder().s("ev2").build()).build());
+ attributeMap.put(":one", AttributeValue.builder().n("1").build());
+ BsonDocument values = DdbAttributesToBsonDocument.getRawBsonDocument(attributeMap);
+
+ Assert.assertEquals(RawBsonDocument.parse(expected),
+ UpdateExpressionDdbToBson.getBsonDocumentForUpdateExpression(ddbUpdateExp, values));
+ }
+
+ /**
+ * Both operands of list_append are if_not_exists. Regression test for the
+ * paren-counting splitter: there are top-level commas at depth 1 followed by
+ * another open-paren on each side, which the old regex-based splitter could not
+ * handle.
+ */
+ @Test
+ public void testListAppendBothOperandsAreIfNotExists() {
+ String ddbUpdateExp =
+ "SET merged = list_append(if_not_exists(left, :emptyList), if_not_exists(right, :emptyList))";
+ String expected = "{\n" +
+ " \"$SET\": {\n" +
+ " \"merged\": {\n" +
+ " \"$LIST_APPEND\": [\n" +
+ " { \"$IF_NOT_EXISTS\": { \"left\": [] } },\n" +
+ " { \"$IF_NOT_EXISTS\": { \"right\": [] } }\n" +
+ " ]\n" +
+ " }\n" +
+ " }\n" +
+ "}";
+ Assert.assertEquals(RawBsonDocument.parse(expected),
+ UpdateExpressionDdbToBson.getBsonDocumentForUpdateExpression(ddbUpdateExp,
+ getListAppendComparisonValuesMap()));
+ }
+
+ /**
+ * Canonical counter-init pattern: increment a counter that may not exist yet, using
+ * if_not_exists as one of the arithmetic operands. Common DDB usage but previously
+ * only covered by IT tests.
+ */
+ @Test
+ public void testArithmeticWithIfNotExistsCounterIncrement() {
+ String ddbUpdateExp = "SET counter = if_not_exists(counter, :zero) + :one";
+ String expected = "{\n" +
+ " \"$SET\": {\n" +
+ " \"counter\": {\n" +
+ " \"$ADD\": [\n" +
+ " { \"$IF_NOT_EXISTS\": { \"counter\": 0 } },\n" +
+ " 1\n" +
+ " ]\n" +
+ " }\n" +
+ " }\n" +
+ "}";
+ Map<String, AttributeValue> attributeMap = new HashMap<>();
+ attributeMap.put(":zero", AttributeValue.builder().n("0").build());
+ attributeMap.put(":one", AttributeValue.builder().n("1").build());
+ BsonDocument values = DdbAttributesToBsonDocument.getRawBsonDocument(attributeMap);
+ Assert.assertEquals(RawBsonDocument.parse(expected),
+ UpdateExpressionDdbToBson.getBsonDocumentForUpdateExpression(ddbUpdateExp, values));
+ }
+
+ /**
+ * Arithmetic with both operands being attribute paths (no placeholders, no
+ * if_not_exists).
+ */
+ @Test
+ public void testArithmeticPathPlusPath() {
+ String ddbUpdateExp = "SET total = subtotal + tax";
+ String expected = "{\n" +
+ " \"$SET\": {\n" +
+ " \"total\": {\n" +
+ " \"$ADD\": [\n" +
+ " \"subtotal\",\n" +
+ " \"tax\"\n" +
+ " ]\n" +
+ " }\n" +
+ " }\n" +
+ "}";
+ Assert.assertEquals(RawBsonDocument.parse(expected),
+ UpdateExpressionDdbToBson.getBsonDocumentForUpdateExpression(ddbUpdateExp,
+ new BsonDocument()));
+ }
+
+ /**
+ * Subtract with placeholder first and attribute path second
+ * (e.g. {@code SET remaining = :limit - used}).
+ */
+ @Test
+ public void testArithmeticLiteralMinusPath() {
+ String ddbUpdateExp = "SET remaining = :limit - used";
+ String expected = "{\n" +
+ " \"$SET\": {\n" +
+ " \"remaining\": {\n" +
+ " \"$SUBTRACT\": [\n" +
+ " 100,\n" +
+ " \"used\"\n" +
+ " ]\n" +
+ " }\n" +
+ " }\n" +
+ "}";
+ Map<String, AttributeValue> attributeMap = new HashMap<>();
+ attributeMap.put(":limit", AttributeValue.builder().n("100").build());
+ BsonDocument values = DdbAttributesToBsonDocument.getRawBsonDocument(attributeMap);
+ Assert.assertEquals(RawBsonDocument.parse(expected),
+ UpdateExpressionDdbToBson.getBsonDocumentForUpdateExpression(ddbUpdateExp, values));
+ }
+
+ /**
+ * Multiple SET clauses interleaving every supported function/operator shape:
+ * plain placeholder, if_not_exists, arithmetic with if_not_exists, list_append,
+ * and a bare nested-path assignment.
+ */
+ @Test
+ public void testSetEveryShapeMixed() {
+ String ddbUpdateExp = "SET title = :newTitle, "
+ + "counter = if_not_exists(counter, :zero) + :one, "
+ + "total = price - :discount, "
+ + "events = list_append(if_not_exists(events, :emptyList), :newEvts), "
+ + "nested.path = :nestedVal";
+ String expected = "{\n" +
+ " \"$SET\": {\n" +
+ " \"title\": \"hello\",\n" +
+ " \"counter\": {\n" +
+ " \"$ADD\": [\n" +
+ " { \"$IF_NOT_EXISTS\": { \"counter\": 0 } },\n" +
+ " 1\n" +
+ " ]\n" +
+ " },\n" +
+ " \"total\": {\n" +
+ " \"$SUBTRACT\": [\n" +
+ " \"price\",\n" +
+ " 5\n" +
+ " ]\n" +
+ " },\n" +
+ " \"events\": {\n" +
+ " \"$LIST_APPEND\": [\n" +
+ " { \"$IF_NOT_EXISTS\": { \"events\": [] } },\n" +
+ " [\"e1\", \"e2\"]\n" +
+ " ]\n" +
+ " },\n" +
+ " \"nested.path\": \"deep\"\n" +
+ " }\n" +
+ "}";
+ Map<String, AttributeValue> attributeMap = new HashMap<>();
+ attributeMap.put(":newTitle", AttributeValue.builder().s("hello").build());
+ attributeMap.put(":zero", AttributeValue.builder().n("0").build());
+ attributeMap.put(":one", AttributeValue.builder().n("1").build());
+ attributeMap.put(":discount", AttributeValue.builder().n("5").build());
+ attributeMap.put(":emptyList",
+ AttributeValue.builder().l(Collections.emptyList()).build());
+ attributeMap.put(":newEvts", AttributeValue.builder().l(
+ AttributeValue.builder().s("e1").build(),
+ AttributeValue.builder().s("e2").build()).build());
+ attributeMap.put(":nestedVal", AttributeValue.builder().s("deep").build());
+ BsonDocument values = DdbAttributesToBsonDocument.getRawBsonDocument(attributeMap);
+ Assert.assertEquals(RawBsonDocument.parse(expected),
+ UpdateExpressionDdbToBson.getBsonDocumentForUpdateExpression(ddbUpdateExp, values));
+ }
+
+ @Test
+ public void testListAppendWrongArityOne() {
+ String ddbUpdateExp = "SET myList = list_append(:listVal)";
+ try {
+ UpdateExpressionDdbToBson.getBsonDocumentForUpdateExpression(ddbUpdateExp,
+ getListAppendComparisonValuesMap());
+ Assert.fail("Expected RuntimeException for wrong arity (1 operand)");
+ } catch (RuntimeException e) {
+ Assert.assertTrue("Unexpected message: " + e.getMessage(),
+ e.getMessage().contains("list_append requires exactly 2 operands"));
+ }
+ }
+
+ @Test
+ public void testListAppendWrongArityThree() {
+ String ddbUpdateExp = "SET myList = list_append(:listVal, :listVal2, myList)";
+ try {
+ UpdateExpressionDdbToBson.getBsonDocumentForUpdateExpression(ddbUpdateExp,
+ getListAppendComparisonValuesMap());
+ Assert.fail("Expected RuntimeException for wrong arity (3 operands)");
+ } catch (RuntimeException e) {
+ Assert.assertTrue("Unexpected message: " + e.getMessage(),
+ e.getMessage().contains("list_append requires exactly 2 operands"));
+ }
+ }
+
+ @Test
+ public void testListAppendMissingPlaceholder() {
+ String ddbUpdateExp = "SET myList = list_append(myList, :missing)";
+ try {
+ UpdateExpressionDdbToBson.getBsonDocumentForUpdateExpression(ddbUpdateExp,
+ getListAppendComparisonValuesMap());
+ Assert.fail("Expected RuntimeException for missing placeholder");
+ } catch (RuntimeException e) {
+ Assert.assertTrue("Unexpected message: " + e.getMessage(),
+ e.getMessage().contains(":missing"));
+ }
+ }
+
+ @Test
+ public void testListAppendNonArrayPlaceholder() {
+ String ddbUpdateExp = "SET myList = list_append(myList, :notList)";
+ try {
+ UpdateExpressionDdbToBson.getBsonDocumentForUpdateExpression(ddbUpdateExp,
+ getListAppendComparisonValuesMap());
+ Assert.fail("Expected RuntimeException for non-list placeholder");
+ } catch (RuntimeException e) {
+ Assert.assertTrue("Unexpected message: " + e.getMessage(),
+ e.getMessage().contains("must resolve to a List type"));
+ }
+ }
+
+ private static BsonDocument getListAppendComparisonValuesMap() {
+ Map<String, AttributeValue> attributeMap = new HashMap<>();
+ attributeMap.put(":listVal", AttributeValue.builder().l(
+ AttributeValue.builder().s("c").build(),
+ AttributeValue.builder().s("d").build()).build());
+ attributeMap.put(":listVal2", AttributeValue.builder().l(
+ AttributeValue.builder().s("e").build(),
+ AttributeValue.builder().s("f").build()).build());
+ attributeMap.put(":emptyList",
+ AttributeValue.builder().l(Collections.emptyList()).build());
+ attributeMap.put(":one", AttributeValue.builder().n("1").build());
+ attributeMap.put(":notList", AttributeValue.builder().s("scalar").build());
+ return DdbAttributesToBsonDocument.getRawBsonDocument(attributeMap);
+ }
+
private static BsonDocument getComparisonValuesMap() {
Map<String, AttributeValue> attributeMap = new HashMap<>();
attributeMap.put(":aCol", AttributeValue.builder().ns("1", "2", "3").build());
diff --git a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/UpdateExpressionValidationIT.java b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/UpdateExpressionValidationIT.java
index e874125..c3752e1 100644
--- a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/UpdateExpressionValidationIT.java
+++ b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/UpdateExpressionValidationIT.java
@@ -529,6 +529,153 @@ public class UpdateExpressionValidationIT {
"DELETE from set same type");
}
+ // list_append Operation Tests
+
+ /**
+ * list_append against an existing list attribute path with a literal list operand.
+ */
+ @Test(timeout = 120000)
+ public void testListAppendExistingList() {
+ Map<String, AttributeValue> item = getKey();
+ item.put("a", AttributeValue.builder()
+ .l(AttributeValue.builder().s("x").build()).build());
+ putTestItem(tableName, item);
+
+ Map<String, AttributeValue> expressionAttributeValues = new HashMap<>();
+ expressionAttributeValues.put(":val", AttributeValue.builder().l(
+ AttributeValue.builder().s("y").build(),
+ AttributeValue.builder().s("z").build()).build());
+
+ testUpdateExpressionSuccess(tableName, "SET a = list_append(a, :val)", null,
+ expressionAttributeValues, "list_append append literal to existing list");
+ }
+
+ /**
+ * list_append with if_not_exists fallback when the target attribute is missing.
+ */
+ @Test(timeout = 120000)
+ public void testListAppendIfNotExistsMissing() {
+ Map<String, AttributeValue> item = getKey();
+ putTestItem(tableName, item);
+
+ Map<String, AttributeValue> expressionAttributeValues = new HashMap<>();
+ expressionAttributeValues.put(":empty",
+ AttributeValue.builder().l(java.util.Collections.emptyList()).build());
+ expressionAttributeValues.put(":val", AttributeValue.builder().l(
+ AttributeValue.builder().s("y").build()).build());
+
+ testUpdateExpressionSuccess(tableName,
+ "SET a = list_append(if_not_exists(a, :empty), :val)", null,
+ expressionAttributeValues, "list_append create-or-append on missing attribute");
+ }
+
+ /**
+ * if_not_exists as the SECOND operand (literal list first). This is the prepend
+ * variant of the canonical create-or-append pattern.
+ */
+ @Test(timeout = 120000)
+ public void testListAppendLiteralAndIfNotExists() {
+ Map<String, AttributeValue> item = getKey();
+ putTestItem(tableName, item);
+
+ Map<String, AttributeValue> expressionAttributeValues = new HashMap<>();
+ expressionAttributeValues.put(":empty",
+ AttributeValue.builder().l(java.util.Collections.emptyList()).build());
+ expressionAttributeValues.put(":val", AttributeValue.builder().l(
+ AttributeValue.builder().s("y").build()).build());
+
+ testUpdateExpressionSuccess(tableName,
+ "SET a = list_append(:val, if_not_exists(a, :empty))", null,
+ expressionAttributeValues, "list_append with literal first and if_not_exists second");
+ }
+
+ /**
+ * Both operands of list_append are literal value placeholders (no paths, no
+ * if_not_exists). Result should be the simple concatenation of the two literals.
+ */
+ @Test(timeout = 120000)
+ public void testListAppendBothLiterals() {
+ Map<String, AttributeValue> item = getKey();
+ putTestItem(tableName, item);
+
+ Map<String, AttributeValue> expressionAttributeValues = new HashMap<>();
+ expressionAttributeValues.put(":v1", AttributeValue.builder().l(
+ AttributeValue.builder().s("a").build(),
+ AttributeValue.builder().s("b").build()).build());
+ expressionAttributeValues.put(":v2", AttributeValue.builder().l(
+ AttributeValue.builder().s("c").build(),
+ AttributeValue.builder().s("d").build()).build());
+
+ testUpdateExpressionSuccess(tableName,
+ "SET combined = list_append(:v1, :v2)", null,
+ expressionAttributeValues, "list_append with two literal-list operands");
+ }
+
+ /**
+ * Both operands of list_append are if_not_exists.
+ */
+ @Test(timeout = 120000)
+ public void testListAppendBothOperandsAreIfNotExists() {
+ // Seed item: 'leftList' exists, 'rightList' does not
+ Map<String, AttributeValue> item = getKey();
+ item.put("leftList", AttributeValue.builder()
+ .l(AttributeValue.builder().s("x").build()).build());
+ putTestItem(tableName, item);
+
+ Map<String, AttributeValue> expressionAttributeValues = new HashMap<>();
+ expressionAttributeValues.put(":emptyLeft",
+ AttributeValue.builder().l(java.util.Collections.emptyList()).build());
+ expressionAttributeValues.put(":emptyRight",
+ AttributeValue.builder().l(java.util.Collections.emptyList()).build());
+
+ testUpdateExpressionSuccess(tableName,
+ "SET merged = list_append("
+ + "if_not_exists(leftList, :emptyLeft), "
+ + "if_not_exists(rightList, :emptyRight))",
+ null, expressionAttributeValues,
+ "list_append with if_not_exists in both operand positions");
+ }
+
+ /**
+ * Nested list_append (list_append inside list_append) is not part of AWS DynamoDB's
+ * documented UpdateExpression grammar -- the only function permitted as an operand is
+ * if_not_exists. Both real DDB and the phoenix-adapter must reject it with HTTP 400.
+ */
+ @Test(timeout = 120000)
+ public void testListAppendNestedRejected() {
+ Map<String, AttributeValue> item = getKey();
+ item.put("a", AttributeValue.builder()
+ .l(AttributeValue.builder().s("x").build()).build());
+ putTestItem(tableName, item);
+
+ Map<String, AttributeValue> expressionAttributeValues = new HashMap<>();
+ expressionAttributeValues.put(":v1", AttributeValue.builder().l(
+ AttributeValue.builder().s("y").build()).build());
+ expressionAttributeValues.put(":v2", AttributeValue.builder().l(
+ AttributeValue.builder().s("z").build()).build());
+
+ testUpdateExpressionFailure(tableName,
+ "SET a = list_append(list_append(a, :v1), :v2)", null,
+ expressionAttributeValues, "nested list_append should be rejected");
+ }
+
+ /**
+ * list_append where the target path resolves to a non-list value should fail.
+ */
+ @Test(timeout = 120000)
+ public void testListAppendTargetNotList() {
+ Map<String, AttributeValue> item = getKey();
+ item.put("a", AttributeValue.builder().n("123").build());
+ putTestItem(tableName, item);
+
+ Map<String, AttributeValue> expressionAttributeValues = new HashMap<>();
+ expressionAttributeValues.put(":val", AttributeValue.builder().l(
+ AttributeValue.builder().s("y").build()).build());
+
+ testUpdateExpressionFailure(tableName, "SET a = list_append(a, :val)", null,
+ expressionAttributeValues, "list_append on attribute that is not a list");
+ }
+
private void putTestItem(String tableName, Map<String, AttributeValue> item) {
PutItemRequest putRequest =
PutItemRequest.builder().tableName(tableName).item(item).build();
diff --git a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/UpdateItemBaseTests.java b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/UpdateItemBaseTests.java
index e2ac382..e758688 100644
--- a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/UpdateItemBaseTests.java
+++ b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/UpdateItemBaseTests.java
@@ -210,6 +210,239 @@ public class UpdateItemBaseTests {
validateItem(tableName, key);
}
+ /**
+ * SET: list_append against an existing list attribute path, plus a parallel
+ * "create-or-append" using if_not_exists for a previously-missing list, plus a
+ * subsequent re-apply that exercises both the existing-path operand and a literal
+ * operand.
+ */
+ @Test(timeout = 120000)
+ public void testListAppend() {
+ final String tableName = testName.getMethodName().replaceAll("[\\[\\]]", "");
+ createTableAndPutItem(tableName, true);
+
+ Map<String, AttributeValue> key = getKey();
+
+ // First update: create NewList via if_not_exists, append literal items to it.
+ UpdateItemRequest.Builder uir1 = UpdateItemRequest.builder().tableName(tableName).key(key);
+ uir1.updateExpression(
+ "SET #newList = list_append(if_not_exists(#newList, :empty), :items1)");
+ Map<String, String> exprAttrNames = new HashMap<>();
+ exprAttrNames.put("#newList", "NewList");
+ uir1.expressionAttributeNames(exprAttrNames);
+ Map<String, AttributeValue> exprAttrVal1 = new HashMap<>();
+ exprAttrVal1.put(":empty", AttributeValue.builder().l(java.util.Collections.emptyList()).build());
+ exprAttrVal1.put(":items1", AttributeValue.builder().l(
+ AttributeValue.builder().s("a").build(),
+ AttributeValue.builder().s("b").build()).build());
+ uir1.expressionAttributeValues(exprAttrVal1);
+ dynamoDbClient.updateItem(uir1.build());
+ phoenixDBClientV2.updateItem(uir1.build());
+
+ validateItem(tableName, key);
+
+ // Second update: append more items to the now-existing list (path operand + literal).
+ UpdateItemRequest.Builder uir2 = UpdateItemRequest.builder().tableName(tableName).key(key);
+ uir2.updateExpression("SET #newList = list_append(#newList, :items2)");
+ uir2.expressionAttributeNames(exprAttrNames);
+ Map<String, AttributeValue> exprAttrVal2 = new HashMap<>();
+ exprAttrVal2.put(":items2", AttributeValue.builder().l(
+ AttributeValue.builder().s("c").build(),
+ AttributeValue.builder().s("d").build()).build());
+ uir2.expressionAttributeValues(exprAttrVal2);
+ dynamoDbClient.updateItem(uir2.build());
+ phoenixDBClientV2.updateItem(uir2.build());
+
+ validateItem(tableName, key);
+
+ // Third update: prepend (literal first, path second).
+ // After this, the final list order is [z, a, b, c, d] — the literal is prepended.
+ UpdateItemRequest.Builder uir3 = UpdateItemRequest.builder().tableName(tableName).key(key);
+ uir3.updateExpression("SET #newList = list_append(:items3, #newList)");
+ uir3.expressionAttributeNames(exprAttrNames);
+ Map<String, AttributeValue> exprAttrVal3 = new HashMap<>();
+ exprAttrVal3.put(":items3", AttributeValue.builder().l(
+ AttributeValue.builder().s("z").build()).build());
+ uir3.expressionAttributeValues(exprAttrVal3);
+ dynamoDbClient.updateItem(uir3.build());
+ phoenixDBClientV2.updateItem(uir3.build());
+
+ validateItem(tableName, key);
+ }
+
+ /**
+ * Create-or-append where the same operation is invoked twice with an overlapping
+ * payload, leaving duplicates in the resulting list (list_append preserves order
+ * and does not de-duplicate).
+ */
+ @Test(timeout = 120000)
+ public void testListAppendCreateOrAppendWithDuplicates() {
+ final String tableName = testName.getMethodName().replaceAll("[\\[\\]]", "");
+ createTableAndPutItem(tableName, true);
+
+ Map<String, AttributeValue> key = getKey();
+
+ UpdateItemRequest.Builder uir = UpdateItemRequest.builder().tableName(tableName).key(key);
+ uir.updateExpression(
+ "SET #newList = list_append(if_not_exists(#newList, :empty), :items)");
+ Map<String, String> exprAttrNames = new HashMap<>();
+ exprAttrNames.put("#newList", "NewList");
+ uir.expressionAttributeNames(exprAttrNames);
+ Map<String, AttributeValue> exprAttrVal = new HashMap<>();
+ exprAttrVal.put(":empty",
+ AttributeValue.builder().l(java.util.Collections.emptyList()).build());
+ // payload contains an internal duplicate ("a" twice) and is then re-applied to itself.
+ exprAttrVal.put(":items", AttributeValue.builder().l(
+ AttributeValue.builder().s("a").build(),
+ AttributeValue.builder().s("b").build(),
+ AttributeValue.builder().s("a").build()).build());
+ uir.expressionAttributeValues(exprAttrVal);
+
+ // First apply: NewList does not exist; create it via if_not_exists fallback and
+ // append the literal payload, yielding [a, b, a].
+ dynamoDbClient.updateItem(uir.build());
+ phoenixDBClientV2.updateItem(uir.build());
+ validateItem(tableName, key);
+
+ // Second apply: NewList now exists; append the same literal payload again,
+ // yielding [a, b, a, a, b, a]. Confirms list_append preserves duplicates and
+ // is not idempotent.
+ UpdateItemResponse dynamoResult = dynamoDbClient.updateItem(
+ uir.returnValues(ALL_NEW).build());
+ UpdateItemResponse phoenixResult = phoenixDBClientV2.updateItem(
+ uir.returnValues(ALL_NEW).build());
+ Assert.assertEquals(dynamoResult.attributes(), phoenixResult.attributes());
+
+ AttributeValue finalList = phoenixResult.attributes().get("NewList");
+ Assert.assertNotNull("NewList should be present in returned attributes", finalList);
+ Assert.assertEquals("list_append preserves order and duplicates",
+ java.util.Arrays.asList("a", "b", "a", "a", "b", "a"),
+ finalList.l().stream().map(AttributeValue::s).collect(java.util.stream.Collectors.toList()));
+
+ validateItem(tableName, key);
+ }
+
+ /**
+ * Prepend variant: list_append with a literal list as the first operand and an
+ * if_not_exists fallback as the second operand. New items end up before the
+ * existing-or-fallback list.
+ */
+ @Test(timeout = 120000)
+ public void testListAppendLiteralAndIfNotExists() {
+ final String tableName = testName.getMethodName().replaceAll("[\\[\\]]", "");
+ createTableAndPutItem(tableName, true);
+
+ Map<String, AttributeValue> key = getKey();
+
+ // First apply: NewList does not exist, so if_not_exists falls back to []. The
+ // result is the literal :items prepended to that empty list -> [a, b].
+ UpdateItemRequest.Builder uir1 = UpdateItemRequest.builder().tableName(tableName).key(key);
+ uir1.updateExpression(
+ "SET #newList = list_append(:items1, if_not_exists(#newList, :empty))");
+ Map<String, String> exprAttrNames = new HashMap<>();
+ exprAttrNames.put("#newList", "NewList");
+ uir1.expressionAttributeNames(exprAttrNames);
+ Map<String, AttributeValue> exprAttrVal1 = new HashMap<>();
+ exprAttrVal1.put(":empty",
+ AttributeValue.builder().l(java.util.Collections.emptyList()).build());
+ exprAttrVal1.put(":items1", AttributeValue.builder().l(
+ AttributeValue.builder().s("a").build(),
+ AttributeValue.builder().s("b").build()).build());
+ uir1.expressionAttributeValues(exprAttrVal1);
+ dynamoDbClient.updateItem(uir1.build());
+ phoenixDBClientV2.updateItem(uir1.build());
+ validateItem(tableName, key);
+
+ // Second apply: NewList now exists; literal :items2 is prepended in front of it.
+ // Result: [c, d, a, b].
+ UpdateItemRequest.Builder uir2 = UpdateItemRequest.builder().tableName(tableName).key(key);
+ uir2.updateExpression(
+ "SET #newList = list_append(:items2, if_not_exists(#newList, :empty))");
+ uir2.expressionAttributeNames(exprAttrNames);
+ Map<String, AttributeValue> exprAttrVal2 = new HashMap<>();
+ exprAttrVal2.put(":empty",
+ AttributeValue.builder().l(java.util.Collections.emptyList()).build());
+ exprAttrVal2.put(":items2", AttributeValue.builder().l(
+ AttributeValue.builder().s("c").build(),
+ AttributeValue.builder().s("d").build()).build());
+ uir2.expressionAttributeValues(exprAttrVal2);
+ UpdateItemResponse dynamoResult = dynamoDbClient.updateItem(uir2.returnValues(ALL_NEW).build());
+ UpdateItemResponse phoenixResult = phoenixDBClientV2.updateItem(uir2.returnValues(ALL_NEW).build());
+ Assert.assertEquals(dynamoResult.attributes(), phoenixResult.attributes());
+
+ AttributeValue finalList = phoenixResult.attributes().get("NewList");
+ Assert.assertNotNull("NewList should be present in returned attributes", finalList);
+ Assert.assertEquals("literal-first list_append prepends to existing list",
+ java.util.Arrays.asList("c", "d", "a", "b"),
+ finalList.l().stream().map(AttributeValue::s)
+ .collect(java.util.stream.Collectors.toList()));
+
+ validateItem(tableName, key);
+ }
+
+ /**
+ * Create-or-append a list while atomically incrementing a counter in the same SET
+ * clause, gated on an updateCounter condition expression.
+ */
+ @Test(timeout = 120000)
+ public void testListAppendWithIfNotExistsAndCounterIncrement() {
+ final String tableName = testName.getMethodName().replaceAll("[\\[\\]]", "");
+ createTableAndPutItem(tableName, true);
+
+ Map<String, AttributeValue> key = getKey();
+
+ // Iteration 1: events does not exist; updateCounter starts unset, so initialize it
+ // first via a plain SET so the condition expression can reference it deterministically.
+ UpdateItemRequest.Builder seed = UpdateItemRequest.builder().tableName(tableName).key(key);
+ seed.updateExpression("SET updateCounter = :zero");
+ Map<String, AttributeValue> seedVals = new HashMap<>();
+ seedVals.put(":zero", AttributeValue.builder().n("0").build());
+ seed.expressionAttributeValues(seedVals);
+ dynamoDbClient.updateItem(seed.build());
+ phoenixDBClientV2.updateItem(seed.build());
+
+ // Iteration 2: list_append(if_not_exists(events, :empty), :evts1) + counter increment,
+ // gated on updateCounter = 0.
+ UpdateItemRequest.Builder uir1 = UpdateItemRequest.builder().tableName(tableName).key(key);
+ uir1.updateExpression(
+ "SET events = list_append(if_not_exists(events, :empty_list), :new_evts), "
+ + "updateCounter = updateCounter + :one");
+ uir1.conditionExpression("updateCounter = :expected");
+ Map<String, AttributeValue> exprAttrVal1 = new HashMap<>();
+ exprAttrVal1.put(":empty_list",
+ AttributeValue.builder().l(java.util.Collections.emptyList()).build());
+ exprAttrVal1.put(":new_evts", AttributeValue.builder().l(
+ AttributeValue.builder().s("ev1").build(),
+ AttributeValue.builder().s("ev2").build()).build());
+ exprAttrVal1.put(":one", AttributeValue.builder().n("1").build());
+ exprAttrVal1.put(":expected", AttributeValue.builder().n("0").build());
+ uir1.expressionAttributeValues(exprAttrVal1);
+ dynamoDbClient.updateItem(uir1.build());
+ phoenixDBClientV2.updateItem(uir1.build());
+
+ validateItem(tableName, key);
+
+ // Iteration 3: append more events; gated on the post-iter-2 counter value.
+ UpdateItemRequest.Builder uir2 = UpdateItemRequest.builder().tableName(tableName).key(key);
+ // Extra whitespace around closing paren and comma to verify parser tolerance.
+ uir2.updateExpression(
+ "SET events = list_append(if_not_exists(events, :empty_list), :new_evts ) , "
+ + "updateCounter = updateCounter + :one");
+ uir2.conditionExpression("updateCounter = :expected");
+ Map<String, AttributeValue> exprAttrVal2 = new HashMap<>();
+ exprAttrVal2.put(":empty_list",
+ AttributeValue.builder().l(java.util.Collections.emptyList()).build());
+ exprAttrVal2.put(":new_evts", AttributeValue.builder().l(
+ AttributeValue.builder().s("ev3").build()).build());
+ exprAttrVal2.put(":one", AttributeValue.builder().n("1").build());
+ exprAttrVal2.put(":expected", AttributeValue.builder().n("1").build());
+ uir2.expressionAttributeValues(exprAttrVal2);
+ dynamoDbClient.updateItem(uir2.build());
+ phoenixDBClientV2.updateItem(uir2.build());
+
+ validateItem(tableName, key);
+ }
+
/**
* REMOVE - Removes one or more attributes from an item.
*/
diff --git a/phoenix-ddb-utils/src/main/java/org/apache/phoenix/ddb/bson/UpdateExpressionDdbToBson.java b/phoenix-ddb-utils/src/main/java/org/apache/phoenix/ddb/bson/UpdateExpressionDdbToBson.java
index 5db52d6..aa404b4 100644
--- a/phoenix-ddb-utils/src/main/java/org/apache/phoenix/ddb/bson/UpdateExpressionDdbToBson.java
+++ b/phoenix-ddb-utils/src/main/java/org/apache/phoenix/ddb/bson/UpdateExpressionDdbToBson.java
@@ -37,11 +37,13 @@ public class UpdateExpressionDdbToBson {
private static final String addRegExPattern = "ADD\\s+(.+?)(?=\\s+(SET|REMOVE|DELETE)\\b|$)";
private static final String deleteRegExPattern = "DELETE\\s+(.+?)(?=\\s+(SET|REMOVE|ADD)\\b|$)";
private static final String ifNotExistsPattern = "if_not_exists\\s*\\(\\s*([^,]+)\\s*,\\s*([^)]+)\\s*\\)";
+ private static final String listAppendPattern = "^list_append\\s*\\(\\s*(.+?)\\s*\\)$";
private static final Pattern SET_PATTERN = Pattern.compile(setRegExPattern);
private static final Pattern REMOVE_PATTERN = Pattern.compile(removeRegExPattern);
private static final Pattern ADD_PATTERN = Pattern.compile(addRegExPattern);
private static final Pattern DELETE_PATTERN = Pattern.compile(deleteRegExPattern);
private static final Pattern IF_NOT_EXISTS_PATTERN = Pattern.compile(ifNotExistsPattern);
+ private static final Pattern LIST_APPEND_PATTERN = Pattern.compile(listAppendPattern);
public static BsonDocument getBsonDocumentForUpdateExpression(
final String updateExpression,
@@ -75,15 +77,17 @@ public class UpdateExpressionDdbToBson {
BsonDocument bsonDocument = new BsonDocument();
if (!setString.isEmpty()) {
BsonDocument setBsonDoc = new BsonDocument();
- // split by comma only if comma is not within ()
- String[] setExpressions = setString.split(",(?![^()]*+\\))");
+ // Split on top-level commas only (commas inside any depth of parens are skipped).
+ String[] setExpressions = splitTopLevelCommas(setString);
for (int i = 0; i < setExpressions.length; i++) {
String setExpression = setExpressions[i].trim();
String[] keyVal = setExpression.split("\\s*=\\s*");
if (keyVal.length == 2) {
String attributeKey = keyVal[0].trim();
String attributeVal = keyVal[1].trim();
- if (attributeVal.contains("+") || attributeVal.contains("-")) {
+ if (attributeVal.startsWith("list_append")) {
+ setBsonDoc.put(attributeKey, getListAppendDoc(attributeVal, comparisonValue));
+ } else if (attributeVal.contains("+") || attributeVal.contains("-")) {
setBsonDoc.put(attributeKey, getArithmeticDoc(attributeVal, comparisonValue));
} else if (attributeVal.startsWith("if_not_exists")) {
setBsonDoc.put(attributeKey, getIfNotExistsDoc(attributeVal, comparisonValue));
@@ -162,7 +166,7 @@ public class UpdateExpressionDdbToBson {
operand = operand.trim();
if (operand.startsWith("if_not_exists")) {
bsonOperands.add(getIfNotExistsDoc(operand, comparisonValue));
- } else if (operand.startsWith(":") || operand.startsWith("$") || operand.startsWith("#")) {
+ } else if (operand.startsWith(":")) {
BsonValue bsonValue = comparisonValue.get(operand);
if (!bsonValue.isNumber() && !bsonValue.isDecimal128()) {
throw new IllegalArgumentException(
@@ -192,4 +196,87 @@ public class UpdateExpressionDdbToBson {
throw new RuntimeException("Invalid format for if_not_exists(path, value)");
}
}
+
+ private static BsonDocument getListAppendDoc(String expr, BsonDocument comparisonValue) {
+ Matcher m = LIST_APPEND_PATTERN.matcher(expr);
+ if (!m.find()) {
+ throw new IllegalArgumentException(
+ "Invalid format for list_append(operand1, operand2): " + expr);
+ }
+ String inner = m.group(1).trim();
+ // Split on top-level commas only (commas inside any depth of parens are skipped).
+ String[] operands = splitTopLevelCommas(inner);
+ if (operands.length != 2) {
+ throw new IllegalArgumentException(
+ "list_append requires exactly 2 operands, got " + operands.length
+ + " in: " + expr);
+ }
+ BsonArray bsonOperands = new BsonArray();
+ for (String operand : operands) {
+ bsonOperands.add(resolveListAppendOperand(operand.trim(), comparisonValue));
+ }
+ BsonDocument listAppendDoc = new BsonDocument();
+ listAppendDoc.put("$LIST_APPEND", bsonOperands);
+ return listAppendDoc;
+ }
+
+ private static BsonValue resolveListAppendOperand(String operand,
+ BsonDocument comparisonValue) {
+ if (operand.startsWith("if_not_exists")) {
+ BsonDocument ifNotExistsDoc = getIfNotExistsDoc(operand, comparisonValue);
+ // Validate that the fallback value inside if_not_exists is a list, matching
+ // the client-side validation already done for literal placeholders.
+ BsonDocument inner = ifNotExistsDoc.getDocument("$IF_NOT_EXISTS");
+ BsonValue fallback = inner.values().iterator().next();
+ if (fallback != null && !fallback.isNull() && !fallback.isArray()) {
+ throw new IllegalArgumentException(
+ "if_not_exists fallback inside list_append must resolve to a List type"
+ + " but got: " + operand);
+ }
+ return ifNotExistsDoc;
+ } else if (operand.matches("^list_append\\s*\\(.*")) {
+ throw new IllegalArgumentException(
+ "Nested list_append is not supported as an operand: " + operand);
+ } else if (operand.startsWith(":")) {
+ BsonValue bsonValue = comparisonValue.get(operand);
+ if (bsonValue == null) {
+ throw new IllegalArgumentException(
+ "Operand " + operand
+ + " not found in expression attribute values for list_append");
+ }
+ if (!bsonValue.isArray()) {
+ throw new IllegalArgumentException(
+ "Operand " + operand + " for list_append must resolve to a List type");
+ }
+ return bsonValue;
+ } else {
+ // bare attribute path (e.g. myList, nested.queue, matrix[0])
+ return new BsonString(operand);
+ }
+ }
+
+ /**
+ * Split the given string on commas that are at paren-depth 0. Commas inside any
+ * level of {@code (...)} are preserved. This correctly handles arbitrarily nested
+ * function calls such as {@code list_append(:v, if_not_exists(a, :empty))} and
+ * {@code if_not_exists(a, :v), b = b + :n} at the SET-clause level.
+ */
+ private static String[] splitTopLevelCommas(String s) {
+ java.util.List<String> parts = new java.util.ArrayList<>();
+ int depth = 0;
+ int last = 0;
+ for (int i = 0; i < s.length(); i++) {
+ char c = s.charAt(i);
+ if (c == '(') {
+ depth++;
+ } else if (c == ')') {
+ depth--;
+ } else if (c == ',' && depth == 0) {
+ parts.add(s.substring(last, i));
+ last = i + 1;
+ }
+ }
+ parts.add(s.substring(last));
+ return parts.toArray(new String[0]);
+ }
}