You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by di...@apache.org on 2025/08/20 17:35:50 UTC
(atlas) branch master updated: ATLAS-5084: provide a way to transform hdfs paths during atlas import (#416)
This is an automated email from the ASF dual-hosted git repository.
dishatalreja pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 2eafacd49 ATLAS-5084: provide a way to transform hdfs paths during atlas import (#416)
2eafacd49 is described below
commit 2eafacd49ebc1acb8aebe7d33cccc50ebde3536b
Author: Disha Talreja <37...@users.noreply.github.com>
AuthorDate: Wed Aug 20 10:35:45 2025 -0700
ATLAS-5084: provide a way to transform hdfs paths during atlas import (#416)
---
.../atlas/repository/impexp/ImportTransformer.java | 13 +++++++++----
.../repository/impexp/AtlasImportRequestTest.java | 4 ++--
.../atlas/repository/impexp/ImportServiceTest.java | 4 ++--
.../impexp/ImportTransformerJSONTest.java | 2 +-
.../repository/impexp/ImportTransformerTest.java | 12 ++++++------
.../impexp/ImportTransformsShaperTest.java | 2 +-
.../repository/impexp/ImportTransformsTest.java | 22 +++++++++++-----------
.../atlas/repository/impexp/ZipSourceTest.java | 2 +-
8 files changed, 33 insertions(+), 28 deletions(-)
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTransformer.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTransformer.java
index c472b9444..7cb5948bc 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTransformer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTransformer.java
@@ -31,7 +31,7 @@ import java.util.Map;
import java.util.Objects;
public abstract class ImportTransformer {
- private static final String TRANSFORMER_PARAMETER_SEPARATOR = "\\:";
+ private static final String TRANSFORMER_PARAMETER_SEPARATOR = "~";
private static final String TRANSFORMER_NAME_ADD = "add";
private static final String TRANSFORMER_NAME_CLEAR_ATTR = "clearAttrValue";
@@ -49,6 +49,11 @@ public abstract class ImportTransformer {
}
public static ImportTransformer getTransformer(String transformerSpec) throws AtlasBaseException {
+ if (transformerSpec.contains(":") && !transformerSpec.contains(TRANSFORMER_PARAMETER_SEPARATOR)) {
+ throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE,
+ "Invalid transformer specification. The parameter separator ':' is no longer supported. Use '~' instead. Transformer specification: " + transformerSpec);
+ }
+
String[] params = StringUtils.split(transformerSpec, TRANSFORMER_PARAMETER_SEPARATOR);
String key = (params == null || params.length < 1) ? transformerSpec : params[0];
@@ -66,13 +71,13 @@ public abstract class ImportTransformer {
} else if (key.equals(TRANSFORMER_NAME_UPPERCASE)) {
ret = new Uppercase();
} else if (key.equals(TRANSFORMER_NAME_REMOVE_CLASSIFICATION)) {
- String name = (params == null || params.length < 1) ? "" : StringUtils.join(params, ":", 1, params.length);
+ String name = (params == null || params.length < 1) ? "" : StringUtils.join(params, "~", 1, params.length);
ret = new RemoveClassification(name);
} else if (key.equals(TRANSFORMER_NAME_ADD)) {
- String name = (params == null || params.length < 1) ? "" : StringUtils.join(params, ":", 1, params.length);
+ String name = (params == null || params.length < 1) ? "" : StringUtils.join(params, "~", 1, params.length);
ret = new AddValueToAttribute(name);
} else if (key.equals(TRANSFORMER_NAME_CLEAR_ATTR)) {
- String name = (params == null || params.length < 1) ? "" : StringUtils.join(params, ":", 1, params.length);
+ String name = (params == null || params.length < 1) ? "" : StringUtils.join(params, "~", 1, params.length);
ret = new ClearAttributes(name);
} else if (key.equals(TRANSFORMER_SET_DELETED)) {
ret = new SetDeleted();
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java
index 0d04d6724..7c91ef441 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java
@@ -102,7 +102,7 @@ public class AtlasImportRequestTest {
@Test
public void serializeAtlasImportRequstFromJsonWith1Transform() {
- String jsonData = "{ \"options\": { \"transforms\": \"{ \\\"hive_db\\\": { \\\"qualifiedName\\\": [ \\\"replace:@cl1:@cl2\\\" ] } }\" } }";
+ String jsonData = "{ \"options\": { \"transforms\": \"{ \\\"hive_db\\\": { \\\"qualifiedName\\\": [ \\\"replace~@cl1~@cl2\\\" ] } }\" } }";
AtlasImportRequest request = AtlasType.fromJson(jsonData, AtlasImportRequest.class);
@@ -123,7 +123,7 @@ public class AtlasImportRequestTest {
@Test
public void serializeAtlasImportRequstFromJson() {
- String jsonData = "{ \"options\": { \"transforms\": \"{ \\\"hive_db\\\": { \\\"qualifiedName\\\": [ \\\"replace:@cl1:@cl2\\\" ] }, \\\"hive_table\\\": { \\\"qualifiedName\\\": [ \\\"lowercase\\\", \\\"replace:@cl1:@cl2\\\" ] } }\" } } }";
+ String jsonData = "{ \"options\": { \"transforms\": \"{ \\\"hive_db\\\": { \\\"qualifiedName\\\": [ \\\"replace~@cl1~@cl2\\\" ] }, \\\"hive_table\\\": { \\\"qualifiedName\\\": [ \\\"lowercase\\\", \\\"replace~@cl1~@cl2\\\" ] } }\" } } }";
AtlasImportRequest request = AtlasType.fromJson(jsonData, AtlasImportRequest.class);
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
index dbf84dbde..66cada7de 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
@@ -844,7 +844,7 @@ public class ImportServiceTest extends AtlasTestBase {
loadBaseModel();
loadHiveModel();
- String transformJSON = "{ \"Asset\": { \"qualifiedName\":[ \"lowercase\", \"replace:@cl1:@cl2\" ] } }";
+ String transformJSON = "{ \"Asset\": { \"qualifiedName\":[ \"lowercase\", \"replace~@cl1~@cl2\" ] } }";
ZipSource zipSource = new ZipSource(inputStream);
importService.setImportTransform(zipSource, transformJSON);
ImportTransforms importTransforms = zipSource.getImportTransform();
@@ -859,7 +859,7 @@ public class ImportServiceTest extends AtlasTestBase {
loadBaseModel();
loadHiveModel();
- String transformJSON = "{ \"Asset\": { \"qualifiedName\":[ \"replace:@cl1:@cl2\" ] }, \"hive_table\": { \"qualifiedName\":[ \"lowercase\" ] } }";
+ String transformJSON = "{ \"Asset\": { \"qualifiedName\":[ \"replace~@cl1~@cl2\" ] }, \"hive_table\": { \"qualifiedName\":[ \"lowercase\" ] } }";
ZipSource zipSource = new ZipSource(inputStream);
importService.setImportTransform(zipSource, transformJSON);
ImportTransforms importTransforms = zipSource.getImportTransform();
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerJSONTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerJSONTest.java
index 4a99f440c..5d34208f8 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerJSONTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerJSONTest.java
@@ -28,7 +28,7 @@ public class ImportTransformerJSONTest {
public void createAtlasImportTransformFromJson() throws Exception {
String hiveTableType = "hive_table";
String qualifiedName = "qualifiedName";
- String jsonTransforms = "{ \"hive_table\": { \"qualifiedName\":[ \"lowercase\", \"replace:@cl1:@cl2\" ] } }";
+ String jsonTransforms = "{ \"hive_table\": { \"qualifiedName\":[ \"lowercase\", \"replace~@cl1~@cl2\" ] } }";
ImportTransforms transforms = ImportTransforms.fromJson(jsonTransforms);
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerTest.java
index 5a2e5c761..819cf2f9c 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerTest.java
@@ -30,7 +30,7 @@ public class ImportTransformerTest {
String param1 = "@cl1";
String param2 = "@cl2";
- ImportTransformer e = ImportTransformer.getTransformer(String.format("%s:%s:%s", "replace", param1, param2));
+ ImportTransformer e = ImportTransformer.getTransformer(String.format("%s~%s~%s", "replace", param1, param2));
assertTrue(e instanceof ImportTransformer.Replace);
assertEquals(((ImportTransformer.Replace) e).getToFindStr(), param1);
@@ -42,8 +42,8 @@ public class ImportTransformerTest {
String param1 = "@cl1";
String param2 = "@cl2";
- ImportTransformer e1 = ImportTransformer.getTransformer(String.format("%s:%s:%s", "replace", param1, param2));
- ImportTransformer e2 = ImportTransformer.getTransformer("replace:tt1:tt2");
+ ImportTransformer e1 = ImportTransformer.getTransformer(String.format("%s~%s~%s", "replace", param1, param2));
+ ImportTransformer e2 = ImportTransformer.getTransformer(String.format("replace~tt1~tt2"));
assertTrue(e1 instanceof ImportTransformer.Replace);
assertEquals(((ImportTransformer.Replace) e1).getToFindStr(), param1);
@@ -56,7 +56,7 @@ public class ImportTransformerTest {
@Test
public void createWithDefaultParameters() throws AtlasBaseException {
- ImportTransformer e1 = ImportTransformer.getTransformer("replace:@cl1");
+ ImportTransformer e1 = ImportTransformer.getTransformer("replace~@cl1");
ImportTransformer e2 = ImportTransformer.getTransformer("replace");
assertTrue(e1 instanceof ImportTransformer.Replace);
@@ -92,7 +92,7 @@ public class ImportTransformerTest {
@Test
public void applyReplaceTransformer1() throws AtlasBaseException {
- ImportTransformer e = ImportTransformer.getTransformer("replace:@cl1:@cl2");
+ ImportTransformer e = ImportTransformer.getTransformer("replace~@cl1~@cl2");
assertEquals(e.apply("@cl1"), "@cl2");
assertEquals(e.apply("default@cl1"), "default@cl2");
@@ -105,7 +105,7 @@ public class ImportTransformerTest {
@Test
public void applyReplaceTransformer2() throws AtlasBaseException {
- ImportTransformer e = ImportTransformer.getTransformer("replace:@cl1");
+ ImportTransformer e = ImportTransformer.getTransformer("replace~@cl1");
assertEquals(e.apply("@cl1"), "");
assertEquals(e.apply("default@cl1"), "default");
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsShaperTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsShaperTest.java
index 54fc4e002..f9f1dcc05 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsShaperTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsShaperTest.java
@@ -100,7 +100,7 @@ public class ImportTransformsShaperTest extends AtlasTestBase {
private AtlasImportRequest getImporRequest() {
AtlasImportRequest request = new AtlasImportRequest();
- request.getOptions().put("transforms", "{ \"Referenceable\": { \"*\":[ \"addClassification:REPLICATED\" ] } }");
+ request.getOptions().put("transforms", "{ \"Referenceable\": { \"*\":[ \"addClassification~REPLICATED\" ] } }");
return request;
}
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsTest.java
index 200bcad86..48ca56bc6 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsTest.java
@@ -47,16 +47,16 @@ public class ImportTransformsTest {
private final String lowerCaseCL1 = "@cl1";
private final String lowerCaseCL2 = "@cl2";
- private final String jsonLowerCaseReplace = "{ \"hive_table\": { \"qualifiedName\":[ \"lowercase\", \"replace:@cl1:@cl2\" ] } }";
- private final String jsonReplaceLowerCase = "{ \"Asset\": { \"qualifiedName\":[ \"replace:@cl1:@cl2\" ] }, \"hive_table\": { \"qualifiedName\":[ \"lowercase\", \"replace:@cl1:@cl2\" ] } }";
- private final String jsonReplaceRemoveClassification = "{ \"hive_table\": { \"qualifiedName\":[ \"replace:@%s:@%s\"], \"*\":[ \"removeClassification:%s_to_%s\" ] } }";
- private final String jsonReplaceAndAddAttrValue = "{ \"hive_table\": { \"qualifiedName\":[ \"replace:@%s:@%s\"], \"*\":[ \"add:%s=list:%s\" ] } }";
- private final String jsonSingleClearAttrValue = "{ \"hive_table\": { \"*\":[ \"clearAttrValue:replicatedToCluster\", \"clearAttrValue:replicatedFromCluster\" ] } }";
- private final String jsonMultipleClearAttrValue = "{ \"hive_table\": { \"*\":[ \"clearAttrValue:replicatedToCluster,replicatedFromCluster\" ] } }";
- private final String jsonSetDeleted = "{ \"hive_table\": { \"*\":[ \"setDeleted\" ] } }";
- private final String jsonAddClasification = "{ \"hive_table\": { \"*\":[ \"addClassification:REPLICATED\" ] } }";
- private final String jsonAddClasification2 = "{ \"hive_table\": { \"*\":[ \"addClassification:REPLICATED_2\" ] } }";
- private final String jsonAddClasificationScoped = "{ \"hive_column\": { \"*\":[ \"addClassification:REPLICATED_2:topLevel\" ] } }";
+ private final String jsonLowerCaseReplace = "{ \"hive_table\": { \"qualifiedName\":[ \"lowercase\", \"replace~@cl1~@cl2\" ] } }";
+ private final String jsonReplaceLowerCase = "{ \"Asset\": { \"qualifiedName\":[ \"replace~@cl1~@cl2\" ] }, \"hive_table\": { \"qualifiedName\":[ \"lowercase\", \"replace~@cl1~@cl2\" ] } }";
+ private final String jsonReplaceRemoveClassification = "{ \"hive_table\": { \"qualifiedName\":[ \"replace~@%s~@%s\"], \"*\":[ \"removeClassification~%s_to_%s\" ] } }";
+ private final String jsonReplaceAndAddAttrValue = "{ \"hive_table\": { \"qualifiedName\":[ \"replace~@%s~@%s\"], \"*\":[ \"add~%s=list:%s\" ] } }";
+ private final String jsonSingleClearAttrValue = "{ \"hive_table\": { \"*\":[ \"clearAttrValue~replicatedToCluster\", \"clearAttrValue~replicatedFromCluster\" ] } }";
+ private final String jsonMultipleClearAttrValue = "{ \"hive_table\": { \"*\":[ \"clearAttrValue~replicatedToCluster,replicatedFromCluster\" ] } }";
+ private final String jsonSetDeleted = "{ \"hive_table\": { \"*\":[ \"setDeleted\" ] } }";
+ private final String jsonAddClasification = "{ \"hive_table\": { \"*\":[ \"addClassification~REPLICATED\" ] } }";
+ private final String jsonAddClasification2 = "{ \"hive_table\": { \"*\":[ \"addClassification~REPLICATED_2\" ] } }";
+ private final String jsonAddClasificationScoped = "{ \"hive_column\": { \"*\":[ \"addClassification~REPLICATED_2~topLevel\" ] } }";
private ImportTransforms transform;
@@ -278,7 +278,7 @@ public class ImportTransformsTest {
Map<String, List<ImportTransformer>> tr = new HashMap<>();
List<ImportTransformer> trList = new ArrayList<>();
- trList.add(ImportTransformer.getTransformer(String.format("replace:%s:%s", lowerCaseCL1, lowerCaseCL2)));
+ trList.add(ImportTransformer.getTransformer(String.format("replace~%s~%s", lowerCaseCL1, lowerCaseCL2)));
tr.put(ATTR_NAME_QUALIFIED_NAME, trList);
transform.getTransforms().put("hive_column", tr);
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java
index 2a9e4f6bf..b948b45c4 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java
@@ -202,6 +202,6 @@ public class ZipSourceTest {
}
private ImportTransforms getTransformForHiveDB() {
- return ImportTransforms.fromJson("{ \"hive_db\": { \"qualifiedName\": [ \"replace:@cl1:@cl2\" ] } }");
+ return ImportTransforms.fromJson("{ \"hive_db\": { \"qualifiedName\": [ \"replace~@cl1~@cl2\" ] } }");
}
}