You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2017/11/05 20:31:48 UTC
[26/40] atlas git commit: ATLAS-2251: Remove TypeSystem and related
implementation, to avoid unncessary duplicate of type details in cache
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java b/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java
deleted file mode 100644
index 7c6b60b..0000000
--- a/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java
+++ /dev/null
@@ -1,428 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.atlas;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import org.apache.atlas.repository.MetadataRepository;
-import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
-import org.apache.atlas.services.MetadataService;
-import org.apache.atlas.type.AtlasTypeRegistry;
-import org.apache.atlas.typesystem.ITypedReferenceableInstance;
-import org.apache.atlas.typesystem.Referenceable;
-import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.json.TypesSerialization;
-import org.apache.atlas.typesystem.persistence.Id;
-import org.apache.atlas.typesystem.types.*;
-import org.apache.atlas.typesystem.types.utils.TypesUtil;
-import org.testng.annotations.Guice;
-
-import javax.inject.Inject;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-
-import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_INPUTS;
-import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS;
-
-/**
- * Base Class to set up hive types and instances for tests
- */
-@Guice(modules = TestModules.TestOnlyModule.class)
-public class BaseRepositoryTest {
-
- @Inject
- protected MetadataService metadataService;
-
- @Inject
- protected MetadataRepository repository;
-
-
- protected void setUp() throws Exception {
- //force graph initialization / built in type registration
- TestUtils.getGraph();
- setUpDefaultTypes();
- setUpTypes();
- TestUtils.getGraph().commit();
- new GraphBackedSearchIndexer(new AtlasTypeRegistry());
- TestUtils.resetRequestContext();
- setupInstances();
- TestUtils.getGraph().commit();
- TestUtils.dumpGraph(TestUtils.getGraph());
- }
-
- protected void tearDown() throws Exception {
- TypeSystem.getInstance().reset();
- }
-
- private void setUpTypes() throws Exception {
- TypesDef typesDef = createTypeDefinitions();
- String typesAsJSON = TypesSerialization.toJson(typesDef);
- metadataService.createType(typesAsJSON);
- }
-
- protected static final String DATABASE_TYPE = "hive_db";
- protected static final String HIVE_TABLE_TYPE = "hive_table";
- private static final String COLUMN_TYPE = "hive_column";
- private static final String HIVE_PROCESS_TYPE = "hive_process";
- private static final String STORAGE_DESC_TYPE = "StorageDesc";
- private static final String VIEW_TYPE = "View";
- private static final String PARTITION_TYPE = "hive_partition";
- protected static final String DATASET_SUBTYPE = "dataset_subtype";
-
- TypesDef createTypeDefinitions() {
- HierarchicalTypeDefinition<ClassType> dbClsDef = TypesUtil
- .createClassTypeDef(DATABASE_TYPE, null,
- TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
- attrDef("description", DataTypes.STRING_TYPE), attrDef("locationUri", DataTypes.STRING_TYPE),
- attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.LONG_TYPE));
-
- HierarchicalTypeDefinition<ClassType> columnClsDef = TypesUtil
- .createClassTypeDef(COLUMN_TYPE, null, attrDef("name", DataTypes.STRING_TYPE),
- attrDef("dataType", DataTypes.STRING_TYPE), attrDef("comment", DataTypes.STRING_TYPE));
-
- HierarchicalTypeDefinition<ClassType> storageDescClsDef = TypesUtil
- .createClassTypeDef(STORAGE_DESC_TYPE, null,
- attrDef("location", DataTypes.STRING_TYPE),
- attrDef("inputFormat", DataTypes.STRING_TYPE), attrDef("outputFormat", DataTypes.STRING_TYPE),
- attrDef("compressed", DataTypes.STRING_TYPE, Multiplicity.REQUIRED, false, null));
-
-
- HierarchicalTypeDefinition<ClassType> tblClsDef = TypesUtil
- .createClassTypeDef(HIVE_TABLE_TYPE, ImmutableSet.of("DataSet"),
- attrDef("owner", DataTypes.STRING_TYPE),
- attrDef("createTime", DataTypes.DATE_TYPE),
- attrDef("lastAccessTime", DataTypes.LONG_TYPE), attrDef("tableType", DataTypes.STRING_TYPE),
- attrDef("temporary", DataTypes.BOOLEAN_TYPE),
- new AttributeDefinition("db", DATABASE_TYPE, Multiplicity.REQUIRED, false, null),
- // todo - uncomment this, something is broken
- new AttributeDefinition("sd", STORAGE_DESC_TYPE, Multiplicity.REQUIRED, true, null),
- new AttributeDefinition("columns", DataTypes.arrayTypeName(COLUMN_TYPE),
- Multiplicity.COLLECTION, true, null));
-
- HierarchicalTypeDefinition<ClassType> loadProcessClsDef = TypesUtil
- .createClassTypeDef(HIVE_PROCESS_TYPE, ImmutableSet.of("Process"),
- attrDef("userName", DataTypes.STRING_TYPE), attrDef("startTime", DataTypes.LONG_TYPE),
- attrDef("endTime", DataTypes.LONG_TYPE),
- attrDef("queryText", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
- attrDef("queryPlan", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
- attrDef("queryId", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
- attrDef("queryGraph", DataTypes.STRING_TYPE, Multiplicity.REQUIRED));
-
- HierarchicalTypeDefinition<ClassType> viewClsDef = TypesUtil
- .createClassTypeDef(VIEW_TYPE, null, attrDef("name", DataTypes.STRING_TYPE),
- new AttributeDefinition("db", DATABASE_TYPE, Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("inputTables", DataTypes.arrayTypeName(HIVE_TABLE_TYPE),
- Multiplicity.COLLECTION, false, null));
-
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition("values", DataTypes.arrayTypeName(DataTypes.STRING_TYPE.getName()),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("table", HIVE_TABLE_TYPE, Multiplicity.REQUIRED, false, null),
- };
- HierarchicalTypeDefinition<ClassType> partClsDef =
- new HierarchicalTypeDefinition<>(ClassType.class, PARTITION_TYPE, null, null,
- attributeDefinitions);
-
- HierarchicalTypeDefinition<ClassType> datasetSubTypeClsDef = TypesUtil
- .createClassTypeDef(DATASET_SUBTYPE, ImmutableSet.of("DataSet"));
-
- HierarchicalTypeDefinition < TraitType > dimTraitDef = TypesUtil.createTraitTypeDef("Dimension", null);
-
- HierarchicalTypeDefinition<TraitType> factTraitDef = TypesUtil.createTraitTypeDef("Fact", null);
-
- HierarchicalTypeDefinition<TraitType> metricTraitDef = TypesUtil.createTraitTypeDef("Metric", null);
-
- HierarchicalTypeDefinition<TraitType> etlTraitDef = TypesUtil.createTraitTypeDef("ETL", null);
-
- HierarchicalTypeDefinition<TraitType> piiTraitDef = TypesUtil.createTraitTypeDef("PII", null);
-
- HierarchicalTypeDefinition<TraitType> jdbcTraitDef = TypesUtil.createTraitTypeDef("JdbcAccess", null);
-
- HierarchicalTypeDefinition<TraitType> logTraitDef = TypesUtil.createTraitTypeDef("Log Data", null);
-
- HierarchicalTypeDefinition<TraitType> isaKeywordTraitDef = TypesUtil.createTraitTypeDef("isa", null);
-
- return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
- ImmutableList.of(dimTraitDef, factTraitDef, piiTraitDef, metricTraitDef, etlTraitDef, jdbcTraitDef, logTraitDef,
- isaKeywordTraitDef),
- ImmutableList.of(dbClsDef, storageDescClsDef, columnClsDef, tblClsDef, loadProcessClsDef, viewClsDef, partClsDef, datasetSubTypeClsDef));
- }
-
- AttributeDefinition attrDef(String name, IDataType dT) {
- return attrDef(name, dT, Multiplicity.OPTIONAL, false, null);
- }
-
- AttributeDefinition attrDef(String name, IDataType dT, Multiplicity m) {
- return attrDef(name, dT, m, false, null);
- }
-
- AttributeDefinition attrDef(String name, IDataType dT, Multiplicity m, boolean isComposite,
- String reverseAttributeName) {
- Preconditions.checkNotNull(name);
- Preconditions.checkNotNull(dT);
- return new AttributeDefinition(name, dT.getName(), m, isComposite, reverseAttributeName);
- }
-
- private void setupInstances() throws Exception {
- Id salesDB = database("Sales", "Sales Database", "John ETL", "hdfs://host:8000/apps/warehouse/sales");
-
- Referenceable sd =
- storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(
- column("time_id", "int", "time id")));
-
- List<Referenceable> salesFactColumns = ImmutableList
- .of(column("time_id", "int", "time id"),
- column("product_id", "int", "product id"),
- column("customer_id", "int", "customer id", "PII"),
- column("sales", "double", "product id", "Metric"));
-
- Id salesFact = table("sales_fact", "sales fact table", salesDB, sd, "Joe", "Managed", salesFactColumns, "Fact");
-
- List<Referenceable> logFactColumns = ImmutableList
- .of(column("time_id", "int", "time id"), column("app_id", "int", "app id"),
- column("machine_id", "int", "machine id"), column("log", "string", "log data", "Log Data"));
-
- List<Referenceable> timeDimColumns = ImmutableList
- .of(column("time_id", "int", "time id"),
- column("dayOfYear", "int", "day Of Year"),
- column("weekDay", "int", "week Day"));
-
- Id timeDim = table("time_dim", "time dimension table", salesDB, sd, "John Doe", "External", timeDimColumns,
- "Dimension");
-
- Id reportingDB =
- database("Reporting", "reporting database", "Jane BI", "hdfs://host:8000/apps/warehouse/reporting");
-
- Id salesFactDaily =
- table("sales_fact_daily_mv", "sales fact daily materialized view", reportingDB, sd, "Joe BI", "Managed",
- salesFactColumns, "Metric");
-
- Id circularLineageTable1 = table("table1", "", reportingDB, sd, "Vimal", "Managed", salesFactColumns, "Metric");
-
- Id circularLineageTable2 = table("table2", "", reportingDB, sd, "Vimal", "Managed", salesFactColumns, "Metric");
-
- loadProcess("circularLineage1", "hive query for daily summary", "John ETL", ImmutableList.of(circularLineageTable1),
- ImmutableList.of(circularLineageTable2), "create table as select ", "plan", "id", "graph", "ETL");
-
- loadProcess("circularLineage2", "hive query for daily summary", "John ETL", ImmutableList.of(circularLineageTable2),
- ImmutableList.of(circularLineageTable1), "create table as select ", "plan", "id", "graph", "ETL");
-
- loadProcess("loadSalesDaily", "hive query for daily summary", "John ETL", ImmutableList.of(salesFact, timeDim),
- ImmutableList.of(salesFactDaily), "create table as select ", "plan", "id", "graph", "ETL");
-
- Id logDB = database("Logging", "logging database", "Tim ETL", "hdfs://host:8000/apps/warehouse/logging");
-
- Id loggingFactDaily =
- table("log_fact_daily_mv", "log fact daily materialized view", logDB, sd, "Tim ETL", "Managed",
- logFactColumns, "Log Data");
-
- List<Referenceable> productDimColumns = ImmutableList
- .of(column("product_id", "int", "product id"),
- column("product_name", "string", "product name"),
- column("brand_name", "int", "brand name"));
-
- Id productDim =
- table("product_dim", "product dimension table", salesDB, sd, "John Doe", "Managed", productDimColumns,
- "Dimension");
-
- view("product_dim_view", reportingDB, ImmutableList.of(productDim), "Dimension", "JdbcAccess");
-
- List<Referenceable> customerDimColumns = ImmutableList.of(
- column("customer_id", "int", "customer id", "PII"),
- column("name", "string", "customer name", "PII"),
- column("address", "string", "customer address", "PII"));
-
- Id customerDim =
- table("customer_dim", "customer dimension table", salesDB, sd, "fetl", "External", customerDimColumns,
- "Dimension");
-
- view("customer_dim_view", reportingDB, ImmutableList.of(customerDim), "Dimension", "JdbcAccess");
-
- Id salesFactMonthly =
- table("sales_fact_monthly_mv", "sales fact monthly materialized view", reportingDB, sd, "Jane BI",
- "Managed", salesFactColumns, "Metric");
-
- loadProcess("loadSalesMonthly", "hive query for monthly summary", "John ETL", ImmutableList.of(salesFactDaily),
- ImmutableList.of(salesFactMonthly), "create table as select ", "plan", "id", "graph", "ETL");
-
- Id loggingFactMonthly =
- table("logging_fact_monthly_mv", "logging fact monthly materialized view", logDB, sd, "Tim ETL",
- "Managed", logFactColumns, "Log Data");
-
- loadProcess("loadLogsMonthly", "hive query for monthly summary", "Tim ETL", ImmutableList.of(loggingFactDaily),
- ImmutableList.of(loggingFactMonthly), "create table as select ", "plan", "id", "graph", "ETL");
-
- partition(new ArrayList() {{ add("2015-01-01"); }}, salesFactDaily);
-
- datasetSubType("dataSetSubTypeInst1", "testOwner");
- }
-
- Id database(String name, String description, String owner, String locationUri, String... traitNames)
- throws Exception {
- Referenceable referenceable = new Referenceable(DATABASE_TYPE, traitNames);
- referenceable.set("name", name);
- referenceable.set("description", description);
- referenceable.set("owner", owner);
- referenceable.set("locationUri", locationUri);
- referenceable.set("createTime", System.currentTimeMillis());
-
- ClassType clsType = TypeSystem.getInstance().getDataType(ClassType.class, DATABASE_TYPE);
- return createInstance(referenceable, clsType);
- }
-
- protected Referenceable storageDescriptor(String location, String inputFormat, String outputFormat, boolean compressed, List<Referenceable> columns)
- throws Exception {
- Referenceable referenceable = new Referenceable(STORAGE_DESC_TYPE);
- referenceable.set("location", location);
- referenceable.set("inputFormat", inputFormat);
- referenceable.set("outputFormat", outputFormat);
- referenceable.set("compressed", compressed);
- referenceable.set("cols", columns);
-
- return referenceable;
- }
-
- protected Referenceable column(String name, String dataType, String comment, String... traitNames) throws Exception {
- Referenceable referenceable = new Referenceable(COLUMN_TYPE, traitNames);
- referenceable.set("name", name);
- referenceable.set("dataType", dataType);
- referenceable.set("comment", comment);
-
- return referenceable;
- }
-
- protected Id table(String name, String description, Id dbId, Referenceable sd, String owner, String tableType,
- List<Referenceable> columns, String... traitNames) throws Exception {
- Referenceable referenceable = new Referenceable(HIVE_TABLE_TYPE, traitNames);
- referenceable.set("name", name);
- referenceable.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, "qualified:" + name);
- referenceable.set("description", description);
- referenceable.set("owner", owner);
- referenceable.set("tableType", tableType);
- referenceable.set("temporary", false);
- referenceable.set("createTime", new Date(System.currentTimeMillis()));
- referenceable.set("lastAccessTime", System.currentTimeMillis());
- referenceable.set("retention", System.currentTimeMillis());
-
- referenceable.set("db", dbId);
- // todo - uncomment this, something is broken
- referenceable.set("sd", sd);
- referenceable.set("columns", columns);
-
- ClassType clsType = TypeSystem.getInstance().getDataType(ClassType.class, HIVE_TABLE_TYPE);
- return createInstance(referenceable, clsType);
- }
-
- protected Id loadProcess(String name, String description, String user, List<Id> inputTables, List<Id> outputTables,
- String queryText, String queryPlan, String queryId, String queryGraph, String... traitNames)
- throws Exception {
- Referenceable referenceable = new Referenceable(HIVE_PROCESS_TYPE, traitNames);
- referenceable.set("name", name);
- referenceable.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name);
- referenceable.set("description", description);
- referenceable.set("user", user);
- referenceable.set("startTime", System.currentTimeMillis());
- referenceable.set("endTime", System.currentTimeMillis() + 10000);
-
- referenceable.set("inputs", inputTables);
- referenceable.set("outputs", outputTables);
-
- referenceable.set("queryText", queryText);
- referenceable.set("queryPlan", queryPlan);
- referenceable.set("queryId", queryId);
- referenceable.set("queryGraph", queryGraph);
-
- ClassType clsType = TypeSystem.getInstance().getDataType(ClassType.class, HIVE_PROCESS_TYPE);
- return createInstance(referenceable, clsType);
- }
-
- Id view(String name, Id dbId, List<Id> inputTables, String... traitNames) throws Exception {
- Referenceable referenceable = new Referenceable(VIEW_TYPE, traitNames);
- referenceable.set("name", name);
- referenceable.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name);
- referenceable.set("db", dbId);
-
- referenceable.set("inputTables", inputTables);
- ClassType clsType = TypeSystem.getInstance().getDataType(ClassType.class, VIEW_TYPE);
- return createInstance(referenceable, clsType);
- }
-
- Id partition(List<String> values, Id table, String... traitNames) throws Exception {
- Referenceable referenceable = new Referenceable(PARTITION_TYPE, traitNames);
- referenceable.set("values", values);
- referenceable.set("table", table);
- ClassType clsType = TypeSystem.getInstance().getDataType(ClassType.class, PARTITION_TYPE);
- return createInstance(referenceable, clsType);
- }
-
- Id datasetSubType(final String name, String owner) throws Exception {
- Referenceable referenceable = new Referenceable(DATASET_SUBTYPE);
- referenceable.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name);
- referenceable.set(AtlasClient.NAME, name);
- referenceable.set("owner", owner);
- ClassType clsType = TypeSystem.getInstance().getDataType(ClassType.class, DATASET_SUBTYPE);
- return createInstance(referenceable, clsType);
- }
- private Id createInstance(Referenceable referenceable, ClassType clsType) throws Exception {
- ITypedReferenceableInstance typedInstance = clsType.convert(referenceable, Multiplicity.REQUIRED);
- List<String> guids = repository.createEntities(typedInstance).getCreatedEntities();
-
- // return the reference to created instance with guid
- return new Id(guids.get(guids.size() - 1), 0, referenceable.getTypeName());
- }
-
- private void setUpDefaultTypes() throws Exception {
- TypesDef typesDef = createDefaultTypeDefinitions();
- String typesAsJSON = TypesSerialization.toJson(typesDef);
- metadataService.createType(typesAsJSON);
- }
-
- TypesDef createDefaultTypeDefinitions() {
- HierarchicalTypeDefinition<ClassType> referenceableType = TypesUtil
- .createClassTypeDef(AtlasClient.REFERENCEABLE_SUPER_TYPE, ImmutableSet.<String>of(),
- new AttributeDefinition(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, true, true, null));
-
- HierarchicalTypeDefinition<ClassType> assetType = TypesUtil
- .createClassTypeDef(AtlasClient.ASSET_TYPE, ImmutableSet.<String>of(),
- new AttributeDefinition(AtlasClient.NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null),
- TypesUtil.createOptionalAttrDef(AtlasClient.DESCRIPTION, DataTypes.STRING_TYPE),
- new AttributeDefinition(AtlasClient.OWNER, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null));
-
- HierarchicalTypeDefinition<ClassType> infraType = TypesUtil
- .createClassTypeDef(AtlasClient.INFRASTRUCTURE_SUPER_TYPE,
- ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE));
-
- HierarchicalTypeDefinition<ClassType> datasetType = TypesUtil
- .createClassTypeDef(AtlasClient.DATA_SET_SUPER_TYPE,
- ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE));
-
- HierarchicalTypeDefinition<ClassType> processType = TypesUtil
- .createClassTypeDef(AtlasClient.PROCESS_SUPER_TYPE,
- ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE),
- new AttributeDefinition(PROCESS_ATTRIBUTE_INPUTS, DataTypes.arrayTypeName(AtlasClient.DATA_SET_SUPER_TYPE),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition(PROCESS_ATTRIBUTE_OUTPUTS, DataTypes.arrayTypeName(AtlasClient.DATA_SET_SUPER_TYPE),
- Multiplicity.OPTIONAL, false, null));
-
- return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
- ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
- ImmutableList.of(referenceableType, assetType, infraType, datasetType, processType));
- }
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/test/java/org/apache/atlas/DBSandboxer.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/DBSandboxer.java b/repository/src/test/java/org/apache/atlas/DBSandboxer.java
deleted file mode 100644
index f4f099a..0000000
--- a/repository/src/test/java/org/apache/atlas/DBSandboxer.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.atlas;
-
-import org.apache.atlas.graph.GraphSandboxUtil;
-import org.apache.atlas.repository.graph.AtlasGraphProvider;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.testng.ITestContext;
-import org.testng.TestListenerAdapter;
-import org.testng.xml.XmlClass;
-
-import java.util.List;
-
-public class DBSandboxer extends TestListenerAdapter {
- @Override
- public void onStart(ITestContext context) {
- // This will only work if each test is run individually (test suite has only one running test)
- // If there are multiple tests the the sandbox folder name is not provided and the GraphSandboxUtil provisions
- // a unique name
- List<XmlClass> testClassesToRun = context.getCurrentXmlTest().getClasses();
- if (CollectionUtils.isNotEmpty(testClassesToRun) && 1 == testClassesToRun.size()) {
- XmlClass currentTestClass = testClassesToRun.get(0);
- if (null != currentTestClass && StringUtils.isNotEmpty(currentTestClass.getName())) {
- GraphSandboxUtil.create(currentTestClass.getName());
- } else {
- GraphSandboxUtil.create();
- }
- } else {
- GraphSandboxUtil.create();
- }
- }
-
- @Override
- public void onFinish(ITestContext context) {
- AtlasGraphProvider.cleanup();
- }
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java b/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java
deleted file mode 100755
index 506d11c..0000000
--- a/repository/src/test/java/org/apache/atlas/RepositoryServiceLoadingTest.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas;
-
-import org.apache.atlas.repository.graph.AtlasGraphProvider;
-import org.testng.Assert;
-import org.testng.annotations.Guice;
-import org.testng.annotations.Test;
-
-/**
- * Unit test for Guice injector service loading
- *
- * Uses TestNG's Guice annotation to load the necessary modules and inject the
- * objects from Guice
- */
-@Guice(modules = TestModules.TestOnlyModule.class)
-public class RepositoryServiceLoadingTest {
-
- @Test
- public void testGetGraphService() throws Exception {
- Assert.assertNotNull(AtlasGraphProvider.getGraphInstance());
- }
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/test/java/org/apache/atlas/TestModules.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/TestModules.java b/repository/src/test/java/org/apache/atlas/TestModules.java
index 144f618..5c9e60f 100644
--- a/repository/src/test/java/org/apache/atlas/TestModules.java
+++ b/repository/src/test/java/org/apache/atlas/TestModules.java
@@ -26,24 +26,14 @@ import com.google.inject.multibindings.Multibinder;
import org.apache.atlas.annotation.GraphTransaction;
import org.apache.atlas.discovery.AtlasDiscoveryService;
import org.apache.atlas.discovery.AtlasLineageService;
-import org.apache.atlas.discovery.DataSetLineageService;
-import org.apache.atlas.discovery.DiscoveryService;
import org.apache.atlas.discovery.EntityDiscoveryService;
import org.apache.atlas.discovery.EntityLineageService;
-import org.apache.atlas.discovery.LineageService;
-import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.graph.GraphSandboxUtil;
import org.apache.atlas.listener.EntityChangeListener;
import org.apache.atlas.listener.TypeDefChangeListener;
-import org.apache.atlas.listener.TypesChangeListener;
-import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.audit.EntityAuditListener;
import org.apache.atlas.repository.audit.EntityAuditRepository;
-import org.apache.atlas.repository.graph.DeleteHandler;
-import org.apache.atlas.repository.graph.GraphBackedMetadataRepository;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
-import org.apache.atlas.repository.graph.HardDeleteHandler;
-import org.apache.atlas.repository.graph.SoftDeleteHandler;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.impexp.ExportService;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
@@ -58,16 +48,9 @@ import org.apache.atlas.repository.store.graph.v1.DeleteHandlerV1;
import org.apache.atlas.repository.store.graph.v1.EntityGraphMapper;
import org.apache.atlas.repository.store.graph.v1.HardDeleteHandlerV1;
import org.apache.atlas.repository.store.graph.v1.SoftDeleteHandlerV1;
-import org.apache.atlas.repository.typestore.GraphBackedTypeStore;
-import org.apache.atlas.repository.typestore.ITypeStore;
-import org.apache.atlas.repository.typestore.StoreBackedTypeCache;
import org.apache.atlas.service.Service;
-import org.apache.atlas.services.DefaultMetadataService;
-import org.apache.atlas.services.MetadataService;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
-import org.apache.atlas.typesystem.types.TypeSystem;
-import org.apache.atlas.typesystem.types.cache.TypeCache;
import org.apache.atlas.util.AtlasRepositoryConfiguration;
import org.apache.atlas.util.SearchTracker;
import org.apache.commons.configuration.Configuration;
@@ -89,13 +72,6 @@ public class TestModules {
private static final Logger LOG = LoggerFactory.getLogger(TestOnlyModule.class);
- static class TypeSystemProvider implements Provider<TypeSystem> {
- @Override
- public TypeSystem get() {
- return TypeSystem.getInstance();
- }
- }
-
static class AtlasConfigurationProvider implements Provider<Configuration> {
@Override
@@ -126,32 +102,19 @@ public class TestModules {
bind(AtlasGraph.class).toProvider(AtlasGraphProvider.class);
- // allow for dynamic binding of the metadata repo & graph service
- // bind the MetadataRepositoryService interface to an implementation
- bind(MetadataRepository.class).to(GraphBackedMetadataRepository.class).asEagerSingleton();
-
- bind(TypeSystem.class).toProvider(TypeSystemProvider.class).in(Singleton.class);
+ // allow for dynamic binding of graph service
bind(Configuration.class).toProvider(AtlasConfigurationProvider.class).in(Singleton.class);
- // bind the ITypeStore interface to an implementation
- bind(ITypeStore.class).to(GraphBackedTypeStore.class).asEagerSingleton();
+ // bind the AtlasTypeDefStore interface to an implementation
bind(AtlasTypeDefStore.class).to(AtlasTypeDefGraphStoreV1.class).asEagerSingleton();
bind(AtlasTypeRegistry.class).asEagerSingleton();
bind(EntityGraphMapper.class).asEagerSingleton();
bind(ExportService.class).asEagerSingleton();
- //GraphBackedSearchIndexer must be an eager singleton to force the search index creation to happen before
- //we try to restore the type system (otherwise we'll end up running queries
- //before we have any indices during the initial graph setup)
- Multibinder<TypesChangeListener> typesChangeListenerBinder =
- Multibinder.newSetBinder(binder(), TypesChangeListener.class);
- typesChangeListenerBinder.addBinding().to(GraphBackedSearchIndexer.class).asEagerSingleton();
-
// New typesdef/instance change listener should also be bound to the corresponding implementation
Multibinder<TypeDefChangeListener> typeDefChangeListenerMultibinder =
Multibinder.newSetBinder(binder(), TypeDefChangeListener.class);
- typeDefChangeListenerMultibinder.addBinding().to(DefaultMetadataService.class);
typeDefChangeListenerMultibinder.addBinding().to(GraphBackedSearchIndexer.class).asEagerSingleton();
bind(SearchTracker.class).asEagerSingleton();
@@ -159,19 +122,12 @@ public class TestModules {
bind(AtlasEntityStore.class).to(AtlasEntityStoreV1.class);
bind(AtlasRelationshipStore.class).to(AtlasRelationshipStoreV1.class);
- // bind the MetadataService interface to an implementation
- bind(MetadataService.class).to(DefaultMetadataService.class).asEagerSingleton();
-
// bind the DiscoveryService interface to an implementation
- bind(DiscoveryService.class).to(GraphBackedDiscoveryService.class).asEagerSingleton();
bind(AtlasDiscoveryService.class).to(EntityDiscoveryService.class).asEagerSingleton();
- bind(LineageService.class).to(DataSetLineageService.class).asEagerSingleton();
bind(AtlasLineageService.class).to(EntityLineageService.class).asEagerSingleton();
bind(BulkImporter.class).to(BulkImporterImpl.class).asEagerSingleton();
- bindTypeCache();
-
//Add EntityAuditListener as EntityChangeListener
Multibinder<EntityChangeListener> entityChangeListenerBinder =
Multibinder.newSetBinder(binder(), EntityChangeListener.class);
@@ -182,12 +138,7 @@ public class TestModules {
bindInterceptor(Matchers.any(), Matchers.annotatedWith(GraphTransaction.class), graphTransactionInterceptor);
}
- protected void bindTypeCache() {
- bind(TypeCache.class).to(AtlasRepositoryConfiguration.getTypeCache()).asEagerSingleton();
- }
-
protected void bindDeleteHandler(Binder binder) {
- binder.bind(DeleteHandler.class).to(AtlasRepositoryConfiguration.getDeleteHandlerImpl()).asEagerSingleton();
binder.bind(DeleteHandlerV1.class).to(AtlasRepositoryConfiguration.getDeleteHandlerV1Impl()).asEagerSingleton();
}
@@ -210,7 +161,6 @@ public class TestModules {
public static class SoftDeleteModule extends TestOnlyModule {
@Override
protected void bindDeleteHandler(Binder binder) {
- bind(DeleteHandler.class).to(SoftDeleteHandler.class).asEagerSingleton();
bind(DeleteHandlerV1.class).to(SoftDeleteHandlerV1.class).asEagerSingleton();
bind(AtlasEntityChangeNotifier.class).toProvider(MockNotifier.class);
}
@@ -219,20 +169,8 @@ public class TestModules {
public static class HardDeleteModule extends TestOnlyModule {
@Override
protected void bindDeleteHandler(Binder binder) {
- bind(DeleteHandler.class).to(HardDeleteHandler.class).asEagerSingleton();
bind(DeleteHandlerV1.class).to(HardDeleteHandlerV1.class).asEagerSingleton();
bind(AtlasEntityChangeNotifier.class).toProvider(MockNotifier.class);
}
}
-
- /**
- * Guice module which sets TypeCache implementation class configuration property to {@link StoreBackedTypeCache}.
- *
- */
- public static class StoreBackedTypeCacheTestModule extends TestOnlyModule {
- @Override
- protected void bindTypeCache() {
- bind(TypeCache.class).to(StoreBackedTypeCache.class).asEagerSingleton();
- }
- }
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/test/java/org/apache/atlas/TestUtils.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/TestUtils.java b/repository/src/test/java/org/apache/atlas/TestUtils.java
deleted file mode 100755
index 56bfb82..0000000
--- a/repository/src/test/java/org/apache/atlas/TestUtils.java
+++ /dev/null
@@ -1,778 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import org.apache.atlas.annotation.GraphTransaction;
-import org.apache.atlas.listener.EntityChangeListener;
-import org.apache.atlas.listener.TypesChangeListener;
-import org.apache.atlas.repository.MetadataRepository;
-import org.apache.atlas.repository.audit.InMemoryEntityAuditRepository;
-import org.apache.atlas.repository.graph.AtlasGraphProvider;
-import org.apache.atlas.repository.graph.GraphBackedMetadataRepository;
-import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
-import org.apache.atlas.repository.graph.GraphHelper;
-import org.apache.atlas.repository.graphdb.AtlasGraph;
-import org.apache.atlas.repository.graphdb.GremlinVersion;
-import org.apache.atlas.repository.typestore.GraphBackedTypeStore;
-import org.apache.atlas.repository.typestore.ITypeStore;
-import org.apache.atlas.services.DefaultMetadataService;
-import org.apache.atlas.services.MetadataService;
-import org.apache.atlas.type.AtlasTypeRegistry;
-import org.apache.atlas.typesystem.IInstance;
-import org.apache.atlas.typesystem.ITypedReferenceableInstance;
-import org.apache.atlas.typesystem.Referenceable;
-import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.json.InstanceSerialization;
-import org.apache.atlas.typesystem.persistence.Id;
-import org.apache.atlas.typesystem.types.*;
-import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
-import org.apache.atlas.typesystem.types.cache.DefaultTypeCache;
-import org.apache.atlas.typesystem.types.cache.TypeCache;
-import org.apache.atlas.typesystem.types.utils.TypesUtil;
-import org.apache.atlas.util.AtlasRepositoryConfiguration;
-import org.apache.commons.configuration.Configuration;
-import org.apache.commons.lang.RandomStringUtils;
-import org.codehaus.jettison.json.JSONArray;
-import org.testng.Assert;
-import org.testng.SkipException;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.lang.reflect.InvocationHandler;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.lang.reflect.Proxy;
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import static org.apache.atlas.typesystem.types.utils.TypesUtil.*;
-import static org.testng.Assert.assertEquals;
-
-/**
- * Test utility class.
- */
-public final class TestUtils {
-
- public static final long TEST_DATE_IN_LONG = 1418265358440L;
-
-
- public static final String EMPLOYEES_ATTR = "employees";
- public static final String DEPARTMENT_ATTR = "department";
- public static final String ASSETS_ATTR = "assets";
-
- public static final String POSITIONS_ATTR = "positions";
- public static final String ASSET_TYPE = "TestAsset";
-
- public static final String DATABASE_TYPE = "hive_database";
- public static final String DATABASE_NAME = "foo";
- public static final String TABLE_TYPE = "hive_table";
- public static final String PROCESS_TYPE = "hive_process";
- public static final String COLUMN_TYPE = "column_type";
- public static final String TABLE_NAME = "bar";
- public static final String CLASSIFICATION = "classification";
- public static final String PII = "PII";
- public static final String SUPER_TYPE_NAME = "Base";
- public static final String STORAGE_DESC_TYPE = "hive_storagedesc";
- public static final String PARTITION_STRUCT_TYPE = "partition_struct_type";
- public static final String PARTITION_CLASS_TYPE = "partition_class_type";
- public static final String SERDE_TYPE = "serdeType";
- public static final String COLUMNS_MAP = "columnsMap";
- public static final String COLUMNS_ATTR_NAME = "columns";
-
- public static final String NAME = "name";
-
- private TestUtils() {
- }
-
- /**
- * Dumps the graph in GSON format in the path returned.
- *
- * @param graph handle to graph
- * @return path to the dump file
- * @throws Exception
- */
- public static String dumpGraph(AtlasGraph<?,?> graph) throws Exception {
- File tempFile = File.createTempFile("graph", ".gson");
- System.out.println("tempFile.getPath() = " + tempFile.getPath());
- GraphHelper.dumpToLog(graph);
- FileOutputStream os = null;
- try {
- os = new FileOutputStream(tempFile);
- graph.exportToGson(os);
- }
- finally {
- if(os != null) {
- try {
- os.close();
- }
- catch(IOException e) {
- e.printStackTrace();
- }
- }
- }
-
- return tempFile.getPath();
- }
-
- /**
- * Class Hierarchy is:
- * Department(name : String, employees : Array[Person])
- * Person(name : String, department : Department, manager : Manager)
- * Manager(subordinates : Array[Person]) extends Person
- * <p/>
- * Persons can have SecurityClearance(level : Int) clearance.
- */
- public static void defineDeptEmployeeTypes(TypeSystem ts) throws AtlasException {
-
- String _description = "_description";
- EnumTypeDefinition orgLevelEnum =
- new EnumTypeDefinition("OrgLevel", "OrgLevel"+_description, new EnumValue("L1", 1), new EnumValue("L2", 2));
-
- StructTypeDefinition addressDetails =
- createStructTypeDef("Address", "Address"+_description, createRequiredAttrDef("street", DataTypes.STRING_TYPE),
- createRequiredAttrDef("city", DataTypes.STRING_TYPE));
-
- HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef(DEPARTMENT_TYPE, "Department"+_description, ImmutableSet.<String>of(),
- createRequiredAttrDef(NAME, DataTypes.STRING_TYPE),
- new AttributeDefinition(EMPLOYEES_ATTR, String.format("array<%s>", "Person"), Multiplicity.OPTIONAL,
- true, DEPARTMENT_ATTR),
- new AttributeDefinition(POSITIONS_ATTR, String.format("map<%s,%s>", DataTypes.STRING_TYPE.getName(), "Person"), Multiplicity.OPTIONAL,
- false, null)
- );
-
- HierarchicalTypeDefinition<ClassType> personTypeDef = createClassTypeDef("Person", "Person"+_description, ImmutableSet.<String>of(),
- createRequiredAttrDef(NAME, DataTypes.STRING_TYPE),
- createOptionalAttrDef("orgLevel", "OrgLevel"),
- createOptionalAttrDef("address", "Address"),
- new AttributeDefinition(DEPARTMENT_ATTR, "Department", Multiplicity.REQUIRED, false, EMPLOYEES_ATTR),
- new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates"),
- new AttributeDefinition("mentor", "Person", Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition(ASSETS_ATTR, String.format("array<%s>", ASSET_TYPE) , Multiplicity.OPTIONAL, false, null),
- createOptionalAttrDef("birthday", DataTypes.DATE_TYPE),
- createOptionalAttrDef("hasPets", DataTypes.BOOLEAN_TYPE),
- createOptionalAttrDef("numberOfCars", DataTypes.BYTE_TYPE),
- createOptionalAttrDef("houseNumber", DataTypes.SHORT_TYPE),
- createOptionalAttrDef("carMileage", DataTypes.INT_TYPE),
- createOptionalAttrDef("shares", DataTypes.LONG_TYPE),
- createOptionalAttrDef("salary", DataTypes.DOUBLE_TYPE),
- createOptionalAttrDef("age", DataTypes.FLOAT_TYPE),
- createOptionalAttrDef("numberOfStarsEstimate", DataTypes.BIGINTEGER_TYPE),
- createOptionalAttrDef("approximationOfPi", DataTypes.BIGDECIMAL_TYPE),
- createOptionalAttrDef("isOrganDonor", DataTypes.BOOLEAN_TYPE)
- );
-
-
- HierarchicalTypeDefinition<ClassType> assetTypeDef = createClassTypeDef(ASSET_TYPE, "Asset"+_description, ImmutableSet.<String>of(),
- createRequiredAttrDef(NAME, DataTypes.STRING_TYPE),
- new AttributeDefinition("childAssets", String.format("array<%s>", ASSET_TYPE) , Multiplicity.OPTIONAL, false, null)
- );
-
- HierarchicalTypeDefinition<ClassType> managerTypeDef = createClassTypeDef("Manager", "Manager"+_description, ImmutableSet.of("Person"),
- new AttributeDefinition("subordinates", String.format("array<%s>", "Person"), Multiplicity.COLLECTION,
- false, "manager"));
-
- HierarchicalTypeDefinition<TraitType> securityClearanceTypeDef =
- createTraitTypeDef("SecurityClearance", "SecurityClearance"+_description, ImmutableSet.<String>of(),
- createRequiredAttrDef("level", DataTypes.INT_TYPE));
-
- ts.defineTypes(ImmutableList.of(orgLevelEnum), ImmutableList.of(addressDetails),
- ImmutableList.of(securityClearanceTypeDef),
- ImmutableList.of(deptTypeDef, personTypeDef, managerTypeDef, assetTypeDef));
- }
-
- public static final String DEPARTMENT_TYPE = "Department";
- public static final String PERSON_TYPE = "Person";
-
- public static ITypedReferenceableInstance createDeptEg1(TypeSystem ts) throws AtlasException {
- Referenceable hrDept = new Referenceable(DEPARTMENT_TYPE);
- Referenceable john = new Referenceable(PERSON_TYPE);
-
- Referenceable jane = new Referenceable("Manager", "SecurityClearance");
- Referenceable johnAddr = new Referenceable("Address");
- Referenceable janeAddr = new Referenceable("Address");
- Referenceable julius = new Referenceable("Manager");
- Referenceable juliusAddr = new Referenceable("Address");
- Referenceable max = new Referenceable("Person");
- Referenceable maxAddr = new Referenceable("Address");
-
- hrDept.set(NAME, "hr");
- john.set(NAME, "John");
- john.set(DEPARTMENT_ATTR, hrDept);
- johnAddr.set("street", "Stewart Drive");
- johnAddr.set("city", "Sunnyvale");
- john.set("address", johnAddr);
-
- john.set("birthday",new Date(1950, 5, 15));
- john.set("isOrganDonor", true);
- john.set("hasPets", true);
- john.set("numberOfCars", 1);
- john.set("houseNumber", 153);
- john.set("carMileage", 13364);
- john.set("shares", 15000);
- john.set("salary", 123345.678);
- john.set("age", 50);
- john.set("numberOfStarsEstimate", new BigInteger("1000000000000000000000"));
- john.set("approximationOfPi", new BigDecimal("3.141592653589793238462643383279502884197169399375105820974944592307816406286"));
-
- jane.set(NAME, "Jane");
- jane.set(DEPARTMENT_ATTR, hrDept);
- janeAddr.set("street", "Great America Parkway");
- janeAddr.set("city", "Santa Clara");
- jane.set("address", janeAddr);
- janeAddr.set("street", "Great America Parkway");
-
- julius.set(NAME, "Julius");
- julius.set(DEPARTMENT_ATTR, hrDept);
- juliusAddr.set("street", "Madison Ave");
- juliusAddr.set("city", "Newtonville");
- julius.set("address", juliusAddr);
- julius.set("subordinates", ImmutableList.<Referenceable>of());
-
- max.set(NAME, "Max");
- max.set(DEPARTMENT_ATTR, hrDept);
- maxAddr.set("street", "Ripley St");
- maxAddr.set("city", "Newton");
- max.set("address", maxAddr);
- max.set("manager", jane);
- max.set("mentor", julius);
- max.set("birthday",new Date(1979, 3, 15));
- max.set("isOrganDonor", true);
- max.set("hasPets", true);
- max.set("age", 36);
- max.set("numberOfCars", 2);
- max.set("houseNumber", 17);
- max.set("carMileage", 13);
- max.set("shares", Long.MAX_VALUE);
- max.set("salary", Double.MAX_VALUE);
- max.set("numberOfStarsEstimate", new BigInteger("1000000000000000000000000000000"));
- max.set("approximationOfPi", new BigDecimal("3.1415926535897932"));
-
- john.set("manager", jane);
- john.set("mentor", max);
- hrDept.set(EMPLOYEES_ATTR, ImmutableList.of(john, jane, julius, max));
-
- jane.set("subordinates", ImmutableList.of(john, max));
-
- jane.getTrait("SecurityClearance").set("level", 1);
-
- ClassType deptType = ts.getDataType(ClassType.class, "Department");
- ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
- Assert.assertNotNull(hrDept2);
-
- return hrDept2;
- }
-
-
-
- public static TypesDef simpleType(){
- HierarchicalTypeDefinition<ClassType> superTypeDefinition =
- createClassTypeDef("h_type", ImmutableSet.<String>of(),
- createOptionalAttrDef("attr", DataTypes.STRING_TYPE));
-
- StructTypeDefinition structTypeDefinition = new StructTypeDefinition("s_type", "structType",
- new AttributeDefinition[]{createRequiredAttrDef(NAME, DataTypes.STRING_TYPE)});
-
- HierarchicalTypeDefinition<TraitType> traitTypeDefinition =
- createTraitTypeDef("t_type", "traitType", ImmutableSet.<String>of());
-
- EnumValue values[] = {new EnumValue("ONE", 1),};
-
- EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("e_type", "enumType", values);
- return TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition), ImmutableList.of(structTypeDefinition),
- ImmutableList.of(traitTypeDefinition), ImmutableList.of(superTypeDefinition));
- }
-
- public static TypesDef simpleTypeUpdated(){
- HierarchicalTypeDefinition<ClassType> superTypeDefinition =
- createClassTypeDef("h_type", ImmutableSet.<String>of(),
- createOptionalAttrDef("attr", DataTypes.STRING_TYPE));
-
- HierarchicalTypeDefinition<ClassType> newSuperTypeDefinition =
- createClassTypeDef("new_h_type", ImmutableSet.<String>of(),
- createOptionalAttrDef("attr", DataTypes.STRING_TYPE));
-
- StructTypeDefinition structTypeDefinition = new StructTypeDefinition("s_type", "structType",
- new AttributeDefinition[]{createRequiredAttrDef(NAME, DataTypes.STRING_TYPE)});
-
- HierarchicalTypeDefinition<TraitType> traitTypeDefinition =
- createTraitTypeDef("t_type", "traitType", ImmutableSet.<String>of());
-
- EnumValue values[] = {new EnumValue("ONE", 1),};
-
- EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("e_type", "enumType", values);
- return TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition), ImmutableList.of(structTypeDefinition),
- ImmutableList.of(traitTypeDefinition), ImmutableList.of(superTypeDefinition, newSuperTypeDefinition));
- }
-
- public static TypesDef simpleTypeUpdatedDiff() {
- HierarchicalTypeDefinition<ClassType> newSuperTypeDefinition =
- createClassTypeDef("new_h_type", ImmutableSet.<String>of(),
- createOptionalAttrDef("attr", DataTypes.STRING_TYPE));
-
- return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
- ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), ImmutableList.of(newSuperTypeDefinition));
- }
-
- public static TypesDef defineHiveTypes() {
- String _description = "_description";
- HierarchicalTypeDefinition<ClassType> superTypeDefinition =
- createClassTypeDef(SUPER_TYPE_NAME, ImmutableSet.<String>of(),
- createOptionalAttrDef("namespace", DataTypes.STRING_TYPE),
- createOptionalAttrDef("cluster", DataTypes.STRING_TYPE),
- createOptionalAttrDef("colo", DataTypes.STRING_TYPE));
- HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
- createClassTypeDef(DATABASE_TYPE, DATABASE_TYPE + _description,ImmutableSet.of(SUPER_TYPE_NAME),
- TypesUtil.createUniqueRequiredAttrDef(NAME, DataTypes.STRING_TYPE),
- createOptionalAttrDef("created", DataTypes.DATE_TYPE),
- createOptionalAttrDef("isReplicated", DataTypes.BOOLEAN_TYPE),
- new AttributeDefinition("parameters", new DataTypes.MapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE).getName(), Multiplicity.OPTIONAL, false, null),
- createRequiredAttrDef("description", DataTypes.STRING_TYPE));
-
-
- StructTypeDefinition structTypeDefinition = new StructTypeDefinition("serdeType", "serdeType" + _description,
- new AttributeDefinition[]{createRequiredAttrDef(NAME, DataTypes.STRING_TYPE),
- createRequiredAttrDef("serde", DataTypes.STRING_TYPE),
- createOptionalAttrDef("description", DataTypes.STRING_TYPE)});
-
- EnumValue values[] = {new EnumValue("MANAGED", 1), new EnumValue("EXTERNAL", 2),};
-
- EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("tableType", "tableType" + _description, values);
-
- HierarchicalTypeDefinition<ClassType> columnsDefinition =
- createClassTypeDef(COLUMN_TYPE, ImmutableSet.<String>of(),
- createUniqueRequiredAttrDef(NAME, DataTypes.STRING_TYPE),
- createRequiredAttrDef("type", DataTypes.STRING_TYPE));
-
- StructTypeDefinition partitionDefinition = new StructTypeDefinition("partition_struct_type", "partition_struct_type" + _description,
- new AttributeDefinition[]{createRequiredAttrDef(NAME, DataTypes.STRING_TYPE),});
-
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition("location", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
- null),
- new AttributeDefinition("inputFormat", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
- null),
- new AttributeDefinition("outputFormat", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
- null),
- new AttributeDefinition("compressed", DataTypes.BOOLEAN_TYPE.getName(), Multiplicity.REQUIRED, false,
- null),
- new AttributeDefinition("numBuckets", DataTypes.INT_TYPE.getName(), Multiplicity.OPTIONAL, false,
- null),
- };
-
- HierarchicalTypeDefinition<ClassType> storageDescClsDef =
- new HierarchicalTypeDefinition<>(ClassType.class, STORAGE_DESC_TYPE, STORAGE_DESC_TYPE + _description,
- ImmutableSet.of(SUPER_TYPE_NAME), attributeDefinitions);
-
- AttributeDefinition[] partClsAttributes = new AttributeDefinition[]{
- new AttributeDefinition("values", DataTypes.arrayTypeName(DataTypes.STRING_TYPE.getName()),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("table", TABLE_TYPE, Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("createTime", DataTypes.LONG_TYPE.getName(), Multiplicity.OPTIONAL, false,
- null),
- new AttributeDefinition("lastAccessTime", DataTypes.LONG_TYPE.getName(), Multiplicity.OPTIONAL, false,
- null),
- new AttributeDefinition("sd", STORAGE_DESC_TYPE, Multiplicity.REQUIRED, true,
- null),
- new AttributeDefinition("columns", DataTypes.arrayTypeName(COLUMN_TYPE),
- Multiplicity.OPTIONAL, true, null),
- new AttributeDefinition("parameters", new DataTypes.MapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE).getName(), Multiplicity.OPTIONAL, false, null),};
-
- HierarchicalTypeDefinition<ClassType> partClsDef =
- new HierarchicalTypeDefinition<>(ClassType.class, "partition_class_type", "partition_class_type" + _description,
- ImmutableSet.of(SUPER_TYPE_NAME), partClsAttributes);
-
- HierarchicalTypeDefinition<ClassType> processClsType =
- new HierarchicalTypeDefinition<>(ClassType.class, PROCESS_TYPE, PROCESS_TYPE + _description,
- ImmutableSet.<String>of(), new AttributeDefinition[]{
- new AttributeDefinition("outputs", "array<" + TABLE_TYPE + ">", Multiplicity.OPTIONAL, false, null)
- });
-
- HierarchicalTypeDefinition<ClassType> tableTypeDefinition =
- createClassTypeDef(TABLE_TYPE, TABLE_TYPE + _description, ImmutableSet.of(SUPER_TYPE_NAME),
- TypesUtil.createUniqueRequiredAttrDef(NAME, DataTypes.STRING_TYPE),
- createRequiredAttrDef("description", DataTypes.STRING_TYPE),
- createRequiredAttrDef("type", DataTypes.STRING_TYPE),
- createOptionalAttrDef("created", DataTypes.DATE_TYPE),
- // enum
- new AttributeDefinition("tableType", "tableType", Multiplicity.REQUIRED, false, null),
- // array of strings
- new AttributeDefinition("columnNames",
- String.format("array<%s>", DataTypes.STRING_TYPE.getName()), Multiplicity.OPTIONAL,
- false, null),
- // array of classes
- new AttributeDefinition("columns", String.format("array<%s>", COLUMN_TYPE),
- Multiplicity.OPTIONAL, true, null),
- // array of structs
- new AttributeDefinition("partitions", String.format("array<%s>", "partition_struct_type"),
- Multiplicity.OPTIONAL, true, null),
- // map of primitives
- new AttributeDefinition("parametersMap",
- DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
- Multiplicity.OPTIONAL, true, null),
- //map of classes -
- new AttributeDefinition(COLUMNS_MAP,
- DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
- COLUMN_TYPE),
- Multiplicity.OPTIONAL, true, null),
- //map of structs
- new AttributeDefinition("partitionsMap",
- DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
- "partition_struct_type"),
- Multiplicity.OPTIONAL, true, null),
- // struct reference
- new AttributeDefinition("serde1", "serdeType", Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("serde2", "serdeType", Multiplicity.OPTIONAL, false, null),
- // class reference
- new AttributeDefinition("database", DATABASE_TYPE, Multiplicity.REQUIRED, false, null),
- //class reference as composite
- new AttributeDefinition("databaseComposite", DATABASE_TYPE, Multiplicity.OPTIONAL, true, null));
-
- HierarchicalTypeDefinition<TraitType> piiTypeDefinition =
- createTraitTypeDef(PII, PII + _description, ImmutableSet.<String>of());
-
- HierarchicalTypeDefinition<TraitType> classificationTypeDefinition =
- createTraitTypeDef(CLASSIFICATION, CLASSIFICATION + _description, ImmutableSet.<String>of(),
- createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
-
- HierarchicalTypeDefinition<TraitType> fetlClassificationTypeDefinition =
- createTraitTypeDef("fetl" + CLASSIFICATION, "fetl" + CLASSIFICATION + _description, ImmutableSet.of(CLASSIFICATION),
- createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
-
- return TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition),
- ImmutableList.of(structTypeDefinition, partitionDefinition),
- ImmutableList.of(classificationTypeDefinition, fetlClassificationTypeDefinition, piiTypeDefinition),
- ImmutableList.of(superTypeDefinition, databaseTypeDefinition, columnsDefinition, tableTypeDefinition,
- storageDescClsDef, partClsDef, processClsType));
- }
-
- public static Collection<IDataType> createHiveTypes(TypeSystem typeSystem) throws Exception {
- if (!typeSystem.isRegistered(TABLE_TYPE)) {
- TypesDef typesDef = defineHiveTypes();
- return typeSystem.defineTypes(typesDef).values();
- }
- return new ArrayList<>();
- }
-
- public static final String randomString() {
- return randomString(10);
- }
-
- public static final String randomString(int count) {
- final String prefix = "r";
-
- return prefix + RandomStringUtils.randomAlphanumeric(count - prefix.length()); // ensure that the string starts with a letter
- }
-
- public static Referenceable createDBEntity() {
- Referenceable entity = new Referenceable(DATABASE_TYPE);
- String dbName = RandomStringUtils.randomAlphanumeric(10);
- entity.set(NAME, dbName);
- entity.set("description", "us db");
- return entity;
- }
-
- public static Referenceable createTableEntity(String dbId) {
- Referenceable entity = new Referenceable(TABLE_TYPE);
- String tableName = RandomStringUtils.randomAlphanumeric(10);
- entity.set(NAME, tableName);
- entity.set("description", "random table");
- entity.set("type", "type");
- entity.set("tableType", "MANAGED");
- entity.set("database", new Id(dbId, 0, DATABASE_TYPE));
- entity.set("created", new Date());
- return entity;
- }
-
- public static Referenceable createColumnEntity() {
- Referenceable entity = new Referenceable(COLUMN_TYPE);
- entity.set(NAME, RandomStringUtils.randomAlphanumeric(10));
- entity.set("type", "VARCHAR(32)");
- return entity;
- }
-
- /**
- * Creates an entity in the graph and does basic validation
- * of the GuidMapping that was created in the process.
- *
- */
- public static String createInstance(MetadataService metadataService, Referenceable entity) throws Exception {
- RequestContext.createContext();
-
- String entityjson = InstanceSerialization.toJson(entity, true);
- JSONArray entitiesJson = new JSONArray();
- entitiesJson.put(entityjson);
- CreateUpdateEntitiesResult creationResult = metadataService.createEntities(entitiesJson.toString());
- Map<String,String> guidMap = creationResult.getGuidMapping().getGuidAssignments();
- Map<Id, Referenceable> referencedObjects = findReferencedObjects(entity);
-
- for(Map.Entry<Id,Referenceable> entry : referencedObjects.entrySet()) {
- Id foundId = entry.getKey();
- if(foundId.isUnassigned()) {
- String guid = guidMap.get(entry.getKey()._getId());
- Referenceable obj = entry.getValue();
- loadAndDoSimpleValidation(guid,obj, metadataService);
- }
- }
- List<String> guids = creationResult.getCreatedEntities();
- if (guids != null && guids.size() > 0) {
- return guids.get(guids.size() - 1);
- }
- return null;
- }
-
- private static Map<Id,Referenceable> findReferencedObjects(Referenceable ref) {
- Map<Id, Referenceable> result = new HashMap<>();
- findReferencedObjects(ref, result);
- return result;
- }
-
- private static void findReferencedObjects(Referenceable ref, Map<Id, Referenceable> seen) {
-
- Id guid = ref.getId();
- if(seen.containsKey(guid)) {
- return;
- }
- seen.put(guid, ref);
- for(Map.Entry<String, Object> attr : ref.getValuesMap().entrySet()) {
- Object value = attr.getValue();
- if(value instanceof Referenceable) {
- findReferencedObjects((Referenceable)value, seen);
- }
- else if(value instanceof List) {
- for(Object o : (List)value) {
- if(o instanceof Referenceable) {
- findReferencedObjects((Referenceable)o, seen);
- }
- }
- }
- else if(value instanceof Map) {
- for(Object o : ((Map)value).values()) {
- if(o instanceof Referenceable) {
- findReferencedObjects((Referenceable)o, seen);
- }
- }
- }
- }
- }
-
- /**
- * Clears the state in the request context.
- *
- */
- public static void resetRequestContext() {
- //reset the context while preserving the user
- String user = RequestContext.get().getUser();
- RequestContext.createContext();
- RequestContext.get().setUser(user);
- }
-
- /**
- * Triggers the Atlas initialization process using the specified MetadataRepository.
- * This causes the built-in types and their indices to be created.
- */
- public static void setupGraphProvider(MetadataRepository repo) throws AtlasException {
- TypeCache typeCache = null;
- try {
- typeCache = AtlasRepositoryConfiguration.getTypeCache().newInstance();
- }
- catch(Throwable t) {
- typeCache = new DefaultTypeCache();
- }
- final GraphBackedSearchIndexer indexer = new GraphBackedSearchIndexer(new AtlasTypeRegistry());
-
- Configuration config = ApplicationProperties.get();
- ITypeStore typeStore = new GraphBackedTypeStore(AtlasGraphProvider.getGraphInstance());
- DefaultMetadataService defaultMetadataService = new DefaultMetadataService(repo,
- typeStore,
- new HashSet<TypesChangeListener>() {{ add(indexer); }},
- new HashSet<EntityChangeListener>(),
- TypeSystem.getInstance(),
- config,
- typeCache,
- // Fixme: Can we work with Noop
- new InMemoryEntityAuditRepository());
-
- //commit the created types
- getGraph().commit();
-
- }
-
- public static AtlasGraph getGraph() {
-
- return AtlasGraphProvider.getGraphInstance();
-
- }
-
- /**
- * Adds a proxy wrapper around the specified MetadataService that automatically
- * resets the request context before every call.
- *
- * @param delegate
- * @return
- */
- public static MetadataService addSessionCleanupWrapper(final MetadataService delegate) {
-
- return (MetadataService)Proxy.newProxyInstance(Thread.currentThread().getContextClassLoader(),
- new Class[]{MetadataService.class}, new InvocationHandler() {
-
- @Override
- public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
-
- try {
- resetRequestContext();
- Object result = method.invoke(delegate, args);
-
- return result;
- }
- catch(InvocationTargetException e) {
- e.getCause().printStackTrace();
- throw e.getCause();
- }
- catch(Throwable t) {
- t.printStackTrace();
- throw t;
- }
- }
-
- });
- }
-
- /**
- * Adds a proxy wrapper around the specified MetadataRepository that automatically
- * resets the request context before every call and either commits or rolls
- * back the graph transaction after every call.
- *
- * @param delegate
- * @return
- */
- public static MetadataRepository addTransactionWrapper(final MetadataRepository delegate) {
- return (MetadataRepository)Proxy.newProxyInstance(Thread.currentThread().getContextClassLoader(),
- new Class[]{MetadataRepository.class}, new InvocationHandler() {
-
- @Override
- public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
- boolean useTransaction = GraphBackedMetadataRepository.class.getMethod(
- method.getName(), method.getParameterTypes())
- .isAnnotationPresent(GraphTransaction.class);
- try {
- resetRequestContext();
- Object result = method.invoke(delegate, args);
- if(useTransaction) {
- System.out.println("Committing changes");
- getGraph().commit();
- System.out.println("Commit succeeded.");
- }
- return result;
- }
- catch(InvocationTargetException e) {
- e.getCause().printStackTrace();
- if(useTransaction) {
- System.out.println("Rolling back changes due to exception.");
- getGraph().rollback();
- }
- throw e.getCause();
- }
- catch(Throwable t) {
- t.printStackTrace();
- if(useTransaction) {
- System.out.println("Rolling back changes due to exception.");
- getGraph().rollback();
- }
- throw t;
- }
- }
-
- });
- }
-
- /**
- * Loads the entity and does sanity testing of the GuidMapping that was
- * created during the operation.
- *
- */
- public static ITypedReferenceableInstance loadAndDoSimpleValidation(String guid, Referenceable original, MetadataRepository repositoryService) throws AtlasException {
- ITypedReferenceableInstance loaded = repositoryService.getEntityDefinition(guid);
- doSimpleValidation(original, loaded);
- return loaded;
- }
-
- /**
- * Loads the entity and does sanity testing of the GuidMapping that was
- * created during the operation.
- *
- */
- public static ITypedReferenceableInstance loadAndDoSimpleValidation(String guid, Referenceable original, MetadataService repositoryService) throws AtlasException {
- ITypedReferenceableInstance loaded = repositoryService.getEntityDefinition(guid);
- doSimpleValidation(original, loaded);
- return loaded;
-
- }
-
- private static void doSimpleValidation(Referenceable original, IInstance loaded) throws AtlasException {
-
- assertEquals(loaded.getTypeName(), original.getTypeName());
- ClassType ct = TypeSystem.getInstance().getDataType(ClassType.class, loaded.getTypeName());
-
- //compare primitive fields
- for(AttributeInfo field : ct.fieldMapping.fields.values()) {
- if(field.dataType().getTypeCategory() == TypeCategory.PRIMITIVE) {
- if(original.get(field.name) != null) {
- Object rawLoadedValue = loaded.get(field.name);
- Object rawProvidedValue = original.get(field.name);
- Object convertedLoadedValue = field.dataType().convert(rawLoadedValue, Multiplicity.REQUIRED);
- Object convertedProvidedValue = field.dataType().convert(rawProvidedValue, Multiplicity.REQUIRED);
-
- assertEquals(convertedLoadedValue, convertedProvidedValue);
- }
- }
- }
- }
-
- /**
- * Validates that the two String Collections contain the same items, without
- * regard to order.
- *
- */
- public static void assertContentsSame(Collection<String> actual, Collection<String> expected) {
- assertEquals(actual.size(), expected.size());
- Set<String> checker = new HashSet<>();
- checker.addAll(expected);
- checker.removeAll(actual);
- assertEquals(checker.size(), 0);
- }
-
- public static void skipForGremlin3EnabledGraphDb() throws SkipException {
- //ATLAS-1579 Currently, some tests are skipped for titan1 backened. As these tests are hard coded to use Gremlin2. See ATLAS-1579, ATLAS-1591 once it is fixed, please remove it.
- if (TestUtils.getGraph().getSupportedGremlinVersion() == GremlinVersion.THREE) {
- throw new SkipException ("This test requires Gremlin2. Skipping test ");
- }
- }
-
-}