You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sh...@apache.org on 2016/08/01 17:56:02 UTC
[1/2] incubator-atlas git commit: ATLAS-1026 StoreBackedTypeCache
issues (dkantor via shwethags)
Repository: incubator-atlas
Updated Branches:
refs/heads/master 1c9b8b419 -> 69f6adfbc
ATLAS-1026 StoreBackedTypeCache issues (dkantor via shwethags)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/96a11675
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/96a11675
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/96a11675
Branch: refs/heads/master
Commit: 96a11675d83a0afc36e6893b6750abbb61a7c0fd
Parents: 1c9b8b4
Author: Shwetha GS <ss...@hortonworks.com>
Authored: Mon Aug 1 22:55:43 2016 +0530
Committer: Shwetha GS <ss...@hortonworks.com>
Committed: Mon Aug 1 22:55:43 2016 +0530
----------------------------------------------------------------------
release-log.txt | 1 +
.../typestore/StoreBackedTypeCache.java | 23 ++----
.../typestore/StoreBackedTypeCacheTest.java | 32 +--------
...StoreBackedTypeCacheMetadataServiceTest.java | 76 +++++++++++++++++---
.../atlas/typesystem/types/TypeSystem.java | 18 ++++-
.../types/cache/DefaultTypeCache.java | 6 ++
.../atlas/typesystem/types/cache/TypeCache.java | 13 ++++
7 files changed, 106 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/96a11675/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index c21b43a..871a70c 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES:
ALL CHANGES:
+ATLAS-1026 StoreBackedTypeCache issues (dkantor via shwethags)
ATLAS-861 1 table out of 50,000 tables is left unimported throwing exception during deserialization (sumasai via shwethags)
ATLAS-1065 UI: Full text search view same as DSL's (kevalbhat18 via shwethags)
ATLAS-1066 Falcon fails to post entity to Atlas due to kafka exception (mneethiraj via shwethags)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/96a11675/repository/src/main/java/org/apache/atlas/repository/typestore/StoreBackedTypeCache.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/StoreBackedTypeCache.java b/repository/src/main/java/org/apache/atlas/repository/typestore/StoreBackedTypeCache.java
index d0c6f6a..600c3b4 100644
--- a/repository/src/main/java/org/apache/atlas/repository/typestore/StoreBackedTypeCache.java
+++ b/repository/src/main/java/org/apache/atlas/repository/typestore/StoreBackedTypeCache.java
@@ -106,12 +106,6 @@ public class StoreBackedTypeCache extends DefaultTypeCache {
}
}
- @Override
- public boolean has(String typeName) throws AtlasException {
-
- return (get(typeName) != null);
- }
-
/**
* Checks whether the specified type is cached in memory and does *not*
* access the type store. Used for testing.
@@ -124,21 +118,12 @@ public class StoreBackedTypeCache extends DefaultTypeCache {
}
/**
- * Gets the requested type from the cache.
- * This implementation will check the type store if the type is
- * not already cached. If found in the type store, the type and
- * any required super and attribute types are loaded from the type store, and
- * added to the cache.
- *
- * @see org.apache.atlas.typesystem.types.cache.DefaultTypeCache#get(java.lang.String)
+ * Check the type store for the requested type.
+ * If found in the type store, the type and any required super and attribute types
+ * are loaded from the type store, and added to the cache.
*/
@Override
- public IDataType get(String typeName) throws AtlasException {
-
- IDataType type = super.get(typeName);
- if (type != null) {
- return type;
- }
+ public IDataType onTypeFault(String typeName) throws AtlasException {
// Type is not cached - check the type store.
// Any super and attribute types needed by the requested type
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/96a11675/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTest.java b/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTest.java
index 2e3a3b1..b7cf7e9 100644
--- a/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTest.java
@@ -49,7 +49,6 @@ import java.util.Map;
* Unit test for {@link StoreBackedTypeCache}
*/
@Guice(modules = RepositoryMetadataModule.class)
-@Test(enabled = false)
public class StoreBackedTypeCacheTest {
@Inject
@@ -106,6 +105,7 @@ public class StoreBackedTypeCacheTest {
ts.reset();
}
+ @Test
public void testGetClassType() throws Exception {
for (Map.Entry<String, ClassType> typeEntry : classTypesToTest.entrySet()) {
// Not cached yet
@@ -122,20 +122,7 @@ public class StoreBackedTypeCacheTest {
}
}
- public void testHasClassType() throws Exception {
- for (Map.Entry<String, ClassType> typeEntry : classTypesToTest.entrySet()) {
- // Not cached yet
- Assert.assertFalse(typeCache.isCachedInMemory(typeEntry.getKey()));
-
- // Calling has() should result in type and its dependencies
- // loaded from the type store and added to the cache.
- Assert.assertTrue(typeCache.has(typeEntry.getKey()));
-
- // Verify the type is now cached in memory.
- Assert.assertTrue(typeCache.isCachedInMemory(typeEntry.getKey()));
- }
- }
-
+ @Test
public void testGetTraitType() throws Exception {
ImmutableList<String> traitNames = ts.getTypeNamesByCategory(TypeCategory.TRAIT);
for (String traitTypeName : traitNames) {
@@ -153,21 +140,6 @@ public class StoreBackedTypeCacheTest {
}
}
- public void testHasTraitType() throws Exception {
- ImmutableList<String> traitNames = ts.getTypeNamesByCategory(TypeCategory.TRAIT);
- for (String traitTypeName : traitNames) {
- // Not cached yet
- Assert.assertFalse(typeCache.isCachedInMemory(traitTypeName));
-
- // Calling has() should result in type and its dependencies
- // loaded from the type store and added to the cache.
- Assert.assertTrue(typeCache.has(traitTypeName));
-
- // Verify the type is now cached.
- Assert.assertTrue(typeCache.isCachedInMemory(traitTypeName));
- }
- }
-
private <T extends HierarchicalType> void verifyHierarchicalType(T dataType, T expectedDataType) throws AtlasException {
Assert.assertEquals(dataType.numFields, expectedDataType.numFields);
Assert.assertEquals(dataType.immediateAttrs.size(), expectedDataType.immediateAttrs.size());
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/96a11675/repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java b/repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java
index 5df1c8d..8fb59c5 100644
--- a/repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java
@@ -23,27 +23,37 @@ import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.repository.typestore.StoreBackedTypeCache;
import org.apache.atlas.repository.typestore.StoreBackedTypeCacheTestModule;
import org.apache.atlas.services.MetadataService;
+import org.apache.atlas.typesystem.TypesDef;
+import org.apache.atlas.typesystem.json.TypesSerialization;
+import org.apache.atlas.typesystem.types.AttributeDefinition;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
+import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.atlas.typesystem.types.TypeUpdateException;
import org.apache.atlas.typesystem.types.cache.TypeCache;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
import com.google.inject.Inject;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
/**
- * Verify MetadataService type lookup triggers StoreBackedTypeCache to load type from the store.
- * StoreBackedTypeCacheTestModule Guice module uses Atlas configuration
- * which has type cache implementation class set to {@link StoreBackedTypeCache}.
+ * Verify MetadataService type operations trigger StoreBackedTypeCache to load non-cached types from the store.
+ * StoreBackedTypeCacheTestModule Guice module sets Atlas configuration
+ * to use {@link StoreBackedTypeCache} as the TypeCache implementation class.
*/
@Guice(modules = StoreBackedTypeCacheTestModule.class)
-@Test(enabled = false)
public class StoreBackedTypeCacheMetadataServiceTest
{
@Inject
@@ -55,13 +65,18 @@ public class StoreBackedTypeCacheMetadataServiceTest
@Inject
TypeCache typeCache;
+ private StoreBackedTypeCache storeBackedTypeCache;
+
@Inject
private GraphProvider<TitanGraph> graphProvider;
private TypeSystem ts;
@BeforeClass
- public void setUp() throws Exception {
+ public void oneTimeSetup() throws Exception {
+ Assert.assertTrue(typeCache instanceof StoreBackedTypeCache);
+ storeBackedTypeCache = (StoreBackedTypeCache) typeCache;
+
ts = TypeSystem.getInstance();
ts.reset();
@@ -70,6 +85,10 @@ public class StoreBackedTypeCacheMetadataServiceTest
TestUtils.createHiveTypes(ts);
ImmutableList<String> typeNames = ts.getTypeNames();
typeStore.store(ts, typeNames);
+ }
+
+ @BeforeMethod
+ public void setUp() throws Exception {
ts.reset();
}
@@ -91,16 +110,51 @@ public class StoreBackedTypeCacheMetadataServiceTest
}
}
- public void testIt() throws Exception {
- Assert.assertTrue(typeCache instanceof StoreBackedTypeCache);
- StoreBackedTypeCache storeBackedCache = (StoreBackedTypeCache) typeCache;
-
+ @Test
+ public void testGetTypeDefinition() throws Exception {
// Cache should be empty
- Assert.assertFalse(storeBackedCache.isCachedInMemory("Manager"));
+ Assert.assertFalse(storeBackedTypeCache.isCachedInMemory("Manager"));
// Type lookup on MetadataService should cause Manager type to be loaded from the type store
// and cached.
Assert.assertNotNull(metadataService.getTypeDefinition("Manager"));
- Assert.assertTrue(storeBackedCache.isCachedInMemory("Manager"));
+ Assert.assertTrue(storeBackedTypeCache.isCachedInMemory("Manager"));
+ }
+
+ @Test
+ public void testValidUpdateType() throws Exception {
+ // Cache should be empty
+ Assert.assertFalse(storeBackedTypeCache.isCachedInMemory(TestUtils.TABLE_TYPE));
+
+ TypesDef typesDef = TestUtils.defineHiveTypes();
+ String json = TypesSerialization.toJson(typesDef);
+
+ // Update types with same definition, which should succeed.
+ metadataService.updateType(json);
+
+ // hive_table type should now be cached.
+ Assert.assertTrue(storeBackedTypeCache.isCachedInMemory(TestUtils.TABLE_TYPE));
+ }
+
+ @Test
+ public void testInvalidUpdateType() throws Exception {
+ // Cache should be empty
+ Assert.assertFalse(storeBackedTypeCache.isCachedInMemory(TestUtils.TABLE_TYPE));
+
+ HierarchicalTypeDefinition<ClassType> classTypeDef = TypesUtil.createClassTypeDef(TestUtils.TABLE_TYPE, ImmutableSet.<String>of(),
+ new AttributeDefinition("attr1", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null));
+ String json = TypesSerialization.toJson(classTypeDef, false);
+
+ // Try to update the type with disallowed changes. Should fail with TypeUpdateException.
+ try {
+ metadataService.updateType(json);
+ Assert.fail(TypeUpdateException.class.getSimpleName() + " was expected but none thrown");
+ }
+ catch(TypeUpdateException e) {
+ // good
+ }
+
+ // hive_table type should now be cached.
+ Assert.assertTrue(storeBackedTypeCache.isCachedInMemory(TestUtils.TABLE_TYPE));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/96a11675/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
index 70f9ea5..70ba89b 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
@@ -148,7 +148,6 @@ public class TypeSystem {
}
public <T> T getDataType(Class<T> cls, String name) throws AtlasException {
-
if (isCoreType(name)) {
return cls.cast(coreTypes.get(name));
}
@@ -180,6 +179,14 @@ public class TypeSystem {
return cls.cast(dT);
}
+ /*
+ * Invoke cache callback to possibly obtain type from other storage.
+ */
+ IDataType dT = typeCache.onTypeFault(name);
+ if (dT != null) {
+ return cls.cast(dT);
+ }
+
throw new TypeNotFoundException(String.format("Unknown datatype: %s", name));
}
@@ -599,8 +606,13 @@ public class TypeSystem {
private void validateUpdateIsPossible() throws TypeUpdateException, AtlasException {
//If the type is modified, validate that update can be done
for (IDataType newType : transientTypes.values()) {
- if (TypeSystem.this.isRegistered(newType.getName())) {
- IDataType oldType = TypeSystem.this.typeCache.get(newType.getName());
+ IDataType oldType = null;
+ try {
+ oldType = TypeSystem.this.getDataType(IDataType.class, newType.getName());
+ } catch (TypeNotFoundException e) {
+ LOG.debug("No existing type %s found - update OK", newType.getName());
+ }
+ if (oldType != null) {
oldType.validateUpdate(newType);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/96a11675/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCache.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCache.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCache.java
index 6c6d5a0..ce750af 100644
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCache.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCache.java
@@ -288,4 +288,10 @@ public class DefaultTypeCache implements TypeCache {
types_.clear();
}
+
+ @Override
+ public IDataType onTypeFault(String typeName) throws AtlasException {
+
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/96a11675/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/TypeCache.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/TypeCache.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/TypeCache.java
index 2bcbdd7..87d83a6 100644
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/TypeCache.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/TypeCache.java
@@ -21,6 +21,7 @@ package org.apache.atlas.typesystem.types.cache;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.TypeSystem;
import java.util.Collection;
import java.util.Map;
@@ -137,4 +138,16 @@ public interface TypeCache {
*
*/
void clear();
+
+ /**
+ * Called when a type lookup request on {@link TypeSystem}
+ * fails because the type is not present in the runtime type information.
+ * Implementations can take action such as retrieving the requested type
+ * from some persistent storage.
+
+ * @param typeName
+ * @throws AtlasException
+ */
+ IDataType onTypeFault(String typeName) throws AtlasException;
+
}
[2/2] incubator-atlas git commit: ATLAS-1048 TestMetadata.py test in
distro project fails on Windows (jnhagelb via shwethags)
Posted by sh...@apache.org.
ATLAS-1048 TestMetadata.py test in distro project fails on Windows (jnhagelb via shwethags)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/69f6adfb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/69f6adfb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/69f6adfb
Branch: refs/heads/master
Commit: 69f6adfbc78e13485db0e20a7b8b2add431333d2
Parents: 96a1167
Author: Shwetha GS <ss...@hortonworks.com>
Authored: Mon Aug 1 22:57:27 2016 +0530
Committer: Shwetha GS <ss...@hortonworks.com>
Committed: Mon Aug 1 22:57:27 2016 +0530
----------------------------------------------------------------------
distro/src/test/python/scripts/TestMetadata.py | 4 ++--
release-log.txt | 1 +
2 files changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/69f6adfb/distro/src/test/python/scripts/TestMetadata.py
----------------------------------------------------------------------
diff --git a/distro/src/test/python/scripts/TestMetadata.py b/distro/src/test/python/scripts/TestMetadata.py
index 90c5b97..f943b8a 100644
--- a/distro/src/test/python/scripts/TestMetadata.py
+++ b/distro/src/test/python/scripts/TestMetadata.py
@@ -64,7 +64,7 @@ class TestMetadata(unittest.TestCase):
self.assertTrue(configure_hbase_mock.called)
if IS_WINDOWS:
- calls = [call(['atlas_home\\hbase\\bin\\start-hbase.cmd', '--config', 'atlas_home\\hbase\\conf', 'start', 'master'], 'atlas_home\\logs', False, True),
+ calls = [call(['atlas_home\\hbase\\bin\\start-hbase.cmd', '--config', 'atlas_home\\hbase\\conf'], 'atlas_home\\logs', False, True),
call(['atlas_home\\solr\\bin\\solr.cmd', 'start', '-z', 'localhost:9838', '-p', '9838'], 'atlas_home\\logs', False, True),
call(['atlas_home\\solr\\bin\\solr.cmd', 'create', '-c', 'vertex_index', '-d', 'atlas_home\\solr\\server\\solr\\configsets\\basic_configs\\conf', '-shards', '1', '-replicationFactor', '1'], 'atlas_home\\logs', False, True),
call(['atlas_home\\solr\\bin\\solr.cmd', 'create', '-c', 'edge_index', '-d', 'atlas_home\\solr\\server\\solr\\configsets\\basic_configs\\conf', '-shards', '1', '-replicationFactor', '1'], 'atlas_home\\logs', False, True),
@@ -130,7 +130,7 @@ class TestMetadata(unittest.TestCase):
self.assertFalse(configure_hbase_mock.called)
if IS_WINDOWS:
- calls = [call(['atlas_home\\hbase\\bin\\start-hbase.cmd', '--config', 'atlas_home\\hbase\\conf', 'start', 'master'], 'atlas_home\\logs', False, True),
+ calls = [call(['atlas_home\\hbase\\bin\\start-hbase.cmd', '--config', 'atlas_home\\hbase\\conf'], 'atlas_home\\logs', False, True),
call(['atlas_home\\solr\\bin\\solr.cmd', 'start', '-z', 'localhost:9838', '-p', '9838'], 'atlas_home\\logs', False, True),
call(['atlas_home\\solr\\bin\\solr.cmd', 'create', '-c', 'vertex_index', '-d', 'atlas_home\\solr\\server\\solr\\configsets\\basic_configs\\conf', '-shards', '1', '-replicationFactor', '1'], 'atlas_home\\logs', False, True),
call(['atlas_home\\solr\\bin\\solr.cmd', 'create', '-c', 'edge_index', '-d', 'atlas_home\\solr\\server\\solr\\configsets\\basic_configs\\conf', '-shards', '1', '-replicationFactor', '1'], 'atlas_home\\logs', False, True),
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/69f6adfb/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 871a70c..c16c62a 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES:
ALL CHANGES:
+ATLAS-1048 TestMetadata.py test in distro project fails on Windows (jnhagelb via shwethags)
ATLAS-1026 StoreBackedTypeCache issues (dkantor via shwethags)
ATLAS-861 1 table out of 50,000 tables is left unimported throwing exception during deserialization (sumasai via shwethags)
ATLAS-1065 UI: Full text search view same as DSL's (kevalbhat18 via shwethags)