You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by dk...@apache.org on 2016/08/05 23:27:31 UTC
[1/9] incubator-atlas git commit: ATLAS-693 Titan 0.5.4
implementation of graph db abstraction. (jnhagelb via dkantor)
Repository: incubator-atlas
Updated Branches:
refs/heads/master 2ef0fc46d -> 4fa10b6ae
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/titan/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueStoreTest.java
----------------------------------------------------------------------
diff --git a/titan/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueStoreTest.java b/titan/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueStoreTest.java
deleted file mode 100644
index 7ed636a..0000000
--- a/titan/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueStoreTest.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.thinkaurelius.titan.diskstorage.hbase;
-
-import com.thinkaurelius.titan.diskstorage.BackendException;
-import com.thinkaurelius.titan.diskstorage.EntryMetaData;
-import com.thinkaurelius.titan.diskstorage.StaticBuffer;
-import com.thinkaurelius.titan.diskstorage.configuration.Configuration;
-import com.thinkaurelius.titan.diskstorage.locking.LocalLockMediator;
-import com.thinkaurelius.titan.diskstorage.locking.PermanentLockingException;
-import com.thinkaurelius.titan.diskstorage.util.KeyColumn;
-import com.thinkaurelius.titan.diskstorage.util.time.StandardDuration;
-import com.thinkaurelius.titan.diskstorage.util.time.Timepoint;
-import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration;
-import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.BeforeTest;
-import org.testng.annotations.Test;
-
-import java.util.concurrent.TimeUnit;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-import static org.testng.Assert.fail;
-
-public class HBaseKeyColumnValueStoreTest {
-
- @Mock
- HBaseStoreManager storeManager;
-
- @Mock
- ConnectionMask connectionMask;
-
- @Mock
- LocalLockMediator localLockMediator;
-
- @Mock
- StaticBuffer key;
-
- @Mock
- StaticBuffer column;
-
- @Mock
- StaticBuffer expectedValue;
-
- @Mock
- HBaseTransaction transaction;
-
- @Mock
- Configuration storageConfig;
-
- @BeforeMethod
- public void setup() {
- MockitoAnnotations.initMocks(this);
- }
-
- @Test
- public void shouldSucceedInLockingIfLockMediatorSucceeds() throws BackendException {
-
- when(storeManager.getMetaDataSchema("hbase")).thenReturn(new EntryMetaData[] {EntryMetaData.TIMESTAMP});
- when(storeManager.getStorageConfig()).thenReturn(storageConfig);
- when(storageConfig.get(GraphDatabaseConfiguration.LOCK_EXPIRE)).thenReturn(
- new StandardDuration(300L, TimeUnit.MILLISECONDS));
- when(storageConfig.get(GraphDatabaseConfiguration.LOCK_WAIT)).thenReturn(
- new StandardDuration(10L, TimeUnit.MILLISECONDS));
- when(storageConfig.get(GraphDatabaseConfiguration.LOCK_RETRY)).thenReturn(3);
- KeyColumn lockID = new KeyColumn(key, column);
- when(localLockMediator.lock(eq(lockID), eq(transaction), any(Timepoint.class))).
- thenReturn(true);
-
- HBaseKeyColumnValueStore hBaseKeyColumnValueStore =
- new HBaseKeyColumnValueStore(storeManager, connectionMask, "titan", "e", "hbase", localLockMediator);
- hBaseKeyColumnValueStore.acquireLock(key, column, expectedValue, transaction);
-
- verify(transaction).updateLocks(lockID, expectedValue);
- verify(localLockMediator, times(1)).lock(eq(lockID), eq(transaction), any(Timepoint.class));
- }
-
- @Test
- public void shouldRetryRightNumberOfTimesIfLockMediationFails() throws BackendException {
- when(storeManager.getMetaDataSchema("hbase")).thenReturn(new EntryMetaData[] {EntryMetaData.TIMESTAMP});
- when(storeManager.getStorageConfig()).thenReturn(storageConfig);
- when(storageConfig.get(GraphDatabaseConfiguration.LOCK_EXPIRE)).thenReturn(
- new StandardDuration(300L, TimeUnit.MILLISECONDS));
- when(storageConfig.get(GraphDatabaseConfiguration.LOCK_WAIT)).thenReturn(
- new StandardDuration(10L, TimeUnit.MILLISECONDS));
- when(storageConfig.get(GraphDatabaseConfiguration.LOCK_RETRY)).thenReturn(3);
- KeyColumn lockID = new KeyColumn(key, column);
- when(localLockMediator.lock(eq(lockID), eq(transaction), any(Timepoint.class))).
- thenReturn(false).thenReturn(false).thenReturn(true);
-
- HBaseKeyColumnValueStore hBaseKeyColumnValueStore =
- new HBaseKeyColumnValueStore(storeManager, connectionMask, "titan", "e", "hbase", localLockMediator);
- hBaseKeyColumnValueStore.acquireLock(key, column, expectedValue, transaction);
-
- verify(transaction).updateLocks(lockID, expectedValue);
- verify(localLockMediator, times(3)).lock(eq(lockID), eq(transaction), any(Timepoint.class));
- }
-
- @Test(expectedExceptions = PermanentLockingException.class)
- public void shouldThrowExceptionAfterConfiguredRetriesIfLockMediationFails() throws BackendException {
- when(storeManager.getMetaDataSchema("hbase")).thenReturn(new EntryMetaData[] {EntryMetaData.TIMESTAMP});
- when(storeManager.getStorageConfig()).thenReturn(storageConfig);
- when(storageConfig.get(GraphDatabaseConfiguration.LOCK_EXPIRE)).thenReturn(
- new StandardDuration(300L, TimeUnit.MILLISECONDS));
- when(storageConfig.get(GraphDatabaseConfiguration.LOCK_WAIT)).thenReturn(
- new StandardDuration(10L, TimeUnit.MILLISECONDS));
- when(storageConfig.get(GraphDatabaseConfiguration.LOCK_RETRY)).thenReturn(3);
- KeyColumn lockID = new KeyColumn(key, column);
- when(localLockMediator.lock(eq(lockID), eq(transaction), any(Timepoint.class))).
- thenReturn(false).thenReturn(false).thenReturn(false);
-
- HBaseKeyColumnValueStore hBaseKeyColumnValueStore =
- new HBaseKeyColumnValueStore(storeManager, connectionMask, "titan", "e", "hbase", localLockMediator);
- hBaseKeyColumnValueStore.acquireLock(key, column, expectedValue, transaction);
-
- fail("Should fail as lock could not be acquired after 3 retries.");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/titan/src/test/java/com/thinkaurelius/titan/diskstorage/locking/LocalLockMediatorTest.java
----------------------------------------------------------------------
diff --git a/titan/src/test/java/com/thinkaurelius/titan/diskstorage/locking/LocalLockMediatorTest.java b/titan/src/test/java/com/thinkaurelius/titan/diskstorage/locking/LocalLockMediatorTest.java
deleted file mode 100644
index d0fd401..0000000
--- a/titan/src/test/java/com/thinkaurelius/titan/diskstorage/locking/LocalLockMediatorTest.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright 2012-2013 Aurelius LLC
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.thinkaurelius.titan.diskstorage.locking;
-
-import com.thinkaurelius.titan.diskstorage.hbase.HBaseTransaction;
-import com.thinkaurelius.titan.diskstorage.util.time.TimestampProvider;
-import com.thinkaurelius.titan.diskstorage.util.time.Timestamps;
-import com.thinkaurelius.titan.diskstorage.StaticBuffer;
-import com.thinkaurelius.titan.diskstorage.util.KeyColumn;
-import com.thinkaurelius.titan.diskstorage.util.StaticArrayBuffer;
-import org.mockito.Mockito;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.util.concurrent.TimeUnit;
-
-public class LocalLockMediatorTest {
-
- private static final String LOCK_NAMESPACE = "test";
- private static final StaticBuffer LOCK_ROW = StaticArrayBuffer.of(new byte[]{1});
- private static final StaticBuffer LOCK_COL = StaticArrayBuffer.of(new byte[]{1});
- private static final KeyColumn kc = new KeyColumn(LOCK_ROW, LOCK_COL);
- private static final HBaseTransaction mockTx1 = Mockito.mock(HBaseTransaction.class);
- private static final HBaseTransaction mockTx2 = Mockito.mock(HBaseTransaction.class);
-
- @Test
- public void testLock() throws InterruptedException {
- TimestampProvider times = Timestamps.MICRO;
- LocalLockMediator<HBaseTransaction> llm =
- new LocalLockMediator<HBaseTransaction>(LOCK_NAMESPACE, times);
-
- //Expire immediately
- Assert.assertTrue(llm.lock(kc, mockTx1, times.getTime(0, TimeUnit.NANOSECONDS)));
- Assert.assertTrue(llm.lock(kc, mockTx2, times.getTime(Long.MAX_VALUE, TimeUnit.NANOSECONDS)));
-
- llm = new LocalLockMediator<HBaseTransaction>(LOCK_NAMESPACE, times);
-
- //Expire later
- Assert.assertTrue(llm.lock(kc, mockTx1, times.getTime(Long.MAX_VALUE, TimeUnit.NANOSECONDS)));
- //So second lock should fail on same keyCol
- Assert.assertFalse(llm.lock(kc, mockTx2, times.getTime(Long.MAX_VALUE, TimeUnit.NANOSECONDS)));
-
- //Unlock
- Assert.assertTrue(llm.unlock(kc, mockTx1));
- //Now locking should succeed
- Assert.assertTrue(llm.lock(kc, mockTx2, times.getTime(Long.MAX_VALUE, TimeUnit.NANOSECONDS)));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/webapp/pom.xml
----------------------------------------------------------------------
diff --git a/webapp/pom.xml b/webapp/pom.xml
index b2cd18c..708a216 100755
--- a/webapp/pom.xml
+++ b/webapp/pom.xml
@@ -105,6 +105,11 @@
<dependency>
<groupId>org.apache.atlas</groupId>
+ <artifactId>atlas-graphdb-titan0</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.atlas</groupId>
<artifactId>atlas-client</artifactId>
</dependency>
@@ -123,7 +128,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</dependency>
-
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minikdc</artifactId>
@@ -133,7 +137,7 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
</dependency>
-
+
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-catalog</artifactId>
@@ -166,16 +170,6 @@
</dependency>
<dependency>
- <groupId>com.tinkerpop.blueprints</groupId>
- <artifactId>blueprints-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.thinkaurelius.titan</groupId>
- <artifactId>titan-core</artifactId>
- </dependency>
-
- <dependency>
<groupId>com.googlecode.json-simple</groupId>
<artifactId>json-simple</artifactId>
</dependency>
[2/9] incubator-atlas git commit: ATLAS-693 Titan 0.5.4
implementation of graph db abstraction. (jnhagelb via dkantor)
Posted by dk...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/locking/LocalLockMediator.java
----------------------------------------------------------------------
diff --git a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/locking/LocalLockMediator.java b/titan/src/main/java/com/thinkaurelius/titan/diskstorage/locking/LocalLockMediator.java
deleted file mode 100644
index 20c59e1..0000000
--- a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/locking/LocalLockMediator.java
+++ /dev/null
@@ -1,345 +0,0 @@
-/*
- * Copyright 2012-2013 Aurelius LLC
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.thinkaurelius.titan.diskstorage.locking;
-
-import com.google.common.base.Preconditions;
-import com.thinkaurelius.titan.diskstorage.util.time.Timepoint;
-import com.thinkaurelius.titan.diskstorage.util.time.TimestampProvider;
-import com.thinkaurelius.titan.diskstorage.locking.consistentkey.ExpectedValueCheckingTransaction;
-import com.thinkaurelius.titan.diskstorage.util.KeyColumn;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.DelayQueue;
-import java.util.concurrent.Delayed;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.TimeUnit;
-
-/**
- * This class resolves lock contention between two transactions on the same JVM.
- * <p/>
- * This is not just an optimization to reduce network traffic. Locks written by
- * Titan to a distributed key-value store contain an identifier, the "Rid",
- * which is unique only to the process level. The Rid can't tell which
- * transaction in a process holds any given lock. This class prevents two
- * transactions in a single process from concurrently writing the same lock to a
- * distributed key-value store.
- *
- * @author Dan LaRocque <da...@hopcount.org>
- */
-
-public class LocalLockMediator<T> {
-
- private static final Logger log = LoggerFactory
- .getLogger(LocalLockMediator.class);
-
- /**
- * Namespace for which this mediator is responsible
- *
- * @see LocalLockMediatorProvider
- */
- private final String name;
-
- private final TimestampProvider times;
-
- private DelayQueue<ExpirableKeyColumn> expiryQueue = new DelayQueue<>();
-
- private ExecutorService lockCleanerService = Executors.newFixedThreadPool(1, new ThreadFactory() {
- @Override
- public Thread newThread(Runnable runnable) {
- Thread thread = Executors.defaultThreadFactory().newThread(runnable);
- thread.setDaemon(true);
- return thread;
- }
- });
-
-
-
- /**
- * Maps a ({@code key}, {@code column}) pair to the local transaction
- * holding a lock on that pair. Values in this map may have already expired
- * according to {@link AuditRecord#expires}, in which case the lock should
- * be considered invalid.
- */
- private final ConcurrentHashMap<KeyColumn, AuditRecord<T>> locks = new ConcurrentHashMap<KeyColumn, AuditRecord<T>>();
-
- public LocalLockMediator(String name, TimestampProvider times) {
- this.name = name;
- this.times = times;
-
- Preconditions.checkNotNull(name);
- Preconditions.checkNotNull(times);
- lockCleanerService.submit(new LockCleaner());
- }
-
- /**
- * Acquire the lock specified by {@code kc}.
- * <p/>
- * <p/>
- * For any particular key-column, whatever value of {@code requestor} is
- * passed to this method must also be passed to the associated later call to
- * {@link #unlock(KeyColumn, ExpectedValueCheckingTransaction)}.
- * <p/>
- * If some requestor {@code r} calls this method on a KeyColumn {@code k}
- * and this method returns true, then subsequent calls to this method by
- * {@code r} on {@code l} merely attempt to update the {@code expiresAt}
- * timestamp. This differs from typical lock reentrance: multiple successful
- * calls to this method do not require an equal number of calls to
- * {@code #unlock()}. One {@code #unlock()} call is enough, no matter how
- * many times a {@code requestor} called {@code lock} beforehand. Note that
- * updating the timestamp may fail, in which case the lock is considered to
- * have expired and the calling context should assume it no longer holds the
- * lock specified by {@code kc}.
- * <p/>
- * The number of nanoseconds elapsed since the UNIX Epoch is not readily
- * available within the JVM. When reckoning expiration times, this method
- * uses the approximation implemented by
- * {@link com.thinkaurelius.titan.diskstorage.util.NanoTime#getApproxNSSinceEpoch(false)}.
- * <p/>
- * The current implementation of this method returns true when given an
- * {@code expiresAt} argument in the past. Future implementations may return
- * false instead.
- *
- * @param kc lock identifier
- * @param requestor the object locking {@code kc}
- * @param expires instant at which this lock will automatically expire
- * @return true if the lock is acquired, false if it was not acquired
- */
- public boolean lock(KeyColumn kc, T requestor, Timepoint expires) {
- assert null != kc;
- assert null != requestor;
-
- AuditRecord<T> audit = new AuditRecord<T>(requestor, expires);
- AuditRecord<T> inmap = locks.putIfAbsent(kc, audit);
-
- boolean success = false;
-
- if (null == inmap) {
- // Uncontended lock succeeded
- if (log.isTraceEnabled()) {
- log.trace("New local lock created: {} namespace={} txn={}",
- new Object[]{kc, name, requestor});
- }
- success = true;
- } else if (inmap.equals(audit)) {
- // requestor has already locked kc; update expiresAt
- success = locks.replace(kc, inmap, audit);
- if (log.isTraceEnabled()) {
- if (success) {
- log.trace(
- "Updated local lock expiration: {} namespace={} txn={} oldexp={} newexp={}",
- new Object[]{kc, name, requestor, inmap.expires,
- audit.expires});
- } else {
- log.trace(
- "Failed to update local lock expiration: {} namespace={} txn={} oldexp={} newexp={}",
- new Object[]{kc, name, requestor, inmap.expires,
- audit.expires});
- }
- }
- } else if (0 > inmap.expires.compareTo(times.getTime())) {
- // the recorded lock has expired; replace it
- success = locks.replace(kc, inmap, audit);
- if (log.isTraceEnabled()) {
- log.trace(
- "Discarding expired lock: {} namespace={} txn={} expired={}",
- new Object[]{kc, name, inmap.holder, inmap.expires});
- }
- } else {
- // we lost to a valid lock
- if (log.isTraceEnabled()) {
- log.trace(
- "Local lock failed: {} namespace={} txn={} (already owned by {})",
- new Object[]{kc, name, requestor, inmap});
- }
- }
-
- if (success) {
- expiryQueue.add(new ExpirableKeyColumn(kc, expires));
- }
- return success;
- }
-
- /**
- * Release the lock specified by {@code kc} and which was previously
- * locked by {@code requestor}, if it is possible to release it.
- *
- * @param kc lock identifier
- * @param requestor the object which previously locked {@code kc}
- */
- public boolean unlock(KeyColumn kc, T requestor) {
-
- if (!locks.containsKey(kc)) {
- log.info("Local unlock failed: no locks found for {}", kc);
- return false;
- }
-
- AuditRecord<T> unlocker = new AuditRecord<T>(requestor, null);
-
- AuditRecord<T> holder = locks.get(kc);
-
- if (!holder.equals(unlocker)) {
- log.error("Local unlock of {} by {} failed: it is held by {}",
- new Object[]{kc, unlocker, holder});
- return false;
- }
-
- boolean removed = locks.remove(kc, unlocker);
-
- if (removed) {
- expiryQueue.remove(kc);
- if (log.isTraceEnabled()) {
- log.trace("Local unlock succeeded: {} namespace={} txn={}",
- new Object[]{kc, name, requestor});
- }
- } else {
- log.warn("Local unlock warning: lock record for {} disappeared "
- + "during removal; this suggests the lock either expired "
- + "while we were removing it, or that it was erroneously "
- + "unlocked multiple times.", kc);
- }
-
- // Even if !removed, we're finished unlocking, so return true
- return true;
- }
-
- public String toString() {
- return "LocalLockMediator [" + name + ", ~" + locks.size()
- + " current locks]";
- }
-
- /**
- * A record containing the local transaction that holds a lock and the
- * lock's expiration time.
- */
- private static class AuditRecord<T> {
-
- /**
- * The local transaction that holds/held the lock.
- */
- private final T holder;
- /**
- * The expiration time of a the lock.
- */
- private final Timepoint expires;
- /**
- * Cached hashCode.
- */
- private int hashCode;
-
- private AuditRecord(T holder, Timepoint expires) {
- this.holder = holder;
- this.expires = expires;
- }
-
- /**
- * This implementation depends only on the lock holder and not on the
- * lock expiration time.
- */
- @Override
- public int hashCode() {
- if (0 == hashCode)
- hashCode = holder.hashCode();
-
- return hashCode;
- }
-
- /**
- * This implementation depends only on the lock holder and not on the
- * lock expiration time.
- */
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- /*
- * This warning suppression is harmless because we are only going to
- * call other.holder.equals(...), and since equals(...) is part of
- * Object, it is guaranteed to be defined no matter the concrete
- * type of parameter T.
- */
- @SuppressWarnings("rawtypes")
- AuditRecord other = (AuditRecord) obj;
- if (holder == null) {
- if (other.holder != null)
- return false;
- } else if (!holder.equals(other.holder))
- return false;
- return true;
- }
-
- @Override
- public String toString() {
- return "AuditRecord [txn=" + holder + ", expires=" + expires + "]";
- }
-
- }
-
- private class LockCleaner implements Runnable {
-
- @Override
- public void run() {
- try {
- while (true) {
- log.debug("Lock Cleaner service started");
- ExpirableKeyColumn lock = expiryQueue.take();
- log.debug("Expiring key column " + lock.getKeyColumn());
- locks.remove(lock.getKeyColumn());
- }
- } catch (InterruptedException e) {
- log.debug("Received interrupt. Exiting");
- }
- }
- }
-
- private static class ExpirableKeyColumn implements Delayed {
-
- private Timepoint expiryTime;
- private KeyColumn kc;
-
- public ExpirableKeyColumn(KeyColumn keyColumn, Timepoint expiryTime) {
- this.kc = keyColumn;
- this.expiryTime = expiryTime;
- }
-
- @Override
- public long getDelay(TimeUnit unit) {
- return expiryTime.getTimestamp(TimeUnit.NANOSECONDS);
- }
-
- @Override
- public int compareTo(Delayed o) {
- if (this.expiryTime.getTimestamp(TimeUnit.NANOSECONDS) < ((ExpirableKeyColumn) o).expiryTime.getTimestamp(TimeUnit.NANOSECONDS)) {
- return -1;
- }
- if (this.expiryTime.getTimestamp(TimeUnit.NANOSECONDS) > ((ExpirableKeyColumn) o).expiryTime.getTimestamp(TimeUnit.NANOSECONDS)) {
- return 1;
- }
- return 0;
- }
-
- public KeyColumn getKeyColumn() {
- return kc;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/solr/Solr5Index.java
----------------------------------------------------------------------
diff --git a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/solr/Solr5Index.java b/titan/src/main/java/com/thinkaurelius/titan/diskstorage/solr/Solr5Index.java
deleted file mode 100644
index 3b5620c..0000000
--- a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/solr/Solr5Index.java
+++ /dev/null
@@ -1,975 +0,0 @@
-/*
- * Copyright 2012-2013 Aurelius LLC
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.thinkaurelius.titan.diskstorage.solr;
-
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Sets;
-import com.thinkaurelius.titan.core.Order;
-import com.thinkaurelius.titan.core.TitanElement;
-import com.thinkaurelius.titan.core.attribute.Cmp;
-import com.thinkaurelius.titan.core.attribute.Geo;
-import com.thinkaurelius.titan.core.attribute.Geoshape;
-import com.thinkaurelius.titan.core.attribute.Text;
-import com.thinkaurelius.titan.core.schema.Mapping;
-import com.thinkaurelius.titan.diskstorage.BackendException;
-import com.thinkaurelius.titan.diskstorage.BaseTransaction;
-import com.thinkaurelius.titan.diskstorage.BaseTransactionConfig;
-import com.thinkaurelius.titan.diskstorage.BaseTransactionConfigurable;
-import com.thinkaurelius.titan.diskstorage.PermanentBackendException;
-import com.thinkaurelius.titan.diskstorage.TemporaryBackendException;
-import com.thinkaurelius.titan.diskstorage.configuration.ConfigNamespace;
-import com.thinkaurelius.titan.diskstorage.configuration.ConfigOption;
-import com.thinkaurelius.titan.diskstorage.configuration.Configuration;
-import com.thinkaurelius.titan.diskstorage.indexing.IndexEntry;
-import com.thinkaurelius.titan.diskstorage.indexing.IndexFeatures;
-import com.thinkaurelius.titan.diskstorage.indexing.IndexMutation;
-import com.thinkaurelius.titan.diskstorage.indexing.IndexProvider;
-import com.thinkaurelius.titan.diskstorage.indexing.IndexQuery;
-import com.thinkaurelius.titan.diskstorage.indexing.KeyInformation;
-import com.thinkaurelius.titan.diskstorage.indexing.RawQuery;
-import com.thinkaurelius.titan.diskstorage.util.DefaultTransaction;
-import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration;
-import com.thinkaurelius.titan.graphdb.configuration.PreInitializeConfigOptions;
-import com.thinkaurelius.titan.graphdb.database.serialize.AttributeUtil;
-import com.thinkaurelius.titan.graphdb.database.serialize.attribute.AbstractDecimal;
-import com.thinkaurelius.titan.graphdb.query.TitanPredicate;
-import com.thinkaurelius.titan.graphdb.query.condition.And;
-import com.thinkaurelius.titan.graphdb.query.condition.Condition;
-import com.thinkaurelius.titan.graphdb.query.condition.Not;
-import com.thinkaurelius.titan.graphdb.query.condition.Or;
-import com.thinkaurelius.titan.graphdb.query.condition.PredicateCondition;
-import com.thinkaurelius.titan.graphdb.types.ParameterType;
-import org.apache.commons.lang.StringUtils;
-import org.apache.http.client.HttpClient;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.HttpClientUtil;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
-import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer;
-import org.apache.solr.client.solrj.impl.LBHttpSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.request.UpdateRequest;
-import org.apache.solr.client.solrj.response.CollectionAdminResponse;
-import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.client.solrj.util.ClientUtils;
-import org.apache.solr.common.SolrDocument;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.ClusterState;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.Slice;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.zookeeper.KeeperException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.TimeZone;
-import java.util.UUID;
-
-import static com.thinkaurelius.titan.core.attribute.Cmp.*;
-import static com.thinkaurelius.titan.core.schema.Mapping.*;
-
-/**
- * NOTE: Copied from titan for supporting sol5. Do not change
- */
-@PreInitializeConfigOptions
-public class Solr5Index implements IndexProvider {
-
- private static final Logger logger = LoggerFactory.getLogger(Solr5Index.class);
-
-
- private static final String DEFAULT_ID_FIELD = "id";
-
- private enum Mode {
- HTTP, CLOUD;
-
- public static Mode parse(String mode) {
- for (Mode m : Mode.values()) {
- if (m.toString().equalsIgnoreCase(mode)) return m;
- }
- throw new IllegalArgumentException("Unrecognized mode: "+mode);
- }
- }
-
- public static final ConfigNamespace SOLR_NS =
- new ConfigNamespace(GraphDatabaseConfiguration.INDEX_NS, "solr", "Solr index configuration");
-
- public static final ConfigOption<String> SOLR_MODE = new ConfigOption<String>(SOLR_NS,"mode",
- "The operation mode for Solr which is either via HTTP (`http`) or using SolrCloud (`cloud`)",
- ConfigOption.Type.GLOBAL_OFFLINE, "cloud");
-
- public static final ConfigOption<Boolean> DYNAMIC_FIELDS = new ConfigOption<Boolean>(SOLR_NS,"dyn-fields",
- "Whether to use dynamic fields (which appends the data type to the field name). If dynamic fields is disabled" +
- "the user must map field names and define them explicitly in the schema.",
- ConfigOption.Type.GLOBAL_OFFLINE, true);
-
- public static final ConfigOption<String[]> KEY_FIELD_NAMES = new ConfigOption<String[]>(SOLR_NS,"key-field-names",
- "Field name that uniquely identifies each document in Solr. Must be specified as a list of `collection=field`.",
- ConfigOption.Type.GLOBAL, String[].class);
-
- public static final ConfigOption<String> TTL_FIELD = new ConfigOption<String>(SOLR_NS,"ttl_field",
- "Name of the TTL field for Solr collections.",
- ConfigOption.Type.GLOBAL_OFFLINE, "ttl");
-
- /** SolrCloud Configuration */
-
- public static final ConfigOption<String> ZOOKEEPER_URL = new ConfigOption<String>(SOLR_NS,"zookeeper-url",
- "URL of the Zookeeper instance coordinating the SolrCloud cluster",
- ConfigOption.Type.MASKABLE, "localhost:2181");
-
- public static final ConfigOption<Integer> NUM_SHARDS = new ConfigOption<Integer>(SOLR_NS,"num-shards",
- "Number of shards for a collection. This applies when creating a new collection which is only supported under the SolrCloud operation mode.",
- ConfigOption.Type.GLOBAL_OFFLINE, 1);
-
- public static final ConfigOption<Integer> MAX_SHARDS_PER_NODE = new ConfigOption<Integer>(SOLR_NS,"max-shards-per-node",
- "Maximum number of shards per node. This applies when creating a new collection which is only supported under the SolrCloud operation mode.",
- ConfigOption.Type.GLOBAL_OFFLINE, 1);
-
- public static final ConfigOption<Integer> REPLICATION_FACTOR = new ConfigOption<Integer>(SOLR_NS,"replication-factor",
- "Replication factor for a collection. This applies when creating a new collection which is only supported under the SolrCloud operation mode.",
- ConfigOption.Type.GLOBAL_OFFLINE, 1);
-
-
- /** HTTP Configuration */
-
- public static final ConfigOption<String[]> HTTP_URLS = new ConfigOption<String[]>(SOLR_NS,"http-urls",
- "List of URLs to use to connect to Solr Servers (LBHttpSolrClient is used), don't add core or collection name to the URL.",
- ConfigOption.Type.MASKABLE, new String[] { "http://localhost:8983/solr" });
-
- public static final ConfigOption<Integer> HTTP_CONNECTION_TIMEOUT = new ConfigOption<Integer>(SOLR_NS,"http-connection-timeout",
- "Solr HTTP connection timeout.",
- ConfigOption.Type.MASKABLE, 5000);
-
- public static final ConfigOption<Boolean> HTTP_ALLOW_COMPRESSION = new ConfigOption<Boolean>(SOLR_NS,"http-compression",
- "Enable/disable compression on the HTTP connections made to Solr.",
- ConfigOption.Type.MASKABLE, false);
-
- public static final ConfigOption<Integer> HTTP_MAX_CONNECTIONS_PER_HOST = new ConfigOption<Integer>(SOLR_NS,"http-max-per-host",
- "Maximum number of HTTP connections per Solr host.",
- ConfigOption.Type.MASKABLE, 20);
-
- public static final ConfigOption<Integer> HTTP_GLOBAL_MAX_CONNECTIONS = new ConfigOption<Integer>(SOLR_NS,"http-max",
- "Maximum number of HTTP connections in total to all Solr servers.",
- ConfigOption.Type.MASKABLE, 100);
-
- public static final ConfigOption<Boolean> WAIT_SEARCHER = new ConfigOption<Boolean>(SOLR_NS, "wait-searcher",
- "When mutating - wait for the index to reflect new mutations before returning. This can have a negative impact on performance.",
- ConfigOption.Type.LOCAL, false);
-
-
-
- private static final IndexFeatures SOLR_FEATURES = new IndexFeatures.Builder().supportsDocumentTTL()
- .setDefaultStringMapping(TEXT).supportedStringMappings(TEXT, STRING).build();
-
- private final SolrClient solrClient;
- private final Configuration configuration;
- private final Mode mode;
- private final boolean dynFields;
- private final Map<String, String> keyFieldIds;
- private final String ttlField;
- private final int maxResults;
- private final boolean waitSearcher;
-
- public Solr5Index(final Configuration config) throws BackendException {
- Preconditions.checkArgument(config!=null);
- configuration = config;
-
- mode = Mode.parse(config.get(SOLR_MODE));
- dynFields = config.get(DYNAMIC_FIELDS);
- keyFieldIds = parseKeyFieldsForCollections(config);
- maxResults = config.get(GraphDatabaseConfiguration.INDEX_MAX_RESULT_SET_SIZE);
- ttlField = config.get(TTL_FIELD);
- waitSearcher = config.get(WAIT_SEARCHER);
-
- if (mode==Mode.CLOUD) {
- HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer());
- String zookeeperUrl = config.get(Solr5Index.ZOOKEEPER_URL);
- CloudSolrClient cloudServer = new CloudSolrClient(zookeeperUrl, true);
- cloudServer.connect();
- solrClient = cloudServer;
- } else if (mode==Mode.HTTP) {
- HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer());
- HttpClient clientParams = HttpClientUtil.createClient(new ModifiableSolrParams() {{
- add(HttpClientUtil.PROP_ALLOW_COMPRESSION, config.get(HTTP_ALLOW_COMPRESSION).toString());
- add(HttpClientUtil.PROP_CONNECTION_TIMEOUT, config.get(HTTP_CONNECTION_TIMEOUT).toString());
- add(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, config.get(HTTP_MAX_CONNECTIONS_PER_HOST).toString());
- add(HttpClientUtil.PROP_MAX_CONNECTIONS, config.get(HTTP_GLOBAL_MAX_CONNECTIONS).toString());
- }});
-
- solrClient = new LBHttpSolrClient(clientParams, config.get(HTTP_URLS));
-
-
- } else {
- throw new IllegalArgumentException("Unsupported Solr operation mode: " + mode);
- }
- }
-
- private Map<String, String> parseKeyFieldsForCollections(Configuration config) throws BackendException {
- Map<String, String> keyFieldNames = new HashMap<String, String>();
- String[] collectionFieldStatements = config.has(KEY_FIELD_NAMES)?config.get(KEY_FIELD_NAMES):new String[0];
- for (String collectionFieldStatement : collectionFieldStatements) {
- String[] parts = collectionFieldStatement.trim().split("=");
- if (parts.length != 2) {
- throw new PermanentBackendException("Unable to parse the collection name / key field name pair. It should be of the format collection=field");
- }
- String collectionName = parts[0];
- String keyFieldName = parts[1];
- keyFieldNames.put(collectionName, keyFieldName);
- }
- return keyFieldNames;
- }
-
- private String getKeyFieldId(String collection) {
- String field = keyFieldIds.get(collection);
- if (field==null) field = DEFAULT_ID_FIELD;
- return field;
- }
-
- /**
- * Unlike the ElasticSearch Index, which is schema free, Solr requires a schema to
- * support searching. This means that you will need to modify the solr schema with the
- * appropriate field definitions in order to work properly. If you have a running instance
- * of Solr and you modify its schema with new fields, don't forget to re-index!
- * @param store Index store
- * @param key New key to register
- * @param information Datatype to register for the key
- * @param tx enclosing transaction
- * @throws com.thinkaurelius.titan.diskstorage.BackendException
- */
- @Override
- public void register(String store, String key, KeyInformation information, BaseTransaction tx) throws BackendException {
- if (mode==Mode.CLOUD) {
- CloudSolrClient client = (CloudSolrClient) solrClient;
- try {
- createCollectionIfNotExists(client, configuration, store);
- } catch (IOException e) {
- throw new PermanentBackendException(e);
- } catch (SolrServerException e) {
- throw new PermanentBackendException(e);
- } catch (InterruptedException e) {
- throw new PermanentBackendException(e);
- } catch (KeeperException e) {
- throw new PermanentBackendException(e);
- }
- }
- //Since all data types must be defined in the schema.xml, pre-registering a type does not work
- }
-
- @Override
- public void mutate(Map<String, Map<String, IndexMutation>> mutations, KeyInformation.IndexRetriever informations, BaseTransaction tx) throws BackendException {
- logger.debug("Mutating SOLR");
- try {
- for (Map.Entry<String, Map<String, IndexMutation>> stores : mutations.entrySet()) {
- String collectionName = stores.getKey();
- String keyIdField = getKeyFieldId(collectionName);
-
- List<String> deleteIds = new ArrayList<String>();
- Collection<SolrInputDocument> changes = new ArrayList<SolrInputDocument>();
-
- for (Map.Entry<String, IndexMutation> entry : stores.getValue().entrySet()) {
- String docId = entry.getKey();
- IndexMutation mutation = entry.getValue();
- Preconditions.checkArgument(!(mutation.isNew() && mutation.isDeleted()));
- Preconditions.checkArgument(!mutation.isNew() || !mutation.hasDeletions());
- Preconditions.checkArgument(!mutation.isDeleted() || !mutation.hasAdditions());
-
- //Handle any deletions
- if (mutation.hasDeletions()) {
- if (mutation.isDeleted()) {
- logger.trace("Deleting entire document {}", docId);
- deleteIds.add(docId);
- } else {
- HashSet<IndexEntry> fieldDeletions = Sets.newHashSet(mutation.getDeletions());
- if (mutation.hasAdditions()) {
- for (IndexEntry indexEntry : mutation.getAdditions()) {
- fieldDeletions.remove(indexEntry);
- }
- }
- deleteIndividualFieldsFromIndex(collectionName, keyIdField, docId, fieldDeletions);
- }
- }
-
- if (mutation.hasAdditions()) {
- int ttl = mutation.determineTTL();
-
- SolrInputDocument doc = new SolrInputDocument();
- doc.setField(keyIdField, docId);
-
- boolean isNewDoc = mutation.isNew();
-
- if (isNewDoc)
- logger.trace("Adding new document {}", docId);
-
- for (IndexEntry e : mutation.getAdditions()) {
- final Object fieldValue = convertValue(e.value);
- doc.setField(e.field, isNewDoc
- ? fieldValue : new HashMap<String, Object>(1) {{ put("set", fieldValue); }});
- }
- if (ttl>0) {
- Preconditions.checkArgument(isNewDoc,"Solr only supports TTL on new documents [%s]",docId);
- doc.setField(ttlField, String.format("+%dSECONDS", ttl));
- }
- changes.add(doc);
- }
- }
-
- commitDeletes(collectionName, deleteIds);
- commitDocumentChanges(collectionName, changes);
- }
- } catch (Exception e) {
- throw storageException(e);
- }
- }
-
- private Object convertValue(Object value) throws BackendException {
- if (value instanceof Geoshape)
- return GeoToWktConverter.convertToWktString((Geoshape) value);
- // in order to serialize/deserialize properly Solr will have to have an
- // access to Titan source which has Decimal type, so for now we simply convert to
- // double and let Solr do the same thing or fail.
- if (value instanceof AbstractDecimal)
- return ((AbstractDecimal) value).doubleValue();
- if (value instanceof UUID)
- return value.toString();
- return value;
- }
-
- @Override
- public void restore(Map<String, Map<String, List<IndexEntry>>> documents, KeyInformation.IndexRetriever informations, BaseTransaction tx) throws BackendException {
- try {
- for (Map.Entry<String, Map<String, List<IndexEntry>>> stores : documents.entrySet()) {
- final String collectionName = stores.getKey();
-
- List<String> deleteIds = new ArrayList<String>();
- List<SolrInputDocument> newDocuments = new ArrayList<SolrInputDocument>();
-
- for (Map.Entry<String, List<IndexEntry>> entry : stores.getValue().entrySet()) {
- final String docID = entry.getKey();
- final List<IndexEntry> content = entry.getValue();
-
- if (content == null || content.isEmpty()) {
- if (logger.isTraceEnabled())
- logger.trace("Deleting document [{}]", docID);
-
- deleteIds.add(docID);
- continue;
- }
-
- newDocuments.add(new SolrInputDocument() {{
- setField(getKeyFieldId(collectionName), docID);
-
- for (IndexEntry addition : content) {
- Object fieldValue = addition.value;
- setField(addition.field, convertValue(fieldValue));
- }
- }});
- }
-
- commitDeletes(collectionName, deleteIds);
- commitDocumentChanges(collectionName, newDocuments);
- }
- } catch (Exception e) {
- throw new TemporaryBackendException("Could not restore Solr index", e);
- }
- }
-
- private void deleteIndividualFieldsFromIndex(String collectionName, String keyIdField, String docId, HashSet<IndexEntry> fieldDeletions) throws SolrServerException, IOException {
- if (fieldDeletions.isEmpty()) return;
-
- Map<String, String> fieldDeletes = new HashMap<String, String>(1) {{ put("set", null); }};
-
- SolrInputDocument doc = new SolrInputDocument();
- doc.addField(keyIdField, docId);
- StringBuilder sb = new StringBuilder();
- for (IndexEntry fieldToDelete : fieldDeletions) {
- doc.addField(fieldToDelete.field, fieldDeletes);
- sb.append(fieldToDelete).append(",");
- }
-
- if (logger.isTraceEnabled())
- logger.trace("Deleting individual fields [{}] for document {}", sb.toString(), docId);
-
- UpdateRequest singleDocument = newUpdateRequest();
- singleDocument.add(doc);
- solrClient.request(singleDocument, collectionName);
- }
-
- private void commitDocumentChanges(String collectionName, Collection<SolrInputDocument> documents) throws SolrServerException, IOException {
- if (documents.size() == 0) return;
-
- try {
- solrClient.request(newUpdateRequest().add(documents), collectionName);
- } catch (HttpSolrClient.RemoteSolrException rse) {
- logger.error("Unable to save documents to Solr as one of the shape objects stored were not compatible with Solr.", rse);
- logger.error("Details in failed document batch: ");
- for (SolrInputDocument d : documents) {
- Collection<String> fieldNames = d.getFieldNames();
- for (String name : fieldNames) {
- logger.error(name + ":" + d.getFieldValue(name).toString());
- }
- }
-
- throw rse;
- }
- }
-
- private void commitDeletes(String collectionName, List<String> deleteIds) throws SolrServerException, IOException {
- if (deleteIds.size() == 0) return;
- solrClient.request(newUpdateRequest().deleteById(deleteIds), collectionName);
- }
-
- @Override
- public List<String> query(IndexQuery query, KeyInformation.IndexRetriever informations, BaseTransaction tx) throws BackendException {
- List<String> result;
- String collection = query.getStore();
- String keyIdField = getKeyFieldId(collection);
- SolrQuery solrQuery = new SolrQuery("*:*");
- String queryFilter = buildQueryFilter(query.getCondition(), informations.get(collection));
- solrQuery.addFilterQuery(queryFilter);
- if (!query.getOrder().isEmpty()) {
- List<IndexQuery.OrderEntry> orders = query.getOrder();
- for (IndexQuery.OrderEntry order1 : orders) {
- String item = order1.getKey();
- SolrQuery.ORDER order = order1.getOrder() == Order.ASC ? SolrQuery.ORDER.asc : SolrQuery.ORDER.desc;
- solrQuery.addSort(new SolrQuery.SortClause(item, order));
- }
- }
- solrQuery.setStart(0);
- if (query.hasLimit()) {
- solrQuery.setRows(query.getLimit());
- } else {
- solrQuery.setRows(maxResults);
- }
- try {
- QueryResponse response = solrClient.query(collection, solrQuery);
-
- if (logger.isDebugEnabled())
- logger.debug("Executed query [{}] in {} ms", query.getCondition(), response.getElapsedTime());
-
- int totalHits = response.getResults().size();
-
- if (!query.hasLimit() && totalHits >= maxResults)
- logger.warn("Query result set truncated to first [{}] elements for query: {}", maxResults, query);
-
- result = new ArrayList<String>(totalHits);
- for (SolrDocument hit : response.getResults()) {
- result.add(hit.getFieldValue(keyIdField).toString());
- }
- } catch (IOException e) {
- logger.error("Query did not complete : ", e);
- throw new PermanentBackendException(e);
- } catch (SolrServerException e) {
- logger.error("Unable to query Solr index.", e);
- throw new PermanentBackendException(e);
- }
- return result;
- }
-
- @Override
- public Iterable<RawQuery.Result<String>> query(RawQuery query, KeyInformation.IndexRetriever informations, BaseTransaction tx) throws BackendException {
- List<RawQuery.Result<String>> result;
- String collection = query.getStore();
- String keyIdField = getKeyFieldId(collection);
- SolrQuery solrQuery = new SolrQuery(query.getQuery())
- .addField(keyIdField)
- .setIncludeScore(true)
- .setStart(query.getOffset())
- .setRows(query.hasLimit() ? query.getLimit() : maxResults);
-
- try {
- QueryResponse response = solrClient.query(collection, solrQuery);
- if (logger.isDebugEnabled())
- logger.debug("Executed query [{}] in {} ms", query.getQuery(), response.getElapsedTime());
-
- int totalHits = response.getResults().size();
- if (!query.hasLimit() && totalHits >= maxResults) {
- logger.warn("Query result set truncated to first [{}] elements for query: {}", maxResults, query);
- }
- result = new ArrayList<RawQuery.Result<String>>(totalHits);
-
- for (SolrDocument hit : response.getResults()) {
- double score = Double.parseDouble(hit.getFieldValue("score").toString());
- result.add(new RawQuery.Result<String>(hit.getFieldValue(keyIdField).toString(), score));
- }
- } catch (IOException e) {
- logger.error("Query did not complete : ", e);
- throw new PermanentBackendException(e);
- } catch (SolrServerException e) {
- logger.error("Unable to query Solr index.", e);
- throw new PermanentBackendException(e);
- }
- return result;
- }
-
- private static String escapeValue(Object value) {
- return ClientUtils.escapeQueryChars(value.toString());
- }
-
- public String buildQueryFilter(Condition<TitanElement> condition, KeyInformation.StoreRetriever informations) {
- if (condition instanceof PredicateCondition) {
- PredicateCondition<String, TitanElement> atom = (PredicateCondition<String, TitanElement>) condition;
- Object value = atom.getValue();
- String key = atom.getKey();
- TitanPredicate titanPredicate = atom.getPredicate();
-
- if (value instanceof Number) {
- String queryValue = escapeValue(value);
- Preconditions.checkArgument(titanPredicate instanceof Cmp, "Relation not supported on numeric types: " + titanPredicate);
- Cmp numRel = (Cmp) titanPredicate;
- switch (numRel) {
- case EQUAL:
- return (key + ":" + queryValue);
- case NOT_EQUAL:
- return ("-" + key + ":" + queryValue);
- case LESS_THAN:
- //use right curly to mean up to but not including value
- return (key + ":[* TO " + queryValue + "}");
- case LESS_THAN_EQUAL:
- return (key + ":[* TO " + queryValue + "]");
- case GREATER_THAN:
- //use left curly to mean greater than but not including value
- return (key + ":{" + queryValue + " TO *]");
- case GREATER_THAN_EQUAL:
- return (key + ":[" + queryValue + " TO *]");
- default: throw new IllegalArgumentException("Unexpected relation: " + numRel);
- }
- } else if (value instanceof String) {
- Mapping map = getStringMapping(informations.get(key));
- assert map== TEXT || map== STRING;
- if (map== TEXT && !titanPredicate.toString().startsWith("CONTAINS"))
- throw new IllegalArgumentException("Text mapped string values only support CONTAINS queries and not: " + titanPredicate);
- if (map== STRING && titanPredicate.toString().startsWith("CONTAINS"))
- throw new IllegalArgumentException("String mapped string values do not support CONTAINS queries: " + titanPredicate);
-
- //Special case
- if (titanPredicate == Text.CONTAINS) {
- //e.g. - if terms tomorrow and world were supplied, and fq=text:(tomorrow world)
- //sample data set would return 2 documents: one where text = Tomorrow is the World,
- //and the second where text = Hello World. Hence, we are decomposing the query string
- //and building an AND query explicitly because we need AND semantics
- value = ((String) value).toLowerCase();
- List<String> terms = Text.tokenize((String) value);
-
- if (terms.isEmpty()) {
- return "";
- } else if (terms.size() == 1) {
- return (key + ":(" + escapeValue(terms.get(0)) + ")");
- } else {
- And<TitanElement> andTerms = new And<TitanElement>();
- for (String term : terms) {
- andTerms.add(new PredicateCondition<String, TitanElement>(key, titanPredicate, term));
- }
- return buildQueryFilter(andTerms, informations);
- }
- }
- if (titanPredicate == Text.PREFIX || titanPredicate == Text.CONTAINS_PREFIX) {
- return (key + ":" + escapeValue(value) + "*");
- } else if (titanPredicate == Text.REGEX || titanPredicate == Text.CONTAINS_REGEX) {
- return (key + ":/" + value + "/");
- } else if (titanPredicate == EQUAL) {
- return (key + ":\"" + escapeValue(value) + "\"");
- } else if (titanPredicate == NOT_EQUAL) {
- return ("-" + key + ":\"" + escapeValue(value) + "\"");
- } else {
- throw new IllegalArgumentException("Relation is not supported for string value: " + titanPredicate);
- }
- } else if (value instanceof Geoshape) {
- Geoshape geo = (Geoshape)value;
- if (geo.getType() == Geoshape.Type.CIRCLE) {
- Geoshape.Point center = geo.getPoint();
- return ("{!geofilt sfield=" + key +
- " pt=" + center.getLatitude() + "," + center.getLongitude() +
- " d=" + geo.getRadius() + "} distErrPct=0"); //distance in kilometers
- } else if (geo.getType() == Geoshape.Type.BOX) {
- Geoshape.Point southwest = geo.getPoint(0);
- Geoshape.Point northeast = geo.getPoint(1);
- return (key + ":[" + southwest.getLatitude() + "," + southwest.getLongitude() +
- " TO " + northeast.getLatitude() + "," + northeast.getLongitude() + "]");
- } else if (geo.getType() == Geoshape.Type.POLYGON) {
- List<Geoshape.Point> coordinates = getPolygonPoints(geo);
- StringBuilder poly = new StringBuilder(key + ":\"IsWithin(POLYGON((");
- for (Geoshape.Point coordinate : coordinates) {
- poly.append(coordinate.getLongitude()).append(" ").append(coordinate.getLatitude()).append(", ");
- }
- //close the polygon with the first coordinate
- poly.append(coordinates.get(0).getLongitude()).append(" ").append(coordinates.get(0).getLatitude());
- poly.append(")))\" distErrPct=0");
- return (poly.toString());
- }
- } else if (value instanceof Date) {
- String queryValue = escapeValue(toIsoDate((Date)value));
- Preconditions.checkArgument(titanPredicate instanceof Cmp, "Relation not supported on date types: " + titanPredicate);
- Cmp numRel = (Cmp) titanPredicate;
-
- switch (numRel) {
- case EQUAL:
- return (key + ":" + queryValue);
- case NOT_EQUAL:
- return ("-" + key + ":" + queryValue);
- case LESS_THAN:
- //use right curly to mean up to but not including value
- return (key + ":[* TO " + queryValue + "}");
- case LESS_THAN_EQUAL:
- return (key + ":[* TO " + queryValue + "]");
- case GREATER_THAN:
- //use left curly to mean greater than but not including value
- return (key + ":{" + queryValue + " TO *]");
- case GREATER_THAN_EQUAL:
- return (key + ":[" + queryValue + " TO *]");
- default: throw new IllegalArgumentException("Unexpected relation: " + numRel);
- }
- } else if (value instanceof Boolean) {
- Cmp numRel = (Cmp) titanPredicate;
- String queryValue = escapeValue(value);
- switch (numRel) {
- case EQUAL:
- return (key + ":" + queryValue);
- case NOT_EQUAL:
- return ("-" + key + ":" + queryValue);
- default:
- throw new IllegalArgumentException("Boolean types only support EQUAL or NOT_EQUAL");
- }
- } else if (value instanceof UUID) {
- if (titanPredicate == EQUAL) {
- return (key + ":\"" + escapeValue(value) + "\"");
- } else if (titanPredicate == NOT_EQUAL) {
- return ("-" + key + ":\"" + escapeValue(value) + "\"");
- } else {
- throw new IllegalArgumentException("Relation is not supported for uuid value: " + titanPredicate);
- }
- } else throw new IllegalArgumentException("Unsupported type: " + value);
- } else if (condition instanceof Not) {
- String sub = buildQueryFilter(((Not)condition).getChild(),informations);
- if (StringUtils.isNotBlank(sub)) return "-("+sub+")";
- else return "";
- } else if (condition instanceof And) {
- int numChildren = ((And) condition).size();
- StringBuilder sb = new StringBuilder();
- for (Condition<TitanElement> c : condition.getChildren()) {
- String sub = buildQueryFilter(c, informations);
-
- if (StringUtils.isBlank(sub))
- continue;
-
- // we don't have to add "+" which means AND iff
- // a. it's a NOT query,
- // b. expression is a single statement in the AND.
- if (!sub.startsWith("-") && numChildren > 1)
- sb.append("+");
-
- sb.append(sub).append(" ");
- }
- return sb.toString();
- } else if (condition instanceof Or) {
- StringBuilder sb = new StringBuilder();
- int element=0;
- for (Condition<TitanElement> c : condition.getChildren()) {
- String sub = buildQueryFilter(c,informations);
- if (StringUtils.isBlank(sub)) continue;
- if (element==0) sb.append("(");
- else sb.append(" OR ");
- sb.append(sub);
- element++;
- }
- if (element>0) sb.append(")");
- return sb.toString();
- } else {
- throw new IllegalArgumentException("Invalid condition: " + condition);
- }
- return null;
- }
-
- private String toIsoDate(Date value) {
- TimeZone tz = TimeZone.getTimeZone("UTC");
- DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
- df.setTimeZone(tz);
- return df.format(value);
- }
-
- private List<Geoshape.Point> getPolygonPoints(Geoshape polygon) {
- List<Geoshape.Point> locations = new ArrayList<Geoshape.Point>();
-
- int index = 0;
- boolean hasCoordinates = true;
- while (hasCoordinates) {
- try {
- locations.add(polygon.getPoint(index));
- } catch (ArrayIndexOutOfBoundsException ignore) {
- //just means we asked for a point past the size of the list
- //of known coordinates
- hasCoordinates = false;
- }
- }
-
- return locations;
- }
-
- /**
- * Solr handles all transactions on the server-side. That means all
- * commit, optimize, or rollback applies since the last commit/optimize/rollback.
- * Solr documentation recommends best way to update Solr is in one process to avoid
- * race conditions.
- *
- * @return New Transaction Handle
- * @throws com.thinkaurelius.titan.diskstorage.BackendException
- */
- @Override
- public BaseTransactionConfigurable beginTransaction(BaseTransactionConfig config) throws BackendException {
- return new DefaultTransaction(config);
- }
-
- @Override
- public void close() throws BackendException {
- logger.trace("Shutting down connection to Solr", solrClient);
- try {
- solrClient.close();
- } catch (IOException e) {
- throw new TemporaryBackendException(e);
- }
- }
-
- @Override
- public void clearStorage() throws BackendException {
- try {
- if (mode!=Mode.CLOUD) throw new UnsupportedOperationException("Operation only supported for SolrCloud");
- logger.debug("Clearing storage from Solr: {}", solrClient);
- ZkStateReader zkStateReader = ((CloudSolrClient) solrClient).getZkStateReader();
- zkStateReader.updateClusterState();
- ClusterState clusterState = zkStateReader.getClusterState();
- for (String collection : clusterState.getCollections()) {
- logger.debug("Clearing collection [{}] in Solr",collection);
- UpdateRequest deleteAll = newUpdateRequest();
- deleteAll.deleteByQuery("*:*");
- solrClient.request(deleteAll, collection);
- }
-
- } catch (SolrServerException e) {
- logger.error("Unable to clear storage from index due to server error on Solr.", e);
- throw new PermanentBackendException(e);
- } catch (IOException e) {
- logger.error("Unable to clear storage from index due to low-level I/O error.", e);
- throw new PermanentBackendException(e);
- } catch (Exception e) {
- logger.error("Unable to clear storage from index due to general error.", e);
- throw new PermanentBackendException(e);
- }
- }
-
- @Override
- public boolean supports(KeyInformation information, TitanPredicate titanPredicate) {
- Class<?> dataType = information.getDataType();
- Mapping mapping = getMapping(information);
- if (mapping!= DEFAULT && !AttributeUtil.isString(dataType)) return false;
-
- if (Number.class.isAssignableFrom(dataType)) {
- return titanPredicate instanceof Cmp;
- } else if (dataType == Geoshape.class) {
- return titanPredicate == Geo.WITHIN;
- } else if (AttributeUtil.isString(dataType)) {
- switch(mapping) {
- case DEFAULT:
- case TEXT:
- return titanPredicate == Text.CONTAINS || titanPredicate == Text.CONTAINS_PREFIX || titanPredicate == Text.CONTAINS_REGEX;
- case STRING:
- return titanPredicate == EQUAL || titanPredicate== NOT_EQUAL || titanPredicate==Text.REGEX || titanPredicate==Text.PREFIX;
- // case TEXTSTRING:
- // return (titanPredicate instanceof Text) || titanPredicate == Cmp.EQUAL || titanPredicate==Cmp.NOT_EQUAL;
- }
- } else if (dataType == Date.class) {
- if (titanPredicate instanceof Cmp) return true;
- } else if (dataType == Boolean.class) {
- return titanPredicate == EQUAL || titanPredicate == NOT_EQUAL;
- } else if (dataType == UUID.class) {
- return titanPredicate == EQUAL || titanPredicate== NOT_EQUAL;
- }
- return false;
- }
-
- @Override
- public boolean supports(KeyInformation information) {
- Class<?> dataType = information.getDataType();
- Mapping mapping = getMapping(information);
- if (Number.class.isAssignableFrom(dataType) || dataType == Geoshape.class || dataType == Date.class || dataType == Boolean.class || dataType == UUID.class) {
- if (mapping== DEFAULT) return true;
- } else if (AttributeUtil.isString(dataType)) {
- if (mapping== DEFAULT || mapping== TEXT || mapping== STRING) return true;
- }
- return false;
- }
-
- @Override
- public String mapKey2Field(String key, KeyInformation keyInfo) {
- Preconditions.checkArgument(!StringUtils.containsAny(key, new char[]{' '}),"Invalid key name provided: %s",key);
- if (!dynFields) return key;
- if (ParameterType.MAPPED_NAME.hasParameter(keyInfo.getParameters())) return key;
- String postfix;
- Class datatype = keyInfo.getDataType();
- if (AttributeUtil.isString(datatype)) {
- Mapping map = getStringMapping(keyInfo);
- switch (map) {
- case TEXT: postfix = "_t"; break;
- case STRING: postfix = "_s"; break;
- default: throw new IllegalArgumentException("Unsupported string mapping: " + map);
- }
- } else if (AttributeUtil.isWholeNumber(datatype)) {
- if (datatype.equals(Long.class)) postfix = "_l";
- else postfix = "_i";
- } else if (AttributeUtil.isDecimal(datatype)) {
- if (datatype.equals(Float.class)) postfix = "_f";
- else postfix = "_d";
- } else if (datatype.equals(Geoshape.class)) {
- postfix = "_g";
- } else if (datatype.equals(Date.class)) {
- postfix = "_dt";
- } else if (datatype.equals(Boolean.class)) {
- postfix = "_b";
- } else if (datatype.equals(UUID.class)) {
- postfix = "_uuid";
- } else throw new IllegalArgumentException("Unsupported data type ["+datatype+"] for field: " + key);
- return key+postfix;
- }
-
- @Override
- public IndexFeatures getFeatures() {
- return SOLR_FEATURES;
- }
-
- /*
- ################# UTILITY METHODS #######################
- */
-
- private static Mapping getStringMapping(KeyInformation information) {
- assert AttributeUtil.isString(information.getDataType());
- Mapping map = getMapping(information);
- if (map== DEFAULT) map = TEXT;
- return map;
- }
-
- private UpdateRequest newUpdateRequest() {
- UpdateRequest req = new UpdateRequest();
- req.setAction(UpdateRequest.ACTION.COMMIT, true, true);
- if (waitSearcher) {
- req.setAction(UpdateRequest.ACTION.COMMIT, true, true);
- }
- return req;
- }
-
- private BackendException storageException(Exception solrException) {
- return new TemporaryBackendException("Unable to complete query on Solr.", solrException);
- }
-
- private static void createCollectionIfNotExists(CloudSolrClient client, Configuration config, String collection)
- throws IOException, SolrServerException, KeeperException, InterruptedException {
- if (!checkIfCollectionExists(client, collection)) {
- Integer numShards = config.get(NUM_SHARDS);
- Integer maxShardsPerNode = config.get(MAX_SHARDS_PER_NODE);
- Integer replicationFactor = config.get(REPLICATION_FACTOR);
-
- CollectionAdminRequest.Create createRequest = new CollectionAdminRequest.Create();
-
- createRequest.setConfigName(collection);
- createRequest.setCollectionName(collection);
- createRequest.setNumShards(numShards);
- createRequest.setMaxShardsPerNode(maxShardsPerNode);
- createRequest.setReplicationFactor(replicationFactor);
-
- CollectionAdminResponse createResponse = createRequest.process(client);
- if (createResponse.isSuccess()) {
- logger.trace("Collection {} successfully created.", collection);
- } else {
- throw new SolrServerException(Joiner.on("\n").join(createResponse.getErrorMessages()));
- }
- }
-
- waitForRecoveriesToFinish(client, collection);
- }
-
- /**
- * Checks if the collection has already been created in Solr.
- */
- private static boolean checkIfCollectionExists(CloudSolrClient server, String collection) throws KeeperException, InterruptedException {
- ZkStateReader zkStateReader = server.getZkStateReader();
- zkStateReader.updateClusterState();
- ClusterState clusterState = zkStateReader.getClusterState();
- return clusterState.getCollectionOrNull(collection) != null;
- }
-
- /**
- * Wait for all the collection shards to be ready.
- */
- private static void waitForRecoveriesToFinish(CloudSolrClient server, String collection) throws KeeperException, InterruptedException {
- ZkStateReader zkStateReader = server.getZkStateReader();
- try {
- boolean cont = true;
-
- while (cont) {
- boolean sawLiveRecovering = false;
- zkStateReader.updateClusterState();
- ClusterState clusterState = zkStateReader.getClusterState();
- Map<String, Slice> slices = clusterState.getSlicesMap(collection);
- Preconditions.checkNotNull("Could not find collection:" + collection, slices);
-
- for (Map.Entry<String, Slice> entry : slices.entrySet()) {
- Map<String, Replica> shards = entry.getValue().getReplicasMap();
- for (Map.Entry<String, Replica> shard : shards.entrySet()) {
- String state = shard.getValue().getStr(ZkStateReader.STATE_PROP);
- if ((state.equals(Replica.State.RECOVERING)
- || state.equals(Replica.State.DOWN))
- && clusterState.liveNodesContain(shard.getValue().getStr(
- ZkStateReader.NODE_NAME_PROP))) {
- sawLiveRecovering = true;
- }
- }
- }
- if (!sawLiveRecovering) {
- cont = false;
- } else {
- Thread.sleep(1000);
- }
- }
- } finally {
- logger.info("Exiting solr wait");
- }
- }
-
- private static class GeoToWktConverter {
- /**
- * {@link com.thinkaurelius.titan.core.attribute.Geoshape} stores Points in the String format: point[X.0,Y.0].
- * Solr needs it to be in Well-Known Text format: POINT(X.0 Y.0)
- */
- static String convertToWktString(Geoshape fieldValue) throws BackendException {
- if (fieldValue.getType() == Geoshape.Type.POINT) {
- Geoshape.Point point = fieldValue.getPoint();
- return "POINT(" + point.getLongitude() + " " + point.getLatitude() + ")";
- } else {
- throw new PermanentBackendException("Cannot index " + fieldValue.getType());
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/titan/src/main/java/com/thinkaurelius/titan/graphdb/query/graph/GraphCentricQueryBuilder.java
----------------------------------------------------------------------
diff --git a/titan/src/main/java/com/thinkaurelius/titan/graphdb/query/graph/GraphCentricQueryBuilder.java b/titan/src/main/java/com/thinkaurelius/titan/graphdb/query/graph/GraphCentricQueryBuilder.java
deleted file mode 100644
index c1a983b..0000000
--- a/titan/src/main/java/com/thinkaurelius/titan/graphdb/query/graph/GraphCentricQueryBuilder.java
+++ /dev/null
@@ -1,457 +0,0 @@
-/*
- * Copyright 2012-2013 Aurelius LLC
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.thinkaurelius.titan.graphdb.query.graph;
-
-import com.google.common.base.Preconditions;
-import com.google.common.base.Predicate;
-import com.google.common.collect.*;
-import com.thinkaurelius.titan.core.*;
-import com.thinkaurelius.titan.core.attribute.Cmp;
-import com.thinkaurelius.titan.core.Cardinality;
-import com.thinkaurelius.titan.core.schema.SchemaStatus;
-import com.thinkaurelius.titan.core.schema.TitanSchemaType;
-import com.thinkaurelius.titan.graphdb.database.IndexSerializer;
-import com.thinkaurelius.titan.graphdb.internal.ElementCategory;
-import com.thinkaurelius.titan.graphdb.internal.InternalRelationType;
-import com.thinkaurelius.titan.graphdb.internal.OrderList;
-import com.thinkaurelius.titan.graphdb.query.*;
-import com.thinkaurelius.titan.graphdb.query.condition.*;
-import com.thinkaurelius.titan.graphdb.transaction.StandardTitanTx;
-import com.thinkaurelius.titan.graphdb.types.*;
-import com.thinkaurelius.titan.graphdb.types.system.ImplicitKey;
-import com.thinkaurelius.titan.util.datastructures.Interval;
-import com.thinkaurelius.titan.util.datastructures.PointInterval;
-import com.tinkerpop.blueprints.Edge;
-import com.tinkerpop.blueprints.Vertex;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.Nullable;
-import java.util.*;
-
-/**
- *
- * Builds a {@link TitanGraphQuery}, optimizes the query and compiles the result into a {@link GraphCentricQuery} which
- * is then executed through a {@link QueryProcessor}.
- * This class from titan-0.5.4 has no major changes except adding a few logs for debugging index usage
- *
- * @author Matthias Broecheler (me@matthiasb.com)
- */
-public class GraphCentricQueryBuilder implements TitanGraphQuery<GraphCentricQueryBuilder> {
-
- private static final Logger log = LoggerFactory.getLogger(GraphCentricQueryBuilder.class);
-
- /**
- * Transaction in which this query is executed.
- */
- private final StandardTitanTx tx;
- /**
- * Serializer used to serialize the query conditions into backend queries.
- */
- private final IndexSerializer serializer;
- /**
- * The constraints added to this query. None by default.
- */
- private List<PredicateCondition<String, TitanElement>> constraints;
- /**
- * The order in which the elements should be returned. None by default.
- */
- private OrderList orders = new OrderList();
- /**
- * The limit of this query. No limit by default.
- */
- private int limit = Query.NO_LIMIT;
-
- public GraphCentricQueryBuilder(StandardTitanTx tx, IndexSerializer serializer) {
- log.debug("Loaded shaded version of class GraphCentricQueryBuilder");
- Preconditions.checkNotNull(tx);
- Preconditions.checkNotNull(serializer);
- this.tx = tx;
- this.serializer = serializer;
- this.constraints = new ArrayList<PredicateCondition<String, TitanElement>>(5);
- }
-
- /* ---------------------------------------------------------------
- * Query Construction
- * ---------------------------------------------------------------
- */
-
- private GraphCentricQueryBuilder has(String key, TitanPredicate predicate, Object condition) {
- Preconditions.checkNotNull(key);
- Preconditions.checkNotNull(predicate);
- Preconditions.checkArgument(predicate.isValidCondition(condition), "Invalid condition: %s", condition);
- constraints.add(new PredicateCondition<String, TitanElement>(key, predicate, condition));
- return this;
- }
-
- @Override
- public GraphCentricQueryBuilder has(String key, com.tinkerpop.blueprints.Predicate predicate, Object condition) {
- Preconditions.checkNotNull(key);
- TitanPredicate titanPredicate = TitanPredicate.Converter.convert(predicate);
- return has(key, titanPredicate, condition);
- }
-
- @Override
- public GraphCentricQueryBuilder has(PropertyKey key, TitanPredicate predicate, Object condition) {
- Preconditions.checkNotNull(key);
- Preconditions.checkNotNull(predicate);
- return has(key.getName(), predicate, condition);
- }
-
- @Override
- public GraphCentricQueryBuilder has(String key) {
- return has(key, Cmp.NOT_EQUAL, (Object) null);
- }
-
- @Override
- public GraphCentricQueryBuilder hasNot(String key) {
- return has(key, Cmp.EQUAL, (Object) null);
- }
-
- @Override
- @Deprecated
- public <T extends Comparable<T>> GraphCentricQueryBuilder has(String s, T t, Compare compare) {
- return has(s, compare, t);
- }
-
- @Override
- public GraphCentricQueryBuilder has(String key, Object value) {
- return has(key, Cmp.EQUAL, value);
- }
-
- @Override
- public GraphCentricQueryBuilder hasNot(String key, Object value) {
- return has(key, Cmp.NOT_EQUAL, value);
- }
-
- @Override
- public <T extends Comparable<?>> GraphCentricQueryBuilder interval(String s, T t1, T t2) {
- has(s, Cmp.GREATER_THAN_EQUAL, t1);
- return has(s, Cmp.LESS_THAN, t2);
- }
-
- @Override
- public GraphCentricQueryBuilder limit(final int limit) {
- Preconditions.checkArgument(limit >= 0, "Non-negative limit expected: %s", limit);
- this.limit = limit;
- return this;
- }
-
- @Override
- public GraphCentricQueryBuilder orderBy(String key, Order order) {
- Preconditions.checkArgument(tx.containsPropertyKey(key),"Provided key does not exist: %s",key);
- return orderBy(tx.getPropertyKey(key), order);
- }
-
- @Override
- public GraphCentricQueryBuilder orderBy(PropertyKey key, Order order) {
- Preconditions.checkArgument(key!=null && order!=null,"Need to specify and key and an order");
- Preconditions.checkArgument(Comparable.class.isAssignableFrom(key.getDataType()),
- "Can only order on keys with comparable data type. [%s] has datatype [%s]", key.getName(), key.getDataType());
- Preconditions.checkArgument(key.getCardinality()== Cardinality.SINGLE, "Ordering is undefined on multi-valued key [%s]", key.getName());
- Preconditions.checkArgument(!orders.containsKey(key));
- orders.add(key, order);
- return this;
- }
-
- /* ---------------------------------------------------------------
- * Query Execution
- * ---------------------------------------------------------------
- */
-
- @Override
- public Iterable<Vertex> vertices() {
- GraphCentricQuery query = constructQuery(ElementCategory.VERTEX);
- return Iterables.filter(new QueryProcessor<GraphCentricQuery, TitanElement, JointIndexQuery>(query, tx.elementProcessor), Vertex.class);
- }
-
- @Override
- public Iterable<Edge> edges() {
- GraphCentricQuery query = constructQuery(ElementCategory.EDGE);
- return Iterables.filter(new QueryProcessor<GraphCentricQuery, TitanElement, JointIndexQuery>(query, tx.elementProcessor), Edge.class);
- }
-
- @Override
- public Iterable<TitanProperty> properties() {
- GraphCentricQuery query = constructQuery(ElementCategory.PROPERTY);
- return Iterables.filter(new QueryProcessor<GraphCentricQuery, TitanElement, JointIndexQuery>(query, tx.elementProcessor), TitanProperty.class);
- }
-
- private QueryDescription describe(ElementCategory category) {
- return new StandardQueryDescription(1,constructQuery(category));
- }
-
- public QueryDescription describeForVertices() {
- return describe(ElementCategory.VERTEX);
- }
-
- public QueryDescription describeForEdges() {
- return describe(ElementCategory.EDGE);
- }
-
- public QueryDescription describeForProperties() {
- return describe(ElementCategory.PROPERTY);
- }
-
- /* ---------------------------------------------------------------
- * Query Construction
- * ---------------------------------------------------------------
- */
-
- private static final int DEFAULT_NO_LIMIT = 1000;
- private static final int MAX_BASE_LIMIT = 20000;
- private static final int HARD_MAX_LIMIT = 100000;
-
- private static final double EQUAL_CONDITION_SCORE = 4;
- private static final double OTHER_CONDITION_SCORE = 1;
- private static final double ORDER_MATCH = 2;
- private static final double ALREADY_MATCHED_ADJUSTOR = 0.1;
- private static final double CARDINALITY_SINGE_SCORE = 1000;
- private static final double CARDINALITY_OTHER_SCORE = 1000;
-
- public GraphCentricQuery constructQuery(final ElementCategory resultType) {
- Preconditions.checkNotNull(resultType);
- if (limit == 0) return GraphCentricQuery.emptyQuery(resultType);
-
- //Prepare constraints
- And<TitanElement> conditions = QueryUtil.constraints2QNF(tx, constraints);
- if (conditions == null) return GraphCentricQuery.emptyQuery(resultType);
-
- //Prepare orders
- orders.makeImmutable();
- if (orders.isEmpty()) orders = OrderList.NO_ORDER;
-
- //Compile all indexes that cover at least one of the query conditions
- final Set<IndexType> indexCandidates = new HashSet<IndexType>();
- ConditionUtil.traversal(conditions,new Predicate<Condition<TitanElement>>() {
- @Override
- public boolean apply(@Nullable Condition<TitanElement> condition) {
- if (condition instanceof PredicateCondition) {
- RelationType type = ((PredicateCondition<RelationType,TitanElement>)condition).getKey();
- Preconditions.checkArgument(type!=null && type.isPropertyKey());
- Iterables.addAll(indexCandidates,Iterables.filter(((InternalRelationType) type).getKeyIndexes(), new Predicate<IndexType>() {
- @Override
- public boolean apply(@Nullable IndexType indexType) {
- return indexType.getElement()==resultType;
- }
- }));
- }
- return true;
- }
- });
-
- /*
- Determine the best join index query to answer this query:
- Iterate over all potential indexes (as compiled above) and compute a score based on how many clauses
- this index covers. The index with the highest score (as long as it covers at least one additional clause)
- is picked and added to the joint query for as long as such exist.
- */
- JointIndexQuery jointQuery = new JointIndexQuery();
- boolean isSorted = orders.isEmpty();
- Set<Condition> coveredClauses = Sets.newHashSet();
- while (true) {
- IndexType bestCandidate = null;
- double candidateScore = 0.0;
- Set<Condition> candidateSubcover = null;
- boolean candidateSupportsSort = false;
- Object candidateSubcondition = null;
-
- for (IndexType index : indexCandidates) {
- Set<Condition> subcover = Sets.newHashSet();
- Object subcondition;
- boolean supportsSort = orders.isEmpty();
- //Check that this index actually applies in case of a schema constraint
- if (index.hasSchemaTypeConstraint()) {
- TitanSchemaType type = index.getSchemaTypeConstraint();
- Map.Entry<Condition,Collection<Object>> equalCon = getEqualityConditionValues(conditions,ImplicitKey.LABEL);
- if (equalCon==null) continue;
- Collection<Object> labels = equalCon.getValue();
- assert labels.size()>=1;
- if (labels.size()>1) {
- log.warn("The query optimizer currently does not support multiple label constraints in query: {}",this);
- continue;
- }
- if (!type.getName().equals((String)Iterables.getOnlyElement(labels))) continue;
- subcover.add(equalCon.getKey());
- }
-
- if (index.isCompositeIndex()) {
- subcondition = indexCover((CompositeIndexType) index,conditions,subcover);
- } else {
- subcondition = indexCover((MixedIndexType) index,conditions,serializer,subcover);
- if (coveredClauses.isEmpty() && !supportsSort
- && indexCoversOrder((MixedIndexType)index,orders)) supportsSort=true;
- }
- if (subcondition==null) continue;
- assert !subcover.isEmpty();
- double score = 0.0;
- boolean coversAdditionalClause = false;
- for (Condition c : subcover) {
- double s = (c instanceof PredicateCondition && ((PredicateCondition)c).getPredicate()==Cmp.EQUAL)?
- EQUAL_CONDITION_SCORE:OTHER_CONDITION_SCORE;
- if (coveredClauses.contains(c)) s=s*ALREADY_MATCHED_ADJUSTOR;
- else coversAdditionalClause = true;
- score+=s;
- if (index.isCompositeIndex())
- score+=((CompositeIndexType)index).getCardinality()==Cardinality.SINGLE?
- CARDINALITY_SINGE_SCORE:CARDINALITY_OTHER_SCORE;
- }
- if (supportsSort) score+=ORDER_MATCH;
- if (coversAdditionalClause && score>candidateScore) {
- candidateScore=score;
- bestCandidate=index;
- candidateSubcover = subcover;
- candidateSubcondition = subcondition;
- candidateSupportsSort = supportsSort;
- }
- }
- if (bestCandidate!=null) {
- if (coveredClauses.isEmpty()) isSorted=candidateSupportsSort;
- coveredClauses.addAll(candidateSubcover);
-
- log.debug("Index chosen for query {} {} " , bestCandidate.isCompositeIndex() ? "COMPOSITE" : "MIXED", coveredClauses);
- if (bestCandidate.isCompositeIndex()) {
- jointQuery.add((CompositeIndexType)bestCandidate,
- serializer.getQuery((CompositeIndexType)bestCandidate,(List<Object[]>)candidateSubcondition));
- } else {
- jointQuery.add((MixedIndexType)bestCandidate,
- serializer.getQuery((MixedIndexType)bestCandidate,(Condition)candidateSubcondition,orders));
- }
- } else {
- break;
- }
- /* TODO: smarter optimization:
- - use in-memory histograms to estimate selectivity of PredicateConditions and filter out low-selectivity ones
- if they would result in an individual index call (better to filter afterwards in memory)
- - move OR's up and extend GraphCentricQuery to allow multiple JointIndexQuery for proper or'ing of queries
- */
- }
-
- BackendQueryHolder<JointIndexQuery> query;
- if (!coveredClauses.isEmpty()) {
- int indexLimit = limit == Query.NO_LIMIT ? HARD_MAX_LIMIT : limit;
- if (tx.getGraph().getConfiguration().adjustQueryLimit()) {
- indexLimit = limit == Query.NO_LIMIT ? DEFAULT_NO_LIMIT : Math.min(MAX_BASE_LIMIT, limit);
- }
- indexLimit = Math.min(HARD_MAX_LIMIT, QueryUtil.adjustLimitForTxModifications(tx, coveredClauses.size(), indexLimit));
- jointQuery.setLimit(indexLimit);
- query = new BackendQueryHolder<JointIndexQuery>(jointQuery, coveredClauses.size()==conditions.numChildren(), isSorted, null);
- } else {
- query = new BackendQueryHolder<JointIndexQuery>(new JointIndexQuery(), false, isSorted, null);
- }
-
- return new GraphCentricQuery(resultType, conditions, orders, query, limit);
- }
-
- public static final boolean indexCoversOrder(MixedIndexType index, OrderList orders) {
- for (int i = 0; i < orders.size(); i++) {
- if (!index.indexesKey(orders.getKey(i))) return false;
- }
- return true;
- }
-
- public static List<Object[]> indexCover(final CompositeIndexType index, Condition<TitanElement> condition, Set<Condition> covered) {
- assert QueryUtil.isQueryNormalForm(condition);
- assert condition instanceof And;
- if (index.getStatus()!= SchemaStatus.ENABLED) return null;
- IndexField[] fields = index.getFieldKeys();
- Object[] indexValues = new Object[fields.length];
- Set<Condition> coveredClauses = new HashSet<Condition>(fields.length);
- List<Object[]> indexCovers = new ArrayList<Object[]>(4);
-
- constructIndexCover(indexValues,0,fields,condition,indexCovers,coveredClauses);
- if (!indexCovers.isEmpty()) {
- covered.addAll(coveredClauses);
- return indexCovers;
- } else return null;
- }
-
- private static void constructIndexCover(Object[] indexValues, int position, IndexField[] fields,
- Condition<TitanElement> condition,
- List<Object[]> indexCovers, Set<Condition> coveredClauses) {
- if (position>=fields.length) {
- indexCovers.add(indexValues);
- } else {
- IndexField field = fields[position];
- Map.Entry<Condition,Collection<Object>> equalCon = getEqualityConditionValues(condition,field.getFieldKey());
- if (equalCon!=null) {
- coveredClauses.add(equalCon.getKey());
- assert equalCon.getValue().size()>0;
- for (Object value : equalCon.getValue()) {
- Object[] newValues = Arrays.copyOf(indexValues,fields.length);
- newValues[position]=value;
- constructIndexCover(newValues,position+1,fields,condition,indexCovers,coveredClauses);
- }
- } else return;
- }
-
- }
-
- private static final Map.Entry<Condition,Collection<Object>> getEqualityConditionValues(Condition<TitanElement> condition, RelationType type) {
- for (Condition c : condition.getChildren()) {
- if (c instanceof Or) {
- Map.Entry<RelationType,Collection> orEqual = QueryUtil.extractOrCondition((Or)c);
- if (orEqual!=null && orEqual.getKey().equals(type) && !orEqual.getValue().isEmpty()) {
- return new AbstractMap.SimpleImmutableEntry(c,orEqual.getValue());
- }
- } else if (c instanceof PredicateCondition) {
- PredicateCondition<RelationType, TitanRelation> atom = (PredicateCondition)c;
- if (atom.getKey().equals(type) && atom.getPredicate()==Cmp.EQUAL && atom.getValue()!=null) {
- return new AbstractMap.SimpleImmutableEntry(c,ImmutableList.of(atom.getValue()));
- }
- }
-
- }
- return null;
- }
-
- public static final Condition<TitanElement> indexCover(final MixedIndexType index, Condition<TitanElement> condition,
- final IndexSerializer indexInfo, final Set<Condition> covered) {
- assert QueryUtil.isQueryNormalForm(condition);
- assert condition instanceof And;
- And<TitanElement> subcondition = new And<TitanElement>(condition.numChildren());
- for (Condition<TitanElement> subclause : condition.getChildren()) {
- if (coversAll(index,subclause,indexInfo)) {
- subcondition.add(subclause);
- covered.add(subclause);
- }
- }
- return subcondition.isEmpty()?null:subcondition;
- }
-
- private static final boolean coversAll(final MixedIndexType index, Condition<TitanElement> condition, IndexSerializer indexInfo) {
- if (condition.getType()==Condition.Type.LITERAL) {
- if (!(condition instanceof PredicateCondition)) return false;
- PredicateCondition<RelationType, TitanElement> atom = (PredicateCondition) condition;
- if (atom.getValue()==null) return false;
-
- Preconditions.checkArgument(atom.getKey().isPropertyKey());
- PropertyKey key = (PropertyKey) atom.getKey();
- ParameterIndexField[] fields = index.getFieldKeys();
- ParameterIndexField match = null;
- for (int i = 0; i < fields.length; i++) {
- if (fields[i].getStatus()!= SchemaStatus.ENABLED) continue;
- if (fields[i].getFieldKey().equals(key)) match = fields[i];
- }
- if (match==null) return false;
- return indexInfo.supports(index,match,atom.getPredicate());
- } else {
- for (Condition<TitanElement> child : condition.getChildren()) {
- if (!coversAll(index,child,indexInfo)) return false;
- }
- return true;
- }
- }
-}
[9/9] incubator-atlas git commit: ATLAS-693 Titan 0.5.4
implementation of graph db abstraction. (jnhagelb via dkantor)
Posted by dk...@apache.org.
ATLAS-693 Titan 0.5.4 implementation of graph db abstraction. (jnhagelb via dkantor)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/4fa10b6a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/4fa10b6a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/4fa10b6a
Branch: refs/heads/master
Commit: 4fa10b6aecb248d09ee2a58b289a1d47f7d24419
Parents: 2ef0fc4
Author: Jeff Hagelberg <jn...@us.ibm.com>
Authored: Fri Aug 5 23:22:41 2016 +0100
Committer: Dave Kantor <dk...@us.ibm.com>
Committed: Fri Aug 5 23:22:41 2016 +0100
----------------------------------------------------------------------
.gitignore | 1 +
catalog/pom.xml | 9 +-
.../org/apache/atlas/repository/Constants.java | 90 ++
distro/src/bin/atlas_start.py | 1 -
distro/src/test/python/scripts/TestMetadata.py | 8 +-
.../atlas/repository/graphdb/AtlasEdge.java | 20 +-
.../repository/graphdb/AtlasEdgeDirection.java | 2 +-
.../atlas/repository/graphdb/AtlasElement.java | 120 ++-
.../atlas/repository/graphdb/AtlasGraph.java | 152 ++-
.../repository/graphdb/AtlasGraphIndex.java | 61 ++
.../graphdb/AtlasGraphManagement.java | 92 +-
.../repository/graphdb/AtlasGraphQuery.java | 71 +-
.../repository/graphdb/AtlasIndexQuery.java | 20 +-
.../repository/graphdb/AtlasPropertyKey.java | 30 +
.../graphdb/AtlasSchemaViolationException.java | 2 +-
.../atlas/repository/graphdb/AtlasVertex.java | 36 +-
.../repository/graphdb/AtlasVertexQuery.java | 16 +-
.../atlas/repository/graphdb/GraphDatabase.java | 22 +-
.../repository/graphdb/GremlinVersion.java | 2 +-
graphdb/common/pom.xml | 64 ++
.../titan/query/NativeTitanGraphQuery.java | 60 ++
.../titan/query/NativeTitanQueryFactory.java | 34 +
.../graphdb/titan/query/TitanGraphQuery.java | 189 ++++
.../graphdb/titan/query/expr/AndCondition.java | 96 ++
.../graphdb/titan/query/expr/HasPredicate.java | 50 +
.../graphdb/titan/query/expr/InPredicate.java | 49 +
.../graphdb/titan/query/expr/OrCondition.java | 136 +++
.../titan/query/expr/QueryPredicate.java | 33 +
graphdb/pom.xml | 2 +
graphdb/titan0/pom.xml | 257 +++++
.../titan/diskstorage/hbase/AdminMask.java | 62 ++
.../titan/diskstorage/hbase/ConnectionMask.java | 30 +
.../titan/diskstorage/hbase/HBaseAdmin0_98.java | 152 +++
.../titan/diskstorage/hbase/HBaseAdmin1_0.java | 135 +++
.../titan/diskstorage/hbase/HBaseCompat.java | 60 ++
.../diskstorage/hbase/HBaseCompat0_98.java | 58 ++
.../titan/diskstorage/hbase/HBaseCompat1_0.java | 58 ++
.../titan/diskstorage/hbase/HBaseCompat1_1.java | 58 ++
.../diskstorage/hbase/HBaseCompatLoader.java | 80 ++
.../hbase/HBaseKeyColumnValueStore.java | 425 ++++++++
.../diskstorage/hbase/HBaseStoreManager.java | 935 ++++++++++++++++++
.../diskstorage/hbase/HBaseTransaction.java | 75 ++
.../diskstorage/hbase/HConnection0_98.java | 49 +
.../titan/diskstorage/hbase/HConnection1_0.java | 50 +
.../titan/diskstorage/hbase/HTable0_98.java | 65 ++
.../titan/diskstorage/hbase/HTable1_0.java | 65 ++
.../titan/diskstorage/hbase/TableMask.java | 41 +
.../diskstorage/locking/LocalLockMediator.java | 345 +++++++
.../titan/diskstorage/solr/Solr5Index.java | 975 +++++++++++++++++++
.../query/graph/GraphCentricQueryBuilder.java | 457 +++++++++
.../graphdb/titan0/GraphDbObjectFactory.java | 100 ++
.../graphdb/titan0/Titan0Database.java | 204 ++++
.../graphdb/titan0/Titan0DatabaseManager.java | 170 ++++
.../repository/graphdb/titan0/Titan0Edge.java | 62 ++
.../graphdb/titan0/Titan0Element.java | 267 +++++
.../repository/graphdb/titan0/Titan0Graph.java | 301 ++++++
.../graphdb/titan0/Titan0GraphIndex.java | 96 ++
.../graphdb/titan0/Titan0IndexQuery.java | 76 ++
.../graphdb/titan0/Titan0PropertyKey.java | 69 ++
.../repository/graphdb/titan0/Titan0Vertex.java | 135 +++
.../graphdb/titan0/Titan0VertexQuery.java | 65 ++
.../graphdb/titan0/TitanObjectFactory.java | 83 ++
.../titan0/query/NativeTitan0GraphQuery.java | 88 ++
.../graphdb/titan0/query/Titan0GraphQuery.java | 56 ++
.../atlas/utils/IteratorToIterableAdapter.java | 38 +
.../hbase/HBaseKeyColumnValueStoreTest.java | 139 +++
.../locking/LocalLockMediatorTest.java | 60 ++
.../titan0/AbstractGraphDatabaseTest.java | 200 ++++
.../graphdb/titan0/GraphQueryTest.java | 447 +++++++++
.../graphdb/titan0/Titan0DatabaseTest.java | 428 ++++++++
.../titan0/Titan0DatabaseValidationTest.java | 77 ++
.../test/resources/atlas-application.properties | 97 ++
pom.xml | 113 +--
release-log.txt | 1 +
repository/pom.xml | 17 +-
.../apache/atlas/RepositoryMetadataModule.java | 5 +-
.../org/apache/atlas/repository/Constants.java | 101 --
.../atlas/repository/graph/GraphHelper.java | 59 +-
.../repository/graph/TitanGraphProvider.java | 115 +--
.../org/apache/atlas/query/TypeUtils.scala | 3 +-
.../graph/TitanGraphProviderTest.java | 78 --
src/build/checkstyle-suppressions.xml | 3 +
titan/pom.xml | 105 --
.../titan/diskstorage/hbase/AdminMask.java | 62 --
.../titan/diskstorage/hbase/ConnectionMask.java | 30 -
.../titan/diskstorage/hbase/HBaseAdmin0_98.java | 152 ---
.../titan/diskstorage/hbase/HBaseAdmin1_0.java | 135 ---
.../titan/diskstorage/hbase/HBaseCompat.java | 60 --
.../diskstorage/hbase/HBaseCompat0_98.java | 58 --
.../titan/diskstorage/hbase/HBaseCompat1_0.java | 58 --
.../titan/diskstorage/hbase/HBaseCompat1_1.java | 58 --
.../diskstorage/hbase/HBaseCompatLoader.java | 80 --
.../hbase/HBaseKeyColumnValueStore.java | 425 --------
.../diskstorage/hbase/HBaseStoreManager.java | 935 ------------------
.../diskstorage/hbase/HBaseTransaction.java | 75 --
.../diskstorage/hbase/HConnection0_98.java | 49 -
.../titan/diskstorage/hbase/HConnection1_0.java | 50 -
.../titan/diskstorage/hbase/HTable0_98.java | 65 --
.../titan/diskstorage/hbase/HTable1_0.java | 65 --
.../titan/diskstorage/hbase/TableMask.java | 41 -
.../diskstorage/locking/LocalLockMediator.java | 345 -------
.../titan/diskstorage/solr/Solr5Index.java | 975 -------------------
.../query/graph/GraphCentricQueryBuilder.java | 457 ---------
.../hbase/HBaseKeyColumnValueStoreTest.java | 139 ---
.../locking/LocalLockMediatorTest.java | 60 --
webapp/pom.xml | 18 +-
106 files changed, 9098 insertions(+), 5052 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 0086a69..e10adbc 100755
--- a/.gitignore
+++ b/.gitignore
@@ -30,6 +30,7 @@ core*.dmp
.cache
.classpath
.project
+.checkstyle
.settings
.externalToolBuilders
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/catalog/pom.xml
----------------------------------------------------------------------
diff --git a/catalog/pom.xml b/catalog/pom.xml
index 5946c10..2f58a8f 100755
--- a/catalog/pom.xml
+++ b/catalog/pom.xml
@@ -30,7 +30,12 @@
<description>Apache Atlas Business Catalog Module</description>
<name>Apache Atlas Business Catalog</name>
<packaging>jar</packaging>
-
+
+ <properties>
+ <tinkerpop.version>2.6.0</tinkerpop.version>
+ <titan.version>0.5.4</titan.version>
+ </properties>
+
<dependencies>
<dependency>
<groupId>org.apache.atlas</groupId>
@@ -82,11 +87,13 @@
<dependency>
<groupId>com.tinkerpop.blueprints</groupId>
<artifactId>blueprints-core</artifactId>
+ <version>${tinkerpop.version}</version>
</dependency>
<dependency>
<groupId>com.tinkerpop.gremlin</groupId>
<artifactId>gremlin-java</artifactId>
+ <version>${tinkerpop.version}</version>
</dependency>
<!-- testing -->
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/common/src/main/java/org/apache/atlas/repository/Constants.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java
new file mode 100644
index 0000000..d7f9c89
--- /dev/null
+++ b/common/src/main/java/org/apache/atlas/repository/Constants.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository;
+
+/**
+ * Repository Constants.
+ *
+ */
+public final class Constants {
+
+ /**
+ * Globally Unique identifier property key.
+ */
+
+ public static final String INTERNAL_PROPERTY_KEY_PREFIX = "__";
+ public static final String GUID_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "guid";
+
+ /**
+ * Entity type name property key.
+ */
+ public static final String ENTITY_TYPE_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "typeName";
+
+ /**
+ * Entity type's super types property key.
+ */
+ public static final String SUPER_TYPES_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "superTypeNames";
+
+ /**
+ * Full-text for the entity for enabling full-text search.
+ */
+ //weird issue in TitanDB if __ added to this property key. Not adding it for now
+ public static final String ENTITY_TEXT_PROPERTY_KEY = "entityText";
+
+ /**
+ * Properties for type store graph.
+ */
+ public static final String TYPE_CATEGORY_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.category";
+ public static final String VERTEX_TYPE_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type";
+ public static final String TYPENAME_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.name";
+ public static final String TYPEDESCRIPTION_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.description";
+
+ /**
+ * Trait names property key and index name.
+ */
+ public static final String TRAIT_NAMES_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "traitNames";
+
+ public static final String VERSION_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "version";
+ public static final String STATE_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "state";
+
+ public static final String TIMESTAMP_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "timestamp";
+
+ public static final String MODIFICATION_TIMESTAMP_PROPERTY_KEY =
+ INTERNAL_PROPERTY_KEY_PREFIX + "modificationTimestamp";
+
+ /**
+ * search backing index name.
+ */
+ public static final String BACKING_INDEX = "search";
+
+ /**
+ * search backing index name for vertex keys.
+ */
+ public static final String VERTEX_INDEX = "vertex_index";
+
+ /**
+ * search backing index name for edge labels.
+ */
+ public static final String EDGE_INDEX = "edge_index";
+
+ public static final String FULLTEXT_INDEX = "fulltext_index";
+
+ private Constants() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/distro/src/bin/atlas_start.py
----------------------------------------------------------------------
diff --git a/distro/src/bin/atlas_start.py b/distro/src/bin/atlas_start.py
index bb61898..4590548 100755
--- a/distro/src/bin/atlas_start.py
+++ b/distro/src/bin/atlas_start.py
@@ -75,7 +75,6 @@ def main():
p = os.pathsep
atlas_classpath = confdir + p \
+ os.path.join(web_app_dir, "atlas", "WEB-INF", "classes" ) + p \
- + os.path.join(web_app_dir, "atlas", "WEB-INF", "lib", "atlas-titan-${project.version}.jar" ) + p \
+ os.path.join(web_app_dir, "atlas", "WEB-INF", "lib", "*" ) + p \
+ os.path.join(atlas_home, "libext", "*")
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/distro/src/test/python/scripts/TestMetadata.py
----------------------------------------------------------------------
diff --git a/distro/src/test/python/scripts/TestMetadata.py b/distro/src/test/python/scripts/TestMetadata.py
index f943b8a..329dc3e 100644
--- a/distro/src/test/python/scripts/TestMetadata.py
+++ b/distro/src/test/python/scripts/TestMetadata.py
@@ -86,14 +86,14 @@ class TestMetadata(unittest.TestCase):
java_mock.assert_called_with(
'org.apache.atlas.Atlas',
['-app', 'atlas_home\\server\\webapp\\atlas'],
- 'atlas_home\\conf;atlas_home\\server\\webapp\\atlas\\WEB-INF\\classes;atlas_home\\server\\webapp\\atlas\\WEB-INF\\lib\\atlas-titan-${project.version}.jar;atlas_home\\server\\webapp\\atlas\\WEB-INF\\lib\\*;atlas_home\\libext\\*;atlas_home\\hbase\\conf',
+ 'atlas_home\\conf;atlas_home\\server\\webapp\\atlas\\WEB-INF\\classes;atlas_home\\server\\webapp\\atlas\\WEB-INF\\lib\\*;atlas_home\\libext\\*;atlas_home\\hbase\\conf',
['-Datlas.log.dir=atlas_home\\logs', '-Datlas.log.file=application.log', '-Datlas.home=atlas_home', '-Datlas.conf=atlas_home\\conf', '-Xmx1024m', '-XX:MaxPermSize=512m', '-Dlog4j.configuration=atlas-log4j.xml', '-Djava.net.preferIPv4Stack=true'], 'atlas_home\\logs')
else:
java_mock.assert_called_with(
'org.apache.atlas.Atlas',
['-app', 'atlas_home/server/webapp/atlas'],
- 'atlas_home/conf:atlas_home/server/webapp/atlas/WEB-INF/classes:atlas_home/server/webapp/atlas/WEB-INF/lib/atlas-titan-${project.version}.jar:atlas_home/server/webapp/atlas/WEB-INF/lib/*:atlas_home/libext/*:atlas_home/hbase/conf',
+ 'atlas_home/conf:atlas_home/server/webapp/atlas/WEB-INF/classes:atlas_home/server/webapp/atlas/WEB-INF/lib/*:atlas_home/libext/*:atlas_home/hbase/conf',
['-Datlas.log.dir=atlas_home/logs', '-Datlas.log.file=application.log', '-Datlas.home=atlas_home', '-Datlas.conf=atlas_home/conf', '-Xmx1024m', '-XX:MaxPermSize=512m', '-Dlog4j.configuration=atlas-log4j.xml', '-Djava.net.preferIPv4Stack=true'], 'atlas_home/logs')
pass
@@ -152,14 +152,14 @@ class TestMetadata(unittest.TestCase):
java_mock.assert_called_with(
'org.apache.atlas.Atlas',
['-app', 'atlas_home\\server\\webapp\\atlas'],
- 'atlas_home\\conf;atlas_home\\server\\webapp\\atlas\\WEB-INF\\classes;atlas_home\\server\\webapp\\atlas\\WEB-INF\\lib\\atlas-titan-${project.version}.jar;atlas_home\\server\\webapp\\atlas\\WEB-INF\\lib\\*;atlas_home\\libext\\*;atlas_home\\hbase\\conf',
+ 'atlas_home\\conf;atlas_home\\server\\webapp\\atlas\\WEB-INF\\classes;atlas_home\\server\\webapp\\atlas\\WEB-INF\\lib\\*;atlas_home\\libext\\*;atlas_home\\hbase\\conf',
['-Datlas.log.dir=atlas_home\\logs', '-Datlas.log.file=application.log', '-Datlas.home=atlas_home', '-Datlas.conf=atlas_home\\conf', '-Xmx1024m', '-XX:MaxPermSize=512m', '-Dlog4j.configuration=atlas-log4j.xml', '-Djava.net.preferIPv4Stack=true'], 'atlas_home\\logs')
else:
java_mock.assert_called_with(
'org.apache.atlas.Atlas',
['-app', 'atlas_home/server/webapp/atlas'],
- 'atlas_home/conf:atlas_home/server/webapp/atlas/WEB-INF/classes:atlas_home/server/webapp/atlas/WEB-INF/lib/atlas-titan-${project.version}.jar:atlas_home/server/webapp/atlas/WEB-INF/lib/*:atlas_home/libext/*:atlas_home/hbase/conf',
+ 'atlas_home/conf:atlas_home/server/webapp/atlas/WEB-INF/classes:atlas_home/server/webapp/atlas/WEB-INF/lib/*:atlas_home/libext/*:atlas_home/hbase/conf',
['-Datlas.log.dir=atlas_home/logs', '-Datlas.log.file=application.log', '-Datlas.home=atlas_home', '-Datlas.conf=atlas_home/conf', '-Xmx1024m', '-XX:MaxPermSize=512m', '-Dlog4j.configuration=atlas-log4j.xml', '-Djava.net.preferIPv4Stack=true'], 'atlas_home/logs')
pass
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasEdge.java
----------------------------------------------------------------------
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasEdge.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasEdge.java
index 71b577b..dd4b7e6 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasEdge.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasEdge.java
@@ -24,37 +24,37 @@ package org.apache.atlas.repository.graphdb;
* @param <V> vertex class used by the graph
* @param <E> edge class used by the graph
*/
-public interface AtlasEdge<V,E> extends AtlasElement {
+public interface AtlasEdge<V, E> extends AtlasElement {
/**
* Gets the incoming vertex for this edge
* @param in
* @return
*/
- AtlasVertex<V,E> getInVertex();
+ AtlasVertex<V, E> getInVertex();
/**
* Gets the outgoing vertex for this edge
- *
+ *
* @param in
* @return
*/
- AtlasVertex<V,E> getOutVertex();
-
+ AtlasVertex<V, E> getOutVertex();
+
/**
* Gets the label associated with this edge.
- *
+ *
* @return
*/
String getLabel();
-
+
/**
* Converts the edge to an instance of the underlying implementation class. This
* is syntactic sugar that allows the graph database implementation code to be strongly typed. This
- * should not be called in other places.
- *
+ * should not be called in other places.
+ *
* @return
*/
- public E getE();
+ public E getE();
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasEdgeDirection.java
----------------------------------------------------------------------
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasEdgeDirection.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasEdgeDirection.java
index e7da1c9..a4b5b93 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasEdgeDirection.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasEdgeDirection.java
@@ -22,6 +22,6 @@ package org.apache.atlas.repository.graphdb;
*/
public enum AtlasEdgeDirection {
IN,
- OUT,
+ OUT,
BOTH
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasElement.java
----------------------------------------------------------------------
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasElement.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasElement.java
index 3c41693..1bc0fc3 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasElement.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasElement.java
@@ -19,19 +19,23 @@
package org.apache.atlas.repository.graphdb;
import java.util.Collection;
+import java.util.List;
import java.util.Set;
+import org.apache.atlas.AtlasException;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
/**
- * Represents a graph element.
+ * Represents a graph element.
*
*/
public interface AtlasElement {
/**
- * Gets the id of this element
+ * Gets the id of this element. If the object has not been physically created in the underlying graph,
+ * calling this method will force the element to be created so that a valid Id can be returned.
+ *
* @return
*/
Object getId();
@@ -43,32 +47,122 @@ public interface AtlasElement {
Collection<? extends String> getPropertyKeys();
/**
- * Gets the value of the element property with the given name
- *
+ * Gets the value of the element property with the given name. The value
+ * returned is guaranteed to be an instance of the specified class (or
+ * an exception will be thrown).
+ *
+ * @param propertyName
+ * @return
+ * @throws IllegalStateException if the property is multi-valued in the graph schema.
+ */
+ <T> T getProperty(String propertyName, Class<T> clazz);
+
+ /**
+ * Gets all of the values of the given property.
* @param propertyName
* @return
*/
- <T> T getProperty(String propertyName);
-
+ <T> Collection<T> getPropertyValues(String propertyName, Class<T> type);
+
+ /**
+ * Gets the value of a multiplicity one property whose value is a String list.
+ * The lists of super types and traits are stored this way. A separate method
+ * is needed for this because special logic is required to handle this situation
+ * in some implementations.
+ */
+ List<String> getListProperty(String propertyName);
+
+ /**
+ * Gets the value of a multiplicity one property whose value is a list. It
+ * attempts to convert the elements in the list to the specified type. Currently
+ * conversion is only supported for subclasses of AtlasElement and String.
+ */
+ <V> List<V> getListProperty(String propertyName, Class<V> elementType);
+
+ /**
+ * Sets a multiplicity one property whose value is a String list.
+ * The lists of super types and traits are stored this way. A separate method
+ * is needed for this because special logic is required to handle this situation
+ * in some implementations.
+ */
+ void setListProperty(String propertyName, List<String> values) throws AtlasException;
+
+
+ /**
+ * Sets a multiplicity one property whose effective value is a String list whose
+ * values consist of the ids of the supplied elements. This is implemented efficiently
+ * so that in many cases the property can be set without requiring new elements
+ * to be immediately created in the graph database. It allows the actual underlying element
+ * creation to be deferred until commit time.
+ */
+ void setPropertyFromElementsIds(String propertyName, List<AtlasElement> values);
+
+ /**
+ * Sets a multiplicity one property whose effective value is a String whose value is the id of the supplied
+ * element. This is implemented efficiently so that in many cases the property can be set without requiring
+ * new elements to be immediately created in the graph database. It allows the actual underlying element
+ * creation to be deferred until commit time.
+ */
+ void setPropertyFromElementId(String propertyName, AtlasElement value);
+
+
/**
* Removes a property from the vertex.
*/
void removeProperty(String propertyName);
/**
- * Sets a single-valued property to the given value.
- *
+ * Sets a single-valued property to the given value. For
+ * properties defined as multiplicty many in the graph schema, the value is added instead
+ * (following set semantics)
+ *
* @param propertyName
* @param value
*/
- <T> void setProperty(String propertyName, T value);
-
-
+ <T> void setProperty(String propertyName, T value);
+
+
/**
* Creates a Jettison JSONObject from this Element
*
* @param propertyKeys The property keys at the root of the element to serialize. If null, then all keys are serialized.
*/
- JSONObject toJson(Set<String> propertyKeys) throws JSONException;
-
+ JSONObject toJson(Set<String> propertyKeys) throws JSONException;
+
+ /**
+ * Determines if this element exists in the graph database. If the element has not
+ * actually been loaded from the underlying graph, this will cause it to be loaded
+ * so that the result is correct.
+ *
+ * @return
+ */
+ boolean exists();
+
+ /**
+ * @param propertyName
+ * @param value
+ */
+ <T> void setJsonProperty(String propertyName, T value);
+
+ /**
+ * @param propertyName
+ * @return
+ */
+ <T> T getJsonProperty(String propertyName);
+
+ /**
+ * Gets a human-readable id without forcing the element to
+ * be created if it does not exist in the graph yet.
+ *
+ * @return
+ */
+ public String getIdForDisplay();
+
+ /**
+ * Whether or not an id has been assigned yet for this Element. This can happen if the element has been created
+ * in memory but has not been actually pushed into the underlying graph yet.
+ *
+ * @return
+ */
+ boolean isIdAssigned();
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
----------------------------------------------------------------------
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
index f312117..995c545 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
@@ -22,70 +22,82 @@ import java.io.OutputStream;
import java.util.List;
import java.util.Set;
-import javax.script.Bindings;
import javax.script.ScriptException;
+import org.apache.atlas.typesystem.types.IDataType;
+
/**
* Represents a graph
- *
+ *
* @param <V> vertex implementation class
* @param <E> edge implementation class
*/
-public interface AtlasGraph<V,E> {
+public interface AtlasGraph<V, E> {
/**
* Adds an edge to the graph
- *
+ *
* @param outVertex
* @param inVertex
* @param label
* @return
*/
- AtlasEdge<V,E> addEdge(AtlasVertex<V,E> outVertex, AtlasVertex<V,E> inVertex, String label);
+ AtlasEdge<V, E> addEdge(AtlasVertex<V, E> outVertex, AtlasVertex<V, E> inVertex, String label);
/**
* Adds a vertex to the graph
- *
+ *
* @return
*/
- AtlasVertex<V,E> addVertex();
+ AtlasVertex<V, E> addVertex();
/**
* Removes the specified edge from the graph
- *
+ *
* @param edge
*/
- void removeEdge(AtlasEdge<V,E> edge);
+ void removeEdge(AtlasEdge<V, E> edge);
/**
* Removes the specified vertex from the graph.
- *
+ *
* @param vertex
*/
- void removeVertex(AtlasVertex<V,E> vertex);
+ void removeVertex(AtlasVertex<V, E> vertex);
/**
- * Retrieves the edge with the specified id
+ * Retrieves the edge with the specified id. As an optimization, a non-null Edge may be
+ * returned by some implementations if the Edge does not exist. In that case,
+ * you can call {@link AtlasElement#exists()} to determine whether the vertex
+ * exists. This allows the retrieval of the Edge information to be deferred
+ * or in come cases avoided altogether in implementations where that might
+ * be an expensive operation.
+ *
* @param edgeId
* @return
*/
- AtlasEdge<V,E> getEdge(String edgeId);
+ AtlasEdge<V, E> getEdge(String edgeId);
/**
* Gets all the edges in the graph.
* @return
*/
- Iterable<AtlasEdge<V,E>> getEdges();
+ Iterable<AtlasEdge<V, E>> getEdges();
/**
* Gets all the vertices in the graph.
* @return
*/
- Iterable<AtlasVertex<V,E>> getVertices();
+ Iterable<AtlasVertex<V, E>> getVertices();
/**
- * Gets the vertex with the specified id
- *
+ * Gets the vertex with the specified id. As an optimization, a non-null vertex may be
+ * returned by some implementations if the Vertex does not exist. In that case,
+ * you can call {@link AtlasElement#exists()} to determine whether the vertex
+ * exists. This allows the retrieval of the Vertex information to be deferred
+ * or in come cases avoided altogether in implementations where that might
+ * be an expensive operation.
+ *
* @param vertexId
* @return
*/
@@ -94,7 +106,7 @@ public interface AtlasGraph<V,E> {
/**
* Gets the names of the indexes on edges
* type.
- *
+ *
* @param type
* @return
*/
@@ -104,40 +116,40 @@ public interface AtlasGraph<V,E> {
/**
* Gets the names of the indexes on vertices.
* type.
- *
+ *
* @param type
* @return
*/
Set<String> getVertexIndexKeys();
-
+
/**
* Finds the vertices where the given property key
* has the specified value. For multi-valued properties,
* finds the vertices where the value list contains
* the specified value.
- *
+ *
* @param key
* @param value
* @return
*/
- Iterable<AtlasVertex<V,E>> getVertices(String key, Object value);
+ Iterable<AtlasVertex<V, E>> getVertices(String key, Object value);
/**
* Creates a graph query
* @return
*/
- AtlasGraphQuery<V,E> query();
+ AtlasGraphQuery<V, E> query();
/**
* Creates an index query
- *
+ *
* @param indexName index name
* @param queryString the query
- *
+ *
* @see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html">Elastic Search Reference</a> for query syntax
*/
- AtlasIndexQuery<V,E> indexQuery(String indexName, String queryString);
+ AtlasIndexQuery<V, E> indexQuery(String indexName, String queryString);
/**
* Gets the management object associated with this graph and opens a transaction
@@ -160,26 +172,30 @@ public interface AtlasGraph<V,E> {
* Unloads and releases any resources associated with the graph.
*/
void shutdown();
-
+
/**
- * deletes everything in the graph. For testing only
+ * Deletes all data in the graph. May or may not delete
+ * the indices, depending on the what the underlying graph supports.
+ *
+ * For testing only.
+ *
*/
void clear();
/**
* Converts the graph to gson and writes it to the specified stream
- *
+ *
* @param os
* @throws IOException
*/
void exportToGson(OutputStream os) throws IOException;
-
+
//the following methods insulate Atlas from the details
//of the interaction with Gremlin
-
-
+
+
/**
- *
+ *
* When we construct Gremlin select queries, the information we request
* is grouped by the vertex the information is coming from. Each vertex
* is assigned a column name which uniquely identifies it. The queries
@@ -197,26 +213,26 @@ public interface AtlasGraph<V,E> {
* <p/>
* If the value found is a vertex or edge, it is automatically converted
* to an AtlasVertex/AtlasEdge.
- *
+ *
* @param rowValue the raw row value that was returned by Gremin
* @param colName the column name to use
- * @param idx the index of the value within the column to retrieve.
- *
+ * @param idx the index of the value within the column to retrieve.
+ *
*/
Object getGremlinColumnValue(Object rowValue, String colName, int idx);
/**
- * When Gremlin queries are executed, they return
+ * When Gremlin queries are executed, they return
* Vertex and Edge objects that are specific to the underlying
* graph database. This method provides a way to convert these
* objects back into the AtlasVertex/AtlasEdge objects that
- * Atlas requires.
- *
+ * Atlas requires.
+ *
* @param rawValue the value that was obtained from Gremlin
* @return either an AtlasEdge, an AtlasVertex, or the original
* value depending on whether the rawValue represents an edge,
* vertex, or something else.
- *
+ *
*/
Object convertGremlinValue(Object rawValue);
@@ -224,27 +240,75 @@ public interface AtlasGraph<V,E> {
* Gremlin 2 and Gremlin 3 represent the results of "path"
* queries differently. This method takes as input the
* path from Gremlin and returns the list of objects in the path.
- *
+ *
* @param rawValue
* @return
*/
List<Object> convertPathQueryResultToList(Object rawValue);
/**
+ * This method is used in the generation of queries. It is used to
+ * convert property values from the value that is stored in the graph
+ * to the value/type that the user expects to get back.
+ *
+ * @param expr - gremlin expr that represents the persistent property value
+ * @param type
+ * @return
+ */
+ String generatePersisentToLogicalConversionExpression(String valueExpr, IDataType<?> type);
+
+ /**
+ * Indicates whether or not stored values with the specified type need to be converted
+ * within generated gremlin queries before they can be compared with literal values.
+ * As an example, a graph database might choose to store Long values as Strings or
+ * List values as a delimited list. In this case, the generated gremlin needs to
+ * convert the stored property value prior to comparing it a literal. In this returns
+ * true, @code{generatePersisentToLogicalConversionExpression} is used to generate a
+ * gremlin expression with the converted value. In addition, this cause the gremlin
+ * 'filter' step to be used to compare the values instead of a 'has' step.
+ */
+ boolean isPropertyValueConversionNeeded(IDataType<?> type);
+
+ /**
* Gets the version of Gremlin that this graph uses.
- *
+ *
* @return
*/
GremlinVersion getSupportedGremlinVersion();
+
+ /**
+ * Whether or not an initial predicate needs to be added to gremlin queries
+ * in order for them to run successfully. This is needed for some graph database where
+ * graph scans are disabled.
+ * @return
+ */
+ boolean requiresInitialIndexedPredicate();
+
+ /**
+ * Some graph database backends have graph scans disabled. In order to execute some queries there,
+ * an initial 'dummy' predicate needs to be added to gremlin queries so that the first
+ * condition uses an index.
+ *
+ * @return
+ */
+ String getInitialIndexedPredicate();
/**
+ * As an optimization, a graph database implementation may want to retrieve additional
+ * information about the query results. For example, in the IBM Graph implementation,
+ * this changes the query to return both matching vertices and their outgoing edges to
+ * avoid the need to make an extra REST API call to look up those edges. For implementations
+ * that do not require any kind of transform, an empty String should be returned.
+ */
+ String getOutputTransformationPredicate(boolean isSelect, boolean isPath);
+
+ /**
* Executes a gremlin query, returns an object with the raw
* result.
- *
+ *
* @param gremlinQuery
* @return
*/
Object executeGremlinScript(String gremlinQuery) throws ScriptException;
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndex.java
----------------------------------------------------------------------
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndex.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndex.java
new file mode 100644
index 0000000..41194d3
--- /dev/null
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndex.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graphdb;
+
+import java.util.Set;
+
+/**
+ * Represents a graph index on the database
+ */
+public interface AtlasGraphIndex {
+
+ /**
+ * Indicates if the index is a mixed index.
+ * @return
+ */
+ boolean isMixedIndex();
+
+
+ /**
+ * Indicates if the index is a composite index.
+ * @return
+ */
+ boolean isCompositeIndex();
+
+ /**
+ * Indicates if the index applies to edges
+ *
+ * @return
+ */
+ boolean isEdgeIndex();
+
+ /**
+ * Indicates if the index applies to vertices
+ *
+ * @return
+ */
+ boolean isVertexIndex();
+
+
+
+ boolean isUnique();
+
+ Set<AtlasPropertyKey> getFieldKeys();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
----------------------------------------------------------------------
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
index 2776b22..c8cd284 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
@@ -18,48 +18,62 @@
package org.apache.atlas.repository.graphdb;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.repository.Constants;
import org.apache.atlas.typesystem.types.Multiplicity;
/**
* Management interface for a graph
- *
+ *
*/
public interface AtlasGraphManagement {
+ public static final Set<String> MULTIPLICITY_MANY_PROPERTY_KEYS =
+ Collections.unmodifiableSet(new HashSet<String>(Arrays.asList(
+ Constants.SUPER_TYPES_PROPERTY_KEY,
+ Constants.TRAIT_NAMES_PROPERTY_KEY )));
+
+
/**
* Checks whether a property with the given key has been defined in the graph schema.
- *
+ *
* @param key
* @return
*/
boolean containsPropertyKey(String key);
/**
- * Creates a mixed Vertex index for the graph
- *
+ * Creates a mixed Vertex index for the graph
+ *
* @param index the name of the index to create
* @param backingIndex the name of the backing index to use
*/
void buildMixedVertexIndex(String index, String backingIndex);
-
-
+
+
/**
- * Creates a mixed Edge index for the graph
- *
+ * Creates a mixed Edge index for the graph
+ *
* @param index the name of the index to create
* @param backingIndex the name of the backing index to use
*/
void buildMixedEdgeIndex(String index, String backingIndex);
-
+
/**
- * Creates a full text index for the given property
- *
+ * Creates a full text index for the given property
+ *
* @param indexName the name of the index to create
- * @param propertyKey the name of the property
+ * @param propertyKey full text property to index
* @param backingIndex the name of the backing index to use
*/
- void createFullTextIndex(String indexName, String propertyKey, String backingIndex);
+ void createFullTextIndex(String indexName, AtlasPropertyKey propertyKey, String backingIndex);
/**
* Rolls back the changes that have been made to the management system.
@@ -73,26 +87,44 @@ public interface AtlasGraphManagement {
void commit();
/**
- * Creates a composite index for the given property.
- *
- * @param propertyName name of the property being indexed
- * @param propertyClass the java class of the property value(s)
- * @param multiplicity the multiplicity of the property
- * @param isUnique whether the property values must be unique
+ * @param propertyName
+ * @param propertyClass
+ * @param cardinality
+ * @return
+ */
+ AtlasPropertyKey makePropertyKey(String propertyName, Class propertyClass, Multiplicity multiplicity);
+
+ /**
+ * @param propertyKey
+ * @return
+ */
+ AtlasPropertyKey getPropertyKey(String propertyName);
+
+ /**
+ * Creates a composite index for the graph.
+ *
+ * @param propertyName
+ * @param propertyKey
+ * @param isUnique
*/
- void createCompositeIndex(String propertyName, Class propertyClass, Multiplicity multiplicity,
- boolean isUnique);
-
+ void createCompositeIndex(String propertyName, AtlasPropertyKey propertyKey, boolean isUnique);
+
/**
- * Creates an index for a property.
- *
- * @param propertyName name of the property being indexed
- * @param vertexIndexName name of the index to create
- * @param propertyClass the java class of the property value(s)
- * @param multiplicity the multiplicity of the property
+ * Adds a property key to the given index in the graph.
+ *
+ * @param vertexIndex
+ * @param propertyKey
+ */
+ void addIndexKey(String vertexIndex, AtlasPropertyKey propertyKey);
+
+ /**
+ * Looks up the index with the specified name in the graph. Returns null if
+ * there is no index with the given name.
+ *
+ * @param edgeIndex
+ * @return
*/
- void createBackingIndex(String propertyName, String vertexIndexName, Class propertyClass,
- Multiplicity multiplicity);
+ AtlasGraphIndex getGraphIndex(String indexName);
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphQuery.java
----------------------------------------------------------------------
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphQuery.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphQuery.java
index f0dca9a..9344749 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphQuery.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphQuery.java
@@ -18,8 +18,14 @@
package org.apache.atlas.repository.graphdb;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.atlas.AtlasException;
+
/**
- * Represents a query against the graph.
+ * Represents a query against the graph within the context of the
+ * current transaction.
*
* @param <V> vertex class used by the graph
* @param <E> edge class used by the graph
@@ -27,10 +33,10 @@ package org.apache.atlas.repository.graphdb;
public interface AtlasGraphQuery<V,E> {
/**
- * Adds a predicate that the returned elements must have the specified
+ * Adds a predicate that the returned vertices must have the specified
* property and that one of the values of the property must be the
* given value.
- *
+ *
* @param propertyKey
* @param value
* @return
@@ -38,34 +44,73 @@ public interface AtlasGraphQuery<V,E> {
AtlasGraphQuery<V,E> has(String propertyKey, Object value);
/**
- * Executes the query and returns the matching vertices.
+ * Adds a predicate that the returned vertices must have the specified
+ * property and that one of the value of the property must be in
+ * the specified list of values.
+ *
+ * @param propertyKey
+ * @param value
* @return
*/
- Iterable<AtlasVertex<V, E>> vertices();
+ AtlasGraphQuery<V,E> in(String propertyKey, Collection<? extends Object> values);
+
-
/**
- * Executes the query and returns the matching edges.
+ * Executes the query and returns the matching vertices.
* @return
+ * @throws AtlasException
*/
- Iterable<AtlasEdge<V, E>> edges();
+ Iterable<AtlasVertex<V, E>> vertices();
-
/**
- * Adds a predicate that the returned elements must have the specified
+ * Adds a predicate that the returned vertices must have the specified
* property and that its value matches the criterion specified.
- *
+ *
* @param propertyKey
* @param value
* @return
*/
- AtlasGraphQuery<V,E> has(String propertyKey, ComparisionOperator compMethod, Object value);
+ AtlasGraphQuery<V,E> has(String propertyKey, ComparisionOperator compMethod, Object values);
+
+ /**
+ * Adds a predicate that the vertices returned must satisfy the
+ * conditions in at least one of the child queries provided.
+ *
+ * @param childQueries
+ * @return
+ */
+ AtlasGraphQuery<V,E> or(List<AtlasGraphQuery<V,E>> childQueries);
+
+ /**
+ * Creates a child query that can be used to add "or" conditions
+ *
+ * @return
+ */
+ AtlasGraphQuery<V,E> createChildQuery();
+
public static enum ComparisionOperator {
GREATER_THAN_EQUAL,
EQUAL,
- LESS_THAN_EQUAL
+ LESS_THAN_EQUAL,
+ NOT_EQUAL
}
+ /**
+ * Adds all of the predicates that have been added to this query to the
+ * specified query.
+ * @param otherQuery
+ * @return
+ */
+ AtlasGraphQuery<V, E> addConditionsFrom(AtlasGraphQuery<V, E> otherQuery);
+
+ /**
+ * Whether or not this is a child query
+ *
+ * @return
+ */
+ boolean isChildQuery();
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java
----------------------------------------------------------------------
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java
index 7ee6b28..e719d30 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java
@@ -26,31 +26,31 @@ import java.util.Iterator;
* @param <V> vertex class used by the graph
* @param <E> edge class used by the graph
*/
-public interface AtlasIndexQuery<V,E> {
+public interface AtlasIndexQuery<V, E> {
/**
* Gets the query results.
- *
+ *
* @return
*/
- Iterator<Result<V,E>> vertices();
+ Iterator<Result<V, E>> vertices();
- /**
+ /**
* Query result from an index query.
- *
+ *
* @param <V>
* @param <E>
*/
- public interface Result<V,E> {
-
+ public interface Result<V, E> {
+
/**
* Gets the vertex for this result
*/
- AtlasVertex<V,E> getVertex();
-
+ AtlasVertex<V, E> getVertex();
+
/**
* Gets the score for this result.
- *
+ *
*/
double getScore();
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasPropertyKey.java
----------------------------------------------------------------------
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasPropertyKey.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasPropertyKey.java
new file mode 100644
index 0000000..315ecdd
--- /dev/null
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasPropertyKey.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graphdb;
+
+/**
+ * Represent a property key
+ *
+ * @param <V> vertex class used by the graph
+ * @param <E> edge class used by the graph
+ */
+public interface AtlasPropertyKey {
+
+ String getName();
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasSchemaViolationException.java
----------------------------------------------------------------------
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasSchemaViolationException.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasSchemaViolationException.java
index fda83b8..048eeaa 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasSchemaViolationException.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasSchemaViolationException.java
@@ -22,7 +22,7 @@ package org.apache.atlas.repository.graphdb;
*
*/
public class AtlasSchemaViolationException extends RuntimeException {
-
+
public AtlasSchemaViolationException(Throwable cause) {
super(cause);
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertex.java
----------------------------------------------------------------------
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertex.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertex.java
index e027b69..fff6fb7 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertex.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertex.java
@@ -17,64 +17,56 @@
*/
package org.apache.atlas.repository.graphdb;
-import java.util.Collection;
-
/**
- * Represents a Vertex
+ * Represents a Vertex
*
* @param <V> vertex class used by the graph
* @param <E> edge class used by the graph
*/
-public interface AtlasVertex<V,E> extends AtlasElement {
-
-
-
+public interface AtlasVertex<V, E> extends AtlasElement {
/**
* Gets the edges incident to this vertex going the
* specified direction that have the specified edgeLabel. If
* the edgeLabel is null, it is ignored.
- *
+ *
* @param in
* @return
*/
- Iterable<AtlasEdge<V,E>> getEdges(AtlasEdgeDirection out, String edgeLabel);
+ Iterable<AtlasEdge<V, E>> getEdges(AtlasEdgeDirection out, String edgeLabel);
/**
* Gets the edges associated with this vertex going the
* specified direction.
- *
+ *
* @param in
* @return
*/
- Iterable<AtlasEdge<V,E>> getEdges(AtlasEdgeDirection in);
-
+ Iterable<AtlasEdge<V, E>> getEdges(AtlasEdgeDirection in);
+
/**
- * Adds a value to a multiplicity many property.
- *
+ * Adds a value to a multiplicity many property. Follows Java set
+ * semantics. If the property is already present, it is not added again,
+ * and no exception is thrown.
+ *
+ *
* @param propertyName
* @param value
*/
<T> void addProperty(String propertyName, T value);
- /**
- * Gets all of the values of the given property.
- * @param propertyName
- * @return
- */
- <T> Collection<T> getPropertyValues(String propertyName);
/**
* Creates a vertex query.
* @return
*/
- AtlasVertexQuery<V,E> query();
+ AtlasVertexQuery<V, E> query();
/**
* Syntactic sugar to get the vertex as an instance of its
* implementation type. This allows the graph database implementation
* code to be strongly typed.
- *
+ *
* @return
*/
V getV();
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertexQuery.java
----------------------------------------------------------------------
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertexQuery.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertexQuery.java
index fd20a65..366cfe1 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertexQuery.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertexQuery.java
@@ -19,33 +19,33 @@
package org.apache.atlas.repository.graphdb;
/**
- * A query against a particular vertex.
+ * A query against a particular vertex.
*
* @param <V> vertex class used by the graph
- * @param <E> edge class used by the graph
+ * @param <E> edge class used by the graph
*/
-public interface AtlasVertexQuery<V,E> {
+public interface AtlasVertexQuery<V, E> {
/**
* Specifies the edge direction that should be query
- *
+ *
* @param queryDirection
* @return
*/
- AtlasVertexQuery<V,E> direction(AtlasEdgeDirection queryDirection);
+ AtlasVertexQuery<V, E> direction(AtlasEdgeDirection queryDirection);
/**
* Returns the vertices that satisfy the query condition.
- *
+ *
* @return
*/
- Iterable<AtlasVertex<V,E>> vertices();
+ Iterable<AtlasVertex<V, E>> vertices();
/**
* Returns the incident edges that satisfy the query condition.
* @return
*/
- Iterable<AtlasEdge<V,E>> edges();
+ Iterable<AtlasEdge<V, E>> edges();
/**
* Returns the number of elements that match the query.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/GraphDatabase.java
----------------------------------------------------------------------
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/GraphDatabase.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/GraphDatabase.java
index a608eb2..37acda5 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/GraphDatabase.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/GraphDatabase.java
@@ -23,22 +23,28 @@ package org.apache.atlas.repository.graphdb;
* @param <V> vertex class used by the graph database
* @param <E> edge class used by the graph database
*/
-public interface GraphDatabase<V,E> {
-
+public interface GraphDatabase<V, E> {
+
/**
* Returns whether the graph has been loaded.
* @return
*/
- boolean isGraphLoaded();
-
+ boolean isGraphLoaded();
+
/**
* Gets the graph, loading it if it has not been loaded already
* @return
*/
- AtlasGraph<V,E> getGraph();
-
+ AtlasGraph<V, E> getGraph();
+
+ /**
+ * Sets things up so that getGraph() will return a graph that can be used for running
+ * tests.
+ */
+ void initializeTestGraph();
+
/**
- * Unloads the graph (used testing)
+ * Removes the test graph that was created.
*/
- void unloadGraph();
+ void removeTestGraph();
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/GremlinVersion.java
----------------------------------------------------------------------
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/GremlinVersion.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/GremlinVersion.java
index e817cba..f328d3c 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/GremlinVersion.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/GremlinVersion.java
@@ -20,7 +20,7 @@ package org.apache.atlas.repository.graphdb;
/**
* Enumeration of the supported versions of Gremlin
- *
+ *
*
*/
public enum GremlinVersion {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/common/pom.xml
----------------------------------------------------------------------
diff --git a/graphdb/common/pom.xml b/graphdb/common/pom.xml
new file mode 100644
index 0000000..f3b1860
--- /dev/null
+++ b/graphdb/common/pom.xml
@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- ~ Licensed to the Apache Software Foundation (ASF) under one ~ or more
+contributor license agreements. See the NOTICE file ~ distributed with this
+work for additional information ~ regarding copyright ownership. The ASF
+licenses this file ~ to you under the Apache License, Version 2.0 (the ~
+"License"); you may not use this file except in compliance ~ with the License.
+You may obtain a copy of the License at ~ ~ http://www.apache.org/licenses/LICENSE-2.0
+~ ~ Unless required by applicable law or agreed to in writing, software ~
+distributed under the License is distributed on an "AS IS" BASIS, ~ WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ~ See the
+License for the specific language governing permissions and ~ limitations
+under the License. -->
+
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns="http://maven.apache.org/POM/4.0.0"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>atlas-graphdb</artifactId>
+ <groupId>org.apache.atlas</groupId>
+ <version>0.8-incubator-SNAPSHOT</version>
+ </parent>
+ <artifactId>atlas-graphdb-common</artifactId>
+
+ <description>Adds IBM Graph Database Support to Apache Atlas</description>
+ <name>Graph Database Common Code</name>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.atlas</groupId>
+ <artifactId>atlas-graphdb-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>18.0</version>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.9</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ <version>6.9.4</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>1.9.5</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/NativeTitanGraphQuery.java
----------------------------------------------------------------------
diff --git a/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/NativeTitanGraphQuery.java b/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/NativeTitanGraphQuery.java
new file mode 100644
index 0000000..b79bf79
--- /dev/null
+++ b/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/NativeTitanGraphQuery.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.titan.query;
+
+import java.util.Collection;
+
+import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+
+/**
+ * Interfaces that provides a thin wrapper around GraphQuery (used by Titan0) and
+ * TitanGraphQuery (used by Titan 1).
+ *
+ * This abstraction allows TitanGraphQuery to work on any version of Titan.
+ *
+ * @param <V>
+ * @param <E>
+ */
+public interface NativeTitanGraphQuery<V,E> {
+
+ /**
+ * Executes the graph query
+ * @return
+ */
+ Iterable<AtlasVertex<V,E>> vertices();
+
+
+ /**
+ * Adds an in condition to the query
+ *
+ * @param propertyName
+ * @param values
+ */
+ void in(String propertyName, Collection<? extends Object> values);
+
+ /**
+ * Adds a has condition to the query
+ *
+ * @param propertyName
+ * @param op
+ * @param value
+ */
+ void has(String propertyName, ComparisionOperator op, Object value);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/NativeTitanQueryFactory.java
----------------------------------------------------------------------
diff --git a/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/NativeTitanQueryFactory.java b/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/NativeTitanQueryFactory.java
new file mode 100644
index 0000000..5c65411
--- /dev/null
+++ b/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/NativeTitanQueryFactory.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.titan.query;
+
+/**
+ * Interface that indicates that something can create instances of
+ * NativeTitanGraphQuery.
+ *
+ * @param <V>
+ * @param <E>
+ */
+public interface NativeTitanQueryFactory<V,E> {
+
+ /**
+ * Creates a NativeTitanGraphQuery
+ * @return
+ */
+ NativeTitanGraphQuery<V,E> createNativeTitanQuery();
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/TitanGraphQuery.java
----------------------------------------------------------------------
diff --git a/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/TitanGraphQuery.java b/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/TitanGraphQuery.java
new file mode 100644
index 0000000..f23847a
--- /dev/null
+++ b/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/TitanGraphQuery.java
@@ -0,0 +1,189 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.titan.query;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.graphdb.titan.query.expr.AndCondition;
+import org.apache.atlas.repository.graphdb.titan.query.expr.HasPredicate;
+import org.apache.atlas.repository.graphdb.titan.query.expr.InPredicate;
+import org.apache.atlas.repository.graphdb.titan.query.expr.OrCondition;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Abstract implementation of AtlasGraphQuery that is used by both Titan 0.5.4
+ * and Titan 1.0.0.
+ *
+ * Represents a graph query as an OrConditions which consists of
+ * 1 or more AndConditions. The query is executed by converting
+ * the AndConditions to native GraphQuery instances that can be executed
+ * directly against Titan. The overall result is obtained by unioning together
+ * the results from those individual GraphQueries.
+ *
+ * Here is a pictoral view of what is going on here. Conceptually,
+ * the query being executed can be though of as the where clause
+ * in a query
+ *
+ *
+ * where (a =1 and b=2) or (a=2 and b=3)
+ *
+ * ||
+ * \||/
+ * \/
+ *
+ * OrCondition
+ * |
+ * +---------+--------+
+ * | |
+ * AndCondition AndCondition
+ * (a=1 and b=2) (a=2 and b=3)
+ *
+ * || ||
+ * \||/ \||/
+ * \/ \/
+ *
+ * GraphQuery GraphQuery
+ *
+ * || ||
+ * \||/ \||/
+ * \/ \/
+ *
+ * vertices vertices
+ * \ /
+ * _\/ \/_
+ * (UNION)
+ *
+ * ||
+ * \||/
+ * \/
+ *
+ * result
+ *
+ *
+ *
+ */
+public abstract class TitanGraphQuery<V,E> implements AtlasGraphQuery<V, E> {
+
+ private final Logger LOG = LoggerFactory.getLogger(TitanGraphQuery.class);
+ protected final AtlasGraph<V,E> graph_;
+ private final OrCondition queryCondition_ = new OrCondition();
+ private final boolean isChildQuery_;
+ protected abstract NativeTitanQueryFactory<V, E> getQueryFactory();
+
+ /**
+ * Creates a TitanGraphQuery
+ *
+ * @param graph
+ */
+ public TitanGraphQuery(AtlasGraph<V,E> graph) {
+ graph_ = graph;
+ isChildQuery_ = false;
+ }
+
+ /**
+ * Creates a TitanGraphQuery
+ *
+ * @param graph
+ * @param isChildQuery
+ */
+ public TitanGraphQuery(AtlasGraph<V,E> graph, boolean isChildQuery) {
+ graph_ = graph;
+ isChildQuery_ = isChildQuery;
+ }
+
+ @Override
+ public AtlasGraphQuery<V, E> has(String propertyKey, Object value) {
+ queryCondition_.andWith(new HasPredicate(propertyKey, ComparisionOperator.EQUAL, value));
+ return this;
+ }
+
+ @Override
+ public Iterable<AtlasVertex<V, E>> vertices() {
+ LOG.debug("Executing: " );
+ LOG.debug(queryCondition_.toString());
+ //compute the overall result by unioning the results from all of the
+ //AndConditions together.
+ Set<AtlasVertex<V, E>> result = new HashSet<>();
+ for(AndCondition andExpr : queryCondition_.getAndTerms()) {
+ NativeTitanGraphQuery<V,E> andQuery = andExpr.create(getQueryFactory());
+ for(AtlasVertex<V,E> vertex : andQuery.vertices()) {
+ result.add(vertex);
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public AtlasGraphQuery<V, E> has(String propertyKey, ComparisionOperator operator,
+ Object value) {
+ queryCondition_.andWith(new HasPredicate(propertyKey, operator, value));
+ return this;
+ }
+
+
+ @Override
+ public AtlasGraphQuery<V, E> in(String propertyKey, Collection<? extends Object> values) {
+ queryCondition_.andWith(new InPredicate(propertyKey, values));
+ return this;
+ }
+
+ @Override
+ public AtlasGraphQuery<V, E> or(List<AtlasGraphQuery<V, E>> childQueries) {
+
+ //Construct an overall OrCondition by combining all of the children for
+ //the OrConditions in all of the childQueries that we passed in. Then, "and" the current
+ //query condition with this overall OrCondition.
+
+ OrCondition overallChildQuery = new OrCondition(false);
+
+ for(AtlasGraphQuery<V, E> atlasChildQuery : childQueries) {
+ if(! atlasChildQuery.isChildQuery()) {
+ throw new IllegalArgumentException(atlasChildQuery + " is not a child query");
+ }
+ TitanGraphQuery<V,E> childQuery = (TitanGraphQuery<V,E>)atlasChildQuery;
+ overallChildQuery.orWith(childQuery.getOrCondition());
+ }
+
+ queryCondition_.andWith(overallChildQuery);
+ return this;
+ }
+
+ private OrCondition getOrCondition() {
+ return queryCondition_;
+ }
+
+ @Override
+ public AtlasGraphQuery<V, E> addConditionsFrom(AtlasGraphQuery<V, E> otherQuery) {
+
+ TitanGraphQuery<V, E> childQuery = (TitanGraphQuery<V, E>)otherQuery;
+ queryCondition_.andWith(childQuery.getOrCondition());
+ return this;
+ }
+
+ @Override
+ public boolean isChildQuery() {
+ return isChildQuery_;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/AndCondition.java
----------------------------------------------------------------------
diff --git a/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/AndCondition.java b/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/AndCondition.java
new file mode 100644
index 0000000..f3996d9
--- /dev/null
+++ b/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/AndCondition.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.titan.query.expr;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.atlas.repository.graphdb.titan.query.NativeTitanGraphQuery;
+import org.apache.atlas.repository.graphdb.titan.query.NativeTitanQueryFactory;
+
+/**
+ * Represents an AndCondition in a graph query. Only vertices that
+ * satisfy the conditions in all of the query predicates will be returned
+ *
+ * Represents a query with predicates that are 'AND'ed together. These can be
+ * executed natively using Titan's GraphQuery mechanism.
+ */
+public class AndCondition {
+
+ private List<QueryPredicate> children_ = new ArrayList<>();
+
+ public AndCondition() {
+
+ }
+
+ /**
+ * Adds a query predicate that must be met by vertices
+ * @param predicate
+ */
+ public void andWith(QueryPredicate predicate) {
+ children_.add(predicate);
+ }
+
+ /**
+ * Adds multiple predicates that much be met by the vertices
+ *
+ * @param predicates
+ */
+ public void andWith(List<QueryPredicate> predicates) {
+ children_.addAll(predicates);
+ }
+
+ /**
+ * Makes a copy of this AndExpr.
+ *
+ * @return
+ */
+ public AndCondition copy() {
+ AndCondition builder = new AndCondition();
+ builder.children_.addAll(children_);
+ return builder;
+ }
+
+ /**
+ * Gets the query predicates
+ *
+ * @return
+ */
+ public List<QueryPredicate> getTerms() {
+ return children_;
+ }
+
+ /**
+ * Creates a NativeTitanGraphQuery that can be used to evaluate this condition.
+ *
+ * @param graph
+ * @return
+ */
+ public <V,E> NativeTitanGraphQuery<V, E> create(NativeTitanQueryFactory<V,E> factory) {
+ NativeTitanGraphQuery<V, E> query = factory.createNativeTitanQuery();
+ for (QueryPredicate predicate : children_) {
+ predicate.addTo(query);
+ }
+ return query;
+ }
+
+ @Override
+ public String toString() {
+ return "AndExpr [predicates=" + children_ + "]";
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/HasPredicate.java
----------------------------------------------------------------------
diff --git a/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/HasPredicate.java b/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/HasPredicate.java
new file mode 100644
index 0000000..8d7bc15
--- /dev/null
+++ b/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/HasPredicate.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.titan.query.expr;
+
+import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator;
+import org.apache.atlas.repository.graphdb.titan.query.NativeTitanGraphQuery;
+
+/**
+ * Query predicate that checks whether the given property has the specified
+ * relationship with the value specified.
+ */
+public class HasPredicate implements QueryPredicate {
+
+ private String propertyName_;
+ private ComparisionOperator op_;
+ private Object value_;
+
+ public HasPredicate(String propertyName, ComparisionOperator op, Object value) {
+ super();
+ propertyName_ = propertyName;
+ op_ = op;
+ value_ = value;
+ }
+
+ @Override
+ public void addTo(NativeTitanGraphQuery query) {
+ query.has(propertyName_, op_, value_);
+ }
+
+ @Override
+ public String toString() {
+ return "HasTerm [propertyName_=" + propertyName_ + ", op_=" + op_ + ", value_=" + value_ + "]";
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/InPredicate.java
----------------------------------------------------------------------
diff --git a/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/InPredicate.java b/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/InPredicate.java
new file mode 100644
index 0000000..5a43a6a
--- /dev/null
+++ b/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/InPredicate.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.titan.query.expr;
+
+import java.util.Collection;
+
+import org.apache.atlas.repository.graphdb.titan.query.NativeTitanGraphQuery;
+
+/**
+ * Query predicate that checks whether the value of a given property is within the
+ * provided set of allowed values.
+ */
+public class InPredicate implements QueryPredicate {
+
+ private String propertyName_;
+ private Collection<? extends Object> values_;
+
+ public InPredicate(String propertyName, Collection<? extends Object> values) {
+ super();
+ propertyName_ = propertyName;
+ values_ = values;
+ }
+
+ @Override
+ public void addTo(NativeTitanGraphQuery query) {
+ query.in(propertyName_, values_);
+ }
+
+ @Override
+ public String toString() {
+ return "InPredicate [propertyName_=" + propertyName_ + ", values_=" + values_ + "]";
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/OrCondition.java
----------------------------------------------------------------------
diff --git a/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/OrCondition.java b/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/OrCondition.java
new file mode 100644
index 0000000..9de4655
--- /dev/null
+++ b/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/OrCondition.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.titan.query.expr;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * Represents an OrCondition that has one or more AndConditions as it children. The OrCondition
+ * matches vertices that meet the criteria in at least one of its children. The overall query result is
+ * computed by executing the TitanGraphQuery queries that correspond to each AndCondition individually and
+ * then unioning their results together. This is needed because the native Titan query mechanism does not
+ * natively support 'OR' conditions. When we execute the query, we accomplish the 'OR' by executing all of the
+ * individual queries and unioning the results together.
+ *
+ */
+public class OrCondition {
+
+ private List<AndCondition> children_;
+
+ public OrCondition() {
+ this(true);
+ }
+
+ private OrCondition(List<AndCondition> children) {
+ children_ = children;
+ }
+
+ public OrCondition(boolean addInitialTerm) {
+ children_ = new ArrayList<AndCondition>();
+ if (addInitialTerm) {
+ children_.add(new AndCondition());
+ }
+ }
+
+ /**
+ /**
+ * Updates this OrCondition in place so that it matches vertices that satisfy the current
+ * OrCondition AND that match the specified predicate.
+ *
+ * @param other
+ */
+ public void andWith(QueryPredicate predicate) {
+
+ for (AndCondition child : children_) {
+ child.andWith(predicate);
+ }
+ }
+
+ public List<AndCondition> getAndTerms() {
+ return children_;
+ }
+
+ /**
+ * Updates this OrCondition in place so that it matches vertices that satisfy the current
+ * OrCondition AND that satisfy the provided OrCondition
+ *
+ * @param other
+ */
+ public void andWith(OrCondition other) {
+
+ //Because Titan does not natively support Or conditions in Graph Queries,
+ //we need to expand out the condition so it is in the form of a single OrCondition
+ //that contains only AndConditions. We do this by following the rules of boolean
+ //algebra. As an example, suppose the current condition is ((a=1 and b=2) or (c=3 and d=4)).
+ //Suppose "other" is ((e=5 and f=6) or (g=7 or h=8)). The overall condition, after applying the
+ //"and" is:
+ //
+ //((a=1 and b=2) or (c=3 and d=4)) and ((e=5 and f=6) or (g=7 and h=8))
+ //
+ //This needs to be expanded out to remove the nested or clauses. The result of this expansion is:
+ //
+ //(a=1 and b=2 and e=5 and f=6) or
+ //(a=1 and b=2 and g=7 and h=8) or
+ //(c=3 and d=4 and e=5 and f=6) or
+ //(c=3 and d=4 and g=7 and h=8)
+
+ //The logic below does this expansion, in a generalized way. It loops through the existing AndConditions
+ //and, in a nested loop, through the AndConditions in "other". For each of those combinations,
+ //it creates a new AndCondition that combines the two AndConditions together. These combined
+ //AndConditions become the new set of AndConditions in this OrCondition.
+
+ List<AndCondition> expandedExpressionChildren = new ArrayList<AndCondition>();
+ for (AndCondition otherExprTerm : other.getAndTerms()) {
+ for (AndCondition currentExpr : children_) {
+ AndCondition currentAndConditionCopy = currentExpr.copy();
+ currentAndConditionCopy.andWith(otherExprTerm.getTerms());
+ expandedExpressionChildren.add(currentAndConditionCopy);
+ }
+ }
+ children_ = expandedExpressionChildren;
+ }
+
+ /**
+ * Updates this OrCondition in place so that it matches vertices that satisfy the current
+ * OrCondition OR that satisfy the provided OrCondition
+ *
+ * @param other
+ */
+ public void orWith(OrCondition other) {
+ children_.addAll(other.getAndTerms());
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("OrCondition [andExprs=");
+ Iterator<AndCondition> it = children_.iterator();
+ while (it.hasNext()) {
+ AndCondition andExpr = it.next();
+ builder.append(andExpr.toString());
+ if (it.hasNext()) {
+ builder.append(",");
+ }
+ }
+ builder.append("]");
+ return builder.toString();
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/QueryPredicate.java
----------------------------------------------------------------------
diff --git a/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/QueryPredicate.java b/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/QueryPredicate.java
new file mode 100644
index 0000000..61c692f
--- /dev/null
+++ b/graphdb/common/src/main/java/org/apache/atlas/repository/graphdb/titan/query/expr/QueryPredicate.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.titan.query.expr;
+
+import org.apache.atlas.repository.graphdb.titan.query.NativeTitanGraphQuery;
+
+/**
+ * Represents a predicate in an AndExpression
+ */
+public interface QueryPredicate {
+
+ /**
+ * Adds the query term to a NativeTitanGraphQuery that is being generated.
+ *
+ * @param query
+ */
+ void addTo(NativeTitanGraphQuery query);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/pom.xml
----------------------------------------------------------------------
diff --git a/graphdb/pom.xml b/graphdb/pom.xml
index ea763c5..ad34617 100644
--- a/graphdb/pom.xml
+++ b/graphdb/pom.xml
@@ -32,5 +32,7 @@
<modules>
<module>api</module>
+ <module>titan0</module>
+ <module>common</module>
</modules>
</project>
[7/9] incubator-atlas git commit: ATLAS-693 Titan 0.5.4
implementation of graph db abstraction. (jnhagelb via dkantor)
Posted by dk...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseStoreManager.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseStoreManager.java b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseStoreManager.java
new file mode 100644
index 0000000..a94a7e4
--- /dev/null
+++ b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseStoreManager.java
@@ -0,0 +1,935 @@
+/*
+ * Copyright 2012-2013 Aurelius LLC
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.thinkaurelius.titan.diskstorage.hbase;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import com.thinkaurelius.titan.diskstorage.Backend;
+import com.thinkaurelius.titan.diskstorage.configuration.ConfigElement;
+import com.thinkaurelius.titan.diskstorage.keycolumnvalue.CustomizeStoreKCVSManager;
+import com.thinkaurelius.titan.diskstorage.locking.LocalLockMediator;
+import com.thinkaurelius.titan.diskstorage.locking.LocalLockMediators;
+import com.thinkaurelius.titan.diskstorage.util.time.Timestamps;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MasterNotRunningException;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableNotEnabledException;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Row;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.VersionInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.BiMap;
+import com.google.common.collect.ImmutableBiMap;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Sets;
+import com.thinkaurelius.titan.core.TitanException;
+import com.thinkaurelius.titan.diskstorage.BackendException;
+import com.thinkaurelius.titan.diskstorage.BaseTransactionConfig;
+import com.thinkaurelius.titan.diskstorage.Entry;
+import com.thinkaurelius.titan.diskstorage.PermanentBackendException;
+import com.thinkaurelius.titan.diskstorage.StaticBuffer;
+import com.thinkaurelius.titan.diskstorage.TemporaryBackendException;
+import com.thinkaurelius.titan.diskstorage.common.DistributedStoreManager;
+import com.thinkaurelius.titan.diskstorage.configuration.ConfigNamespace;
+import com.thinkaurelius.titan.diskstorage.configuration.ConfigOption;
+import com.thinkaurelius.titan.diskstorage.configuration.Configuration;
+import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KCVMutation;
+import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStore;
+import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStoreManager;
+import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyRange;
+import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StandardStoreFeatures;
+import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreFeatures;
+import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreTransaction;
+import com.thinkaurelius.titan.diskstorage.util.BufferUtil;
+import com.thinkaurelius.titan.diskstorage.util.StaticArrayBuffer;
+import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration;
+import com.thinkaurelius.titan.graphdb.configuration.PreInitializeConfigOptions;
+import com.thinkaurelius.titan.util.system.IOUtils;
+import com.thinkaurelius.titan.util.system.NetworkUtil;
+
+/**
+ * Storage Manager for HBase
+ *
+ * @author Dan LaRocque <da...@hopcount.org>
+ */
+@PreInitializeConfigOptions
+public class HBaseStoreManager extends DistributedStoreManager implements KeyColumnValueStoreManager, CustomizeStoreKCVSManager {
+
+ private static final Logger logger = LoggerFactory.getLogger(HBaseStoreManager.class);
+
+ public static final ConfigNamespace HBASE_NS =
+ new ConfigNamespace(GraphDatabaseConfiguration.STORAGE_NS, "hbase", "HBase storage options");
+
+ public static final ConfigOption<Boolean> SHORT_CF_NAMES =
+ new ConfigOption<Boolean>(HBASE_NS, "short-cf-names",
+ "Whether to shorten the names of Titan's column families to one-character mnemonics " +
+ "to conserve storage space", ConfigOption.Type.FIXED, true);
+
+ public static final String COMPRESSION_DEFAULT = "-DEFAULT-";
+
+ public static final ConfigOption<String> COMPRESSION =
+ new ConfigOption<String>(HBASE_NS, "compression-algorithm",
+ "An HBase Compression.Algorithm enum string which will be applied to newly created column families. " +
+ "The compression algorithm must be installed and available on the HBase cluster. Titan cannot install " +
+ "and configure new compression algorithms on the HBase cluster by itself.",
+ ConfigOption.Type.MASKABLE, "GZ");
+
+ public static final ConfigOption<Boolean> SKIP_SCHEMA_CHECK =
+ new ConfigOption<Boolean>(HBASE_NS, "skip-schema-check",
+ "Assume that Titan's HBase table and column families already exist. " +
+ "When this is true, Titan will not check for the existence of its table/CFs, " +
+ "nor will it attempt to create them under any circumstances. This is useful " +
+ "when running Titan without HBase admin privileges.",
+ ConfigOption.Type.MASKABLE, false);
+
+ public static final ConfigOption<String> HBASE_TABLE =
+ new ConfigOption<String>(HBASE_NS, "table",
+ "The name of the table Titan will use. When " + ConfigElement.getPath(SKIP_SCHEMA_CHECK) +
+ " is false, Titan will automatically create this table if it does not already exist.",
+ ConfigOption.Type.LOCAL, "titan");
+
+ /**
+ * Related bug fixed in 0.98.0, 0.94.7, 0.95.0:
+ *
+ * https://issues.apache.org/jira/browse/HBASE-8170
+ */
+ public static final int MIN_REGION_COUNT = 3;
+
+ /**
+ * The total number of HBase regions to create with Titan's table. This
+ * setting only effects table creation; this normally happens just once when
+ * Titan connects to an HBase backend for the first time.
+ */
+ public static final ConfigOption<Integer> REGION_COUNT =
+ new ConfigOption<Integer>(HBASE_NS, "region-count",
+ "The number of initial regions set when creating Titan's HBase table",
+ ConfigOption.Type.MASKABLE, Integer.class, new Predicate<Integer>() {
+ @Override
+ public boolean apply(Integer input) {
+ return null != input && MIN_REGION_COUNT <= input;
+ }
+ }
+ );
+
+ /**
+ * This setting is used only when {@link #REGION_COUNT} is unset.
+ * <p/>
+ * If Titan's HBase table does not exist, then it will be created with total
+ * region count = (number of servers reported by ClusterStatus) * (this
+ * value).
+ * <p/>
+ * The Apache HBase manual suggests an order-of-magnitude range of potential
+ * values for this setting:
+ *
+ * <ul>
+ * <li>
+ * <a href="https://hbase.apache.org/book/important_configurations.html#disable.splitting">2.5.2.7. Managed Splitting</a>:
+ * <blockquote>
+ * What's the optimal number of pre-split regions to create? Mileage will
+ * vary depending upon your application. You could start low with 10
+ * pre-split regions / server and watch as data grows over time. It's
+ * better to err on the side of too little regions and rolling split later.
+ * </blockquote>
+ * </li>
+ * <li>
+ * <a href="https://hbase.apache.org/book/regions.arch.html">9.7 Regions</a>:
+ * <blockquote>
+ * In general, HBase is designed to run with a small (20-200) number of
+ * relatively large (5-20Gb) regions per server... Typically you want to
+ * keep your region count low on HBase for numerous reasons. Usually
+ * right around 100 regions per RegionServer has yielded the best results.
+ * </blockquote>
+ * </li>
+ * </ul>
+ *
+ * These considerations may differ for other HBase implementations (e.g. MapR).
+ */
+ public static final ConfigOption<Integer> REGIONS_PER_SERVER =
+ new ConfigOption<Integer>(HBASE_NS, "regions-per-server",
+ "The number of regions per regionserver to set when creating Titan's HBase table",
+ ConfigOption.Type.MASKABLE, Integer.class);
+
+ /**
+ * If this key is present in either the JVM system properties or the process
+ * environment (checked in the listed order, first hit wins), then its value
+ * must be the full package and class name of an implementation of
+ * {@link HBaseCompat} that has a no-arg public constructor.
+ * <p>
+ * When this <b>is not</b> set, Titan attempts to automatically detect the
+ * HBase runtime version by calling {@link VersionInfo#getVersion()}. Titan
+ * then checks the returned version string against a hard-coded list of
+ * supported version prefixes and instantiates the associated compat layer
+ * if a match is found.
+ * <p>
+ * When this <b>is</b> set, Titan will not call
+ * {@code VersionInfo.getVersion()} or read its hard-coded list of supported
+ * version prefixes. Titan will instead attempt to instantiate the class
+ * specified (via the no-arg constructor which must exist) and then attempt
+ * to cast it to HBaseCompat and use it as such. Titan will assume the
+ * supplied implementation is compatible with the runtime HBase version and
+ * make no attempt to verify that assumption.
+ * <p>
+ * Setting this key incorrectly could cause runtime exceptions at best or
+ * silent data corruption at worst. This setting is intended for users
+ * running exotic HBase implementations that don't support VersionInfo or
+ * implementations which return values from {@code VersionInfo.getVersion()}
+ * that are inconsistent with Apache's versioning convention. It may also be
+ * useful to users who want to run against a new release of HBase that Titan
+ * doesn't yet officially support.
+ *
+ */
+ public static final ConfigOption<String> COMPAT_CLASS =
+ new ConfigOption<String>(HBASE_NS, "compat-class",
+ "The package and class name of the HBaseCompat implementation. HBaseCompat masks version-specific HBase API differences. " +
+ "When this option is unset, Titan calls HBase's VersionInfo.getVersion() and loads the matching compat class " +
+ "at runtime. Setting this option forces Titan to instead reflectively load and instantiate the specified class.",
+ ConfigOption.Type.MASKABLE, String.class);
+
+ public static final int PORT_DEFAULT = 9160;
+
+ public static final Timestamps PREFERRED_TIMESTAMPS = Timestamps.MILLI;
+
+ public static final ConfigNamespace HBASE_CONFIGURATION_NAMESPACE =
+ new ConfigNamespace(HBASE_NS, "ext", "Overrides for hbase-{site,default}.xml options", true);
+
+ private static final BiMap<String, String> SHORT_CF_NAME_MAP =
+ ImmutableBiMap.<String, String>builder()
+ .put(Backend.INDEXSTORE_NAME, "g")
+ .put(Backend.INDEXSTORE_NAME + Backend.LOCK_STORE_SUFFIX, "h")
+ .put(Backend.ID_STORE_NAME, "i")
+ .put(Backend.EDGESTORE_NAME, "e")
+ .put(Backend.EDGESTORE_NAME + Backend.LOCK_STORE_SUFFIX, "f")
+ .put(GraphDatabaseConfiguration.SYSTEM_PROPERTIES_STORE_NAME, "s")
+ .put(GraphDatabaseConfiguration.SYSTEM_PROPERTIES_STORE_NAME + Backend.LOCK_STORE_SUFFIX, "t")
+ .put(Backend.SYSTEM_MGMT_LOG_NAME, "m")
+ .put(Backend.SYSTEM_TX_LOG_NAME, "l")
+ .build();
+
+ private static final StaticBuffer FOUR_ZERO_BYTES = BufferUtil.zeroBuffer(4);
+
+ static {
+ // Verify that shortCfNameMap is injective
+ // Should be guaranteed by Guava BiMap, but it doesn't hurt to check
+ Preconditions.checkArgument(null != SHORT_CF_NAME_MAP);
+ Collection<String> shorts = SHORT_CF_NAME_MAP.values();
+ Preconditions.checkArgument(Sets.newHashSet(shorts).size() == shorts.size());
+ }
+
+ // Immutable instance fields
+ private final String tableName;
+ private final String compression;
+ private final int regionCount;
+ private final int regionsPerServer;
+ private final ConnectionMask cnx;
+ private final org.apache.hadoop.conf.Configuration hconf;
+ private final boolean shortCfNames;
+ private final boolean skipSchemaCheck;
+ private final String compatClass;
+ private final HBaseCompat compat;
+
+ private static final ConcurrentHashMap<HBaseStoreManager, Throwable> openManagers =
+ new ConcurrentHashMap<HBaseStoreManager, Throwable>();
+
+ // Mutable instance state
+ private final ConcurrentMap<String, HBaseKeyColumnValueStore> openStores;
+
+ private LocalLockMediator<StoreTransaction> llm;
+
+ public HBaseStoreManager(com.thinkaurelius.titan.diskstorage.configuration.Configuration config) throws BackendException {
+ super(config, PORT_DEFAULT);
+
+ checkConfigDeprecation(config);
+
+ this.tableName = config.get(HBASE_TABLE);
+ this.compression = config.get(COMPRESSION);
+ this.regionCount = config.has(REGION_COUNT) ? config.get(REGION_COUNT) : -1;
+ this.regionsPerServer = config.has(REGIONS_PER_SERVER) ? config.get(REGIONS_PER_SERVER) : -1;
+ this.skipSchemaCheck = config.get(SKIP_SCHEMA_CHECK);
+ this.compatClass = config.has(COMPAT_CLASS) ? config.get(COMPAT_CLASS) : null;
+ this.compat = HBaseCompatLoader.getCompat(compatClass);
+
+ /*
+ * Specifying both region count options is permitted but may be
+ * indicative of a misunderstanding, so issue a warning.
+ */
+ if (config.has(REGIONS_PER_SERVER) && config.has(REGION_COUNT)) {
+ logger.warn("Both {} and {} are set in Titan's configuration, but "
+ + "the former takes precedence and the latter will be ignored.",
+ REGION_COUNT, REGIONS_PER_SERVER);
+ }
+
+ /* This static factory calls HBaseConfiguration.addHbaseResources(),
+ * which in turn applies the contents of hbase-default.xml and then
+ * applies the contents of hbase-site.xml.
+ */
+ this.hconf = HBaseConfiguration.create();
+
+ // Copy a subset of our commons config into a Hadoop config
+ int keysLoaded=0;
+ Map<String,Object> configSub = config.getSubset(HBASE_CONFIGURATION_NAMESPACE);
+ for (Map.Entry<String,Object> entry : configSub.entrySet()) {
+ logger.info("HBase configuration: setting {}={}", entry.getKey(), entry.getValue());
+ if (entry.getValue()==null) continue;
+ hconf.set(entry.getKey(), entry.getValue().toString());
+ keysLoaded++;
+ }
+
+ // Special case for STORAGE_HOSTS
+ if (config.has(GraphDatabaseConfiguration.STORAGE_HOSTS)) {
+ String zkQuorumKey = "hbase.zookeeper.quorum";
+ String csHostList = Joiner.on(",").join(config.get(GraphDatabaseConfiguration.STORAGE_HOSTS));
+ hconf.set(zkQuorumKey, csHostList);
+ logger.info("Copied host list from {} to {}: {}", GraphDatabaseConfiguration.STORAGE_HOSTS, zkQuorumKey, csHostList);
+ }
+
+ logger.debug("HBase configuration: set a total of {} configuration values", keysLoaded);
+
+ this.shortCfNames = config.get(SHORT_CF_NAMES);
+
+ try {
+ //this.cnx = HConnectionManager.createConnection(hconf);
+ this.cnx = compat.createConnection(hconf);
+ } catch (IOException e) {
+ throw new PermanentBackendException(e);
+ }
+
+ if (logger.isTraceEnabled()) {
+ openManagers.put(this, new Throwable("Manager Opened"));
+ dumpOpenManagers();
+ }
+
+ logger.debug("Dumping HBase config key=value pairs");
+ for (Map.Entry<String, String> entry : hconf) {
+ logger.debug("[HBaseConfig] " + entry.getKey() + "=" + entry.getValue());
+ }
+ logger.debug("End of HBase config key=value pairs");
+
+ openStores = new ConcurrentHashMap<String, HBaseKeyColumnValueStore>();
+ }
+
+ @Override
+ public Deployment getDeployment() {
+ return Deployment.REMOTE;
+
+ /* If just one of the regions for titan table is in the localhost,
+ * this method returns Deployment.LOCAL - which does not sound right.
+ *
+ List<KeyRange> local;
+ try {
+ local = getLocalKeyPartition();
+ return null != local && !local.isEmpty() ? Deployment.LOCAL : Deployment.REMOTE;
+ } catch (BackendException e) {
+ // propagating StorageException might be a better approach
+ throw new RuntimeException(e);
+ }
+ *
+ */
+ }
+
+ @Override
+ public String toString() {
+ return "hbase[" + tableName + "@" + super.toString() + "]";
+ }
+
+ public void dumpOpenManagers() {
+ int estimatedSize = openManagers.size();
+ logger.trace("---- Begin open HBase store manager list ({} managers) ----", estimatedSize);
+ for (HBaseStoreManager m : openManagers.keySet()) {
+ logger.trace("Manager {} opened at:", m, openManagers.get(m));
+ }
+ logger.trace("---- End open HBase store manager list ({} managers) ----", estimatedSize);
+ }
+
+ @Override
+ public void close() {
+ openStores.clear();
+ if (logger.isTraceEnabled())
+ openManagers.remove(this);
+ IOUtils.closeQuietly(cnx);
+ }
+
+ @Override
+ public StoreFeatures getFeatures() {
+
+ Configuration c = GraphDatabaseConfiguration.buildConfiguration();
+
+ StandardStoreFeatures.Builder fb = new StandardStoreFeatures.Builder()
+ .orderedScan(true).unorderedScan(true).batchMutation(true)
+ .multiQuery(true).distributed(true).keyOrdered(true).storeTTL(true)
+ .timestamps(true).preferredTimestamps(PREFERRED_TIMESTAMPS)
+ .locking(true)
+ .keyConsistent(c);
+
+ try {
+ fb.localKeyPartition(getDeployment() == Deployment.LOCAL);
+ } catch (Exception e) {
+ logger.warn("Unexpected exception during getDeployment()", e);
+ }
+
+ return fb.build();
+ }
+
+ @Override
+ public void mutateMany(Map<String, Map<StaticBuffer, KCVMutation>> mutations, StoreTransaction txh) throws BackendException {
+ logger.debug("Enter mutateMany");
+ final MaskedTimestamp commitTime = new MaskedTimestamp(txh);
+ // In case of an addition and deletion with identical timestamps, the
+ // deletion tombstone wins.
+ // http://hbase.apache.org/book/versions.html#d244e4250
+ Map<StaticBuffer, Pair<Put, Delete>> commandsPerKey =
+ convertToCommands(
+ mutations,
+ commitTime.getAdditionTime(times.getUnit()),
+ commitTime.getDeletionTime(times.getUnit()));
+
+ List<Row> batch = new ArrayList<Row>(commandsPerKey.size()); // actual batch operation
+
+ // convert sorted commands into representation required for 'batch' operation
+ for (Pair<Put, Delete> commands : commandsPerKey.values()) {
+ if (commands.getFirst() != null)
+ batch.add(commands.getFirst());
+
+ if (commands.getSecond() != null)
+ batch.add(commands.getSecond());
+ }
+
+ try {
+ TableMask table = null;
+
+ try {
+ table = cnx.getTable(tableName);
+ logger.debug("mutateMany : batch mutate started size {} ", batch.size());
+ table.batch(batch, new Object[batch.size()]);
+ logger.debug("mutateMany : batch mutate finished {} ", batch.size());
+ } finally {
+ IOUtils.closeQuietly(table);
+ }
+ } catch (IOException e) {
+ throw new TemporaryBackendException(e);
+ } catch (InterruptedException e) {
+ throw new TemporaryBackendException(e);
+ }
+
+ sleepAfterWrite(txh, commitTime);
+ }
+
+ @Override
+ public KeyColumnValueStore openDatabase(String longName) throws BackendException {
+
+ return openDatabase(longName, -1);
+ }
+
+ @Override
+ public KeyColumnValueStore openDatabase(final String longName, int ttlInSeconds) throws BackendException {
+
+ HBaseKeyColumnValueStore store = openStores.get(longName);
+
+ if (store == null) {
+ final String cfName = shortCfNames ? shortenCfName(longName) : longName;
+
+ final String llmPrefix = getName();
+ llm = LocalLockMediators.INSTANCE.<StoreTransaction>get(llmPrefix, times);
+ HBaseKeyColumnValueStore newStore = new HBaseKeyColumnValueStore(this, cnx, tableName, cfName, longName, llm);
+
+ store = openStores.putIfAbsent(longName, newStore); // nothing bad happens if we loose to other thread
+
+ if (store == null) {
+ if (!skipSchemaCheck)
+ ensureColumnFamilyExists(tableName, cfName, ttlInSeconds);
+
+ store = newStore;
+ }
+ logger.info("Loaded 1.x Hbase Client Store Manager");
+ }
+
+ return store;
+ }
+
+
+ @Override
+ public StoreTransaction beginTransaction(final BaseTransactionConfig config) throws BackendException {
+ return new HBaseTransaction(config, llm);
+ }
+
+ @Override
+ public String getName() {
+ return tableName;
+ }
+
+ /**
+ * Deletes the specified table with all its columns.
+ * ATTENTION: Invoking this method will delete the table if it exists and therefore causes data loss.
+ */
+ @Override
+ public void clearStorage() throws BackendException {
+ try (AdminMask adm = getAdminInterface()) {
+ adm.clearTable(tableName, times.getTime().getNativeTimestamp());
+ } catch (IOException e)
+ {
+ throw new TemporaryBackendException(e);
+ }
+ }
+
+ @Override
+ public List<KeyRange> getLocalKeyPartition() throws BackendException {
+
+ List<KeyRange> result = new LinkedList<KeyRange>();
+
+ TableMask table = null;
+ try {
+ ensureTableExists(tableName, getCfNameForStoreName(GraphDatabaseConfiguration.SYSTEM_PROPERTIES_STORE_NAME), 0);
+
+ table = cnx.getTable(tableName);
+
+ HTable hTable = (HTable)table.getTableObject();
+
+ Map<KeyRange, ServerName> normed =
+ normalizeKeyBounds(hTable.getRegionLocations());
+
+ for (Map.Entry<KeyRange, ServerName> e : normed.entrySet()) {
+ if (NetworkUtil.isLocalConnection(e.getValue().getHostname())) {
+ result.add(e.getKey());
+ logger.debug("Found local key/row partition {} on host {}", e.getKey(), e.getValue());
+ } else {
+ logger.debug("Discarding remote {}", e.getValue());
+ }
+ }
+ } catch (MasterNotRunningException e) {
+ logger.warn("Unexpected MasterNotRunningException", e);
+ } catch (ZooKeeperConnectionException e) {
+ logger.warn("Unexpected ZooKeeperConnectionException", e);
+ } catch (IOException e) {
+ logger.warn("Unexpected IOException", e);
+ } finally {
+ IOUtils.closeQuietly(table);
+ }
+ return result;
+ }
+
+ /**
+ * Given a map produced by {@link HTable#getRegionLocations()}, transform
+ * each key from an {@link HRegionInfo} to a {@link KeyRange} expressing the
+ * region's start and end key bounds using Titan-partitioning-friendly
+ * conventions (start inclusive, end exclusive, zero bytes appended where
+ * necessary to make all keys at least 4 bytes long).
+ * <p/>
+ * This method iterates over the entries in its map parameter and performs
+ * the following conditional conversions on its keys. "Require" below means
+ * either a {@link Preconditions} invocation or an assertion. HRegionInfo
+ * sometimes returns start and end keys of zero length; this method replaces
+ * zero length keys with null before doing any of the checks described
+ * below. The parameter map and the values it contains are only read and
+ * never modified.
+ *
+ * <ul>
+ * <li>If an entry's HRegionInfo has null start and end keys, then first
+ * require that the parameter map is a singleton, and then return a
+ * single-entry map whose {@code KeyRange} has start and end buffers that
+ * are both four bytes of zeros.</li>
+ * <li>If the entry has a null end key (but non-null start key), put an
+ * equivalent entry in the result map with a start key identical to the
+ * input, except that zeros are appended to values less than 4 bytes long,
+ * and an end key that is four bytes of zeros.
+ * <li>If the entry has a null start key (but non-null end key), put an
+ * equivalent entry in the result map where the start key is four bytes of
+ * zeros, and the end key has zeros appended, if necessary, to make it at
+ * least 4 bytes long, after which one is added to the padded value in
+ * unsigned 32-bit arithmetic with overflow allowed.</li>
+ * <li>Any entry which matches none of the above criteria results in an
+ * equivalent entry in the returned map, except that zeros are appended to
+ * both keys to make each at least 4 bytes long, and the end key is then
+ * incremented as described in the last bullet point.</li>
+ * </ul>
+ *
+ * After iterating over the parameter map, this method checks that it either
+ * saw no entries with null keys, one entry with a null start key and a
+ * different entry with a null end key, or one entry with both start and end
+ * keys null. If any null keys are observed besides these three cases, the
+ * method will die with a precondition failure.
+ *
+ * @param raw
+ * A map of HRegionInfo and ServerName from HBase
+ * @return Titan-friendly expression of each region's rowkey boundaries
+ */
+ private Map<KeyRange, ServerName> normalizeKeyBounds(NavigableMap<HRegionInfo, ServerName> raw) {
+
+ Map.Entry<HRegionInfo, ServerName> nullStart = null;
+ Map.Entry<HRegionInfo, ServerName> nullEnd = null;
+
+ ImmutableMap.Builder<KeyRange, ServerName> b = ImmutableMap.builder();
+
+ for (Map.Entry<HRegionInfo, ServerName> e : raw.entrySet()) {
+ HRegionInfo regionInfo = e.getKey();
+ byte startKey[] = regionInfo.getStartKey();
+ byte endKey[] = regionInfo.getEndKey();
+
+ if (0 == startKey.length) {
+ startKey = null;
+ logger.trace("Converted zero-length HBase startKey byte array to null");
+ }
+
+ if (0 == endKey.length) {
+ endKey = null;
+ logger.trace("Converted zero-length HBase endKey byte array to null");
+ }
+
+ if (null == startKey && null == endKey) {
+ Preconditions.checkState(1 == raw.size());
+ logger.debug("HBase table {} has a single region {}", tableName, regionInfo);
+ // Choose arbitrary shared value = startKey = endKey
+ return b.put(new KeyRange(FOUR_ZERO_BYTES, FOUR_ZERO_BYTES), e.getValue()).build();
+ } else if (null == startKey) {
+ logger.debug("Found HRegionInfo with null startKey on server {}: {}", e.getValue(), regionInfo);
+ Preconditions.checkState(null == nullStart);
+ nullStart = e;
+ // I thought endBuf would be inclusive from the HBase javadoc, but in practice it is exclusive
+ StaticBuffer endBuf = StaticArrayBuffer.of(zeroExtend(endKey));
+ // Replace null start key with zeroes
+ b.put(new KeyRange(FOUR_ZERO_BYTES, endBuf), e.getValue());
+ } else if (null == endKey) {
+ logger.debug("Found HRegionInfo with null endKey on server {}: {}", e.getValue(), regionInfo);
+ Preconditions.checkState(null == nullEnd);
+ nullEnd = e;
+ // Replace null end key with zeroes
+ b.put(new KeyRange(StaticArrayBuffer.of(zeroExtend(startKey)), FOUR_ZERO_BYTES), e.getValue());
+ } else {
+ Preconditions.checkState(null != startKey);
+ Preconditions.checkState(null != endKey);
+
+ // Convert HBase's inclusive end keys into exclusive Titan end keys
+ StaticBuffer startBuf = StaticArrayBuffer.of(zeroExtend(startKey));
+ StaticBuffer endBuf = StaticArrayBuffer.of(zeroExtend(endKey));
+
+ KeyRange kr = new KeyRange(startBuf, endBuf);
+ b.put(kr, e.getValue());
+ logger.debug("Found HRegionInfo with non-null end and start keys on server {}: {}", e.getValue(), regionInfo);
+ }
+ }
+
+ // Require either no null key bounds or a pair of them
+ Preconditions.checkState(!(null == nullStart ^ null == nullEnd));
+
+ // Check that every key in the result is at least 4 bytes long
+ Map<KeyRange, ServerName> result = b.build();
+ for (KeyRange kr : result.keySet()) {
+ Preconditions.checkState(4 <= kr.getStart().length());
+ Preconditions.checkState(4 <= kr.getEnd().length());
+ }
+
+ return result;
+ }
+
+ /**
+ * If the parameter is shorter than 4 bytes, then create and return a new 4
+ * byte array with the input array's bytes followed by zero bytes. Otherwise
+ * return the parameter.
+ *
+ * @param dataToPad non-null but possibly zero-length byte array
+ * @return either the parameter or a new array
+ */
+ private final byte[] zeroExtend(byte[] dataToPad) {
+ assert null != dataToPad;
+
+ final int targetLength = 4;
+
+ if (targetLength <= dataToPad.length)
+ return dataToPad;
+
+ byte padded[] = new byte[targetLength];
+
+ for (int i = 0; i < dataToPad.length; i++)
+ padded[i] = dataToPad[i];
+
+ for (int i = dataToPad.length; i < padded.length; i++)
+ padded[i] = (byte)0;
+
+ return padded;
+ }
+
+ public static String shortenCfName(String longName) throws PermanentBackendException {
+ final String s;
+ if (SHORT_CF_NAME_MAP.containsKey(longName)) {
+ s = SHORT_CF_NAME_MAP.get(longName);
+ Preconditions.checkNotNull(s);
+ logger.debug("Substituted default CF name \"{}\" with short form \"{}\" to reduce HBase KeyValue size", longName, s);
+ } else {
+ if (SHORT_CF_NAME_MAP.containsValue(longName)) {
+ String fmt = "Must use CF long-form name \"%s\" instead of the short-form name \"%s\" when configured with %s=true";
+ String msg = String.format(fmt, SHORT_CF_NAME_MAP.inverse().get(longName), longName, SHORT_CF_NAMES.getName());
+ throw new PermanentBackendException(msg);
+ }
+ s = longName;
+ logger.debug("Kept default CF name \"{}\" because it has no associated short form", s);
+ }
+ return s;
+ }
+
+ private HTableDescriptor ensureTableExists(String tableName, String initialCFName, int ttlInSeconds) throws BackendException {
+ AdminMask adm = null;
+
+ HTableDescriptor desc;
+
+ try { // Create our table, if necessary
+ adm = getAdminInterface();
+ /*
+ * Some HBase versions/impls respond badly to attempts to create a
+ * table without at least one CF. See #661. Creating a CF along with
+ * the table avoids HBase carping.
+ */
+ if (adm.tableExists(tableName)) {
+ desc = adm.getTableDescriptor(tableName);
+ } else {
+ desc = createTable(tableName, initialCFName, ttlInSeconds, adm);
+ }
+ } catch (IOException e) {
+ throw new TemporaryBackendException(e);
+ } finally {
+ IOUtils.closeQuietly(adm);
+ }
+
+ return desc;
+ }
+
+ private HTableDescriptor createTable(String tableName, String cfName, int ttlInSeconds, AdminMask adm) throws IOException {
+ HTableDescriptor desc = compat.newTableDescriptor(tableName);
+
+ HColumnDescriptor cdesc = new HColumnDescriptor(cfName);
+ setCFOptions(cdesc, ttlInSeconds);
+
+ compat.addColumnFamilyToTableDescriptor(desc, cdesc);
+
+ int count; // total regions to create
+ String src;
+
+ if (MIN_REGION_COUNT <= (count = regionCount)) {
+ src = "region count configuration";
+ } else if (0 < regionsPerServer &&
+ MIN_REGION_COUNT <= (count = regionsPerServer * adm.getEstimatedRegionServerCount())) {
+ src = "ClusterStatus server count";
+ } else {
+ count = -1;
+ src = "default";
+ }
+
+ if (MIN_REGION_COUNT < count) {
+ adm.createTable(desc, getStartKey(count), getEndKey(count), count);
+ logger.debug("Created table {} with region count {} from {}", tableName, count, src);
+ } else {
+ adm.createTable(desc);
+ logger.debug("Created table {} with default start key, end key, and region count", tableName);
+ }
+
+ return desc;
+ }
+
+ /**
+ * This method generates the second argument to
+ * {@link HBaseAdmin#createTable(HTableDescriptor, byte[], byte[], int)}.
+ * <p/>
+ * From the {@code createTable} javadoc:
+ * "The start key specified will become the end key of the first region of
+ * the table, and the end key specified will become the start key of the
+ * last region of the table (the first region has a null start key and
+ * the last region has a null end key)"
+ * <p/>
+ * To summarize, the {@code createTable} argument called "startKey" is
+ * actually the end key of the first region.
+ */
+ private byte[] getStartKey(int regionCount) {
+ ByteBuffer regionWidth = ByteBuffer.allocate(4);
+ regionWidth.putInt((int)(((1L << 32) - 1L) / regionCount)).flip();
+ return StaticArrayBuffer.of(regionWidth).getBytes(0, 4);
+ }
+
+ /**
+ * Companion to {@link #getStartKey(int)}. See its javadoc for details.
+ */
+ private byte[] getEndKey(int regionCount) {
+ ByteBuffer regionWidth = ByteBuffer.allocate(4);
+ regionWidth.putInt((int)(((1L << 32) - 1L) / regionCount * (regionCount - 1))).flip();
+ return StaticArrayBuffer.of(regionWidth).getBytes(0, 4);
+ }
+
+ private void ensureColumnFamilyExists(String tableName, String columnFamily, int ttlInSeconds) throws BackendException {
+ AdminMask adm = null;
+ try {
+ adm = getAdminInterface();
+ HTableDescriptor desc = ensureTableExists(tableName, columnFamily, ttlInSeconds);
+
+ Preconditions.checkNotNull(desc);
+
+ HColumnDescriptor cf = desc.getFamily(columnFamily.getBytes());
+
+ // Create our column family, if necessary
+ if (cf == null) {
+ try {
+ if (!adm.isTableDisabled(tableName)) {
+ adm.disableTable(tableName);
+ }
+ } catch (TableNotEnabledException e) {
+ logger.debug("Table {} already disabled", tableName);
+ } catch (IOException e) {
+ throw new TemporaryBackendException(e);
+ }
+
+ try {
+ HColumnDescriptor cdesc = new HColumnDescriptor(columnFamily);
+
+ setCFOptions(cdesc, ttlInSeconds);
+
+ adm.addColumn(tableName, cdesc);
+
+ logger.debug("Added HBase ColumnFamily {}, waiting for 1 sec. to propogate.", columnFamily);
+
+ adm.enableTable(tableName);
+ } catch (TableNotFoundException ee) {
+ logger.error("TableNotFoundException", ee);
+ throw new PermanentBackendException(ee);
+ } catch (org.apache.hadoop.hbase.TableExistsException ee) {
+ logger.debug("Swallowing exception {}", ee);
+ } catch (IOException ee) {
+ throw new TemporaryBackendException(ee);
+ }
+ }
+ } finally {
+ IOUtils.closeQuietly(adm);
+ }
+ }
+
+ private void setCFOptions(HColumnDescriptor cdesc, int ttlInSeconds) {
+ if (null != compression && !compression.equals(COMPRESSION_DEFAULT))
+ compat.setCompression(cdesc, compression);
+
+ if (ttlInSeconds > 0)
+ cdesc.setTimeToLive(ttlInSeconds);
+
+ cdesc.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF);
+ }
+
+ /**
+ * Convert Titan internal Mutation representation into HBase native commands.
+ *
+ * @param mutations Mutations to convert into HBase commands.
+ * @param putTimestamp The timestamp to use for Put commands.
+ * @param delTimestamp The timestamp to use for Delete commands.
+ * @return Commands sorted by key converted from Titan internal representation.
+ * @throws com.thinkaurelius.titan.diskstorage.PermanentBackendException
+ */
+ private Map<StaticBuffer, Pair<Put, Delete>> convertToCommands(Map<String, Map<StaticBuffer, KCVMutation>> mutations,
+ final long putTimestamp,
+ final long delTimestamp) throws PermanentBackendException {
+ Map<StaticBuffer, Pair<Put, Delete>> commandsPerKey = new HashMap<StaticBuffer, Pair<Put, Delete>>();
+
+ for (Map.Entry<String, Map<StaticBuffer, KCVMutation>> entry : mutations.entrySet()) {
+
+ String cfString = getCfNameForStoreName(entry.getKey());
+ byte[] cfName = cfString.getBytes();
+
+ for (Map.Entry<StaticBuffer, KCVMutation> m : entry.getValue().entrySet()) {
+ byte[] key = m.getKey().as(StaticBuffer.ARRAY_FACTORY);
+ KCVMutation mutation = m.getValue();
+
+ Pair<Put, Delete> commands = commandsPerKey.get(m.getKey());
+
+ if (commands == null) {
+ commands = new Pair<Put, Delete>();
+ commandsPerKey.put(m.getKey(), commands);
+ }
+
+ if (mutation.hasDeletions()) {
+ if (commands.getSecond() == null) {
+ Delete d = new Delete(key);
+ compat.setTimestamp(d, delTimestamp);
+ commands.setSecond(d);
+ }
+
+ for (StaticBuffer b : mutation.getDeletions()) {
+ commands.getSecond().deleteColumns(cfName, b.as(StaticBuffer.ARRAY_FACTORY), delTimestamp);
+ }
+ }
+
+ if (mutation.hasAdditions()) {
+ if (commands.getFirst() == null) {
+ Put p = new Put(key, putTimestamp);
+ commands.setFirst(p);
+ }
+
+ for (Entry e : mutation.getAdditions()) {
+ commands.getFirst().add(cfName,
+ e.getColumnAs(StaticBuffer.ARRAY_FACTORY),
+ putTimestamp,
+ e.getValueAs(StaticBuffer.ARRAY_FACTORY));
+ }
+ }
+ }
+ }
+
+ return commandsPerKey;
+ }
+
+ private String getCfNameForStoreName(String storeName) throws PermanentBackendException {
+ return shortCfNames ? shortenCfName(storeName) : storeName;
+ }
+
+ private void checkConfigDeprecation(com.thinkaurelius.titan.diskstorage.configuration.Configuration config) {
+ if (config.has(GraphDatabaseConfiguration.STORAGE_PORT)) {
+ logger.warn("The configuration property {} is ignored for HBase. Set hbase.zookeeper.property.clientPort in hbase-site.xml or {}.hbase.zookeeper.property.clientPort in Titan's configuration file.",
+ ConfigElement.getPath(GraphDatabaseConfiguration.STORAGE_PORT), ConfigElement.getPath(HBASE_CONFIGURATION_NAMESPACE));
+ }
+ }
+
+ private AdminMask getAdminInterface() {
+ try {
+ return cnx.getAdmin();
+ } catch (IOException e) {
+ throw new TitanException(e);
+ }
+ }
+
+ /**
+ * Similar to {@link Function}, except that the {@code apply} method is allowed
+ * to throw {@link BackendException}.
+ */
+ private static interface BackendFunction<F, T> {
+
+ T apply(F input) throws BackendException;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseTransaction.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseTransaction.java b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseTransaction.java
new file mode 100644
index 0000000..e13593f
--- /dev/null
+++ b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseTransaction.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2012-2013 Aurelius LLC
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.thinkaurelius.titan.diskstorage.hbase;
+
+import com.thinkaurelius.titan.diskstorage.BackendException;
+import com.thinkaurelius.titan.diskstorage.BaseTransactionConfig;
+import com.thinkaurelius.titan.diskstorage.StaticBuffer;
+import com.thinkaurelius.titan.diskstorage.common.AbstractStoreTransaction;
+import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreTransaction;
+import com.thinkaurelius.titan.diskstorage.locking.LocalLockMediator;
+import com.thinkaurelius.titan.diskstorage.util.KeyColumn;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+/**
+ * This class overrides and adds nothing compared with
+ * {@link com.thinkaurelius.titan.diskstorage.locking.consistentkey.ExpectedValueCheckingTransaction}; however, it creates a transaction type specific
+ * to HBase, which lets us check for user errors like passing a Cassandra
+ * transaction into a HBase method.
+ *
+ * @author Dan LaRocque <da...@hopcount.org>
+ */
+public class HBaseTransaction extends AbstractStoreTransaction {
+
+ private static final Logger log = LoggerFactory.getLogger(HBaseTransaction.class);
+
+ LocalLockMediator<StoreTransaction> llm;
+
+ Set<KeyColumn> keyColumnLocks = new LinkedHashSet<>();
+
+ public HBaseTransaction(final BaseTransactionConfig config, LocalLockMediator<StoreTransaction> llm) {
+ super(config);
+ this.llm = llm;
+ }
+
+ @Override
+ public synchronized void rollback() throws BackendException {
+ super.rollback();
+ log.debug("Rolled back transaction");
+ deleteAllLocks();
+ }
+
+ @Override
+ public synchronized void commit() throws BackendException {
+ super.commit();
+ log.debug("Committed transaction");
+ deleteAllLocks();
+ }
+
+ public void updateLocks(KeyColumn lockID, StaticBuffer expectedValue) {
+ keyColumnLocks.add(lockID);
+ }
+
+ private void deleteAllLocks() {
+ for(KeyColumn kc : keyColumnLocks) {
+ log.debug("Removed lock {} ", kc);
+ llm.unlock(kc, this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection0_98.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection0_98.java b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection0_98.java
new file mode 100644
index 0000000..8660644
--- /dev/null
+++ b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection0_98.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2012-2013 Aurelius LLC
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.thinkaurelius.titan.diskstorage.hbase;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnection;
+
+public class HConnection0_98 implements ConnectionMask
+{
+
+ private final HConnection cnx;
+
+ public HConnection0_98(HConnection cnx)
+ {
+ this.cnx = cnx;
+ }
+
+ @Override
+ public TableMask getTable(String name) throws IOException
+ {
+ return new HTable0_98(cnx.getTable(name));
+ }
+
+ @Override
+ public AdminMask getAdmin() throws IOException
+ {
+ return new HBaseAdmin0_98(new HBaseAdmin(cnx));
+ }
+
+ @Override
+ public void close() throws IOException
+ {
+ cnx.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection1_0.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection1_0.java b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection1_0.java
new file mode 100644
index 0000000..91e5026
--- /dev/null
+++ b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection1_0.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2012-2013 Aurelius LLC
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.thinkaurelius.titan.diskstorage.hbase;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+
+public class HConnection1_0 implements ConnectionMask
+{
+
+ private final Connection cnx;
+
+ public HConnection1_0(Connection cnx)
+ {
+ this.cnx = cnx;
+ }
+
+ @Override
+ public TableMask getTable(String name) throws IOException
+ {
+ return new HTable1_0(cnx.getTable(TableName.valueOf(name)));
+ }
+
+ @Override
+ public AdminMask getAdmin() throws IOException
+ {
+ return new HBaseAdmin1_0(new HBaseAdmin(cnx));
+ }
+
+ @Override
+ public void close() throws IOException
+ {
+ cnx.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable0_98.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable0_98.java b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable0_98.java
new file mode 100644
index 0000000..b11532a
--- /dev/null
+++ b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable0_98.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2012-2013 Aurelius LLC
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.thinkaurelius.titan.diskstorage.hbase;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Row;
+import org.apache.hadoop.hbase.client.Scan;
+
+public class HTable0_98 implements TableMask
+{
+ private final HTableInterface table;
+
+ public HTable0_98(HTableInterface table)
+ {
+ this.table = table;
+ }
+
+ @Override
+ public ResultScanner getScanner(Scan filter) throws IOException
+ {
+ return table.getScanner(filter);
+ }
+
+ @Override
+ public Result[] get(List<Get> gets) throws IOException
+ {
+ return table.get(gets);
+ }
+
+ @Override
+ public void batch(List<Row> writes, Object[] results) throws IOException, InterruptedException
+ {
+ table.batch(writes, results);
+ table.flushCommits();
+ }
+
+ @Override
+ public void close() throws IOException
+ {
+ table.close();
+ }
+
+ @Override
+ public Object getTableObject() {
+ return table;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable1_0.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable1_0.java b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable1_0.java
new file mode 100644
index 0000000..5c90617
--- /dev/null
+++ b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable1_0.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2012-2013 Aurelius LLC
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.thinkaurelius.titan.diskstorage.hbase;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Row;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+
+public class HTable1_0 implements TableMask
+{
+ private final Table table;
+
+ public HTable1_0(Table table)
+ {
+ this.table = table;
+ }
+
+ @Override
+ public ResultScanner getScanner(Scan filter) throws IOException
+ {
+ return table.getScanner(filter);
+ }
+
+ @Override
+ public Result[] get(List<Get> gets) throws IOException
+ {
+ return table.get(gets);
+ }
+
+ @Override
+ public void batch(List<Row> writes, Object[] results) throws IOException, InterruptedException
+ {
+ table.batch(writes, results);
+ /* table.flushCommits(); not needed anymore */
+ }
+
+ @Override
+ public void close() throws IOException
+ {
+ table.close();
+ }
+
+ @Override
+ public Object getTableObject() {
+ return table;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/TableMask.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/TableMask.java b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/TableMask.java
new file mode 100644
index 0000000..54f8743
--- /dev/null
+++ b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/TableMask.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2012-2013 Aurelius LLC
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.thinkaurelius.titan.diskstorage.hbase;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Row;
+import org.apache.hadoop.hbase.client.Scan;
+
+/**
+ * This interface hides ABI/API breaking changes that HBase has made to its Table/HTableInterface over the course
+ * of development from 0.94 to 1.0 and beyond.
+ */
+public interface TableMask extends Closeable
+{
+
+ ResultScanner getScanner(Scan filter) throws IOException;
+
+ Result[] get(List<Get> gets) throws IOException;
+
+ void batch(List<Row> writes, Object[] results) throws IOException, InterruptedException;
+
+ Object getTableObject();
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/locking/LocalLockMediator.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/locking/LocalLockMediator.java b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/locking/LocalLockMediator.java
new file mode 100644
index 0000000..20c59e1
--- /dev/null
+++ b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/locking/LocalLockMediator.java
@@ -0,0 +1,345 @@
+/*
+ * Copyright 2012-2013 Aurelius LLC
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.thinkaurelius.titan.diskstorage.locking;
+
+import com.google.common.base.Preconditions;
+import com.thinkaurelius.titan.diskstorage.util.time.Timepoint;
+import com.thinkaurelius.titan.diskstorage.util.time.TimestampProvider;
+import com.thinkaurelius.titan.diskstorage.locking.consistentkey.ExpectedValueCheckingTransaction;
+import com.thinkaurelius.titan.diskstorage.util.KeyColumn;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.DelayQueue;
+import java.util.concurrent.Delayed;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * This class resolves lock contention between two transactions on the same JVM.
+ * <p/>
+ * This is not just an optimization to reduce network traffic. Locks written by
+ * Titan to a distributed key-value store contain an identifier, the "Rid",
+ * which is unique only to the process level. The Rid can't tell which
+ * transaction in a process holds any given lock. This class prevents two
+ * transactions in a single process from concurrently writing the same lock to a
+ * distributed key-value store.
+ *
+ * @author Dan LaRocque <da...@hopcount.org>
+ */
+
+public class LocalLockMediator<T> {
+
+ private static final Logger log = LoggerFactory
+ .getLogger(LocalLockMediator.class);
+
+ /**
+ * Namespace for which this mediator is responsible
+ *
+ * @see LocalLockMediatorProvider
+ */
+ private final String name;
+
+ private final TimestampProvider times;
+
+ private DelayQueue<ExpirableKeyColumn> expiryQueue = new DelayQueue<>();
+
+ private ExecutorService lockCleanerService = Executors.newFixedThreadPool(1, new ThreadFactory() {
+ @Override
+ public Thread newThread(Runnable runnable) {
+ Thread thread = Executors.defaultThreadFactory().newThread(runnable);
+ thread.setDaemon(true);
+ return thread;
+ }
+ });
+
+
+
+ /**
+ * Maps a ({@code key}, {@code column}) pair to the local transaction
+ * holding a lock on that pair. Values in this map may have already expired
+ * according to {@link AuditRecord#expires}, in which case the lock should
+ * be considered invalid.
+ */
+ private final ConcurrentHashMap<KeyColumn, AuditRecord<T>> locks = new ConcurrentHashMap<KeyColumn, AuditRecord<T>>();
+
+ public LocalLockMediator(String name, TimestampProvider times) {
+ this.name = name;
+ this.times = times;
+
+ Preconditions.checkNotNull(name);
+ Preconditions.checkNotNull(times);
+ lockCleanerService.submit(new LockCleaner());
+ }
+
+ /**
+ * Acquire the lock specified by {@code kc}.
+ * <p/>
+ * <p/>
+ * For any particular key-column, whatever value of {@code requestor} is
+ * passed to this method must also be passed to the associated later call to
+ * {@link #unlock(KeyColumn, ExpectedValueCheckingTransaction)}.
+ * <p/>
+ * If some requestor {@code r} calls this method on a KeyColumn {@code k}
+ * and this method returns true, then subsequent calls to this method by
+ * {@code r} on {@code l} merely attempt to update the {@code expiresAt}
+ * timestamp. This differs from typical lock reentrance: multiple successful
+ * calls to this method do not require an equal number of calls to
+ * {@code #unlock()}. One {@code #unlock()} call is enough, no matter how
+ * many times a {@code requestor} called {@code lock} beforehand. Note that
+ * updating the timestamp may fail, in which case the lock is considered to
+ * have expired and the calling context should assume it no longer holds the
+ * lock specified by {@code kc}.
+ * <p/>
+ * The number of nanoseconds elapsed since the UNIX Epoch is not readily
+ * available within the JVM. When reckoning expiration times, this method
+ * uses the approximation implemented by
+ * {@link com.thinkaurelius.titan.diskstorage.util.NanoTime#getApproxNSSinceEpoch(false)}.
+ * <p/>
+ * The current implementation of this method returns true when given an
+ * {@code expiresAt} argument in the past. Future implementations may return
+ * false instead.
+ *
+ * @param kc lock identifier
+ * @param requestor the object locking {@code kc}
+ * @param expires instant at which this lock will automatically expire
+ * @return true if the lock is acquired, false if it was not acquired
+ */
+ public boolean lock(KeyColumn kc, T requestor, Timepoint expires) {
+ assert null != kc;
+ assert null != requestor;
+
+ AuditRecord<T> audit = new AuditRecord<T>(requestor, expires);
+ AuditRecord<T> inmap = locks.putIfAbsent(kc, audit);
+
+ boolean success = false;
+
+ if (null == inmap) {
+ // Uncontended lock succeeded
+ if (log.isTraceEnabled()) {
+ log.trace("New local lock created: {} namespace={} txn={}",
+ new Object[]{kc, name, requestor});
+ }
+ success = true;
+ } else if (inmap.equals(audit)) {
+ // requestor has already locked kc; update expiresAt
+ success = locks.replace(kc, inmap, audit);
+ if (log.isTraceEnabled()) {
+ if (success) {
+ log.trace(
+ "Updated local lock expiration: {} namespace={} txn={} oldexp={} newexp={}",
+ new Object[]{kc, name, requestor, inmap.expires,
+ audit.expires});
+ } else {
+ log.trace(
+ "Failed to update local lock expiration: {} namespace={} txn={} oldexp={} newexp={}",
+ new Object[]{kc, name, requestor, inmap.expires,
+ audit.expires});
+ }
+ }
+ } else if (0 > inmap.expires.compareTo(times.getTime())) {
+ // the recorded lock has expired; replace it
+ success = locks.replace(kc, inmap, audit);
+ if (log.isTraceEnabled()) {
+ log.trace(
+ "Discarding expired lock: {} namespace={} txn={} expired={}",
+ new Object[]{kc, name, inmap.holder, inmap.expires});
+ }
+ } else {
+ // we lost to a valid lock
+ if (log.isTraceEnabled()) {
+ log.trace(
+ "Local lock failed: {} namespace={} txn={} (already owned by {})",
+ new Object[]{kc, name, requestor, inmap});
+ }
+ }
+
+ if (success) {
+ expiryQueue.add(new ExpirableKeyColumn(kc, expires));
+ }
+ return success;
+ }
+
+ /**
+ * Release the lock specified by {@code kc} and which was previously
+ * locked by {@code requestor}, if it is possible to release it.
+ *
+ * @param kc lock identifier
+ * @param requestor the object which previously locked {@code kc}
+ */
+ public boolean unlock(KeyColumn kc, T requestor) {
+
+ if (!locks.containsKey(kc)) {
+ log.info("Local unlock failed: no locks found for {}", kc);
+ return false;
+ }
+
+ AuditRecord<T> unlocker = new AuditRecord<T>(requestor, null);
+
+ AuditRecord<T> holder = locks.get(kc);
+
+ if (!holder.equals(unlocker)) {
+ log.error("Local unlock of {} by {} failed: it is held by {}",
+ new Object[]{kc, unlocker, holder});
+ return false;
+ }
+
+ boolean removed = locks.remove(kc, unlocker);
+
+ if (removed) {
+ expiryQueue.remove(kc);
+ if (log.isTraceEnabled()) {
+ log.trace("Local unlock succeeded: {} namespace={} txn={}",
+ new Object[]{kc, name, requestor});
+ }
+ } else {
+ log.warn("Local unlock warning: lock record for {} disappeared "
+ + "during removal; this suggests the lock either expired "
+ + "while we were removing it, or that it was erroneously "
+ + "unlocked multiple times.", kc);
+ }
+
+ // Even if !removed, we're finished unlocking, so return true
+ return true;
+ }
+
+ public String toString() {
+ return "LocalLockMediator [" + name + ", ~" + locks.size()
+ + " current locks]";
+ }
+
+ /**
+ * A record containing the local transaction that holds a lock and the
+ * lock's expiration time.
+ */
+ private static class AuditRecord<T> {
+
+ /**
+ * The local transaction that holds/held the lock.
+ */
+ private final T holder;
+ /**
+ * The expiration time of a the lock.
+ */
+ private final Timepoint expires;
+ /**
+ * Cached hashCode.
+ */
+ private int hashCode;
+
+ private AuditRecord(T holder, Timepoint expires) {
+ this.holder = holder;
+ this.expires = expires;
+ }
+
+ /**
+ * This implementation depends only on the lock holder and not on the
+ * lock expiration time.
+ */
+ @Override
+ public int hashCode() {
+ if (0 == hashCode)
+ hashCode = holder.hashCode();
+
+ return hashCode;
+ }
+
+ /**
+ * This implementation depends only on the lock holder and not on the
+ * lock expiration time.
+ */
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ /*
+ * This warning suppression is harmless because we are only going to
+ * call other.holder.equals(...), and since equals(...) is part of
+ * Object, it is guaranteed to be defined no matter the concrete
+ * type of parameter T.
+ */
+ @SuppressWarnings("rawtypes")
+ AuditRecord other = (AuditRecord) obj;
+ if (holder == null) {
+ if (other.holder != null)
+ return false;
+ } else if (!holder.equals(other.holder))
+ return false;
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ return "AuditRecord [txn=" + holder + ", expires=" + expires + "]";
+ }
+
+ }
+
+ private class LockCleaner implements Runnable {
+
+ @Override
+ public void run() {
+ try {
+ while (true) {
+ log.debug("Lock Cleaner service started");
+ ExpirableKeyColumn lock = expiryQueue.take();
+ log.debug("Expiring key column " + lock.getKeyColumn());
+ locks.remove(lock.getKeyColumn());
+ }
+ } catch (InterruptedException e) {
+ log.debug("Received interrupt. Exiting");
+ }
+ }
+ }
+
+ private static class ExpirableKeyColumn implements Delayed {
+
+ private Timepoint expiryTime;
+ private KeyColumn kc;
+
+ public ExpirableKeyColumn(KeyColumn keyColumn, Timepoint expiryTime) {
+ this.kc = keyColumn;
+ this.expiryTime = expiryTime;
+ }
+
+ @Override
+ public long getDelay(TimeUnit unit) {
+ return expiryTime.getTimestamp(TimeUnit.NANOSECONDS);
+ }
+
+ @Override
+ public int compareTo(Delayed o) {
+ if (this.expiryTime.getTimestamp(TimeUnit.NANOSECONDS) < ((ExpirableKeyColumn) o).expiryTime.getTimestamp(TimeUnit.NANOSECONDS)) {
+ return -1;
+ }
+ if (this.expiryTime.getTimestamp(TimeUnit.NANOSECONDS) > ((ExpirableKeyColumn) o).expiryTime.getTimestamp(TimeUnit.NANOSECONDS)) {
+ return 1;
+ }
+ return 0;
+ }
+
+ public KeyColumn getKeyColumn() {
+ return kc;
+ }
+ }
+}
[6/9] incubator-atlas git commit: ATLAS-693 Titan 0.5.4
implementation of graph db abstraction. (jnhagelb via dkantor)
Posted by dk...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/solr/Solr5Index.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/solr/Solr5Index.java b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/solr/Solr5Index.java
new file mode 100644
index 0000000..3b5620c
--- /dev/null
+++ b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/solr/Solr5Index.java
@@ -0,0 +1,975 @@
+/*
+ * Copyright 2012-2013 Aurelius LLC
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.thinkaurelius.titan.diskstorage.solr;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
+import com.thinkaurelius.titan.core.Order;
+import com.thinkaurelius.titan.core.TitanElement;
+import com.thinkaurelius.titan.core.attribute.Cmp;
+import com.thinkaurelius.titan.core.attribute.Geo;
+import com.thinkaurelius.titan.core.attribute.Geoshape;
+import com.thinkaurelius.titan.core.attribute.Text;
+import com.thinkaurelius.titan.core.schema.Mapping;
+import com.thinkaurelius.titan.diskstorage.BackendException;
+import com.thinkaurelius.titan.diskstorage.BaseTransaction;
+import com.thinkaurelius.titan.diskstorage.BaseTransactionConfig;
+import com.thinkaurelius.titan.diskstorage.BaseTransactionConfigurable;
+import com.thinkaurelius.titan.diskstorage.PermanentBackendException;
+import com.thinkaurelius.titan.diskstorage.TemporaryBackendException;
+import com.thinkaurelius.titan.diskstorage.configuration.ConfigNamespace;
+import com.thinkaurelius.titan.diskstorage.configuration.ConfigOption;
+import com.thinkaurelius.titan.diskstorage.configuration.Configuration;
+import com.thinkaurelius.titan.diskstorage.indexing.IndexEntry;
+import com.thinkaurelius.titan.diskstorage.indexing.IndexFeatures;
+import com.thinkaurelius.titan.diskstorage.indexing.IndexMutation;
+import com.thinkaurelius.titan.diskstorage.indexing.IndexProvider;
+import com.thinkaurelius.titan.diskstorage.indexing.IndexQuery;
+import com.thinkaurelius.titan.diskstorage.indexing.KeyInformation;
+import com.thinkaurelius.titan.diskstorage.indexing.RawQuery;
+import com.thinkaurelius.titan.diskstorage.util.DefaultTransaction;
+import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration;
+import com.thinkaurelius.titan.graphdb.configuration.PreInitializeConfigOptions;
+import com.thinkaurelius.titan.graphdb.database.serialize.AttributeUtil;
+import com.thinkaurelius.titan.graphdb.database.serialize.attribute.AbstractDecimal;
+import com.thinkaurelius.titan.graphdb.query.TitanPredicate;
+import com.thinkaurelius.titan.graphdb.query.condition.And;
+import com.thinkaurelius.titan.graphdb.query.condition.Condition;
+import com.thinkaurelius.titan.graphdb.query.condition.Not;
+import com.thinkaurelius.titan.graphdb.query.condition.Or;
+import com.thinkaurelius.titan.graphdb.query.condition.PredicateCondition;
+import com.thinkaurelius.titan.graphdb.types.ParameterType;
+import org.apache.commons.lang.StringUtils;
+import org.apache.http.client.HttpClient;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpClientUtil;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer;
+import org.apache.solr.client.solrj.impl.LBHttpSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.util.ClientUtils;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.TimeZone;
+import java.util.UUID;
+
+import static com.thinkaurelius.titan.core.attribute.Cmp.*;
+import static com.thinkaurelius.titan.core.schema.Mapping.*;
+
+/**
+ * NOTE: Copied from titan for supporting sol5. Do not change
+ */
+@PreInitializeConfigOptions
+public class Solr5Index implements IndexProvider {
+
+ private static final Logger logger = LoggerFactory.getLogger(Solr5Index.class);
+
+
+ private static final String DEFAULT_ID_FIELD = "id";
+
+ private enum Mode {
+ HTTP, CLOUD;
+
+ public static Mode parse(String mode) {
+ for (Mode m : Mode.values()) {
+ if (m.toString().equalsIgnoreCase(mode)) return m;
+ }
+ throw new IllegalArgumentException("Unrecognized mode: "+mode);
+ }
+ }
+
+ public static final ConfigNamespace SOLR_NS =
+ new ConfigNamespace(GraphDatabaseConfiguration.INDEX_NS, "solr", "Solr index configuration");
+
+ public static final ConfigOption<String> SOLR_MODE = new ConfigOption<String>(SOLR_NS,"mode",
+ "The operation mode for Solr which is either via HTTP (`http`) or using SolrCloud (`cloud`)",
+ ConfigOption.Type.GLOBAL_OFFLINE, "cloud");
+
+ public static final ConfigOption<Boolean> DYNAMIC_FIELDS = new ConfigOption<Boolean>(SOLR_NS,"dyn-fields",
+ "Whether to use dynamic fields (which appends the data type to the field name). If dynamic fields is disabled" +
+ "the user must map field names and define them explicitly in the schema.",
+ ConfigOption.Type.GLOBAL_OFFLINE, true);
+
+ public static final ConfigOption<String[]> KEY_FIELD_NAMES = new ConfigOption<String[]>(SOLR_NS,"key-field-names",
+ "Field name that uniquely identifies each document in Solr. Must be specified as a list of `collection=field`.",
+ ConfigOption.Type.GLOBAL, String[].class);
+
+ public static final ConfigOption<String> TTL_FIELD = new ConfigOption<String>(SOLR_NS,"ttl_field",
+ "Name of the TTL field for Solr collections.",
+ ConfigOption.Type.GLOBAL_OFFLINE, "ttl");
+
+ /** SolrCloud Configuration */
+
+ public static final ConfigOption<String> ZOOKEEPER_URL = new ConfigOption<String>(SOLR_NS,"zookeeper-url",
+ "URL of the Zookeeper instance coordinating the SolrCloud cluster",
+ ConfigOption.Type.MASKABLE, "localhost:2181");
+
+ public static final ConfigOption<Integer> NUM_SHARDS = new ConfigOption<Integer>(SOLR_NS,"num-shards",
+ "Number of shards for a collection. This applies when creating a new collection which is only supported under the SolrCloud operation mode.",
+ ConfigOption.Type.GLOBAL_OFFLINE, 1);
+
+ public static final ConfigOption<Integer> MAX_SHARDS_PER_NODE = new ConfigOption<Integer>(SOLR_NS,"max-shards-per-node",
+ "Maximum number of shards per node. This applies when creating a new collection which is only supported under the SolrCloud operation mode.",
+ ConfigOption.Type.GLOBAL_OFFLINE, 1);
+
+ public static final ConfigOption<Integer> REPLICATION_FACTOR = new ConfigOption<Integer>(SOLR_NS,"replication-factor",
+ "Replication factor for a collection. This applies when creating a new collection which is only supported under the SolrCloud operation mode.",
+ ConfigOption.Type.GLOBAL_OFFLINE, 1);
+
+
+ /** HTTP Configuration */
+
+ public static final ConfigOption<String[]> HTTP_URLS = new ConfigOption<String[]>(SOLR_NS,"http-urls",
+ "List of URLs to use to connect to Solr Servers (LBHttpSolrClient is used), don't add core or collection name to the URL.",
+ ConfigOption.Type.MASKABLE, new String[] { "http://localhost:8983/solr" });
+
+ public static final ConfigOption<Integer> HTTP_CONNECTION_TIMEOUT = new ConfigOption<Integer>(SOLR_NS,"http-connection-timeout",
+ "Solr HTTP connection timeout.",
+ ConfigOption.Type.MASKABLE, 5000);
+
+ public static final ConfigOption<Boolean> HTTP_ALLOW_COMPRESSION = new ConfigOption<Boolean>(SOLR_NS,"http-compression",
+ "Enable/disable compression on the HTTP connections made to Solr.",
+ ConfigOption.Type.MASKABLE, false);
+
+ public static final ConfigOption<Integer> HTTP_MAX_CONNECTIONS_PER_HOST = new ConfigOption<Integer>(SOLR_NS,"http-max-per-host",
+ "Maximum number of HTTP connections per Solr host.",
+ ConfigOption.Type.MASKABLE, 20);
+
+ public static final ConfigOption<Integer> HTTP_GLOBAL_MAX_CONNECTIONS = new ConfigOption<Integer>(SOLR_NS,"http-max",
+ "Maximum number of HTTP connections in total to all Solr servers.",
+ ConfigOption.Type.MASKABLE, 100);
+
+ public static final ConfigOption<Boolean> WAIT_SEARCHER = new ConfigOption<Boolean>(SOLR_NS, "wait-searcher",
+ "When mutating - wait for the index to reflect new mutations before returning. This can have a negative impact on performance.",
+ ConfigOption.Type.LOCAL, false);
+
+
+
+ private static final IndexFeatures SOLR_FEATURES = new IndexFeatures.Builder().supportsDocumentTTL()
+ .setDefaultStringMapping(TEXT).supportedStringMappings(TEXT, STRING).build();
+
+ private final SolrClient solrClient;
+ private final Configuration configuration;
+ private final Mode mode;
+ private final boolean dynFields;
+ private final Map<String, String> keyFieldIds;
+ private final String ttlField;
+ private final int maxResults;
+ private final boolean waitSearcher;
+
+ public Solr5Index(final Configuration config) throws BackendException {
+ Preconditions.checkArgument(config!=null);
+ configuration = config;
+
+ mode = Mode.parse(config.get(SOLR_MODE));
+ dynFields = config.get(DYNAMIC_FIELDS);
+ keyFieldIds = parseKeyFieldsForCollections(config);
+ maxResults = config.get(GraphDatabaseConfiguration.INDEX_MAX_RESULT_SET_SIZE);
+ ttlField = config.get(TTL_FIELD);
+ waitSearcher = config.get(WAIT_SEARCHER);
+
+ if (mode==Mode.CLOUD) {
+ HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer());
+ String zookeeperUrl = config.get(Solr5Index.ZOOKEEPER_URL);
+ CloudSolrClient cloudServer = new CloudSolrClient(zookeeperUrl, true);
+ cloudServer.connect();
+ solrClient = cloudServer;
+ } else if (mode==Mode.HTTP) {
+ HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer());
+ HttpClient clientParams = HttpClientUtil.createClient(new ModifiableSolrParams() {{
+ add(HttpClientUtil.PROP_ALLOW_COMPRESSION, config.get(HTTP_ALLOW_COMPRESSION).toString());
+ add(HttpClientUtil.PROP_CONNECTION_TIMEOUT, config.get(HTTP_CONNECTION_TIMEOUT).toString());
+ add(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, config.get(HTTP_MAX_CONNECTIONS_PER_HOST).toString());
+ add(HttpClientUtil.PROP_MAX_CONNECTIONS, config.get(HTTP_GLOBAL_MAX_CONNECTIONS).toString());
+ }});
+
+ solrClient = new LBHttpSolrClient(clientParams, config.get(HTTP_URLS));
+
+
+ } else {
+ throw new IllegalArgumentException("Unsupported Solr operation mode: " + mode);
+ }
+ }
+
+ private Map<String, String> parseKeyFieldsForCollections(Configuration config) throws BackendException {
+ Map<String, String> keyFieldNames = new HashMap<String, String>();
+ String[] collectionFieldStatements = config.has(KEY_FIELD_NAMES)?config.get(KEY_FIELD_NAMES):new String[0];
+ for (String collectionFieldStatement : collectionFieldStatements) {
+ String[] parts = collectionFieldStatement.trim().split("=");
+ if (parts.length != 2) {
+ throw new PermanentBackendException("Unable to parse the collection name / key field name pair. It should be of the format collection=field");
+ }
+ String collectionName = parts[0];
+ String keyFieldName = parts[1];
+ keyFieldNames.put(collectionName, keyFieldName);
+ }
+ return keyFieldNames;
+ }
+
+ private String getKeyFieldId(String collection) {
+ String field = keyFieldIds.get(collection);
+ if (field==null) field = DEFAULT_ID_FIELD;
+ return field;
+ }
+
+ /**
+ * Unlike the ElasticSearch Index, which is schema free, Solr requires a schema to
+ * support searching. This means that you will need to modify the solr schema with the
+ * appropriate field definitions in order to work properly. If you have a running instance
+ * of Solr and you modify its schema with new fields, don't forget to re-index!
+ * @param store Index store
+ * @param key New key to register
+ * @param information Datatype to register for the key
+ * @param tx enclosing transaction
+ * @throws com.thinkaurelius.titan.diskstorage.BackendException
+ */
+ @Override
+ public void register(String store, String key, KeyInformation information, BaseTransaction tx) throws BackendException {
+ if (mode==Mode.CLOUD) {
+ CloudSolrClient client = (CloudSolrClient) solrClient;
+ try {
+ createCollectionIfNotExists(client, configuration, store);
+ } catch (IOException e) {
+ throw new PermanentBackendException(e);
+ } catch (SolrServerException e) {
+ throw new PermanentBackendException(e);
+ } catch (InterruptedException e) {
+ throw new PermanentBackendException(e);
+ } catch (KeeperException e) {
+ throw new PermanentBackendException(e);
+ }
+ }
+ //Since all data types must be defined in the schema.xml, pre-registering a type does not work
+ }
+
+ @Override
+ public void mutate(Map<String, Map<String, IndexMutation>> mutations, KeyInformation.IndexRetriever informations, BaseTransaction tx) throws BackendException {
+ logger.debug("Mutating SOLR");
+ try {
+ for (Map.Entry<String, Map<String, IndexMutation>> stores : mutations.entrySet()) {
+ String collectionName = stores.getKey();
+ String keyIdField = getKeyFieldId(collectionName);
+
+ List<String> deleteIds = new ArrayList<String>();
+ Collection<SolrInputDocument> changes = new ArrayList<SolrInputDocument>();
+
+ for (Map.Entry<String, IndexMutation> entry : stores.getValue().entrySet()) {
+ String docId = entry.getKey();
+ IndexMutation mutation = entry.getValue();
+ Preconditions.checkArgument(!(mutation.isNew() && mutation.isDeleted()));
+ Preconditions.checkArgument(!mutation.isNew() || !mutation.hasDeletions());
+ Preconditions.checkArgument(!mutation.isDeleted() || !mutation.hasAdditions());
+
+ //Handle any deletions
+ if (mutation.hasDeletions()) {
+ if (mutation.isDeleted()) {
+ logger.trace("Deleting entire document {}", docId);
+ deleteIds.add(docId);
+ } else {
+ HashSet<IndexEntry> fieldDeletions = Sets.newHashSet(mutation.getDeletions());
+ if (mutation.hasAdditions()) {
+ for (IndexEntry indexEntry : mutation.getAdditions()) {
+ fieldDeletions.remove(indexEntry);
+ }
+ }
+ deleteIndividualFieldsFromIndex(collectionName, keyIdField, docId, fieldDeletions);
+ }
+ }
+
+ if (mutation.hasAdditions()) {
+ int ttl = mutation.determineTTL();
+
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.setField(keyIdField, docId);
+
+ boolean isNewDoc = mutation.isNew();
+
+ if (isNewDoc)
+ logger.trace("Adding new document {}", docId);
+
+ for (IndexEntry e : mutation.getAdditions()) {
+ final Object fieldValue = convertValue(e.value);
+ doc.setField(e.field, isNewDoc
+ ? fieldValue : new HashMap<String, Object>(1) {{ put("set", fieldValue); }});
+ }
+ if (ttl>0) {
+ Preconditions.checkArgument(isNewDoc,"Solr only supports TTL on new documents [%s]",docId);
+ doc.setField(ttlField, String.format("+%dSECONDS", ttl));
+ }
+ changes.add(doc);
+ }
+ }
+
+ commitDeletes(collectionName, deleteIds);
+ commitDocumentChanges(collectionName, changes);
+ }
+ } catch (Exception e) {
+ throw storageException(e);
+ }
+ }
+
+ private Object convertValue(Object value) throws BackendException {
+ if (value instanceof Geoshape)
+ return GeoToWktConverter.convertToWktString((Geoshape) value);
+ // in order to serialize/deserialize properly Solr will have to have an
+ // access to Titan source which has Decimal type, so for now we simply convert to
+ // double and let Solr do the same thing or fail.
+ if (value instanceof AbstractDecimal)
+ return ((AbstractDecimal) value).doubleValue();
+ if (value instanceof UUID)
+ return value.toString();
+ return value;
+ }
+
+ @Override
+ public void restore(Map<String, Map<String, List<IndexEntry>>> documents, KeyInformation.IndexRetriever informations, BaseTransaction tx) throws BackendException {
+ try {
+ for (Map.Entry<String, Map<String, List<IndexEntry>>> stores : documents.entrySet()) {
+ final String collectionName = stores.getKey();
+
+ List<String> deleteIds = new ArrayList<String>();
+ List<SolrInputDocument> newDocuments = new ArrayList<SolrInputDocument>();
+
+ for (Map.Entry<String, List<IndexEntry>> entry : stores.getValue().entrySet()) {
+ final String docID = entry.getKey();
+ final List<IndexEntry> content = entry.getValue();
+
+ if (content == null || content.isEmpty()) {
+ if (logger.isTraceEnabled())
+ logger.trace("Deleting document [{}]", docID);
+
+ deleteIds.add(docID);
+ continue;
+ }
+
+ newDocuments.add(new SolrInputDocument() {{
+ setField(getKeyFieldId(collectionName), docID);
+
+ for (IndexEntry addition : content) {
+ Object fieldValue = addition.value;
+ setField(addition.field, convertValue(fieldValue));
+ }
+ }});
+ }
+
+ commitDeletes(collectionName, deleteIds);
+ commitDocumentChanges(collectionName, newDocuments);
+ }
+ } catch (Exception e) {
+ throw new TemporaryBackendException("Could not restore Solr index", e);
+ }
+ }
+
+ private void deleteIndividualFieldsFromIndex(String collectionName, String keyIdField, String docId, HashSet<IndexEntry> fieldDeletions) throws SolrServerException, IOException {
+ if (fieldDeletions.isEmpty()) return;
+
+ Map<String, String> fieldDeletes = new HashMap<String, String>(1) {{ put("set", null); }};
+
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField(keyIdField, docId);
+ StringBuilder sb = new StringBuilder();
+ for (IndexEntry fieldToDelete : fieldDeletions) {
+ doc.addField(fieldToDelete.field, fieldDeletes);
+ sb.append(fieldToDelete).append(",");
+ }
+
+ if (logger.isTraceEnabled())
+ logger.trace("Deleting individual fields [{}] for document {}", sb.toString(), docId);
+
+ UpdateRequest singleDocument = newUpdateRequest();
+ singleDocument.add(doc);
+ solrClient.request(singleDocument, collectionName);
+ }
+
+ private void commitDocumentChanges(String collectionName, Collection<SolrInputDocument> documents) throws SolrServerException, IOException {
+ if (documents.size() == 0) return;
+
+ try {
+ solrClient.request(newUpdateRequest().add(documents), collectionName);
+ } catch (HttpSolrClient.RemoteSolrException rse) {
+ logger.error("Unable to save documents to Solr as one of the shape objects stored were not compatible with Solr.", rse);
+ logger.error("Details in failed document batch: ");
+ for (SolrInputDocument d : documents) {
+ Collection<String> fieldNames = d.getFieldNames();
+ for (String name : fieldNames) {
+ logger.error(name + ":" + d.getFieldValue(name).toString());
+ }
+ }
+
+ throw rse;
+ }
+ }
+
+ private void commitDeletes(String collectionName, List<String> deleteIds) throws SolrServerException, IOException {
+ if (deleteIds.size() == 0) return;
+ solrClient.request(newUpdateRequest().deleteById(deleteIds), collectionName);
+ }
+
+ @Override
+ public List<String> query(IndexQuery query, KeyInformation.IndexRetriever informations, BaseTransaction tx) throws BackendException {
+ List<String> result;
+ String collection = query.getStore();
+ String keyIdField = getKeyFieldId(collection);
+ SolrQuery solrQuery = new SolrQuery("*:*");
+ String queryFilter = buildQueryFilter(query.getCondition(), informations.get(collection));
+ solrQuery.addFilterQuery(queryFilter);
+ if (!query.getOrder().isEmpty()) {
+ List<IndexQuery.OrderEntry> orders = query.getOrder();
+ for (IndexQuery.OrderEntry order1 : orders) {
+ String item = order1.getKey();
+ SolrQuery.ORDER order = order1.getOrder() == Order.ASC ? SolrQuery.ORDER.asc : SolrQuery.ORDER.desc;
+ solrQuery.addSort(new SolrQuery.SortClause(item, order));
+ }
+ }
+ solrQuery.setStart(0);
+ if (query.hasLimit()) {
+ solrQuery.setRows(query.getLimit());
+ } else {
+ solrQuery.setRows(maxResults);
+ }
+ try {
+ QueryResponse response = solrClient.query(collection, solrQuery);
+
+ if (logger.isDebugEnabled())
+ logger.debug("Executed query [{}] in {} ms", query.getCondition(), response.getElapsedTime());
+
+ int totalHits = response.getResults().size();
+
+ if (!query.hasLimit() && totalHits >= maxResults)
+ logger.warn("Query result set truncated to first [{}] elements for query: {}", maxResults, query);
+
+ result = new ArrayList<String>(totalHits);
+ for (SolrDocument hit : response.getResults()) {
+ result.add(hit.getFieldValue(keyIdField).toString());
+ }
+ } catch (IOException e) {
+ logger.error("Query did not complete : ", e);
+ throw new PermanentBackendException(e);
+ } catch (SolrServerException e) {
+ logger.error("Unable to query Solr index.", e);
+ throw new PermanentBackendException(e);
+ }
+ return result;
+ }
+
+ @Override
+ public Iterable<RawQuery.Result<String>> query(RawQuery query, KeyInformation.IndexRetriever informations, BaseTransaction tx) throws BackendException {
+ List<RawQuery.Result<String>> result;
+ String collection = query.getStore();
+ String keyIdField = getKeyFieldId(collection);
+ SolrQuery solrQuery = new SolrQuery(query.getQuery())
+ .addField(keyIdField)
+ .setIncludeScore(true)
+ .setStart(query.getOffset())
+ .setRows(query.hasLimit() ? query.getLimit() : maxResults);
+
+ try {
+ QueryResponse response = solrClient.query(collection, solrQuery);
+ if (logger.isDebugEnabled())
+ logger.debug("Executed query [{}] in {} ms", query.getQuery(), response.getElapsedTime());
+
+ int totalHits = response.getResults().size();
+ if (!query.hasLimit() && totalHits >= maxResults) {
+ logger.warn("Query result set truncated to first [{}] elements for query: {}", maxResults, query);
+ }
+ result = new ArrayList<RawQuery.Result<String>>(totalHits);
+
+ for (SolrDocument hit : response.getResults()) {
+ double score = Double.parseDouble(hit.getFieldValue("score").toString());
+ result.add(new RawQuery.Result<String>(hit.getFieldValue(keyIdField).toString(), score));
+ }
+ } catch (IOException e) {
+ logger.error("Query did not complete : ", e);
+ throw new PermanentBackendException(e);
+ } catch (SolrServerException e) {
+ logger.error("Unable to query Solr index.", e);
+ throw new PermanentBackendException(e);
+ }
+ return result;
+ }
+
+ private static String escapeValue(Object value) {
+ return ClientUtils.escapeQueryChars(value.toString());
+ }
+
+ public String buildQueryFilter(Condition<TitanElement> condition, KeyInformation.StoreRetriever informations) {
+ if (condition instanceof PredicateCondition) {
+ PredicateCondition<String, TitanElement> atom = (PredicateCondition<String, TitanElement>) condition;
+ Object value = atom.getValue();
+ String key = atom.getKey();
+ TitanPredicate titanPredicate = atom.getPredicate();
+
+ if (value instanceof Number) {
+ String queryValue = escapeValue(value);
+ Preconditions.checkArgument(titanPredicate instanceof Cmp, "Relation not supported on numeric types: " + titanPredicate);
+ Cmp numRel = (Cmp) titanPredicate;
+ switch (numRel) {
+ case EQUAL:
+ return (key + ":" + queryValue);
+ case NOT_EQUAL:
+ return ("-" + key + ":" + queryValue);
+ case LESS_THAN:
+ //use right curly to mean up to but not including value
+ return (key + ":[* TO " + queryValue + "}");
+ case LESS_THAN_EQUAL:
+ return (key + ":[* TO " + queryValue + "]");
+ case GREATER_THAN:
+ //use left curly to mean greater than but not including value
+ return (key + ":{" + queryValue + " TO *]");
+ case GREATER_THAN_EQUAL:
+ return (key + ":[" + queryValue + " TO *]");
+ default: throw new IllegalArgumentException("Unexpected relation: " + numRel);
+ }
+ } else if (value instanceof String) {
+ Mapping map = getStringMapping(informations.get(key));
+ assert map== TEXT || map== STRING;
+ if (map== TEXT && !titanPredicate.toString().startsWith("CONTAINS"))
+ throw new IllegalArgumentException("Text mapped string values only support CONTAINS queries and not: " + titanPredicate);
+ if (map== STRING && titanPredicate.toString().startsWith("CONTAINS"))
+ throw new IllegalArgumentException("String mapped string values do not support CONTAINS queries: " + titanPredicate);
+
+ //Special case
+ if (titanPredicate == Text.CONTAINS) {
+ //e.g. - if terms tomorrow and world were supplied, and fq=text:(tomorrow world)
+ //sample data set would return 2 documents: one where text = Tomorrow is the World,
+ //and the second where text = Hello World. Hence, we are decomposing the query string
+ //and building an AND query explicitly because we need AND semantics
+ value = ((String) value).toLowerCase();
+ List<String> terms = Text.tokenize((String) value);
+
+ if (terms.isEmpty()) {
+ return "";
+ } else if (terms.size() == 1) {
+ return (key + ":(" + escapeValue(terms.get(0)) + ")");
+ } else {
+ And<TitanElement> andTerms = new And<TitanElement>();
+ for (String term : terms) {
+ andTerms.add(new PredicateCondition<String, TitanElement>(key, titanPredicate, term));
+ }
+ return buildQueryFilter(andTerms, informations);
+ }
+ }
+ if (titanPredicate == Text.PREFIX || titanPredicate == Text.CONTAINS_PREFIX) {
+ return (key + ":" + escapeValue(value) + "*");
+ } else if (titanPredicate == Text.REGEX || titanPredicate == Text.CONTAINS_REGEX) {
+ return (key + ":/" + value + "/");
+ } else if (titanPredicate == EQUAL) {
+ return (key + ":\"" + escapeValue(value) + "\"");
+ } else if (titanPredicate == NOT_EQUAL) {
+ return ("-" + key + ":\"" + escapeValue(value) + "\"");
+ } else {
+ throw new IllegalArgumentException("Relation is not supported for string value: " + titanPredicate);
+ }
+ } else if (value instanceof Geoshape) {
+ Geoshape geo = (Geoshape)value;
+ if (geo.getType() == Geoshape.Type.CIRCLE) {
+ Geoshape.Point center = geo.getPoint();
+ return ("{!geofilt sfield=" + key +
+ " pt=" + center.getLatitude() + "," + center.getLongitude() +
+ " d=" + geo.getRadius() + "} distErrPct=0"); //distance in kilometers
+ } else if (geo.getType() == Geoshape.Type.BOX) {
+ Geoshape.Point southwest = geo.getPoint(0);
+ Geoshape.Point northeast = geo.getPoint(1);
+ return (key + ":[" + southwest.getLatitude() + "," + southwest.getLongitude() +
+ " TO " + northeast.getLatitude() + "," + northeast.getLongitude() + "]");
+ } else if (geo.getType() == Geoshape.Type.POLYGON) {
+ List<Geoshape.Point> coordinates = getPolygonPoints(geo);
+ StringBuilder poly = new StringBuilder(key + ":\"IsWithin(POLYGON((");
+ for (Geoshape.Point coordinate : coordinates) {
+ poly.append(coordinate.getLongitude()).append(" ").append(coordinate.getLatitude()).append(", ");
+ }
+ //close the polygon with the first coordinate
+ poly.append(coordinates.get(0).getLongitude()).append(" ").append(coordinates.get(0).getLatitude());
+ poly.append(")))\" distErrPct=0");
+ return (poly.toString());
+ }
+ } else if (value instanceof Date) {
+ String queryValue = escapeValue(toIsoDate((Date)value));
+ Preconditions.checkArgument(titanPredicate instanceof Cmp, "Relation not supported on date types: " + titanPredicate);
+ Cmp numRel = (Cmp) titanPredicate;
+
+ switch (numRel) {
+ case EQUAL:
+ return (key + ":" + queryValue);
+ case NOT_EQUAL:
+ return ("-" + key + ":" + queryValue);
+ case LESS_THAN:
+ //use right curly to mean up to but not including value
+ return (key + ":[* TO " + queryValue + "}");
+ case LESS_THAN_EQUAL:
+ return (key + ":[* TO " + queryValue + "]");
+ case GREATER_THAN:
+ //use left curly to mean greater than but not including value
+ return (key + ":{" + queryValue + " TO *]");
+ case GREATER_THAN_EQUAL:
+ return (key + ":[" + queryValue + " TO *]");
+ default: throw new IllegalArgumentException("Unexpected relation: " + numRel);
+ }
+ } else if (value instanceof Boolean) {
+ Cmp numRel = (Cmp) titanPredicate;
+ String queryValue = escapeValue(value);
+ switch (numRel) {
+ case EQUAL:
+ return (key + ":" + queryValue);
+ case NOT_EQUAL:
+ return ("-" + key + ":" + queryValue);
+ default:
+ throw new IllegalArgumentException("Boolean types only support EQUAL or NOT_EQUAL");
+ }
+ } else if (value instanceof UUID) {
+ if (titanPredicate == EQUAL) {
+ return (key + ":\"" + escapeValue(value) + "\"");
+ } else if (titanPredicate == NOT_EQUAL) {
+ return ("-" + key + ":\"" + escapeValue(value) + "\"");
+ } else {
+ throw new IllegalArgumentException("Relation is not supported for uuid value: " + titanPredicate);
+ }
+ } else throw new IllegalArgumentException("Unsupported type: " + value);
+ } else if (condition instanceof Not) {
+ String sub = buildQueryFilter(((Not)condition).getChild(),informations);
+ if (StringUtils.isNotBlank(sub)) return "-("+sub+")";
+ else return "";
+ } else if (condition instanceof And) {
+ int numChildren = ((And) condition).size();
+ StringBuilder sb = new StringBuilder();
+ for (Condition<TitanElement> c : condition.getChildren()) {
+ String sub = buildQueryFilter(c, informations);
+
+ if (StringUtils.isBlank(sub))
+ continue;
+
+ // we don't have to add "+" which means AND iff
+ // a. it's a NOT query,
+ // b. expression is a single statement in the AND.
+ if (!sub.startsWith("-") && numChildren > 1)
+ sb.append("+");
+
+ sb.append(sub).append(" ");
+ }
+ return sb.toString();
+ } else if (condition instanceof Or) {
+ StringBuilder sb = new StringBuilder();
+ int element=0;
+ for (Condition<TitanElement> c : condition.getChildren()) {
+ String sub = buildQueryFilter(c,informations);
+ if (StringUtils.isBlank(sub)) continue;
+ if (element==0) sb.append("(");
+ else sb.append(" OR ");
+ sb.append(sub);
+ element++;
+ }
+ if (element>0) sb.append(")");
+ return sb.toString();
+ } else {
+ throw new IllegalArgumentException("Invalid condition: " + condition);
+ }
+ return null;
+ }
+
+ private String toIsoDate(Date value) {
+ TimeZone tz = TimeZone.getTimeZone("UTC");
+ DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
+ df.setTimeZone(tz);
+ return df.format(value);
+ }
+
+ private List<Geoshape.Point> getPolygonPoints(Geoshape polygon) {
+ List<Geoshape.Point> locations = new ArrayList<Geoshape.Point>();
+
+ int index = 0;
+ boolean hasCoordinates = true;
+ while (hasCoordinates) {
+ try {
+ locations.add(polygon.getPoint(index));
+ } catch (ArrayIndexOutOfBoundsException ignore) {
+ //just means we asked for a point past the size of the list
+ //of known coordinates
+ hasCoordinates = false;
+ }
+ }
+
+ return locations;
+ }
+
+ /**
+ * Solr handles all transactions on the server-side. That means all
+ * commit, optimize, or rollback applies since the last commit/optimize/rollback.
+ * Solr documentation recommends best way to update Solr is in one process to avoid
+ * race conditions.
+ *
+ * @return New Transaction Handle
+ * @throws com.thinkaurelius.titan.diskstorage.BackendException
+ */
+ @Override
+ public BaseTransactionConfigurable beginTransaction(BaseTransactionConfig config) throws BackendException {
+ return new DefaultTransaction(config);
+ }
+
+ @Override
+ public void close() throws BackendException {
+ logger.trace("Shutting down connection to Solr", solrClient);
+ try {
+ solrClient.close();
+ } catch (IOException e) {
+ throw new TemporaryBackendException(e);
+ }
+ }
+
+ @Override
+ public void clearStorage() throws BackendException {
+ try {
+ if (mode!=Mode.CLOUD) throw new UnsupportedOperationException("Operation only supported for SolrCloud");
+ logger.debug("Clearing storage from Solr: {}", solrClient);
+ ZkStateReader zkStateReader = ((CloudSolrClient) solrClient).getZkStateReader();
+ zkStateReader.updateClusterState();
+ ClusterState clusterState = zkStateReader.getClusterState();
+ for (String collection : clusterState.getCollections()) {
+ logger.debug("Clearing collection [{}] in Solr",collection);
+ UpdateRequest deleteAll = newUpdateRequest();
+ deleteAll.deleteByQuery("*:*");
+ solrClient.request(deleteAll, collection);
+ }
+
+ } catch (SolrServerException e) {
+ logger.error("Unable to clear storage from index due to server error on Solr.", e);
+ throw new PermanentBackendException(e);
+ } catch (IOException e) {
+ logger.error("Unable to clear storage from index due to low-level I/O error.", e);
+ throw new PermanentBackendException(e);
+ } catch (Exception e) {
+ logger.error("Unable to clear storage from index due to general error.", e);
+ throw new PermanentBackendException(e);
+ }
+ }
+
+ @Override
+ public boolean supports(KeyInformation information, TitanPredicate titanPredicate) {
+ Class<?> dataType = information.getDataType();
+ Mapping mapping = getMapping(information);
+ if (mapping!= DEFAULT && !AttributeUtil.isString(dataType)) return false;
+
+ if (Number.class.isAssignableFrom(dataType)) {
+ return titanPredicate instanceof Cmp;
+ } else if (dataType == Geoshape.class) {
+ return titanPredicate == Geo.WITHIN;
+ } else if (AttributeUtil.isString(dataType)) {
+ switch(mapping) {
+ case DEFAULT:
+ case TEXT:
+ return titanPredicate == Text.CONTAINS || titanPredicate == Text.CONTAINS_PREFIX || titanPredicate == Text.CONTAINS_REGEX;
+ case STRING:
+ return titanPredicate == EQUAL || titanPredicate== NOT_EQUAL || titanPredicate==Text.REGEX || titanPredicate==Text.PREFIX;
+ // case TEXTSTRING:
+ // return (titanPredicate instanceof Text) || titanPredicate == Cmp.EQUAL || titanPredicate==Cmp.NOT_EQUAL;
+ }
+ } else if (dataType == Date.class) {
+ if (titanPredicate instanceof Cmp) return true;
+ } else if (dataType == Boolean.class) {
+ return titanPredicate == EQUAL || titanPredicate == NOT_EQUAL;
+ } else if (dataType == UUID.class) {
+ return titanPredicate == EQUAL || titanPredicate== NOT_EQUAL;
+ }
+ return false;
+ }
+
+ @Override
+ public boolean supports(KeyInformation information) {
+ Class<?> dataType = information.getDataType();
+ Mapping mapping = getMapping(information);
+ if (Number.class.isAssignableFrom(dataType) || dataType == Geoshape.class || dataType == Date.class || dataType == Boolean.class || dataType == UUID.class) {
+ if (mapping== DEFAULT) return true;
+ } else if (AttributeUtil.isString(dataType)) {
+ if (mapping== DEFAULT || mapping== TEXT || mapping== STRING) return true;
+ }
+ return false;
+ }
+
+ @Override
+ public String mapKey2Field(String key, KeyInformation keyInfo) {
+ Preconditions.checkArgument(!StringUtils.containsAny(key, new char[]{' '}),"Invalid key name provided: %s",key);
+ if (!dynFields) return key;
+ if (ParameterType.MAPPED_NAME.hasParameter(keyInfo.getParameters())) return key;
+ String postfix;
+ Class datatype = keyInfo.getDataType();
+ if (AttributeUtil.isString(datatype)) {
+ Mapping map = getStringMapping(keyInfo);
+ switch (map) {
+ case TEXT: postfix = "_t"; break;
+ case STRING: postfix = "_s"; break;
+ default: throw new IllegalArgumentException("Unsupported string mapping: " + map);
+ }
+ } else if (AttributeUtil.isWholeNumber(datatype)) {
+ if (datatype.equals(Long.class)) postfix = "_l";
+ else postfix = "_i";
+ } else if (AttributeUtil.isDecimal(datatype)) {
+ if (datatype.equals(Float.class)) postfix = "_f";
+ else postfix = "_d";
+ } else if (datatype.equals(Geoshape.class)) {
+ postfix = "_g";
+ } else if (datatype.equals(Date.class)) {
+ postfix = "_dt";
+ } else if (datatype.equals(Boolean.class)) {
+ postfix = "_b";
+ } else if (datatype.equals(UUID.class)) {
+ postfix = "_uuid";
+ } else throw new IllegalArgumentException("Unsupported data type ["+datatype+"] for field: " + key);
+ return key+postfix;
+ }
+
+ @Override
+ public IndexFeatures getFeatures() {
+ return SOLR_FEATURES;
+ }
+
+ /*
+ ################# UTILITY METHODS #######################
+ */
+
+ private static Mapping getStringMapping(KeyInformation information) {
+ assert AttributeUtil.isString(information.getDataType());
+ Mapping map = getMapping(information);
+ if (map== DEFAULT) map = TEXT;
+ return map;
+ }
+
+ private UpdateRequest newUpdateRequest() {
+ UpdateRequest req = new UpdateRequest();
+ req.setAction(UpdateRequest.ACTION.COMMIT, true, true);
+ if (waitSearcher) {
+ req.setAction(UpdateRequest.ACTION.COMMIT, true, true);
+ }
+ return req;
+ }
+
+ private BackendException storageException(Exception solrException) {
+ return new TemporaryBackendException("Unable to complete query on Solr.", solrException);
+ }
+
+ private static void createCollectionIfNotExists(CloudSolrClient client, Configuration config, String collection)
+ throws IOException, SolrServerException, KeeperException, InterruptedException {
+ if (!checkIfCollectionExists(client, collection)) {
+ Integer numShards = config.get(NUM_SHARDS);
+ Integer maxShardsPerNode = config.get(MAX_SHARDS_PER_NODE);
+ Integer replicationFactor = config.get(REPLICATION_FACTOR);
+
+ CollectionAdminRequest.Create createRequest = new CollectionAdminRequest.Create();
+
+ createRequest.setConfigName(collection);
+ createRequest.setCollectionName(collection);
+ createRequest.setNumShards(numShards);
+ createRequest.setMaxShardsPerNode(maxShardsPerNode);
+ createRequest.setReplicationFactor(replicationFactor);
+
+ CollectionAdminResponse createResponse = createRequest.process(client);
+ if (createResponse.isSuccess()) {
+ logger.trace("Collection {} successfully created.", collection);
+ } else {
+ throw new SolrServerException(Joiner.on("\n").join(createResponse.getErrorMessages()));
+ }
+ }
+
+ waitForRecoveriesToFinish(client, collection);
+ }
+
+ /**
+ * Checks if the collection has already been created in Solr.
+ */
+ private static boolean checkIfCollectionExists(CloudSolrClient server, String collection) throws KeeperException, InterruptedException {
+ ZkStateReader zkStateReader = server.getZkStateReader();
+ zkStateReader.updateClusterState();
+ ClusterState clusterState = zkStateReader.getClusterState();
+ return clusterState.getCollectionOrNull(collection) != null;
+ }
+
+ /**
+ * Wait for all the collection shards to be ready.
+ */
+ private static void waitForRecoveriesToFinish(CloudSolrClient server, String collection) throws KeeperException, InterruptedException {
+ ZkStateReader zkStateReader = server.getZkStateReader();
+ try {
+ boolean cont = true;
+
+ while (cont) {
+ boolean sawLiveRecovering = false;
+ zkStateReader.updateClusterState();
+ ClusterState clusterState = zkStateReader.getClusterState();
+ Map<String, Slice> slices = clusterState.getSlicesMap(collection);
+ Preconditions.checkNotNull("Could not find collection:" + collection, slices);
+
+ for (Map.Entry<String, Slice> entry : slices.entrySet()) {
+ Map<String, Replica> shards = entry.getValue().getReplicasMap();
+ for (Map.Entry<String, Replica> shard : shards.entrySet()) {
+ String state = shard.getValue().getStr(ZkStateReader.STATE_PROP);
+ if ((state.equals(Replica.State.RECOVERING)
+ || state.equals(Replica.State.DOWN))
+ && clusterState.liveNodesContain(shard.getValue().getStr(
+ ZkStateReader.NODE_NAME_PROP))) {
+ sawLiveRecovering = true;
+ }
+ }
+ }
+ if (!sawLiveRecovering) {
+ cont = false;
+ } else {
+ Thread.sleep(1000);
+ }
+ }
+ } finally {
+ logger.info("Exiting solr wait");
+ }
+ }
+
+ private static class GeoToWktConverter {
+ /**
+ * {@link com.thinkaurelius.titan.core.attribute.Geoshape} stores Points in the String format: point[X.0,Y.0].
+ * Solr needs it to be in Well-Known Text format: POINT(X.0 Y.0)
+ */
+ static String convertToWktString(Geoshape fieldValue) throws BackendException {
+ if (fieldValue.getType() == Geoshape.Type.POINT) {
+ Geoshape.Point point = fieldValue.getPoint();
+ return "POINT(" + point.getLongitude() + " " + point.getLatitude() + ")";
+ } else {
+ throw new PermanentBackendException("Cannot index " + fieldValue.getType());
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/graphdb/query/graph/GraphCentricQueryBuilder.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/graphdb/query/graph/GraphCentricQueryBuilder.java b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/graphdb/query/graph/GraphCentricQueryBuilder.java
new file mode 100644
index 0000000..c1a983b
--- /dev/null
+++ b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/graphdb/query/graph/GraphCentricQueryBuilder.java
@@ -0,0 +1,457 @@
+/*
+ * Copyright 2012-2013 Aurelius LLC
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.thinkaurelius.titan.graphdb.query.graph;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.*;
+import com.thinkaurelius.titan.core.*;
+import com.thinkaurelius.titan.core.attribute.Cmp;
+import com.thinkaurelius.titan.core.Cardinality;
+import com.thinkaurelius.titan.core.schema.SchemaStatus;
+import com.thinkaurelius.titan.core.schema.TitanSchemaType;
+import com.thinkaurelius.titan.graphdb.database.IndexSerializer;
+import com.thinkaurelius.titan.graphdb.internal.ElementCategory;
+import com.thinkaurelius.titan.graphdb.internal.InternalRelationType;
+import com.thinkaurelius.titan.graphdb.internal.OrderList;
+import com.thinkaurelius.titan.graphdb.query.*;
+import com.thinkaurelius.titan.graphdb.query.condition.*;
+import com.thinkaurelius.titan.graphdb.transaction.StandardTitanTx;
+import com.thinkaurelius.titan.graphdb.types.*;
+import com.thinkaurelius.titan.graphdb.types.system.ImplicitKey;
+import com.thinkaurelius.titan.util.datastructures.Interval;
+import com.thinkaurelius.titan.util.datastructures.PointInterval;
+import com.tinkerpop.blueprints.Edge;
+import com.tinkerpop.blueprints.Vertex;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+import java.util.*;
+
+/**
+ *
+ * Builds a {@link TitanGraphQuery}, optimizes the query and compiles the result into a {@link GraphCentricQuery} which
+ * is then executed through a {@link QueryProcessor}.
+ * This class from titan-0.5.4 has no major changes except adding a few logs for debugging index usage
+ *
+ * @author Matthias Broecheler (me@matthiasb.com)
+ */
+public class GraphCentricQueryBuilder implements TitanGraphQuery<GraphCentricQueryBuilder> {
+
+ private static final Logger log = LoggerFactory.getLogger(GraphCentricQueryBuilder.class);
+
+ /**
+ * Transaction in which this query is executed.
+ */
+ private final StandardTitanTx tx;
+ /**
+ * Serializer used to serialize the query conditions into backend queries.
+ */
+ private final IndexSerializer serializer;
+ /**
+ * The constraints added to this query. None by default.
+ */
+ private List<PredicateCondition<String, TitanElement>> constraints;
+ /**
+ * The order in which the elements should be returned. None by default.
+ */
+ private OrderList orders = new OrderList();
+ /**
+ * The limit of this query. No limit by default.
+ */
+ private int limit = Query.NO_LIMIT;
+
+ public GraphCentricQueryBuilder(StandardTitanTx tx, IndexSerializer serializer) {
+ log.debug("Loaded shaded version of class GraphCentricQueryBuilder");
+ Preconditions.checkNotNull(tx);
+ Preconditions.checkNotNull(serializer);
+ this.tx = tx;
+ this.serializer = serializer;
+ this.constraints = new ArrayList<PredicateCondition<String, TitanElement>>(5);
+ }
+
+ /* ---------------------------------------------------------------
+ * Query Construction
+ * ---------------------------------------------------------------
+ */
+
+ private GraphCentricQueryBuilder has(String key, TitanPredicate predicate, Object condition) {
+ Preconditions.checkNotNull(key);
+ Preconditions.checkNotNull(predicate);
+ Preconditions.checkArgument(predicate.isValidCondition(condition), "Invalid condition: %s", condition);
+ constraints.add(new PredicateCondition<String, TitanElement>(key, predicate, condition));
+ return this;
+ }
+
+ @Override
+ public GraphCentricQueryBuilder has(String key, com.tinkerpop.blueprints.Predicate predicate, Object condition) {
+ Preconditions.checkNotNull(key);
+ TitanPredicate titanPredicate = TitanPredicate.Converter.convert(predicate);
+ return has(key, titanPredicate, condition);
+ }
+
+ @Override
+ public GraphCentricQueryBuilder has(PropertyKey key, TitanPredicate predicate, Object condition) {
+ Preconditions.checkNotNull(key);
+ Preconditions.checkNotNull(predicate);
+ return has(key.getName(), predicate, condition);
+ }
+
+ @Override
+ public GraphCentricQueryBuilder has(String key) {
+ return has(key, Cmp.NOT_EQUAL, (Object) null);
+ }
+
+ @Override
+ public GraphCentricQueryBuilder hasNot(String key) {
+ return has(key, Cmp.EQUAL, (Object) null);
+ }
+
+ @Override
+ @Deprecated
+ public <T extends Comparable<T>> GraphCentricQueryBuilder has(String s, T t, Compare compare) {
+ return has(s, compare, t);
+ }
+
+ @Override
+ public GraphCentricQueryBuilder has(String key, Object value) {
+ return has(key, Cmp.EQUAL, value);
+ }
+
+ @Override
+ public GraphCentricQueryBuilder hasNot(String key, Object value) {
+ return has(key, Cmp.NOT_EQUAL, value);
+ }
+
+ @Override
+ public <T extends Comparable<?>> GraphCentricQueryBuilder interval(String s, T t1, T t2) {
+ has(s, Cmp.GREATER_THAN_EQUAL, t1);
+ return has(s, Cmp.LESS_THAN, t2);
+ }
+
+ @Override
+ public GraphCentricQueryBuilder limit(final int limit) {
+ Preconditions.checkArgument(limit >= 0, "Non-negative limit expected: %s", limit);
+ this.limit = limit;
+ return this;
+ }
+
+ @Override
+ public GraphCentricQueryBuilder orderBy(String key, Order order) {
+ Preconditions.checkArgument(tx.containsPropertyKey(key),"Provided key does not exist: %s",key);
+ return orderBy(tx.getPropertyKey(key), order);
+ }
+
+ @Override
+ public GraphCentricQueryBuilder orderBy(PropertyKey key, Order order) {
+ Preconditions.checkArgument(key!=null && order!=null,"Need to specify and key and an order");
+ Preconditions.checkArgument(Comparable.class.isAssignableFrom(key.getDataType()),
+ "Can only order on keys with comparable data type. [%s] has datatype [%s]", key.getName(), key.getDataType());
+ Preconditions.checkArgument(key.getCardinality()== Cardinality.SINGLE, "Ordering is undefined on multi-valued key [%s]", key.getName());
+ Preconditions.checkArgument(!orders.containsKey(key));
+ orders.add(key, order);
+ return this;
+ }
+
+ /* ---------------------------------------------------------------
+ * Query Execution
+ * ---------------------------------------------------------------
+ */
+
+ @Override
+ public Iterable<Vertex> vertices() {
+ GraphCentricQuery query = constructQuery(ElementCategory.VERTEX);
+ return Iterables.filter(new QueryProcessor<GraphCentricQuery, TitanElement, JointIndexQuery>(query, tx.elementProcessor), Vertex.class);
+ }
+
+ @Override
+ public Iterable<Edge> edges() {
+ GraphCentricQuery query = constructQuery(ElementCategory.EDGE);
+ return Iterables.filter(new QueryProcessor<GraphCentricQuery, TitanElement, JointIndexQuery>(query, tx.elementProcessor), Edge.class);
+ }
+
+ @Override
+ public Iterable<TitanProperty> properties() {
+ GraphCentricQuery query = constructQuery(ElementCategory.PROPERTY);
+ return Iterables.filter(new QueryProcessor<GraphCentricQuery, TitanElement, JointIndexQuery>(query, tx.elementProcessor), TitanProperty.class);
+ }
+
+ private QueryDescription describe(ElementCategory category) {
+ return new StandardQueryDescription(1,constructQuery(category));
+ }
+
+ public QueryDescription describeForVertices() {
+ return describe(ElementCategory.VERTEX);
+ }
+
+ public QueryDescription describeForEdges() {
+ return describe(ElementCategory.EDGE);
+ }
+
+ public QueryDescription describeForProperties() {
+ return describe(ElementCategory.PROPERTY);
+ }
+
+ /* ---------------------------------------------------------------
+ * Query Construction
+ * ---------------------------------------------------------------
+ */
+
+ private static final int DEFAULT_NO_LIMIT = 1000;
+ private static final int MAX_BASE_LIMIT = 20000;
+ private static final int HARD_MAX_LIMIT = 100000;
+
+ private static final double EQUAL_CONDITION_SCORE = 4;
+ private static final double OTHER_CONDITION_SCORE = 1;
+ private static final double ORDER_MATCH = 2;
+ private static final double ALREADY_MATCHED_ADJUSTOR = 0.1;
+ private static final double CARDINALITY_SINGE_SCORE = 1000;
+ private static final double CARDINALITY_OTHER_SCORE = 1000;
+
+ public GraphCentricQuery constructQuery(final ElementCategory resultType) {
+ Preconditions.checkNotNull(resultType);
+ if (limit == 0) return GraphCentricQuery.emptyQuery(resultType);
+
+ //Prepare constraints
+ And<TitanElement> conditions = QueryUtil.constraints2QNF(tx, constraints);
+ if (conditions == null) return GraphCentricQuery.emptyQuery(resultType);
+
+ //Prepare orders
+ orders.makeImmutable();
+ if (orders.isEmpty()) orders = OrderList.NO_ORDER;
+
+ //Compile all indexes that cover at least one of the query conditions
+ final Set<IndexType> indexCandidates = new HashSet<IndexType>();
+ ConditionUtil.traversal(conditions,new Predicate<Condition<TitanElement>>() {
+ @Override
+ public boolean apply(@Nullable Condition<TitanElement> condition) {
+ if (condition instanceof PredicateCondition) {
+ RelationType type = ((PredicateCondition<RelationType,TitanElement>)condition).getKey();
+ Preconditions.checkArgument(type!=null && type.isPropertyKey());
+ Iterables.addAll(indexCandidates,Iterables.filter(((InternalRelationType) type).getKeyIndexes(), new Predicate<IndexType>() {
+ @Override
+ public boolean apply(@Nullable IndexType indexType) {
+ return indexType.getElement()==resultType;
+ }
+ }));
+ }
+ return true;
+ }
+ });
+
+ /*
+ Determine the best join index query to answer this query:
+ Iterate over all potential indexes (as compiled above) and compute a score based on how many clauses
+ this index covers. The index with the highest score (as long as it covers at least one additional clause)
+ is picked and added to the joint query for as long as such exist.
+ */
+ JointIndexQuery jointQuery = new JointIndexQuery();
+ boolean isSorted = orders.isEmpty();
+ Set<Condition> coveredClauses = Sets.newHashSet();
+ while (true) {
+ IndexType bestCandidate = null;
+ double candidateScore = 0.0;
+ Set<Condition> candidateSubcover = null;
+ boolean candidateSupportsSort = false;
+ Object candidateSubcondition = null;
+
+ for (IndexType index : indexCandidates) {
+ Set<Condition> subcover = Sets.newHashSet();
+ Object subcondition;
+ boolean supportsSort = orders.isEmpty();
+ //Check that this index actually applies in case of a schema constraint
+ if (index.hasSchemaTypeConstraint()) {
+ TitanSchemaType type = index.getSchemaTypeConstraint();
+ Map.Entry<Condition,Collection<Object>> equalCon = getEqualityConditionValues(conditions,ImplicitKey.LABEL);
+ if (equalCon==null) continue;
+ Collection<Object> labels = equalCon.getValue();
+ assert labels.size()>=1;
+ if (labels.size()>1) {
+ log.warn("The query optimizer currently does not support multiple label constraints in query: {}",this);
+ continue;
+ }
+ if (!type.getName().equals((String)Iterables.getOnlyElement(labels))) continue;
+ subcover.add(equalCon.getKey());
+ }
+
+ if (index.isCompositeIndex()) {
+ subcondition = indexCover((CompositeIndexType) index,conditions,subcover);
+ } else {
+ subcondition = indexCover((MixedIndexType) index,conditions,serializer,subcover);
+ if (coveredClauses.isEmpty() && !supportsSort
+ && indexCoversOrder((MixedIndexType)index,orders)) supportsSort=true;
+ }
+ if (subcondition==null) continue;
+ assert !subcover.isEmpty();
+ double score = 0.0;
+ boolean coversAdditionalClause = false;
+ for (Condition c : subcover) {
+ double s = (c instanceof PredicateCondition && ((PredicateCondition)c).getPredicate()==Cmp.EQUAL)?
+ EQUAL_CONDITION_SCORE:OTHER_CONDITION_SCORE;
+ if (coveredClauses.contains(c)) s=s*ALREADY_MATCHED_ADJUSTOR;
+ else coversAdditionalClause = true;
+ score+=s;
+ if (index.isCompositeIndex())
+ score+=((CompositeIndexType)index).getCardinality()==Cardinality.SINGLE?
+ CARDINALITY_SINGE_SCORE:CARDINALITY_OTHER_SCORE;
+ }
+ if (supportsSort) score+=ORDER_MATCH;
+ if (coversAdditionalClause && score>candidateScore) {
+ candidateScore=score;
+ bestCandidate=index;
+ candidateSubcover = subcover;
+ candidateSubcondition = subcondition;
+ candidateSupportsSort = supportsSort;
+ }
+ }
+ if (bestCandidate!=null) {
+ if (coveredClauses.isEmpty()) isSorted=candidateSupportsSort;
+ coveredClauses.addAll(candidateSubcover);
+
+ log.debug("Index chosen for query {} {} " , bestCandidate.isCompositeIndex() ? "COMPOSITE" : "MIXED", coveredClauses);
+ if (bestCandidate.isCompositeIndex()) {
+ jointQuery.add((CompositeIndexType)bestCandidate,
+ serializer.getQuery((CompositeIndexType)bestCandidate,(List<Object[]>)candidateSubcondition));
+ } else {
+ jointQuery.add((MixedIndexType)bestCandidate,
+ serializer.getQuery((MixedIndexType)bestCandidate,(Condition)candidateSubcondition,orders));
+ }
+ } else {
+ break;
+ }
+ /* TODO: smarter optimization:
+ - use in-memory histograms to estimate selectivity of PredicateConditions and filter out low-selectivity ones
+ if they would result in an individual index call (better to filter afterwards in memory)
+ - move OR's up and extend GraphCentricQuery to allow multiple JointIndexQuery for proper or'ing of queries
+ */
+ }
+
+ BackendQueryHolder<JointIndexQuery> query;
+ if (!coveredClauses.isEmpty()) {
+ int indexLimit = limit == Query.NO_LIMIT ? HARD_MAX_LIMIT : limit;
+ if (tx.getGraph().getConfiguration().adjustQueryLimit()) {
+ indexLimit = limit == Query.NO_LIMIT ? DEFAULT_NO_LIMIT : Math.min(MAX_BASE_LIMIT, limit);
+ }
+ indexLimit = Math.min(HARD_MAX_LIMIT, QueryUtil.adjustLimitForTxModifications(tx, coveredClauses.size(), indexLimit));
+ jointQuery.setLimit(indexLimit);
+ query = new BackendQueryHolder<JointIndexQuery>(jointQuery, coveredClauses.size()==conditions.numChildren(), isSorted, null);
+ } else {
+ query = new BackendQueryHolder<JointIndexQuery>(new JointIndexQuery(), false, isSorted, null);
+ }
+
+ return new GraphCentricQuery(resultType, conditions, orders, query, limit);
+ }
+
+ public static final boolean indexCoversOrder(MixedIndexType index, OrderList orders) {
+ for (int i = 0; i < orders.size(); i++) {
+ if (!index.indexesKey(orders.getKey(i))) return false;
+ }
+ return true;
+ }
+
+ public static List<Object[]> indexCover(final CompositeIndexType index, Condition<TitanElement> condition, Set<Condition> covered) {
+ assert QueryUtil.isQueryNormalForm(condition);
+ assert condition instanceof And;
+ if (index.getStatus()!= SchemaStatus.ENABLED) return null;
+ IndexField[] fields = index.getFieldKeys();
+ Object[] indexValues = new Object[fields.length];
+ Set<Condition> coveredClauses = new HashSet<Condition>(fields.length);
+ List<Object[]> indexCovers = new ArrayList<Object[]>(4);
+
+ constructIndexCover(indexValues,0,fields,condition,indexCovers,coveredClauses);
+ if (!indexCovers.isEmpty()) {
+ covered.addAll(coveredClauses);
+ return indexCovers;
+ } else return null;
+ }
+
+ private static void constructIndexCover(Object[] indexValues, int position, IndexField[] fields,
+ Condition<TitanElement> condition,
+ List<Object[]> indexCovers, Set<Condition> coveredClauses) {
+ if (position>=fields.length) {
+ indexCovers.add(indexValues);
+ } else {
+ IndexField field = fields[position];
+ Map.Entry<Condition,Collection<Object>> equalCon = getEqualityConditionValues(condition,field.getFieldKey());
+ if (equalCon!=null) {
+ coveredClauses.add(equalCon.getKey());
+ assert equalCon.getValue().size()>0;
+ for (Object value : equalCon.getValue()) {
+ Object[] newValues = Arrays.copyOf(indexValues,fields.length);
+ newValues[position]=value;
+ constructIndexCover(newValues,position+1,fields,condition,indexCovers,coveredClauses);
+ }
+ } else return;
+ }
+
+ }
+
+ private static final Map.Entry<Condition,Collection<Object>> getEqualityConditionValues(Condition<TitanElement> condition, RelationType type) {
+ for (Condition c : condition.getChildren()) {
+ if (c instanceof Or) {
+ Map.Entry<RelationType,Collection> orEqual = QueryUtil.extractOrCondition((Or)c);
+ if (orEqual!=null && orEqual.getKey().equals(type) && !orEqual.getValue().isEmpty()) {
+ return new AbstractMap.SimpleImmutableEntry(c,orEqual.getValue());
+ }
+ } else if (c instanceof PredicateCondition) {
+ PredicateCondition<RelationType, TitanRelation> atom = (PredicateCondition)c;
+ if (atom.getKey().equals(type) && atom.getPredicate()==Cmp.EQUAL && atom.getValue()!=null) {
+ return new AbstractMap.SimpleImmutableEntry(c,ImmutableList.of(atom.getValue()));
+ }
+ }
+
+ }
+ return null;
+ }
+
+ public static final Condition<TitanElement> indexCover(final MixedIndexType index, Condition<TitanElement> condition,
+ final IndexSerializer indexInfo, final Set<Condition> covered) {
+ assert QueryUtil.isQueryNormalForm(condition);
+ assert condition instanceof And;
+ And<TitanElement> subcondition = new And<TitanElement>(condition.numChildren());
+ for (Condition<TitanElement> subclause : condition.getChildren()) {
+ if (coversAll(index,subclause,indexInfo)) {
+ subcondition.add(subclause);
+ covered.add(subclause);
+ }
+ }
+ return subcondition.isEmpty()?null:subcondition;
+ }
+
+ private static final boolean coversAll(final MixedIndexType index, Condition<TitanElement> condition, IndexSerializer indexInfo) {
+ if (condition.getType()==Condition.Type.LITERAL) {
+ if (!(condition instanceof PredicateCondition)) return false;
+ PredicateCondition<RelationType, TitanElement> atom = (PredicateCondition) condition;
+ if (atom.getValue()==null) return false;
+
+ Preconditions.checkArgument(atom.getKey().isPropertyKey());
+ PropertyKey key = (PropertyKey) atom.getKey();
+ ParameterIndexField[] fields = index.getFieldKeys();
+ ParameterIndexField match = null;
+ for (int i = 0; i < fields.length; i++) {
+ if (fields[i].getStatus()!= SchemaStatus.ENABLED) continue;
+ if (fields[i].getFieldKey().equals(key)) match = fields[i];
+ }
+ if (match==null) return false;
+ return indexInfo.supports(index,match,atom.getPredicate());
+ } else {
+ for (Condition<TitanElement> child : condition.getChildren()) {
+ if (!coversAll(index,child,indexInfo)) return false;
+ }
+ return true;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/GraphDbObjectFactory.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/GraphDbObjectFactory.java b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/GraphDbObjectFactory.java
new file mode 100644
index 0000000..89de23d
--- /dev/null
+++ b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/GraphDbObjectFactory.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graphdb.titan0;
+
+import org.apache.atlas.repository.graphdb.AtlasGraphIndex;
+import org.apache.atlas.repository.graphdb.titan0.query.Titan0GraphQuery;
+
+import com.thinkaurelius.titan.core.PropertyKey;
+import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
+import com.tinkerpop.blueprints.Edge;
+import com.tinkerpop.blueprints.Vertex;
+
+/**
+ * Factory that serves up instances of graph database abstraction layer classes
+ * that correspond to Titan/Tinkerpop classes.
+ */
+public final class GraphDbObjectFactory {
+
+ private GraphDbObjectFactory() {
+
+ }
+
+ /**
+ * Creates a Titan0Edge that corresponds to the given Gremlin Edge.
+ *
+ * @param source
+ * @return
+ */
+ public static Titan0Edge createEdge(Titan0Graph graph, Edge source) {
+
+ if (source == null) {
+ return null;
+ }
+ return new Titan0Edge(graph, source);
+ }
+
+ /**
+ * Creates a Titan0GraphQuery that corresponds to the given GraphQuery.
+ * @param source
+ *
+ * @return
+ */
+ public static Titan0GraphQuery createQuery(Titan0Graph graph) {
+
+ return new Titan0GraphQuery(graph);
+ }
+
+ /**
+ * Creates a Titan0Vertex that corresponds to the given Gremlin Vertex.
+ *
+ * @param source
+ * @return
+ */
+ public static Titan0Vertex createVertex(Titan0Graph graph, Vertex source) {
+
+ if (source == null) {
+ return null;
+ }
+ return new Titan0Vertex(graph, source);
+ }
+
+ /**
+ * @param propertyKey
+ * @return
+ */
+ public static Titan0PropertyKey createPropertyKey(PropertyKey propertyKey) {
+ if (propertyKey == null) {
+ return null;
+ }
+ return new Titan0PropertyKey(propertyKey);
+ }
+
+ /**
+ * @param index
+ * @return
+ */
+ public static AtlasGraphIndex createGraphIndex(TitanGraphIndex index) {
+ if (index == null) {
+ return null;
+ }
+ return new Titan0GraphIndex(index);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Database.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Database.java b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Database.java
new file mode 100644
index 0000000..56b1664
--- /dev/null
+++ b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Database.java
@@ -0,0 +1,204 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graphdb.titan0;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.atlas.ApplicationProperties;
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.GraphDatabase;
+import org.apache.commons.configuration.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.ImmutableMap;
+import com.thinkaurelius.titan.core.TitanFactory;
+import com.thinkaurelius.titan.core.TitanGraph;
+import com.thinkaurelius.titan.core.schema.TitanManagement;
+import com.thinkaurelius.titan.core.util.TitanCleanup;
+import com.thinkaurelius.titan.diskstorage.StandardIndexProvider;
+import com.thinkaurelius.titan.diskstorage.solr.Solr5Index;
+
+/**
+ * Titan 0.5.4 implementation of GraphDatabase.
+ */
+public class Titan0Database implements GraphDatabase<Titan0Vertex, Titan0Edge> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Titan0Database.class);
+
+ /**
+ * Constant for the configuration property that indicates the prefix.
+ */
+ public static final String GRAPH_PREFIX = "atlas.graph";
+
+ public static final String INDEX_BACKEND_CONF = "index.search.backend";
+
+ public static final String INDEX_BACKEND_LUCENE = "lucene";
+
+ public static final String INDEX_BACKEND_ES = "elasticsearch";
+
+ private static volatile TitanGraph graphInstance;
+
+ public static Configuration getConfiguration() throws AtlasException {
+ Configuration configProperties = ApplicationProperties.get();
+ return ApplicationProperties.getSubsetConfiguration(configProperties, GRAPH_PREFIX);
+ }
+
+ static {
+ addSolr5Index();
+ }
+
+ /**
+ * Titan loads index backend name to implementation using
+ * StandardIndexProvider.ALL_MANAGER_CLASSES But
+ * StandardIndexProvider.ALL_MANAGER_CLASSES is a private static final
+ * ImmutableMap Only way to inject Solr5Index is to modify this field. So,
+ * using hacky reflection to add Sol5Index
+ */
+ private static void addSolr5Index() {
+ try {
+ Field field = StandardIndexProvider.class.getDeclaredField("ALL_MANAGER_CLASSES");
+ field.setAccessible(true);
+
+ Field modifiersField = Field.class.getDeclaredField("modifiers");
+ modifiersField.setAccessible(true);
+ modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
+
+ Map<String, String> customMap = new HashMap<>(StandardIndexProvider.getAllProviderClasses());
+ customMap.put("solr", Solr5Index.class.getName()); // for
+ // consistency
+ // with Titan
+ // 1.0.0
+ customMap.put("solr5", Solr5Index.class.getName()); // for backward
+ // compatibility
+ ImmutableMap<String, String> immap = ImmutableMap.copyOf(customMap);
+ field.set(null, immap);
+
+ LOG.debug("Injected solr5 index - {}", Solr5Index.class.getName());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static TitanGraph getGraphInstance() {
+ if (graphInstance == null) {
+ synchronized (Titan0Database.class) {
+ if (graphInstance == null) {
+ Configuration config;
+ try {
+ config = getConfiguration();
+ } catch (AtlasException e) {
+ throw new RuntimeException(e);
+ }
+
+ graphInstance = TitanFactory.open(config);
+ validateIndexBackend(config);
+ }
+ }
+ }
+ return graphInstance;
+ }
+
+ public static void unload() {
+
+ synchronized (Titan0Database.class) {
+ if (graphInstance == null) {
+ return;
+ }
+
+ graphInstance.commit();
+ //shutdown invalidates the graph instance
+ graphInstance.shutdown();
+ graphInstance = null;
+ }
+ }
+
+ static void validateIndexBackend(Configuration config) {
+ String configuredIndexBackend = config.getString(INDEX_BACKEND_CONF);
+ TitanManagement managementSystem = null;
+
+ try {
+ managementSystem = getGraphInstance().getManagementSystem();
+ String currentIndexBackend = managementSystem.get(INDEX_BACKEND_CONF);
+
+ if (!equals(configuredIndexBackend, currentIndexBackend)) {
+ throw new RuntimeException("Configured Index Backend " + configuredIndexBackend
+ + " differs from earlier configured Index Backend " + currentIndexBackend + ". Aborting!");
+ }
+
+ } finally {
+ if (managementSystem != null) {
+ managementSystem.commit();
+ }
+ }
+
+
+ }
+
+ private static boolean equals(Object o1, Object o2) {
+ if (o1 == null) {
+ return o2 == null;
+ }
+ return o1.equals(o2);
+ }
+
+ @Override
+ public AtlasGraph<Titan0Vertex, Titan0Edge> getGraph() {
+ // force graph loading up front to avoid bootstrapping
+ // issues
+ getGraphInstance();
+ return new Titan0Graph();
+ }
+
+ @Override
+ public boolean isGraphLoaded() {
+ return graphInstance != null;
+ }
+
+
+ @Override
+ public void initializeTestGraph() {
+
+ //nothing to do
+ }
+
+ @Override
+ public void removeTestGraph() {
+ try {
+ getGraphInstance().shutdown();
+ }
+ catch(Throwable t) {
+ LOG.warn("Could not shutdown test TitanGraph", t);
+ t.printStackTrace();
+ }
+
+ try {
+ TitanCleanup.clear(getGraphInstance());
+ }
+ catch(Throwable t) {
+ LOG.warn("Could not clear test TitanGraph", t);
+ t.printStackTrace();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0DatabaseManager.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0DatabaseManager.java b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0DatabaseManager.java
new file mode 100644
index 0000000..b4234d7
--- /dev/null
+++ b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0DatabaseManager.java
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.titan0;
+
+import org.apache.atlas.repository.graphdb.AtlasGraphIndex;
+import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
+import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.thinkaurelius.titan.core.Cardinality;
+import com.thinkaurelius.titan.core.PropertyKey;
+import com.thinkaurelius.titan.core.schema.Mapping;
+import com.thinkaurelius.titan.core.schema.PropertyKeyMaker;
+import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
+import com.thinkaurelius.titan.core.schema.TitanManagement;
+import com.tinkerpop.blueprints.Edge;
+import com.tinkerpop.blueprints.Element;
+import com.tinkerpop.blueprints.Vertex;
+
+/**
+ * Titan 0.5.4 implementation of AtlasGraphManagement.
+ */
+public class Titan0DatabaseManager implements AtlasGraphManagement {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Titan0DatabaseManager.class);
+
+ private TitanManagement management;
+
+ public Titan0DatabaseManager(TitanManagement managementSystem) {
+
+ management = managementSystem;
+ }
+
+ @Override
+ public void buildMixedVertexIndex(String index, String backingIndex) {
+ buildMixedIndex(index, Vertex.class, backingIndex);
+ }
+
+ @Override
+ public void buildMixedEdgeIndex(String index, String backingIndex) {
+ buildMixedIndex(index, Edge.class, backingIndex);
+ }
+
+ private void buildMixedIndex(String index, Class<? extends Element> titanClass, String backingIndex) {
+
+ management.buildIndex(index, titanClass).buildMixedIndex(backingIndex);
+ }
+
+ @Override
+ public void createFullTextIndex(String indexName, AtlasPropertyKey propertyKey, String backingIndex) {
+
+ PropertyKey fullText = TitanObjectFactory.createPropertyKey(propertyKey);
+
+ management.buildIndex(indexName, Vertex.class)
+ .addKey(fullText, com.thinkaurelius.titan.core.schema.Parameter.of("mapping", Mapping.TEXT))
+ .buildMixedIndex(backingIndex);
+ }
+
+ @Override
+ public boolean containsPropertyKey(String propertyKey) {
+ return management.containsPropertyKey(propertyKey);
+ }
+
+ @Override
+ public void rollback() {
+ management.rollback();
+
+ }
+
+ @Override
+ public void commit() {
+ management.commit();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.atlas.repository.graphdb.AtlasGraphManagement#makePropertyKey(
+ * java.lang.String, java.lang.Class,
+ * org.apache.atlas.typesystem.types.Multiplicity)
+ */
+ @Override
+ public AtlasPropertyKey makePropertyKey(String propertyName, Class propertyClass, Multiplicity multiplicity) {
+
+ PropertyKeyMaker propertyKeyBuilder = management.makePropertyKey(propertyName).dataType(propertyClass);
+
+ if (multiplicity != null) {
+ Cardinality cardinality = TitanObjectFactory.createCardinality(multiplicity);
+ propertyKeyBuilder.cardinality(cardinality);
+ }
+ PropertyKey propertyKey = propertyKeyBuilder.make();
+ return GraphDbObjectFactory.createPropertyKey(propertyKey);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.atlas.repository.graphdb.AtlasGraphManagement#getPropertyKey(
+ * java.lang.String)
+ */
+ @Override
+ public AtlasPropertyKey getPropertyKey(String propertyName) {
+
+ return GraphDbObjectFactory.createPropertyKey(management.getPropertyKey(propertyName));
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.atlas.repository.graphdb.AtlasGraphManagement#
+ * createCompositeIndex(java.lang.String,
+ * org.apache.atlas.repository.graphdb.AtlasPropertyKey, boolean)
+ */
+ @Override
+ public void createCompositeIndex(String propertyName, AtlasPropertyKey propertyKey, boolean enforceUniqueness) {
+ PropertyKey titanKey = TitanObjectFactory.createPropertyKey(propertyKey);
+ TitanManagement.IndexBuilder indexBuilder = management.buildIndex(propertyName, Vertex.class).addKey(titanKey);
+ if (enforceUniqueness) {
+ indexBuilder.unique();
+ }
+ indexBuilder.buildCompositeIndex();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.atlas.repository.graphdb.AtlasGraphManagement#addIndexKey(java
+ * .lang.String, org.apache.atlas.repository.graphdb.AtlasPropertyKey)
+ */
+ @Override
+ public void addIndexKey(String indexName, AtlasPropertyKey propertyKey) {
+ PropertyKey titanKey = TitanObjectFactory.createPropertyKey(propertyKey);
+ TitanGraphIndex vertexIndex = management.getGraphIndex(indexName);
+ management.addIndexKey(vertexIndex, titanKey);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.atlas.repository.graphdb.AtlasGraphManagement#getGraphIndex(
+ * java.lang.String)
+ */
+ @Override
+ public AtlasGraphIndex getGraphIndex(String indexName) {
+ TitanGraphIndex index = management.getGraphIndex(indexName);
+ return GraphDbObjectFactory.createGraphIndex(index);
+ }
+
+}
[4/9] incubator-atlas git commit: ATLAS-693 Titan 0.5.4
implementation of graph db abstraction. (jnhagelb via dkantor)
Posted by dk...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/Titan0DatabaseTest.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/Titan0DatabaseTest.java b/graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/Titan0DatabaseTest.java
new file mode 100644
index 0000000..6c2ea26
--- /dev/null
+++ b/graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/Titan0DatabaseTest.java
@@ -0,0 +1,428 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graphdb.titan0;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.AtlasEdge;
+import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
+import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
+import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator;
+import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.Test;
+
+/**
+ * Sanity test of basic graph operations using the Titan 0.5.4 graphdb
+ * abstraction layer implementation.
+ */
+public class Titan0DatabaseTest {
+
+ private AtlasGraph<?, ?> atlasGraph;
+
+ private <V, E> AtlasGraph<V, E> getGraph() {
+ if (atlasGraph == null) {
+ Titan0Database db = new Titan0Database();
+ atlasGraph = db.getGraph();
+ AtlasGraphManagement mgmt = atlasGraph.getManagementSystem();
+ // create the index (which defines these properties as being mult
+ // many)
+ for (String propertyName : AtlasGraphManagement.MULTIPLICITY_MANY_PROPERTY_KEYS) {
+ AtlasPropertyKey propertyKey = mgmt.getPropertyKey(propertyName);
+ if (propertyKey == null) {
+ propertyKey = mgmt.makePropertyKey(propertyName, String.class, Multiplicity.SET);
+ mgmt.createCompositeIndex(propertyName, propertyKey, false);
+ }
+ }
+ mgmt.commit();
+ }
+ return (AtlasGraph<V, E>) atlasGraph;
+ }
+
+ @AfterClass
+ public void cleanup() {
+ atlasGraph.clear();
+ atlasGraph = null;
+
+ }
+
+ @Test
+ public <V, E> void testPropertyDataTypes() {
+
+ // primitives
+ AtlasGraph<V, E> graph = getGraph();
+
+ testProperty(graph, "booleanProperty", Boolean.TRUE);
+ testProperty(graph, "booleanProperty", Boolean.FALSE);
+ testProperty(graph, "booleanProperty", new Boolean(Boolean.TRUE));
+ testProperty(graph, "booleanProperty", new Boolean(Boolean.FALSE));
+
+ testProperty(graph, "byteProperty", Byte.MAX_VALUE);
+ testProperty(graph, "byteProperty", Byte.MIN_VALUE);
+ testProperty(graph, "byteProperty", new Byte(Byte.MAX_VALUE));
+ testProperty(graph, "byteProperty", new Byte(Byte.MIN_VALUE));
+
+ testProperty(graph, "shortProperty", Short.MAX_VALUE);
+ testProperty(graph, "shortProperty", Short.MIN_VALUE);
+ testProperty(graph, "shortProperty", new Short(Short.MAX_VALUE));
+ testProperty(graph, "shortProperty", new Short(Short.MIN_VALUE));
+
+ testProperty(graph, "intProperty", Integer.MAX_VALUE);
+ testProperty(graph, "intProperty", Integer.MIN_VALUE);
+ testProperty(graph, "intProperty", new Integer(Integer.MAX_VALUE));
+ testProperty(graph, "intProperty", new Integer(Integer.MIN_VALUE));
+
+ testProperty(graph, "longProperty", Long.MIN_VALUE);
+ testProperty(graph, "longProperty", Long.MAX_VALUE);
+ testProperty(graph, "longProperty", new Long(Long.MIN_VALUE));
+ testProperty(graph, "longProperty", new Long(Long.MAX_VALUE));
+
+ testProperty(graph, "doubleProperty", Double.MAX_VALUE);
+ testProperty(graph, "doubleProperty", Double.MIN_VALUE);
+ testProperty(graph, "doubleProperty", new Double(Double.MAX_VALUE));
+ testProperty(graph, "doubleProperty", new Double(Double.MIN_VALUE));
+
+ testProperty(graph, "floatProperty", Float.MAX_VALUE);
+ testProperty(graph, "floatProperty", Float.MIN_VALUE);
+ testProperty(graph, "floatProperty", new Float(Float.MAX_VALUE));
+ testProperty(graph, "floatProperty", new Float(Float.MIN_VALUE));
+
+ // enumerations - TypeCategory
+ testProperty(graph, "typeCategoryProperty", TypeCategory.CLASS);
+
+ // biginteger
+ testProperty(graph, "bigIntegerProperty",
+ new BigInteger(String.valueOf(Long.MAX_VALUE)).multiply(BigInteger.TEN));
+
+ // bigdecimal
+ BigDecimal bigDecimal = new BigDecimal(Double.MAX_VALUE);
+ testProperty(graph, "bigDecimalProperty", bigDecimal.multiply(bigDecimal));
+ }
+
+ private <V, E> void testProperty(AtlasGraph<V, E> graph, String name, Object value) {
+
+ AtlasVertex<V, E> vertex = graph.addVertex();
+ vertex.setProperty(name, value);
+ assertEquals(value, vertex.getProperty(name, value.getClass()));
+ AtlasVertex<V, E> loaded = graph.getVertex(vertex.getId().toString());
+ assertEquals(value, loaded.getProperty(name, value.getClass()));
+ }
+
+ @Test
+ public <V, E> void testMultiplicityOnePropertySupport() {
+
+ AtlasGraph<V, E> graph = (AtlasGraph<V, E>) getGraph();
+
+ AtlasVertex<V, E> vertex = graph.addVertex();
+ vertex.setProperty("name", "Jeff");
+ vertex.setProperty("location", "Littleton");
+ assertEquals("Jeff", vertex.getProperty("name", String.class));
+ assertEquals("Littleton", vertex.getProperty("location", String.class));
+
+ AtlasVertex<V, E> vertexCopy = graph.getVertex(vertex.getId().toString());
+
+ assertEquals("Jeff", vertexCopy.getProperty("name", String.class));
+ assertEquals("Littleton", vertexCopy.getProperty("location", String.class));
+
+ assertTrue(vertexCopy.getPropertyKeys().contains("name"));
+ assertTrue(vertexCopy.getPropertyKeys().contains("location"));
+
+ assertTrue(vertexCopy.getPropertyValues("name", String.class).contains("Jeff"));
+ assertTrue(vertexCopy.getPropertyValues("location", String.class).contains("Littleton"));
+ assertTrue(vertexCopy.getPropertyValues("test", String.class).isEmpty());
+ assertNull(vertexCopy.getProperty("test", String.class));
+
+ vertex.removeProperty("name");
+ assertFalse(vertex.getPropertyKeys().contains("name"));
+ assertNull(vertex.getProperty("name", String.class));
+ assertTrue(vertex.getPropertyValues("name", String.class).isEmpty());
+
+ vertexCopy = graph.getVertex(vertex.getId().toString());
+ assertFalse(vertexCopy.getPropertyKeys().contains("name"));
+ assertNull(vertexCopy.getProperty("name", String.class));
+ assertTrue(vertexCopy.getPropertyValues("name", String.class).isEmpty());
+
+ }
+
+ @Test
+ public <V, E> void testRemoveEdge() {
+
+ AtlasGraph<V, E> graph = (AtlasGraph<V, E>) getGraph();
+ AtlasVertex<V, E> v1 = graph.addVertex();
+ AtlasVertex<V, E> v2 = graph.addVertex();
+
+ AtlasEdge<V, E> edge = graph.addEdge(v1, v2, "knows");
+
+ // make sure the edge exists
+ AtlasEdge<V, E> edgeCopy = graph.getEdge(edge.getId().toString());
+ assertNotNull(edgeCopy);
+ assertEquals(edgeCopy, edge);
+
+ graph.removeEdge(edge);
+
+ edgeCopy = graph.getEdge(edge.getId().toString());
+ // should return null now, since edge was deleted
+ assertNull(edgeCopy);
+
+ }
+
+ @Test
+ public <V, E> void testRemoveVertex() {
+
+ AtlasGraph<V, E> graph = (AtlasGraph<V, E>) getGraph();
+
+ AtlasVertex<V, E> v1 = graph.addVertex();
+
+ assertNotNull(graph.getVertex(v1.getId().toString()));
+
+ graph.removeVertex(v1);
+
+ assertNull(graph.getVertex(v1.getId().toString()));
+ }
+
+ @Test
+ public <V, E> void testGetEdges() {
+
+ AtlasGraph<V, E> graph = (AtlasGraph<V, E>) getGraph();
+ AtlasVertex<V, E> v1 = graph.addVertex();
+ AtlasVertex<V, E> v2 = graph.addVertex();
+ AtlasVertex<V, E> v3 = graph.addVertex();
+
+ AtlasEdge<V, E> knows = graph.addEdge(v2, v1, "knows");
+ AtlasEdge<V, E> eats = graph.addEdge(v3, v1, "eats");
+ AtlasEdge<V, E> drives = graph.addEdge(v3, v2, "drives");
+ AtlasEdge<V, E> sleeps = graph.addEdge(v2, v3, "sleeps");
+
+ assertEdgesMatch(v1.getEdges(AtlasEdgeDirection.IN), knows, eats);
+ assertEdgesMatch(v1.getEdges(AtlasEdgeDirection.OUT));
+ assertEdgesMatch(v1.getEdges(AtlasEdgeDirection.BOTH), knows, eats);
+
+ assertEdgesMatch(v1.getEdges(AtlasEdgeDirection.IN, "knows"), knows);
+ assertEdgesMatch(v1.getEdges(AtlasEdgeDirection.OUT, "knows"));
+ assertEdgesMatch(v1.getEdges(AtlasEdgeDirection.BOTH, "knows"), knows);
+
+ assertEdgesMatch(v2.getEdges(AtlasEdgeDirection.IN), drives);
+ assertEdgesMatch(v2.getEdges(AtlasEdgeDirection.OUT), knows, sleeps);
+ assertEdgesMatch(v2.getEdges(AtlasEdgeDirection.BOTH), knows, sleeps, drives);
+
+ assertEdgesMatch(v2.getEdges(AtlasEdgeDirection.BOTH, "delivers"));
+ }
+
+ private <V, E> void assertEdgesMatch(Iterable<AtlasEdge<V, E>> edgesIt, AtlasEdge<V, E>... expected) {
+ List<AtlasEdge<V, E>> edges = toList(edgesIt);
+ assertEquals(expected.length, edges.size());
+ for (AtlasEdge<V, E> edge : expected) {
+ assertTrue(edges.contains(edge));
+ }
+ }
+
+ @Test
+ public <V, E> void testMultiplictyManyPropertySupport() {
+
+ AtlasGraph<V, E> graph = getGraph();
+
+ AtlasVertex<V, E> vertex = graph.addVertex();
+ String vertexId = vertex.getId().toString();
+ vertex.setProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait1");
+ vertex.setProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait2");
+ assertEquals(vertex.getPropertyValues(Constants.TRAIT_NAMES_PROPERTY_KEY, String.class).size(), 2);
+ vertex.addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait3");
+ vertex.addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait4");
+
+ assertTrue(vertex.getPropertyKeys().contains(Constants.TRAIT_NAMES_PROPERTY_KEY));
+ validateMultManyPropertiesInVertex(vertex);
+ // fetch a copy of the vertex, make sure result
+ // is the same
+
+ validateMultManyPropertiesInVertex(graph.getVertex(vertexId));
+
+ }
+
+ private <V, E> void validateMultManyPropertiesInVertex(AtlasVertex<V, E> vertex) {
+
+ assertTrue(vertex.getPropertyKeys().contains(Constants.TRAIT_NAMES_PROPERTY_KEY));
+ Collection<String> traitNames = vertex.getPropertyValues(Constants.TRAIT_NAMES_PROPERTY_KEY, String.class);
+ assertTrue(traitNames.contains("trait1"));
+ assertTrue(traitNames.contains("trait2"));
+ assertTrue(traitNames.contains("trait3"));
+ assertTrue(traitNames.contains("trait4"));
+
+ try {
+ vertex.getProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, String.class);
+ fail("Expected exception not thrown");
+ } catch (IllegalStateException expected) {
+ // multiple property values exist
+ }
+ }
+
+ @Test
+ public <V, E> void testListProperties() throws AtlasException {
+
+ AtlasGraph<V, E> graph = getGraph();
+ AtlasVertex<V, E> vertex = graph.addVertex();
+ vertex.setListProperty("colors", Arrays.asList(new String[] { "red", "blue", "green" }));
+ List<String> colors = vertex.getListProperty("colors");
+ assertTrue(colors.contains("red"));
+ assertTrue(colors.contains("blue"));
+ assertTrue(colors.contains("green"));
+
+ AtlasVertex<V, E> vertexCopy = graph.getVertex(vertex.getId().toString());
+ colors = vertexCopy.getListProperty("colors");
+ assertTrue(colors.contains("red"));
+ assertTrue(colors.contains("blue"));
+ assertTrue(colors.contains("green"));
+
+ }
+
+ @Test
+ public <V, E> void testRemoveProperty() {
+
+ AtlasGraph<V, E> graph = getGraph();
+ AtlasVertex<V, E> vertex = graph.addVertex();
+ vertex.addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait1");
+ vertex.addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait1");
+ vertex.setProperty("name", "Jeff");
+
+ // remove existing property - multiplicity one
+ vertex.removeProperty("jeff");
+
+ assertFalse(vertex.getPropertyKeys().contains("jeff"));
+
+ // remove existing property - multiplicity many
+ vertex.removeProperty(Constants.TRAIT_NAMES_PROPERTY_KEY);
+ assertFalse(vertex.getPropertyKeys().contains(Constants.TRAIT_NAMES_PROPERTY_KEY));
+
+ AtlasVertex<V, E> vertexCopy = graph.getVertex(vertex.getId().toString());
+ assertFalse(vertexCopy.getPropertyKeys().contains("jeff"));
+ assertFalse(vertexCopy.getPropertyKeys().contains(Constants.TRAIT_NAMES_PROPERTY_KEY));
+
+ // remove non-existing property
+ vertex.removeProperty(Constants.TRAIT_NAMES_PROPERTY_KEY);
+ vertex.removeProperty("jeff");
+
+ }
+
+ @Test
+ public <V, E> void getGetGraphQueryForVertices() {
+
+ AtlasGraph<V, E> graph = getGraph();
+
+ AtlasVertex<V, E> v1 = graph.addVertex();
+ AtlasVertex<V, E> v2 = graph.addVertex();
+ AtlasVertex<V, E> v3 = graph.addVertex();
+
+ v1.setProperty("name", "Jeff");
+ v1.setProperty("weight", 1);
+
+ v2.setProperty("name", "Fred");
+ v2.setProperty("weight", 2);
+
+ v3.setProperty("name", "Chris");
+ v3.setProperty("weight", 3);
+
+ AtlasEdge<V, E> knows = graph.addEdge(v2, v1, "knows");
+ knows.setProperty("weight", 1);
+ AtlasEdge<V, E> eats = graph.addEdge(v3, v1, "eats");
+ eats.setProperty("weight", 2);
+ AtlasEdge<V, E> drives = graph.addEdge(v3, v2, "drives");
+ drives.setProperty("weight", 3);
+
+ AtlasEdge<V, E> sleeps = graph.addEdge(v2, v3, "sleeps");
+ sleeps.setProperty("weight", 4);
+
+ testExecuteGraphQuery("name", null, "Jeff", v1);
+ testExecuteGraphQuery("weight", ComparisionOperator.EQUAL, 2, v2);
+ testExecuteGraphQuery("weight", ComparisionOperator.GREATER_THAN_EQUAL, 2, v2, v3);
+ testExecuteGraphQuery("weight", ComparisionOperator.LESS_THAN_EQUAL, 2, v2, v1);
+
+ }
+
+ private <V, E> void testExecuteGraphQuery(String property, ComparisionOperator op, Object value,
+ AtlasVertex<V, E>... expected) {
+ AtlasGraph<V, E> graph = getGraph();
+ AtlasGraphQuery<V, E> query = graph.query();
+ if (op != null) {
+ query.has(property, op, value);
+ } else {
+ query.has(property, value);
+ }
+ Iterable<? extends AtlasVertex<V, E>> result = query.vertices();
+ List<AtlasVertex<V, E>> list = toList(result);
+ assertEquals(expected.length, list.size());
+ for (AtlasVertex<V, E> vertex : expected) {
+ assertTrue(list.contains(vertex));
+ }
+ }
+
+ @Test
+ public <V, E> void testAddMultManyPropertyValueTwice() {
+
+ AtlasGraph<V, E> graph = getGraph();
+ String vertexId;
+
+ AtlasVertex<V, E> vertex = graph.addVertex();
+ vertexId = vertex.getId().toString();
+ vertex.setProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait1");
+ vertex.setProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait1");
+ vertex.addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait2");
+ vertex.addProperty(Constants.TRAIT_NAMES_PROPERTY_KEY, "trait2");
+
+ validateDuplicatePropertyVertex(vertex);
+
+ // fetch a copy of the vertex, make sure result is the same
+
+ validateDuplicatePropertyVertex(graph.getVertex(vertexId));
+ }
+
+ private <V, E> void validateDuplicatePropertyVertex(AtlasVertex<V, E> vertex) {
+ assertEquals(2, vertex.getPropertyValues(Constants.TRAIT_NAMES_PROPERTY_KEY, String.class).size());
+ assertTrue(vertex.getPropertyKeys().contains(Constants.TRAIT_NAMES_PROPERTY_KEY));
+ Collection<String> traitNames = vertex.getPropertyValues(Constants.TRAIT_NAMES_PROPERTY_KEY, String.class);
+ assertTrue(traitNames.contains("trait1"));
+ assertTrue(traitNames.contains("trait2"));
+ }
+
+ private static <T> List<T> toList(Iterable<? extends T> iterable) {
+ List<T> result = new ArrayList<T>();
+ for (T item : iterable) {
+ result.add(item);
+ }
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/Titan0DatabaseValidationTest.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/Titan0DatabaseValidationTest.java b/graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/Titan0DatabaseValidationTest.java
new file mode 100644
index 0000000..341c064
--- /dev/null
+++ b/graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/Titan0DatabaseValidationTest.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.titan0;
+
+import org.apache.atlas.ApplicationProperties;
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.commons.configuration.Configuration;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+@Test
+public class Titan0DatabaseValidationTest {
+
+ private Configuration configuration;
+ private AtlasGraph<?, ?> graph;
+
+ @BeforeTest
+ public void setUp() throws AtlasException {
+ // First get Instance
+ graph = new Titan0Graph();
+ configuration = ApplicationProperties.getSubsetConfiguration(ApplicationProperties.get(),
+ Titan0Database.GRAPH_PREFIX);
+ }
+
+ @AfterClass
+ public void tearDown() throws Exception {
+ try {
+ graph.shutdown();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ try {
+ graph.clear();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Test
+ public void testValidate() throws AtlasException {
+ try {
+ Titan0Database.validateIndexBackend(configuration);
+ } catch (Exception e) {
+ Assert.fail("Unexpected exception ", e);
+ }
+
+ // Change backend
+ configuration.setProperty(Titan0Database.INDEX_BACKEND_CONF, Titan0Database.INDEX_BACKEND_LUCENE);
+ try {
+ Titan0Database.validateIndexBackend(configuration);
+ Assert.fail("Expected exception");
+ } catch (Exception e) {
+ Assert.assertEquals(e.getMessage(),
+ "Configured Index Backend lucene differs from earlier configured "
+ + "Index Backend elasticsearch. Aborting!");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/test/resources/atlas-application.properties
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/test/resources/atlas-application.properties b/graphdb/titan0/src/test/resources/atlas-application.properties
new file mode 100644
index 0000000..1e8963e
--- /dev/null
+++ b/graphdb/titan0/src/test/resources/atlas-application.properties
@@ -0,0 +1,97 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+######### Graph Database to Use #########
+atlas.graphdb.backend=org.apache.atlas.repository.graphdb.titan0.Titan0Database
+
+######### Atlas Server Configs #########
+atlas.rest.address=http://localhost:31000
+
+######### Graph Database Configs #########
+# Graph Storage
+atlas.graph.storage.backend=${titan.storage.backend}
+
+# Graph Search Index Backend
+atlas.graph.index.search.backend=${titan.index.backend}
+
+#Berkeley storage directory
+atlas.graph.storage.directory=${sys:atlas.data}/berkley
+
+#hbase
+#For standalone mode , specify localhost
+#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
+
+atlas.graph.storage.hostname=${titan.storage.hostname}
+atlas.graph.storage.hbase.regions-per-server=1
+atlas.graph.storage.lock.wait-time=10000
+
+#ElasticSearch
+atlas.graph.index.search.directory=${sys:atlas.data}/es
+atlas.graph.index.search.elasticsearch.client-only=false
+atlas.graph.index.search.elasticsearch.local-mode=true
+atlas.graph.index.search.elasticsearch.create.sleep=2000
+
+# Solr cloud mode properties
+atlas.graph.index.search.solr.mode=cloud
+atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address}
+
+
+######### Hive Lineage Configs #########
+# This models reflects the base super types for Data and Process
+#atlas.lineage.hive.table.type.name=DataSet
+#atlas.lineage.hive.process.type.name=Process
+#atlas.lineage.hive.process.inputs.name=inputs
+#atlas.lineage.hive.process.outputs.name=outputs
+
+## Schema
+atlas.lineage.hive.table.schema.query.hive_table=hive_table where name='%s'\, columns
+
+######### Notification Configs #########
+atlas.notification.embedded=true
+
+atlas.kafka.zookeeper.connect=localhost:19026
+atlas.kafka.bootstrap.servers=localhost:19027
+atlas.kafka.data=${sys:atlas.data}/kafka
+atlas.kafka.zookeeper.session.timeout.ms=4000
+atlas.kafka.zookeeper.sync.time.ms=20
+atlas.kafka.consumer.timeout.ms=100
+atlas.kafka.auto.commit.interval.ms=100
+atlas.kafka.hook.group.id=atlas
+atlas.kafka.entities.group.id=atlas_entities
+
+######### Entity Audit Configs #########
+atlas.audit.hbase.tablename=ATLAS_ENTITY_AUDIT_EVENTS
+atlas.audit.zookeeper.session.timeout.ms=1000
+atlas.audit.hbase.zookeeper.quorum=localhost
+atlas.audit.hbase.zookeeper.property.clientPort=19026
+
+######### Security Properties #########
+
+# SSL config
+atlas.enableTLS=false
+atlas.server.https.port=31443
+
+######### Security Properties #########
+
+hbase.security.authentication=simple
+
+atlas.hook.falcon.synchronous=true
+######### High Availability Configuration ########
+atlas.server.ha.enabled=false
+#atlas.server.ids=id1
+#atlas.server.address.id1=localhost:21000
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 2384494..ac984e0 100755
--- a/pom.xml
+++ b/pom.xml
@@ -434,7 +434,8 @@
<titan.storage.backend>berkeleyje</titan.storage.backend>
<titan.index.backend>elasticsearch</titan.index.backend>
- <entity.repository.impl>org.apache.atlas.repository.audit.InMemoryEntityAuditRepository</entity.repository.impl>
+ <entity.repository.impl>org.apache.atlas.repository.audit.InMemoryEntityAuditRepository</entity.repository.impl>
+ <atlas.surefire.options></atlas.surefire.options>
</properties>
<profiles>
@@ -446,10 +447,10 @@
</activation>
<properties>
<titan.storage.backend>hbase</titan.storage.backend>
- <titan.index.backend>solr5</titan.index.backend>
+ <titan.index.backend>solr</titan.index.backend>
<solr.zk.address>localhost:9983</solr.zk.address>
<titan.storage.hostname>localhost</titan.storage.hostname>
- <entity.repository.impl>org.apache.atlas.repository.audit.HBaseBasedAuditRepository</entity.repository.impl>
+ <entity.repository.impl>org.apache.atlas.repository.audit.HBaseBasedAuditRepository</entity.repository.impl>
</properties>
</profile>
@@ -487,7 +488,6 @@
<module>notification</module>
<module>client</module>
<module>graphdb</module>
- <module>titan</module>
<module>repository</module>
<module>authorization</module>
<module>catalog</module>
@@ -683,6 +683,11 @@
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -874,58 +879,7 @@
</dependency>
<!-- Graph DB -->
- <dependency>
- <groupId>com.tinkerpop.blueprints</groupId>
- <artifactId>blueprints-core</artifactId>
- <version>${tinkerpop.version}</version>
- </dependency>
-
- <dependency>
- <groupId>com.thinkaurelius.titan</groupId>
- <artifactId>titan-core</artifactId>
- <version>${titan.version}</version>
- <exclusions>
- <!-- rexster does not work with servlet-api -->
- <exclusion>
- <groupId>com.tinkerpop.rexster</groupId>
- <artifactId>rexster-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.tinkerpop.rexster</groupId>
- <artifactId>rexster-server</artifactId>
- </exclusion>
- <!-- asm 4.0 does not work with jersey asm 3.1 -->
- <exclusion>
- <groupId>com.tinkerpop</groupId>
- <artifactId>frames</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.esotericsoftware.reflectasm</groupId>
- <artifactId>reflectasm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.ow2.asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion> <!-- GPL license imported from ganglia -->
- <groupId>org.acplt</groupId>
- <artifactId>oncrpc</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>com.thinkaurelius.titan</groupId>
- <artifactId>titan-berkeleyje</artifactId>
- <version>${titan.version}</version>
- </dependency>
-
- <dependency>
- <groupId>com.thinkaurelius.titan</groupId>
- <artifactId>titan-hbase</artifactId>
- <version>${titan.version}</version>
- </dependency>
-
+
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
@@ -980,12 +934,6 @@
</dependency>
<dependency>
- <groupId>com.thinkaurelius.titan</groupId>
- <artifactId>titan-es</artifactId>
- <version>${titan.version}</version>
- </dependency>
-
- <dependency>
<groupId>com.vividsolutions</groupId>
<artifactId>jts</artifactId>
<version>1.13</version>
@@ -1016,6 +964,10 @@
<artifactId>*</artifactId>
<groupId>org.apache.lucene</groupId>
</exclusion>
+ <exclusion>
+ <artifactId>*</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
</exclusions>
</dependency>
@@ -1025,19 +977,6 @@
<version>${solr.version}</version>
</dependency>
- <dependency>
- <groupId>com.thinkaurelius.titan</groupId>
- <artifactId>titan-lucene</artifactId>
- <version>${titan.version}</version>
- <!--<scope>test</scope>-->
- </dependency>
-
- <dependency>
- <groupId>com.tinkerpop.gremlin</groupId>
- <artifactId>gremlin-java</artifactId>
- <version>${tinkerpop.version}</version>
- </dependency>
-
<!-- atlas modules -->
<dependency>
<groupId>org.apache.atlas</groupId>
@@ -1055,7 +994,13 @@
<dependency>
<groupId>org.apache.atlas</groupId>
- <artifactId>atlas-titan</artifactId>
+ <artifactId>atlas-graphdb-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.atlas</groupId>
+ <artifactId>atlas-graphdb-titan0</artifactId>
<version>${project.version}</version>
</dependency>
@@ -1576,6 +1521,14 @@
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-remote-resources-plugin</artifactId>
+ <version>1.5</version>
+ <configuration>
+ <excludeGroupIds>org.restlet.jee</excludeGroupIds>
+ </configuration>
+ </plugin>
</plugins>
</pluginManagement>
@@ -1628,7 +1581,7 @@
<redirectTestOutputToFile>true</redirectTestOutputToFile>
<argLine>-Djava.awt.headless=true -Dproject.version=${project.version}
-Dhadoop.tmp.dir="${project.build.directory}/tmp-hadoop-${user.name}"
- -Xmx1024m -XX:MaxPermSize=512m -Djava.net.preferIPv4Stack=true
+ -Xmx1024m -XX:MaxPermSize=512m -Djava.net.preferIPv4Stack=true ${atlas.surefire.options}
</argLine>
<skip>${skipUTs}</skip>
</configuration>
@@ -1654,7 +1607,7 @@
<redirectTestOutputToFile>true</redirectTestOutputToFile>
<argLine>-Djava.awt.headless=true -Dproject.version=${project.version}
-Dhadoop.tmp.dir="${project.build.directory}/tmp-hadoop-${user.name}"
- -Xmx1024m -XX:MaxPermSize=512m
+ -Xmx1024m -XX:MaxPermSize=512m ${atlas.surefire.options}
</argLine>
<skip>${skipITs}</skip>
<parallel>none</parallel>
@@ -1727,7 +1680,11 @@
<useEclipseDefaultExcludes>true</useEclipseDefaultExcludes>
<excludeSubProjects>true</excludeSubProjects>
<excludes>
+ <exclude>**/dependency-reduced-pom.xml</exclude>
+ <exclude>.reviewboardrc</exclude>
<exclude>3party-licenses/**</exclude>
+ <exclude>**/.cache-main</exclude>
+ <exclude>**/.checkstyle</exclude>
<exclude>*.txt</exclude>
<exclude>**/*.json</exclude>
<exclude>.pc/**</exclude>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 885afd4..7c8e008 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES:
ATLAS-1060 Add composite indexes for exact match performance improvements for all attributes (sumasai via shwethags)
ALL CHANGES:
+ATLAS-693 Titan 0.5.4 implementation of the graph db abstraction {jnhagelb via dkantor)
ATLAS-1099 UI : multiple tag assign button hides wrongly (Kalyanikashikar via sumasai)
ATLAS-1087 Provide an option to turn off persisting entity definition in audits (sumasai, shwethags)
ATLAS-1097 Fix a potential NPE issue flagged by Coverity scan (mneethiraj via shwethags)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/repository/pom.xml
----------------------------------------------------------------------
diff --git a/repository/pom.xml b/repository/pom.xml
index 3d525e0..663ac87 100755
--- a/repository/pom.xml
+++ b/repository/pom.xml
@@ -49,7 +49,12 @@
<dependency>
<groupId>org.apache.atlas</groupId>
- <artifactId>atlas-titan</artifactId>
+ <artifactId>atlas-graphdb-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.atlas</groupId>
+ <artifactId>atlas-graphdb-titan0</artifactId>
</dependency>
<dependency>
@@ -83,16 +88,6 @@
</dependency>
<dependency>
- <groupId>com.tinkerpop.blueprints</groupId>
- <artifactId>blueprints-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.tinkerpop.gremlin</groupId>
- <artifactId>gremlin-java</artifactId>
- </dependency>
-
- <dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-reflect</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
index 3486436..f1ef140 100755
--- a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
+++ b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
@@ -75,9 +75,12 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
// bind the ITypeStore interface to an implementation
bind(ITypeStore.class).to(GraphBackedTypeStore.class).asEagerSingleton();
+ //GraphBackedSearchIndexer must be an eager singleton to force the search index creation to happen before
+ //we try to restore the type system (otherwise we'll end up running queries
+ //before we have any indices during the initial graph setup)
Multibinder<TypesChangeListener> typesChangeListenerBinder =
Multibinder.newSetBinder(binder(), TypesChangeListener.class);
- typesChangeListenerBinder.addBinding().to(GraphBackedSearchIndexer.class);
+ typesChangeListenerBinder.addBinding().to(GraphBackedSearchIndexer.class).asEagerSingleton();
// bind the MetadataService interface to an implementation
bind(MetadataService.class).to(DefaultMetadataService.class).asEagerSingleton();
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/repository/src/main/java/org/apache/atlas/repository/Constants.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/Constants.java b/repository/src/main/java/org/apache/atlas/repository/Constants.java
deleted file mode 100755
index 893f1b6..0000000
--- a/repository/src/main/java/org/apache/atlas/repository/Constants.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.repository;
-
-import org.apache.atlas.typesystem.types.AttributeInfo;
-import org.apache.atlas.typesystem.types.DataTypes;
-import org.apache.atlas.typesystem.types.utils.TypesUtil;
-
-public final class Constants {
-
- /**
- * Globally Unique identifier property key.
- */
-
- public static final String INTERNAL_PROPERTY_KEY_PREFIX = "__";
- public static final String GUID_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "guid";
-
- /**
- * Entity type name property key.
- */
- public static final String ENTITY_TYPE_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "typeName";
-
- /**
- * Entity type's super types property key.
- */
- public static final String SUPER_TYPES_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "superTypeNames";
-
- /**
- * Full-text for the entity for enabling full-text search.
- */
- //weird issue in TitanDB if __ added to this property key. Not adding it for now
- public static final String ENTITY_TEXT_PROPERTY_KEY = "entityText";
-
- /**
- * Properties for type store graph
- */
- public static final String TYPE_CATEGORY_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.category";
- public static final String VERTEX_TYPE_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type";
- public static final String TYPENAME_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.name";
- public static final String TYPEDESCRIPTION_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "type.description";
-
- /**
- * Trait names property key and index name.
- */
- public static final String TRAIT_NAMES_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "traitNames";
-
- public static final String VERSION_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "version";
- public static final String STATE_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "state";
-
- public static final String TIMESTAMP_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "timestamp";
- public static final String MODIFICATION_TIMESTAMP_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "modificationTimestamp";
-
- public static AttributeInfo getAttributeInfoForSystemAttributes(String field) {
- switch (field) {
- case STATE_PROPERTY_KEY:
- case GUID_PROPERTY_KEY:
- return TypesUtil.newAttributeInfo(field, DataTypes.STRING_TYPE);
-
- case TIMESTAMP_PROPERTY_KEY:
- case MODIFICATION_TIMESTAMP_PROPERTY_KEY:
- return TypesUtil.newAttributeInfo(field, DataTypes.LONG_TYPE);
- }
- return null;
- }
-
- /**
- * search backing index name.
- */
- public static final String BACKING_INDEX = "search";
-
- /**
- * search backing index name for vertex keys.
- */
- public static final String VERTEX_INDEX = "vertex_index";
-
- /**
- * search backing index name for edge labels.
- */
- public static final String EDGE_INDEX = "edge_index";
-
- public static final String FULLTEXT_INDEX = "fulltext_index";
-
- private Constants() {
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
index 81fb76e..0a90bda 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
@@ -18,18 +18,14 @@
package org.apache.atlas.repository.graph;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.BiMap;
-import com.google.common.collect.HashBiMap;
-import com.thinkaurelius.titan.core.TitanGraph;
-import com.thinkaurelius.titan.core.TitanProperty;
-import com.thinkaurelius.titan.core.TitanVertex;
-import com.tinkerpop.blueprints.Direction;
-import com.tinkerpop.blueprints.Edge;
-import com.tinkerpop.blueprints.Element;
-import com.tinkerpop.blueprints.Graph;
-import com.tinkerpop.blueprints.GraphQuery;
-import com.tinkerpop.blueprints.Vertex;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+
import org.apache.atlas.AtlasException;
import org.apache.atlas.RequestContext;
import org.apache.atlas.repository.Constants;
@@ -44,17 +40,23 @@ import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.HierarchicalType;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-import java.util.UUID;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.BiMap;
+import com.google.common.collect.HashBiMap;
+import com.thinkaurelius.titan.core.TitanGraph;
+import com.thinkaurelius.titan.core.TitanProperty;
+import com.thinkaurelius.titan.core.TitanVertex;
+import com.tinkerpop.blueprints.Direction;
+import com.tinkerpop.blueprints.Edge;
+import com.tinkerpop.blueprints.Element;
+import com.tinkerpop.blueprints.Graph;
+import com.tinkerpop.blueprints.GraphQuery;
+import com.tinkerpop.blueprints.Vertex;
/**
* Utility class for graph operations.
@@ -295,7 +297,7 @@ public final class GraphHelper {
/**
* Remove the specified edge from the graph.
- *
+ *
* @param edge
*/
public void removeEdge(Edge edge) {
@@ -304,10 +306,10 @@ public final class GraphHelper {
titanGraph.removeEdge(edge);
LOG.info("Removed {}", edgeString);
}
-
+
/**
* Remove the specified vertex from the graph.
- *
+ *
* @param vertex
*/
public void removeVertex(Vertex vertex) {
@@ -488,4 +490,17 @@ public final class GraphHelper {
}
return key;
}
+ public static AttributeInfo getAttributeInfoForSystemAttributes(String field) {
+ switch (field) {
+ case Constants.STATE_PROPERTY_KEY:
+ case Constants.GUID_PROPERTY_KEY:
+ return TypesUtil.newAttributeInfo(field, DataTypes.STRING_TYPE);
+
+ case Constants.TIMESTAMP_PROPERTY_KEY:
+ case Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY:
+ return TypesUtil.newAttributeInfo(field, DataTypes.LONG_TYPE);
+ }
+ return null;
+ }
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/repository/src/main/java/org/apache/atlas/repository/graph/TitanGraphProvider.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/TitanGraphProvider.java b/repository/src/main/java/org/apache/atlas/repository/graph/TitanGraphProvider.java
index 2cc1a50..7a5e6a9 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/TitanGraphProvider.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/TitanGraphProvider.java
@@ -18,122 +18,33 @@
package org.apache.atlas.repository.graph;
-import com.google.common.collect.ImmutableMap;
-import com.google.inject.Provides;
-import com.thinkaurelius.titan.core.TitanFactory;
-import com.thinkaurelius.titan.core.TitanGraph;
-import com.thinkaurelius.titan.core.schema.TitanManagement;
-import com.thinkaurelius.titan.diskstorage.StandardIndexProvider;
-import com.thinkaurelius.titan.diskstorage.solr.Solr5Index;
-import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
+import org.apache.atlas.repository.graphdb.titan0.Titan0Database;
import org.apache.commons.configuration.Configuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import javax.inject.Singleton;
-import java.lang.reflect.Field;
-import java.lang.reflect.Modifier;
-import java.util.HashMap;
-import java.util.Map;
+import com.thinkaurelius.titan.core.TitanGraph;
/**
- * Default implementation for Graph Provider that doles out Titan Graph.
+ * Temporary TitanGraphProvider to use until the graph database abstraction
+ * layer is fully in place. Delegates to the Titan 0.5.4 implementation. This
+ * will be removed once the abstraction layer is being used.
*/
public class TitanGraphProvider implements GraphProvider<TitanGraph> {
- private static final Logger LOG = LoggerFactory.getLogger(TitanGraphProvider.class);
-
- /**
- * Constant for the configuration property that indicates the prefix.
- */
- public static final String GRAPH_PREFIX = "atlas.graph";
-
- public static final String INDEX_BACKEND_CONF = "index.search.backend";
-
- public static final String INDEX_BACKEND_LUCENE = "lucene";
-
- public static final String INDEX_BACKEND_ES = "elasticsearch";
-
- private static volatile TitanGraph graphInstance;
-
- public static Configuration getConfiguration() throws AtlasException {
- Configuration configProperties = ApplicationProperties.get();
- return ApplicationProperties.getSubsetConfiguration(configProperties, GRAPH_PREFIX);
- }
-
- static {
- addSolr5Index();
- }
-
- /**
- * Titan loads index backend name to implementation using StandardIndexProvider.ALL_MANAGER_CLASSES
- * But StandardIndexProvider.ALL_MANAGER_CLASSES is a private static final ImmutableMap
- * Only way to inject Solr5Index is to modify this field. So, using hacky reflection to add Sol5Index
+ /* (non-Javadoc)
+ * @see org.apache.atlas.repository.graph.GraphProvider#get()
*/
- private static void addSolr5Index() {
- try {
- Field field = StandardIndexProvider.class.getDeclaredField("ALL_MANAGER_CLASSES");
- field.setAccessible(true);
-
- Field modifiersField = Field.class.getDeclaredField("modifiers");
- modifiersField.setAccessible(true);
- modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
-
- Map<String, String> customMap = new HashMap(StandardIndexProvider.getAllProviderClasses());
- customMap.put("solr5", Solr5Index.class.getName());
- ImmutableMap<String, String> immap = ImmutableMap.copyOf(customMap);
- field.set(null, immap);
-
- LOG.debug("Injected solr5 index - {}", Solr5Index.class.getName());
- } catch(Exception e) {
- throw new RuntimeException(e);
- }
+ @Override
+ public TitanGraph get() {
+ return Titan0Database.getGraphInstance();
}
public static TitanGraph getGraphInstance() {
- if (graphInstance == null) {
- synchronized (TitanGraphProvider.class) {
- if (graphInstance == null) {
- Configuration config;
- try {
- config = getConfiguration();
- } catch (AtlasException e) {
- throw new RuntimeException(e);
- }
-
- graphInstance = TitanFactory.open(config);
- validateIndexBackend(config);
- }
- }
- }
- return graphInstance;
- }
-
- public static void clear() {
- synchronized (TitanGraphProvider.class) {
- graphInstance.shutdown();
- graphInstance = null;
- }
+ return Titan0Database.getGraphInstance();
}
- static void validateIndexBackend(Configuration config) {
- String configuredIndexBackend = config.getString(INDEX_BACKEND_CONF);
-
- TitanManagement managementSystem = graphInstance.getManagementSystem();
- String currentIndexBackend = managementSystem.get(INDEX_BACKEND_CONF);
- managementSystem.commit();
-
- if(!configuredIndexBackend.equals(currentIndexBackend)) {
- throw new RuntimeException("Configured Index Backend " + configuredIndexBackend + " differs from earlier configured Index Backend " + currentIndexBackend + ". Aborting!");
- }
-
+ public static Configuration getConfiguration() throws AtlasException {
+ return Titan0Database.getConfiguration();
}
- @Override
- @Singleton
- @Provides
- public TitanGraph get() {
- return getGraphInstance();
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala
----------------------------------------------------------------------
diff --git a/repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala b/repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala
index ddcc106..dfa7093 100755
--- a/repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala
+++ b/repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala
@@ -24,6 +24,7 @@ import java.util.concurrent.atomic.AtomicInteger
import org.apache.atlas.AtlasException
import org.apache.atlas.query.Expressions.{LimitExpression, PathExpression, SelectExpression}
import org.apache.atlas.repository.Constants
+import org.apache.atlas.repository.graph.GraphHelper
import org.apache.atlas.typesystem.types.DataTypes.{ArrayType, PrimitiveType, TypeCategory}
import org.apache.atlas.typesystem.types._
@@ -204,7 +205,7 @@ object TypeUtils {
return Some(FieldInfo(typ,fMap.get.fields.get(id)))
}
- val systemField = Constants.getAttributeInfoForSystemAttributes(id)
+ val systemField = GraphHelper.getAttributeInfoForSystemAttributes(id)
if (systemField != null) {
return Some(FieldInfo(systemField.dataType(), systemField))
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/repository/src/test/java/org/apache/atlas/repository/graph/TitanGraphProviderTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/TitanGraphProviderTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/TitanGraphProviderTest.java
deleted file mode 100644
index 6fc7008..0000000
--- a/repository/src/test/java/org/apache/atlas/repository/graph/TitanGraphProviderTest.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.atlas.repository.graph;
-
-import com.thinkaurelius.titan.core.TitanGraph;
-import com.thinkaurelius.titan.core.util.TitanCleanup;
-import com.thinkaurelius.titan.diskstorage.Backend;
-import com.thinkaurelius.titan.graphdb.database.StandardTitanGraph;
-import org.apache.atlas.ApplicationProperties;
-import org.apache.atlas.AtlasException;
-import org.apache.commons.configuration.Configuration;
-import org.mockito.Mockito;
-import org.testng.Assert;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeTest;
-import org.testng.annotations.Test;
-
-@Test
-public class TitanGraphProviderTest {
-
- private Configuration configuration;
- private TitanGraph graph;
-
- @BeforeTest
- public void setUp() throws AtlasException {
- //First get Instance
- graph = TitanGraphProvider.getGraphInstance();
- configuration = ApplicationProperties.getSubsetConfiguration(ApplicationProperties.get(), TitanGraphProvider.GRAPH_PREFIX);
- }
-
- @AfterClass
- public void tearDown() throws Exception {
- try {
- graph.shutdown();
- } catch (Exception e) {
- e.printStackTrace();
- }
-
- try {
- TitanCleanup.clear(graph);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- @Test
- public void testValidate() throws AtlasException {
- try {
- TitanGraphProvider.validateIndexBackend(configuration);
- } catch(Exception e){
- Assert.fail("Unexpected exception ", e);
- }
-
- //Change backend
- configuration.setProperty(TitanGraphProvider.INDEX_BACKEND_CONF, TitanGraphProvider.INDEX_BACKEND_LUCENE);
- try {
- TitanGraphProvider.validateIndexBackend(configuration);
- Assert.fail("Expected exception");
- } catch(Exception e){
- Assert.assertEquals(e.getMessage(), "Configured Index Backend lucene differs from earlier configured Index Backend elasticsearch. Aborting!");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/src/build/checkstyle-suppressions.xml
----------------------------------------------------------------------
diff --git a/src/build/checkstyle-suppressions.xml b/src/build/checkstyle-suppressions.xml
index 0025360..bf16a9d 100644
--- a/src/build/checkstyle-suppressions.xml
+++ b/src/build/checkstyle-suppressions.xml
@@ -23,4 +23,7 @@
<suppressions>
<suppress checks="JavadocType" files="[/\\]src[/\\]test[/\\]java[/\\]"/>
+
+ <!-- skip checks on customized titan 0.5.4 files -->
+ <suppress checks="[a-zA-Z0-9]*" files="[/\\]com[/\\]thinkaurelius[/\\]titan[/\\]"/>
</suppressions>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/titan/pom.xml
----------------------------------------------------------------------
diff --git a/titan/pom.xml b/titan/pom.xml
deleted file mode 100644
index ae7894e..0000000
--- a/titan/pom.xml
+++ /dev/null
@@ -1,105 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>apache-atlas</artifactId>
- <groupId>org.apache.atlas</groupId>
- <version>0.8-incubating-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
- <artifactId>atlas-titan</artifactId>
- <description>Apache Atlas Titan Overrides</description>
- <name>Apache Atlas Titan</name>
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>com.thinkaurelius.titan</groupId>
- <artifactId>titan-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.vividsolutions</groupId>
- <artifactId>jts</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.solr</groupId>
- <artifactId>solr-core</artifactId>
- <version>${solr.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.solr</groupId>
- <artifactId>solr-solrj</artifactId>
- <version>${solr.version}</version>
- </dependency>
-
- <dependency>
- <groupId>com.thinkaurelius.titan</groupId>
- <artifactId>titan-es</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.thinkaurelius.titan</groupId>
- <artifactId>titan-berkeleyje</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.thinkaurelius.titan</groupId>
- <artifactId>titan-lucene</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.testng</groupId>
- <artifactId>testng</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- </dependency>
-
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <version>2.4</version>
- <configuration>
- <excludes>
- <exclude>**/log4j.xml</exclude>
- </excludes>
- </configuration>
- </plugin>
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- </plugin>
- </plugins>
- </build>
-
-
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/AdminMask.java
----------------------------------------------------------------------
diff --git a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/AdminMask.java b/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/AdminMask.java
deleted file mode 100644
index e255f1b..0000000
--- a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/AdminMask.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Copyright 2012-2013 Aurelius LLC
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.thinkaurelius.titan.diskstorage.hbase;
-
-import java.io.Closeable;
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.ClusterStatus;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-
-/**
- * This interface hides ABI/API breaking changes that HBase has made to its Admin/HBaseAdmin over the course
- * of development from 0.94 to 1.0 and beyond.
- */
-public interface AdminMask extends Closeable
-{
-
- void clearTable(String tableName, long timestamp) throws IOException;
-
- HTableDescriptor getTableDescriptor(String tableName) throws TableNotFoundException, IOException;
-
- boolean tableExists(String tableName) throws IOException;
-
- void createTable(HTableDescriptor desc) throws IOException;
-
- void createTable(HTableDescriptor desc, byte[] startKey, byte[] endKey, int numRegions) throws IOException;
-
- /**
- * Estimate the number of regionservers in the HBase cluster.
- *
- * This is usually implemented by calling
- * {@link HBaseAdmin#getClusterStatus()} and then
- * {@link ClusterStatus#getServers()} and finally {@code size()} on the
- * returned server list.
- *
- * @return the number of servers in the cluster or -1 if it could not be determined
- */
- int getEstimatedRegionServerCount();
-
- void disableTable(String tableName) throws IOException;
-
- void enableTable(String tableName) throws IOException;
-
- boolean isTableDisabled(String tableName) throws IOException;
-
- void addColumn(String tableName, HColumnDescriptor columnDescriptor) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/ConnectionMask.java
----------------------------------------------------------------------
diff --git a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/ConnectionMask.java b/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/ConnectionMask.java
deleted file mode 100644
index feb578b..0000000
--- a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/ConnectionMask.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright 2012-2013 Aurelius LLC
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.thinkaurelius.titan.diskstorage.hbase;
-
-import java.io.Closeable;
-import java.io.IOException;
-
-/**
- * This interface hides ABI/API breaking changes that HBase has made to its (H)Connection class over the course
- * of development from 0.94 to 1.0 and beyond.
- */
-public interface ConnectionMask extends Closeable
-{
-
- TableMask getTable(String name) throws IOException;
-
- AdminMask getAdmin() throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin0_98.java
----------------------------------------------------------------------
diff --git a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin0_98.java b/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin0_98.java
deleted file mode 100644
index 0cd4795..0000000
--- a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin0_98.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Copyright 2012-2013 Aurelius LLC
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.thinkaurelius.titan.diskstorage.hbase;
-
-import java.io.IOException;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.thinkaurelius.titan.util.system.IOUtils;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-
-public class HBaseAdmin0_98 implements AdminMask
-{
-
- private static final Logger log = LoggerFactory.getLogger(HBaseAdmin0_98.class);
-
- private final HBaseAdmin adm;
-
- public HBaseAdmin0_98(HBaseAdmin adm)
- {
- this.adm = adm;
- }
-
- @Override
- public void clearTable(String tableName, long timestamp) throws IOException
- {
- if (!adm.tableExists(tableName)) {
- log.debug("clearStorage() called before table {} was created, skipping.", tableName);
- return;
- }
-
- // Unfortunately, linear scanning and deleting tables is faster in HBase < 1 when running integration tests than
- // disabling and deleting tables.
- HTable table = null;
-
- try {
- table = new HTable(adm.getConfiguration(), tableName);
-
- Scan scan = new Scan();
- scan.setBatch(100);
- scan.setCacheBlocks(false);
- scan.setCaching(2000);
- scan.setTimeRange(0, Long.MAX_VALUE);
- scan.setMaxVersions(1);
-
- ResultScanner scanner = null;
-
- try {
- scanner = table.getScanner(scan);
-
- for (Result res : scanner) {
- Delete d = new Delete(res.getRow());
-
- d.setTimestamp(timestamp);
- table.delete(d);
- }
- } finally {
- IOUtils.closeQuietly(scanner);
- }
- } finally {
- IOUtils.closeQuietly(table);
- }
- }
-
- @Override
- public HTableDescriptor getTableDescriptor(String tableName) throws TableNotFoundException, IOException
- {
- return adm.getTableDescriptor(tableName.getBytes());
- }
-
- @Override
- public boolean tableExists(String tableName) throws IOException
- {
- return adm.tableExists(tableName);
- }
-
- @Override
- public void createTable(HTableDescriptor desc) throws IOException
- {
- adm.createTable(desc);
- }
-
- @Override
- public void createTable(HTableDescriptor desc, byte[] startKey, byte[] endKey, int numRegions) throws IOException
- {
- adm.createTable(desc, startKey, endKey, numRegions);
- }
-
- @Override
- public int getEstimatedRegionServerCount()
- {
- int serverCount = -1;
- try {
- serverCount = adm.getClusterStatus().getServers().size();
- log.debug("Read {} servers from HBase ClusterStatus", serverCount);
- } catch (IOException e) {
- log.debug("Unable to retrieve HBase cluster status", e);
- }
- return serverCount;
- }
-
- @Override
- public void disableTable(String tableName) throws IOException
- {
- adm.disableTable(tableName);
- }
-
- @Override
- public void enableTable(String tableName) throws IOException
- {
- adm.enableTable(tableName);
- }
-
- @Override
- public boolean isTableDisabled(String tableName) throws IOException
- {
- return adm.isTableDisabled(tableName);
- }
-
- @Override
- public void addColumn(String tableName, HColumnDescriptor columnDescriptor) throws IOException
- {
- adm.addColumn(tableName, columnDescriptor);
- }
-
- @Override
- public void close() throws IOException
- {
- adm.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin1_0.java
----------------------------------------------------------------------
diff --git a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin1_0.java b/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin1_0.java
deleted file mode 100644
index 7e8f72d..0000000
--- a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin1_0.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Copyright 2012-2013 Aurelius LLC
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.thinkaurelius.titan.diskstorage.hbase;
-
-import java.io.IOException;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.TableNotDisabledException;
-import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-
-public class HBaseAdmin1_0 implements AdminMask
-{
-
- private static final Logger log = LoggerFactory.getLogger(HBaseAdmin1_0.class);
-
- private final Admin adm;
-
- public HBaseAdmin1_0(HBaseAdmin adm)
- {
- this.adm = adm;
- }
- @Override
- public void clearTable(String tableString, long timestamp) throws IOException
- {
- TableName tableName = TableName.valueOf(tableString);
-
- if (!adm.tableExists(tableName)) {
- log.debug("Attempted to clear table {} before it exists (noop)", tableString);
- return;
- }
-
- if (!adm.isTableDisabled(tableName))
- adm.disableTable(tableName);
-
- if (!adm.isTableDisabled(tableName))
- throw new RuntimeException("Unable to disable table " + tableName);
-
- // This API call appears to both truncate and reenable the table.
- log.info("Truncating table {}", tableName);
- adm.truncateTable(tableName, true /* preserve splits */);
-
- try {
- adm.enableTable(tableName);
- } catch (TableNotDisabledException e) {
- // This triggers seemingly every time in testing with 1.0.2.
- log.debug("Table automatically reenabled by truncation: {}", tableName, e);
- }
- }
-
- @Override
- public HTableDescriptor getTableDescriptor(String tableString) throws TableNotFoundException, IOException
- {
- return adm.getTableDescriptor(TableName.valueOf(tableString));
- }
-
- @Override
- public boolean tableExists(String tableString) throws IOException
- {
- return adm.tableExists(TableName.valueOf(tableString));
- }
-
- @Override
- public void createTable(HTableDescriptor desc) throws IOException
- {
- adm.createTable(desc);
- }
-
- @Override
- public void createTable(HTableDescriptor desc, byte[] startKey, byte[] endKey, int numRegions) throws IOException
- {
- adm.createTable(desc, startKey, endKey, numRegions);
- }
-
- @Override
- public int getEstimatedRegionServerCount()
- {
- int serverCount = -1;
- try {
- serverCount = adm.getClusterStatus().getServers().size();
- log.debug("Read {} servers from HBase ClusterStatus", serverCount);
- } catch (IOException e) {
- log.debug("Unable to retrieve HBase cluster status", e);
- }
- return serverCount;
- }
-
- @Override
- public void disableTable(String tableString) throws IOException
- {
- adm.disableTable(TableName.valueOf(tableString));
- }
-
- @Override
- public void enableTable(String tableString) throws IOException
- {
- adm.enableTable(TableName.valueOf(tableString));
- }
-
- @Override
- public boolean isTableDisabled(String tableString) throws IOException
- {
- return adm.isTableDisabled(TableName.valueOf(tableString));
- }
-
- @Override
- public void addColumn(String tableString, HColumnDescriptor columnDescriptor) throws IOException
- {
- adm.addColumn(TableName.valueOf(tableString), columnDescriptor);
- }
-
- @Override
- public void close() throws IOException
- {
- adm.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat.java
----------------------------------------------------------------------
diff --git a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat.java b/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat.java
deleted file mode 100644
index c9b03aa..0000000
--- a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright 2012-2013 Aurelius LLC
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.thinkaurelius.titan.diskstorage.hbase;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.Delete;
-
-public interface HBaseCompat {
-
- /**
- * Configure the compression scheme {@code algo} on a column family
- * descriptor {@code cd}. The {@code algo} parameter is a string value
- * corresponding to one of the values of HBase's Compression enum. The
- * Compression enum has moved between packages as HBase has evolved, which
- * is why this method has a String argument in the signature instead of the
- * enum itself.
- *
- * @param cd
- * column family to configure
- * @param algo
- * compression type to use
- */
- public void setCompression(HColumnDescriptor cd, String algo);
-
- /**
- * Create and return a HTableDescriptor instance with the given name. The
- * constructors on this method have remained stable over HBase development
- * so far, but the old HTableDescriptor(String) constructor & byte[] friends
- * are now marked deprecated and may eventually be removed in favor of the
- * HTableDescriptor(TableName) constructor. That constructor (and the
- * TableName type) only exists in newer HBase versions. Hence this method.
- *
- * @param tableName
- * HBase table name
- * @return a new table descriptor instance
- */
- public HTableDescriptor newTableDescriptor(String tableName);
-
- ConnectionMask createConnection(Configuration conf) throws IOException;
-
- void addColumnFamilyToTableDescriptor(HTableDescriptor tdesc, HColumnDescriptor cdesc);
-
- void setTimestamp(Delete d, long timestamp);
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat0_98.java
----------------------------------------------------------------------
diff --git a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat0_98.java b/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat0_98.java
deleted file mode 100644
index 2c0f3b4..0000000
--- a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat0_98.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Copyright 2012-2013 Aurelius LLC
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.thinkaurelius.titan.diskstorage.hbase;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.io.compress.Compression;
-
-public class HBaseCompat0_98 implements HBaseCompat {
-
- @Override
- public void setCompression(HColumnDescriptor cd, String algo) {
- cd.setCompressionType(Compression.Algorithm.valueOf(algo));
- }
-
- @Override
- public HTableDescriptor newTableDescriptor(String tableName) {
- TableName tn = TableName.valueOf(tableName);
- return new HTableDescriptor(tn);
- }
-
- @Override
- public ConnectionMask createConnection(Configuration conf) throws IOException
- {
- return new HConnection0_98(HConnectionManager.createConnection(conf));
- }
-
- @Override
- public void addColumnFamilyToTableDescriptor(HTableDescriptor tdesc, HColumnDescriptor cdesc)
- {
- tdesc.addFamily(cdesc);
- }
-
- @Override
- public void setTimestamp(Delete d, long timestamp)
- {
- d.setTimestamp(timestamp);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_0.java
----------------------------------------------------------------------
diff --git a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_0.java b/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_0.java
deleted file mode 100644
index bb3fb3b..0000000
--- a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_0.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Copyright 2012-2013 Aurelius LLC
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.thinkaurelius.titan.diskstorage.hbase;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.io.compress.Compression;
-
-public class HBaseCompat1_0 implements HBaseCompat {
-
- @Override
- public void setCompression(HColumnDescriptor cd, String algo) {
- cd.setCompressionType(Compression.Algorithm.valueOf(algo));
- }
-
- @Override
- public HTableDescriptor newTableDescriptor(String tableName) {
- TableName tn = TableName.valueOf(tableName);
- return new HTableDescriptor(tn);
- }
-
- @Override
- public ConnectionMask createConnection(Configuration conf) throws IOException
- {
- return new HConnection1_0(ConnectionFactory.createConnection(conf));
- }
-
- @Override
- public void addColumnFamilyToTableDescriptor(HTableDescriptor tdesc, HColumnDescriptor cdesc)
- {
- tdesc.addFamily(cdesc);
- }
-
- @Override
- public void setTimestamp(Delete d, long timestamp)
- {
- d.setTimestamp(timestamp);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_1.java
----------------------------------------------------------------------
diff --git a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_1.java b/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_1.java
deleted file mode 100644
index e5c3d31..0000000
--- a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_1.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Copyright 2012-2013 Aurelius LLC
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.thinkaurelius.titan.diskstorage.hbase;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.io.compress.Compression;
-
-import java.io.IOException;
-
-public class HBaseCompat1_1 implements HBaseCompat {
-
- @Override
- public void setCompression(HColumnDescriptor cd, String algo) {
- cd.setCompressionType(Compression.Algorithm.valueOf(algo));
- }
-
- @Override
- public HTableDescriptor newTableDescriptor(String tableName) {
- TableName tn = TableName.valueOf(tableName);
- return new HTableDescriptor(tn);
- }
-
- @Override
- public ConnectionMask createConnection(Configuration conf) throws IOException
- {
- return new HConnection1_0(ConnectionFactory.createConnection(conf));
- }
-
- @Override
- public void addColumnFamilyToTableDescriptor(HTableDescriptor tdesc, HColumnDescriptor cdesc)
- {
- tdesc.addFamily(cdesc);
- }
-
- @Override
- public void setTimestamp(Delete d, long timestamp)
- {
- d.setTimestamp(timestamp);
- }
-
-}
[3/9] incubator-atlas git commit: ATLAS-693 Titan 0.5.4
implementation of graph db abstraction. (jnhagelb via dkantor)
Posted by dk...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompatLoader.java
----------------------------------------------------------------------
diff --git a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompatLoader.java b/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompatLoader.java
deleted file mode 100644
index 2c0d6fe..0000000
--- a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompatLoader.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Copyright 2012-2013 Aurelius LLC
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.thinkaurelius.titan.diskstorage.hbase;
-
-import java.util.Arrays;
-
-import org.apache.hadoop.hbase.util.VersionInfo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class HBaseCompatLoader {
-
- private static final Logger log = LoggerFactory.getLogger(HBaseCompatLoader.class);
-
- private static final String DEFAULT_HBASE_COMPAT_VERSION = "1.1";
-
- private static final String DEFAULT_HBASE_CLASS_NAME = "com.thinkaurelius.titan.diskstorage.hbase.HBaseCompat1_1";
-
- private static HBaseCompat cachedCompat;
-
- public synchronized static HBaseCompat getCompat(String classOverride) {
-
- if (null != cachedCompat) {
- log.debug("Returning cached HBase compatibility layer: {}", cachedCompat);
- return cachedCompat;
- }
-
- HBaseCompat compat;
- String className = null;
- String classNameSource = null;
-
- if (null != classOverride) {
- className = classOverride;
- classNameSource = "from explicit configuration";
- } else {
- String hbaseVersion = VersionInfo.getVersion();
- for (String supportedVersion : Arrays.asList("0.94", "0.96", "0.98", "1.0", "1.1")) {
- if (hbaseVersion.startsWith(supportedVersion + ".")) {
- className = "com.thinkaurelius.titan.diskstorage.hbase.HBaseCompat" + supportedVersion.replaceAll("\\.", "_");
- classNameSource = "supporting runtime HBase version " + hbaseVersion;
- break;
- }
- }
- if (null == className) {
- log.info("The HBase version {} is not explicitly supported by Titan. " +
- "Loading Titan's compatibility layer for its most recent supported HBase version ({})",
- hbaseVersion, DEFAULT_HBASE_COMPAT_VERSION);
- className = DEFAULT_HBASE_CLASS_NAME;
- classNameSource = " by default";
- }
- }
-
- final String errTemplate = " when instantiating HBase compatibility class " + className;
-
- try {
- compat = (HBaseCompat)Class.forName(className).newInstance();
- log.info("Instantiated HBase compatibility layer {}: {}", classNameSource, compat.getClass().getCanonicalName());
- } catch (IllegalAccessException e) {
- throw new RuntimeException(e.getClass().getSimpleName() + errTemplate, e);
- } catch (InstantiationException e) {
- throw new RuntimeException(e.getClass().getSimpleName() + errTemplate, e);
- } catch (ClassNotFoundException e) {
- throw new RuntimeException(e.getClass().getSimpleName() + errTemplate, e);
- }
-
- return cachedCompat = compat;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueStore.java
----------------------------------------------------------------------
diff --git a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueStore.java b/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueStore.java
deleted file mode 100644
index c5f6e0d..0000000
--- a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueStore.java
+++ /dev/null
@@ -1,425 +0,0 @@
-/*
- * Copyright 2012-2013 Aurelius LLC
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.thinkaurelius.titan.diskstorage.hbase;
-
-import com.google.common.base.Predicate;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Iterators;
-import com.thinkaurelius.titan.core.attribute.Duration;
-import com.thinkaurelius.titan.diskstorage.*;
-import com.thinkaurelius.titan.diskstorage.configuration.Configuration;
-import com.thinkaurelius.titan.diskstorage.keycolumnvalue.*;
-import com.thinkaurelius.titan.diskstorage.locking.LocalLockMediator;
-import com.thinkaurelius.titan.diskstorage.locking.PermanentLockingException;
-import com.thinkaurelius.titan.diskstorage.util.KeyColumn;
-import com.thinkaurelius.titan.diskstorage.util.RecordIterator;
-import com.thinkaurelius.titan.diskstorage.util.StaticArrayBuffer;
-import com.thinkaurelius.titan.diskstorage.util.StaticArrayEntry;
-import com.thinkaurelius.titan.diskstorage.util.StaticArrayEntryList;
-import com.thinkaurelius.titan.diskstorage.util.time.Timepoint;
-import com.thinkaurelius.titan.diskstorage.util.time.Timestamps;
-import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration;
-import com.thinkaurelius.titan.util.system.IOUtils;
-
-import org.apache.hadoop.hbase.client.*;
-import org.apache.hadoop.hbase.filter.ColumnPaginationFilter;
-import org.apache.hadoop.hbase.filter.ColumnRangeFilter;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterList;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.Nullable;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.*;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Here are some areas that might need work:
- * <p/>
- * - batching? (consider HTable#batch, HTable#setAutoFlush(false)
- * - tuning HTable#setWriteBufferSize (?)
- * - writing a server-side filter to replace ColumnCountGetFilter, which drops
- * all columns on the row where it reaches its limit. This requires getSlice,
- * currently, to impose its limit on the client side. That obviously won't
- * scale.
- * - RowMutations for combining Puts+Deletes (need a newer HBase than 0.92 for this)
- * - (maybe) fiddle with HTable#setRegionCachePrefetch and/or #prewarmRegionCache
- * <p/>
- * There may be other problem areas. These are just the ones of which I'm aware.
- */
-public class HBaseKeyColumnValueStore implements KeyColumnValueStore {
-
- private static final Logger logger = LoggerFactory.getLogger(HBaseKeyColumnValueStore.class);
-
- private final String tableName;
- private final HBaseStoreManager storeManager;
-
- // When using shortened CF names, columnFamily is the shortname and storeName is the longname
- // When not using shortened CF names, they are the same
- //private final String columnFamily;
- private final String storeName;
- // This is columnFamily.getBytes()
- private final byte[] columnFamilyBytes;
- private final HBaseGetter entryGetter;
-
- private final ConnectionMask cnx;
-
- private LocalLockMediator<StoreTransaction> localLockMediator;
-
- private final Duration lockExpiryTimeMs;
- private final Duration lockMaxWaitTimeMs;
- private final Integer lockMaxRetries;
-
- HBaseKeyColumnValueStore(HBaseStoreManager storeManager, ConnectionMask cnx, String tableName, String columnFamily, String storeName, LocalLockMediator<StoreTransaction> llm) {
- this.storeManager = storeManager;
- this.cnx = cnx;
- this.tableName = tableName;
- //this.columnFamily = columnFamily;
- this.storeName = storeName;
- this.columnFamilyBytes = columnFamily.getBytes();
- this.entryGetter = new HBaseGetter(storeManager.getMetaDataSchema(storeName));
- this.localLockMediator = llm;
- Configuration storageConfig = storeManager.getStorageConfig();
- this.lockExpiryTimeMs = storageConfig.get(GraphDatabaseConfiguration.LOCK_EXPIRE);
- this.lockMaxWaitTimeMs = storageConfig.get(GraphDatabaseConfiguration.LOCK_WAIT);
- this.lockMaxRetries = storageConfig.get(GraphDatabaseConfiguration.LOCK_RETRY);
- }
-
- @Override
- public void close() throws BackendException {
- }
-
- @Override
- public EntryList getSlice(KeySliceQuery query, StoreTransaction txh) throws BackendException {
- Map<StaticBuffer, EntryList> result = getHelper(Arrays.asList(query.getKey()), getFilter(query));
- return Iterables.getOnlyElement(result.values(), EntryList.EMPTY_LIST);
- }
-
- @Override
- public Map<StaticBuffer,EntryList> getSlice(List<StaticBuffer> keys, SliceQuery query, StoreTransaction txh) throws BackendException {
- return getHelper(keys, getFilter(query));
- }
-
- @Override
- public void mutate(StaticBuffer key, List<Entry> additions, List<StaticBuffer> deletions, StoreTransaction txh) throws BackendException {
- Map<StaticBuffer, KCVMutation> mutations = ImmutableMap.of(key, new KCVMutation(additions, deletions));
- mutateMany(mutations, txh);
- }
-
- @Override
- public void acquireLock(StaticBuffer key,
- StaticBuffer column,
- StaticBuffer expectedValue,
- StoreTransaction txh) throws BackendException {
-
- KeyColumn lockID = new KeyColumn(key, column);
- logger.debug("Attempting to acquireLock on {} ", lockID);
- int trialCount = 0;
- boolean locked;
- while (trialCount < lockMaxRetries) {
- final Timepoint lockStartTime = Timestamps.MILLI.getTime(System.currentTimeMillis(), TimeUnit.MILLISECONDS);
- locked = localLockMediator.lock(lockID, txh, lockStartTime.add(lockExpiryTimeMs));
- trialCount++;
- if (!locked) {
- handleLockFailure(txh, lockID, trialCount);
- } else {
- logger.debug("Acquired lock on {}, {}", lockID, txh);
- break;
- }
- }
- ((HBaseTransaction) txh).updateLocks(lockID, expectedValue);
- }
-
- void handleLockFailure(StoreTransaction txh, KeyColumn lockID, int trialCount) throws PermanentLockingException {
- if (trialCount < lockMaxRetries) {
- try {
- Thread.sleep(lockMaxWaitTimeMs.getLength(TimeUnit.DAYS.MILLISECONDS));
- } catch (InterruptedException e) {
- throw new PermanentLockingException(
- "Interrupted while waiting for acquiring lock for transaction "
- + txh + " lockID " + lockID + " on retry " + trialCount, e);
- }
- } else {
- throw new PermanentLockingException("Could not lock the keyColumn " +
- lockID + " on CF {} " + Bytes.toString(columnFamilyBytes));
- }
- }
-
- @Override
- public KeyIterator getKeys(KeyRangeQuery query, StoreTransaction txh) throws BackendException {
- return executeKeySliceQuery(query.getKeyStart().as(StaticBuffer.ARRAY_FACTORY),
- query.getKeyEnd().as(StaticBuffer.ARRAY_FACTORY),
- new FilterList(FilterList.Operator.MUST_PASS_ALL),
- query);
- }
-
- @Override
- public String getName() {
- return storeName;
- }
-
- @Override
- public KeyIterator getKeys(SliceQuery query, StoreTransaction txh) throws BackendException {
- return executeKeySliceQuery(new FilterList(FilterList.Operator.MUST_PASS_ALL), query);
- }
-
- public static Filter getFilter(SliceQuery query) {
- byte[] colStartBytes = query.getSliceEnd().length() > 0 ? query.getSliceStart().as(StaticBuffer.ARRAY_FACTORY) : null;
- byte[] colEndBytes = query.getSliceEnd().length() > 0 ? query.getSliceEnd().as(StaticBuffer.ARRAY_FACTORY) : null;
-
- Filter filter = new ColumnRangeFilter(colStartBytes, true, colEndBytes, false);
-
- if (query.hasLimit()) {
- filter = new FilterList(FilterList.Operator.MUST_PASS_ALL,
- filter,
- new ColumnPaginationFilter(query.getLimit(), 0));
- }
-
- logger.debug("Generated HBase Filter {}", filter);
-
- return filter;
- }
-
- private Map<StaticBuffer,EntryList> getHelper(List<StaticBuffer> keys, Filter getFilter) throws BackendException {
- List<Get> requests = new ArrayList<Get>(keys.size());
- {
- for (StaticBuffer key : keys) {
- Get g = new Get(key.as(StaticBuffer.ARRAY_FACTORY)).addFamily(columnFamilyBytes).setFilter(getFilter);
- try {
- g.setTimeRange(0, Long.MAX_VALUE);
- } catch (IOException e) {
- throw new PermanentBackendException(e);
- }
- requests.add(g);
- }
- }
-
- Map<StaticBuffer,EntryList> resultMap = new HashMap<StaticBuffer,EntryList>(keys.size());
-
- try {
- TableMask table = null;
- Result[] results = null;
-
- try {
- table = cnx.getTable(tableName);
- logger.debug("Get requests {} {} ", Bytes.toString(columnFamilyBytes), requests.size());
- results = table.get(requests);
- logger.debug("Get requests finished {} {} ", Bytes.toString(columnFamilyBytes), requests.size());
- } finally {
- IOUtils.closeQuietly(table);
- }
-
- if (results == null)
- return KCVSUtil.emptyResults(keys);
-
- assert results.length==keys.size();
-
- for (int i = 0; i < results.length; i++) {
- Result result = results[i];
- NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> f = result.getMap();
-
- if (f == null) { // no result for this key
- resultMap.put(keys.get(i), EntryList.EMPTY_LIST);
- continue;
- }
-
- // actual key with <timestamp, value>
- NavigableMap<byte[], NavigableMap<Long, byte[]>> r = f.get(columnFamilyBytes);
- resultMap.put(keys.get(i), (r == null)
- ? EntryList.EMPTY_LIST
- : StaticArrayEntryList.ofBytes(r.entrySet(), entryGetter));
- }
-
- return resultMap;
- } catch (IOException e) {
- throw new TemporaryBackendException(e);
- }
- }
-
- private void mutateMany(Map<StaticBuffer, KCVMutation> mutations, StoreTransaction txh) throws BackendException {
- storeManager.mutateMany(ImmutableMap.of(storeName, mutations), txh);
- }
-
- private KeyIterator executeKeySliceQuery(FilterList filters, @Nullable SliceQuery columnSlice) throws BackendException {
- return executeKeySliceQuery(null, null, filters, columnSlice);
- }
-
- private KeyIterator executeKeySliceQuery(@Nullable byte[] startKey,
- @Nullable byte[] endKey,
- FilterList filters,
- @Nullable SliceQuery columnSlice) throws BackendException {
- Scan scan = new Scan().addFamily(columnFamilyBytes);
-
- try {
- scan.setTimeRange(0, Long.MAX_VALUE);
- } catch (IOException e) {
- throw new PermanentBackendException(e);
- }
-
- if (startKey != null)
- scan.setStartRow(startKey);
-
- if (endKey != null)
- scan.setStopRow(endKey);
-
- if (columnSlice != null) {
- filters.addFilter(getFilter(columnSlice));
- }
-
- TableMask table = null;
-
- logger.debug("Scan for row keys {} {} ", Bytes.toString(startKey), Bytes.toString(endKey));
- try {
- table = cnx.getTable(tableName);
- return new RowIterator(table, table.getScanner(scan.setFilter(filters)), columnFamilyBytes);
- } catch (IOException e) {
- IOUtils.closeQuietly(table);
- throw new PermanentBackendException(e);
- }
- }
-
- private class RowIterator implements KeyIterator {
- private final Closeable table;
- private final Iterator<Result> rows;
- private final byte[] columnFamilyBytes;
-
- private Result currentRow;
- private boolean isClosed;
-
- public RowIterator(Closeable table, ResultScanner rows, byte[] columnFamilyBytes) {
- this.table = table;
- this.columnFamilyBytes = Arrays.copyOf(columnFamilyBytes, columnFamilyBytes.length);
- this.rows = Iterators.filter(rows.iterator(), new Predicate<Result>() {
- @Override
- public boolean apply(@Nullable Result result) {
- if (result == null)
- return false;
-
- try {
- StaticBuffer id = StaticArrayBuffer.of(result.getRow());
- id.getLong(0);
- } catch (NumberFormatException e) {
- return false;
- }
-
- return true;
- }
- });
- }
-
- @Override
- public RecordIterator<Entry> getEntries() {
- ensureOpen();
-
- return new RecordIterator<Entry>() {
- private final NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> currentMap = currentRow.getMap();
- private final Iterator<Map.Entry<byte[], NavigableMap<Long, byte[]>>> kv = currentMap == null ? null : currentMap.get(columnFamilyBytes).entrySet().iterator();
-
- @Override
- public boolean hasNext() {
- ensureOpen();
- return kv == null ? false : kv.hasNext();
- }
-
- @Override
- public Entry next() {
- ensureOpen();
- return kv == null ? null : StaticArrayEntry.ofBytes(kv.next(), entryGetter);
- }
-
- @Override
- public void close() {
- isClosed = true;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- };
- }
-
- @Override
- public boolean hasNext() {
- ensureOpen();
- return rows.hasNext();
- }
-
- @Override
- public StaticBuffer next() {
- ensureOpen();
-
- currentRow = rows.next();
- return StaticArrayBuffer.of(currentRow.getRow());
- }
-
- @Override
- public void close() {
- IOUtils.closeQuietly(table);
- isClosed = true;
- logger.debug("RowIterator closed table {}", table);
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- private void ensureOpen() {
- if (isClosed)
- throw new IllegalStateException("Iterator has been closed.");
- }
- }
-
- private static class HBaseGetter implements StaticArrayEntry.GetColVal<Map.Entry<byte[], NavigableMap<Long, byte[]>>, byte[]> {
-
- private final EntryMetaData[] schema;
-
- private HBaseGetter(EntryMetaData[] schema) {
- this.schema = schema;
- }
-
- @Override
- public byte[] getColumn(Map.Entry<byte[], NavigableMap<Long, byte[]>> element) {
- return element.getKey();
- }
-
- @Override
- public byte[] getValue(Map.Entry<byte[], NavigableMap<Long, byte[]>> element) {
- return element.getValue().lastEntry().getValue();
- }
-
- @Override
- public EntryMetaData[] getMetaSchema(Map.Entry<byte[], NavigableMap<Long, byte[]>> element) {
- return schema;
- }
-
- @Override
- public Object getMetaData(Map.Entry<byte[], NavigableMap<Long, byte[]>> element, EntryMetaData meta) {
- switch(meta) {
- case TIMESTAMP:
- return element.getValue().lastEntry().getKey();
- default:
- throw new UnsupportedOperationException("Unsupported meta data: " + meta);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseStoreManager.java
----------------------------------------------------------------------
diff --git a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseStoreManager.java b/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseStoreManager.java
deleted file mode 100644
index a94a7e4..0000000
--- a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseStoreManager.java
+++ /dev/null
@@ -1,935 +0,0 @@
-/*
- * Copyright 2012-2013 Aurelius LLC
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.thinkaurelius.titan.diskstorage.hbase;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-
-import com.thinkaurelius.titan.diskstorage.Backend;
-import com.thinkaurelius.titan.diskstorage.configuration.ConfigElement;
-import com.thinkaurelius.titan.diskstorage.keycolumnvalue.CustomizeStoreKCVSManager;
-import com.thinkaurelius.titan.diskstorage.locking.LocalLockMediator;
-import com.thinkaurelius.titan.diskstorage.locking.LocalLockMediators;
-import com.thinkaurelius.titan.diskstorage.util.time.Timestamps;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.MasterNotRunningException;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.TableNotEnabledException;
-import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.ZooKeeperConnectionException;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Row;
-import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.util.VersionInfo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Function;
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Predicate;
-import com.google.common.collect.BiMap;
-import com.google.common.collect.ImmutableBiMap;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Sets;
-import com.thinkaurelius.titan.core.TitanException;
-import com.thinkaurelius.titan.diskstorage.BackendException;
-import com.thinkaurelius.titan.diskstorage.BaseTransactionConfig;
-import com.thinkaurelius.titan.diskstorage.Entry;
-import com.thinkaurelius.titan.diskstorage.PermanentBackendException;
-import com.thinkaurelius.titan.diskstorage.StaticBuffer;
-import com.thinkaurelius.titan.diskstorage.TemporaryBackendException;
-import com.thinkaurelius.titan.diskstorage.common.DistributedStoreManager;
-import com.thinkaurelius.titan.diskstorage.configuration.ConfigNamespace;
-import com.thinkaurelius.titan.diskstorage.configuration.ConfigOption;
-import com.thinkaurelius.titan.diskstorage.configuration.Configuration;
-import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KCVMutation;
-import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStore;
-import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStoreManager;
-import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyRange;
-import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StandardStoreFeatures;
-import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreFeatures;
-import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreTransaction;
-import com.thinkaurelius.titan.diskstorage.util.BufferUtil;
-import com.thinkaurelius.titan.diskstorage.util.StaticArrayBuffer;
-import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration;
-import com.thinkaurelius.titan.graphdb.configuration.PreInitializeConfigOptions;
-import com.thinkaurelius.titan.util.system.IOUtils;
-import com.thinkaurelius.titan.util.system.NetworkUtil;
-
-/**
- * Storage Manager for HBase
- *
- * @author Dan LaRocque <da...@hopcount.org>
- */
-@PreInitializeConfigOptions
-public class HBaseStoreManager extends DistributedStoreManager implements KeyColumnValueStoreManager, CustomizeStoreKCVSManager {
-
- private static final Logger logger = LoggerFactory.getLogger(HBaseStoreManager.class);
-
- public static final ConfigNamespace HBASE_NS =
- new ConfigNamespace(GraphDatabaseConfiguration.STORAGE_NS, "hbase", "HBase storage options");
-
- public static final ConfigOption<Boolean> SHORT_CF_NAMES =
- new ConfigOption<Boolean>(HBASE_NS, "short-cf-names",
- "Whether to shorten the names of Titan's column families to one-character mnemonics " +
- "to conserve storage space", ConfigOption.Type.FIXED, true);
-
- public static final String COMPRESSION_DEFAULT = "-DEFAULT-";
-
- public static final ConfigOption<String> COMPRESSION =
- new ConfigOption<String>(HBASE_NS, "compression-algorithm",
- "An HBase Compression.Algorithm enum string which will be applied to newly created column families. " +
- "The compression algorithm must be installed and available on the HBase cluster. Titan cannot install " +
- "and configure new compression algorithms on the HBase cluster by itself.",
- ConfigOption.Type.MASKABLE, "GZ");
-
- public static final ConfigOption<Boolean> SKIP_SCHEMA_CHECK =
- new ConfigOption<Boolean>(HBASE_NS, "skip-schema-check",
- "Assume that Titan's HBase table and column families already exist. " +
- "When this is true, Titan will not check for the existence of its table/CFs, " +
- "nor will it attempt to create them under any circumstances. This is useful " +
- "when running Titan without HBase admin privileges.",
- ConfigOption.Type.MASKABLE, false);
-
- public static final ConfigOption<String> HBASE_TABLE =
- new ConfigOption<String>(HBASE_NS, "table",
- "The name of the table Titan will use. When " + ConfigElement.getPath(SKIP_SCHEMA_CHECK) +
- " is false, Titan will automatically create this table if it does not already exist.",
- ConfigOption.Type.LOCAL, "titan");
-
- /**
- * Related bug fixed in 0.98.0, 0.94.7, 0.95.0:
- *
- * https://issues.apache.org/jira/browse/HBASE-8170
- */
- public static final int MIN_REGION_COUNT = 3;
-
- /**
- * The total number of HBase regions to create with Titan's table. This
- * setting only effects table creation; this normally happens just once when
- * Titan connects to an HBase backend for the first time.
- */
- public static final ConfigOption<Integer> REGION_COUNT =
- new ConfigOption<Integer>(HBASE_NS, "region-count",
- "The number of initial regions set when creating Titan's HBase table",
- ConfigOption.Type.MASKABLE, Integer.class, new Predicate<Integer>() {
- @Override
- public boolean apply(Integer input) {
- return null != input && MIN_REGION_COUNT <= input;
- }
- }
- );
-
- /**
- * This setting is used only when {@link #REGION_COUNT} is unset.
- * <p/>
- * If Titan's HBase table does not exist, then it will be created with total
- * region count = (number of servers reported by ClusterStatus) * (this
- * value).
- * <p/>
- * The Apache HBase manual suggests an order-of-magnitude range of potential
- * values for this setting:
- *
- * <ul>
- * <li>
- * <a href="https://hbase.apache.org/book/important_configurations.html#disable.splitting">2.5.2.7. Managed Splitting</a>:
- * <blockquote>
- * What's the optimal number of pre-split regions to create? Mileage will
- * vary depending upon your application. You could start low with 10
- * pre-split regions / server and watch as data grows over time. It's
- * better to err on the side of too little regions and rolling split later.
- * </blockquote>
- * </li>
- * <li>
- * <a href="https://hbase.apache.org/book/regions.arch.html">9.7 Regions</a>:
- * <blockquote>
- * In general, HBase is designed to run with a small (20-200) number of
- * relatively large (5-20Gb) regions per server... Typically you want to
- * keep your region count low on HBase for numerous reasons. Usually
- * right around 100 regions per RegionServer has yielded the best results.
- * </blockquote>
- * </li>
- * </ul>
- *
- * These considerations may differ for other HBase implementations (e.g. MapR).
- */
- public static final ConfigOption<Integer> REGIONS_PER_SERVER =
- new ConfigOption<Integer>(HBASE_NS, "regions-per-server",
- "The number of regions per regionserver to set when creating Titan's HBase table",
- ConfigOption.Type.MASKABLE, Integer.class);
-
- /**
- * If this key is present in either the JVM system properties or the process
- * environment (checked in the listed order, first hit wins), then its value
- * must be the full package and class name of an implementation of
- * {@link HBaseCompat} that has a no-arg public constructor.
- * <p>
- * When this <b>is not</b> set, Titan attempts to automatically detect the
- * HBase runtime version by calling {@link VersionInfo#getVersion()}. Titan
- * then checks the returned version string against a hard-coded list of
- * supported version prefixes and instantiates the associated compat layer
- * if a match is found.
- * <p>
- * When this <b>is</b> set, Titan will not call
- * {@code VersionInfo.getVersion()} or read its hard-coded list of supported
- * version prefixes. Titan will instead attempt to instantiate the class
- * specified (via the no-arg constructor which must exist) and then attempt
- * to cast it to HBaseCompat and use it as such. Titan will assume the
- * supplied implementation is compatible with the runtime HBase version and
- * make no attempt to verify that assumption.
- * <p>
- * Setting this key incorrectly could cause runtime exceptions at best or
- * silent data corruption at worst. This setting is intended for users
- * running exotic HBase implementations that don't support VersionInfo or
- * implementations which return values from {@code VersionInfo.getVersion()}
- * that are inconsistent with Apache's versioning convention. It may also be
- * useful to users who want to run against a new release of HBase that Titan
- * doesn't yet officially support.
- *
- */
- public static final ConfigOption<String> COMPAT_CLASS =
- new ConfigOption<String>(HBASE_NS, "compat-class",
- "The package and class name of the HBaseCompat implementation. HBaseCompat masks version-specific HBase API differences. " +
- "When this option is unset, Titan calls HBase's VersionInfo.getVersion() and loads the matching compat class " +
- "at runtime. Setting this option forces Titan to instead reflectively load and instantiate the specified class.",
- ConfigOption.Type.MASKABLE, String.class);
-
- public static final int PORT_DEFAULT = 9160;
-
- public static final Timestamps PREFERRED_TIMESTAMPS = Timestamps.MILLI;
-
- public static final ConfigNamespace HBASE_CONFIGURATION_NAMESPACE =
- new ConfigNamespace(HBASE_NS, "ext", "Overrides for hbase-{site,default}.xml options", true);
-
- private static final BiMap<String, String> SHORT_CF_NAME_MAP =
- ImmutableBiMap.<String, String>builder()
- .put(Backend.INDEXSTORE_NAME, "g")
- .put(Backend.INDEXSTORE_NAME + Backend.LOCK_STORE_SUFFIX, "h")
- .put(Backend.ID_STORE_NAME, "i")
- .put(Backend.EDGESTORE_NAME, "e")
- .put(Backend.EDGESTORE_NAME + Backend.LOCK_STORE_SUFFIX, "f")
- .put(GraphDatabaseConfiguration.SYSTEM_PROPERTIES_STORE_NAME, "s")
- .put(GraphDatabaseConfiguration.SYSTEM_PROPERTIES_STORE_NAME + Backend.LOCK_STORE_SUFFIX, "t")
- .put(Backend.SYSTEM_MGMT_LOG_NAME, "m")
- .put(Backend.SYSTEM_TX_LOG_NAME, "l")
- .build();
-
- private static final StaticBuffer FOUR_ZERO_BYTES = BufferUtil.zeroBuffer(4);
-
- static {
- // Verify that shortCfNameMap is injective
- // Should be guaranteed by Guava BiMap, but it doesn't hurt to check
- Preconditions.checkArgument(null != SHORT_CF_NAME_MAP);
- Collection<String> shorts = SHORT_CF_NAME_MAP.values();
- Preconditions.checkArgument(Sets.newHashSet(shorts).size() == shorts.size());
- }
-
- // Immutable instance fields
- private final String tableName;
- private final String compression;
- private final int regionCount;
- private final int regionsPerServer;
- private final ConnectionMask cnx;
- private final org.apache.hadoop.conf.Configuration hconf;
- private final boolean shortCfNames;
- private final boolean skipSchemaCheck;
- private final String compatClass;
- private final HBaseCompat compat;
-
- private static final ConcurrentHashMap<HBaseStoreManager, Throwable> openManagers =
- new ConcurrentHashMap<HBaseStoreManager, Throwable>();
-
- // Mutable instance state
- private final ConcurrentMap<String, HBaseKeyColumnValueStore> openStores;
-
- private LocalLockMediator<StoreTransaction> llm;
-
- public HBaseStoreManager(com.thinkaurelius.titan.diskstorage.configuration.Configuration config) throws BackendException {
- super(config, PORT_DEFAULT);
-
- checkConfigDeprecation(config);
-
- this.tableName = config.get(HBASE_TABLE);
- this.compression = config.get(COMPRESSION);
- this.regionCount = config.has(REGION_COUNT) ? config.get(REGION_COUNT) : -1;
- this.regionsPerServer = config.has(REGIONS_PER_SERVER) ? config.get(REGIONS_PER_SERVER) : -1;
- this.skipSchemaCheck = config.get(SKIP_SCHEMA_CHECK);
- this.compatClass = config.has(COMPAT_CLASS) ? config.get(COMPAT_CLASS) : null;
- this.compat = HBaseCompatLoader.getCompat(compatClass);
-
- /*
- * Specifying both region count options is permitted but may be
- * indicative of a misunderstanding, so issue a warning.
- */
- if (config.has(REGIONS_PER_SERVER) && config.has(REGION_COUNT)) {
- logger.warn("Both {} and {} are set in Titan's configuration, but "
- + "the former takes precedence and the latter will be ignored.",
- REGION_COUNT, REGIONS_PER_SERVER);
- }
-
- /* This static factory calls HBaseConfiguration.addHbaseResources(),
- * which in turn applies the contents of hbase-default.xml and then
- * applies the contents of hbase-site.xml.
- */
- this.hconf = HBaseConfiguration.create();
-
- // Copy a subset of our commons config into a Hadoop config
- int keysLoaded=0;
- Map<String,Object> configSub = config.getSubset(HBASE_CONFIGURATION_NAMESPACE);
- for (Map.Entry<String,Object> entry : configSub.entrySet()) {
- logger.info("HBase configuration: setting {}={}", entry.getKey(), entry.getValue());
- if (entry.getValue()==null) continue;
- hconf.set(entry.getKey(), entry.getValue().toString());
- keysLoaded++;
- }
-
- // Special case for STORAGE_HOSTS
- if (config.has(GraphDatabaseConfiguration.STORAGE_HOSTS)) {
- String zkQuorumKey = "hbase.zookeeper.quorum";
- String csHostList = Joiner.on(",").join(config.get(GraphDatabaseConfiguration.STORAGE_HOSTS));
- hconf.set(zkQuorumKey, csHostList);
- logger.info("Copied host list from {} to {}: {}", GraphDatabaseConfiguration.STORAGE_HOSTS, zkQuorumKey, csHostList);
- }
-
- logger.debug("HBase configuration: set a total of {} configuration values", keysLoaded);
-
- this.shortCfNames = config.get(SHORT_CF_NAMES);
-
- try {
- //this.cnx = HConnectionManager.createConnection(hconf);
- this.cnx = compat.createConnection(hconf);
- } catch (IOException e) {
- throw new PermanentBackendException(e);
- }
-
- if (logger.isTraceEnabled()) {
- openManagers.put(this, new Throwable("Manager Opened"));
- dumpOpenManagers();
- }
-
- logger.debug("Dumping HBase config key=value pairs");
- for (Map.Entry<String, String> entry : hconf) {
- logger.debug("[HBaseConfig] " + entry.getKey() + "=" + entry.getValue());
- }
- logger.debug("End of HBase config key=value pairs");
-
- openStores = new ConcurrentHashMap<String, HBaseKeyColumnValueStore>();
- }
-
- @Override
- public Deployment getDeployment() {
- return Deployment.REMOTE;
-
- /* If just one of the regions for titan table is in the localhost,
- * this method returns Deployment.LOCAL - which does not sound right.
- *
- List<KeyRange> local;
- try {
- local = getLocalKeyPartition();
- return null != local && !local.isEmpty() ? Deployment.LOCAL : Deployment.REMOTE;
- } catch (BackendException e) {
- // propagating StorageException might be a better approach
- throw new RuntimeException(e);
- }
- *
- */
- }
-
- @Override
- public String toString() {
- return "hbase[" + tableName + "@" + super.toString() + "]";
- }
-
- public void dumpOpenManagers() {
- int estimatedSize = openManagers.size();
- logger.trace("---- Begin open HBase store manager list ({} managers) ----", estimatedSize);
- for (HBaseStoreManager m : openManagers.keySet()) {
- logger.trace("Manager {} opened at:", m, openManagers.get(m));
- }
- logger.trace("---- End open HBase store manager list ({} managers) ----", estimatedSize);
- }
-
- @Override
- public void close() {
- openStores.clear();
- if (logger.isTraceEnabled())
- openManagers.remove(this);
- IOUtils.closeQuietly(cnx);
- }
-
- @Override
- public StoreFeatures getFeatures() {
-
- Configuration c = GraphDatabaseConfiguration.buildConfiguration();
-
- StandardStoreFeatures.Builder fb = new StandardStoreFeatures.Builder()
- .orderedScan(true).unorderedScan(true).batchMutation(true)
- .multiQuery(true).distributed(true).keyOrdered(true).storeTTL(true)
- .timestamps(true).preferredTimestamps(PREFERRED_TIMESTAMPS)
- .locking(true)
- .keyConsistent(c);
-
- try {
- fb.localKeyPartition(getDeployment() == Deployment.LOCAL);
- } catch (Exception e) {
- logger.warn("Unexpected exception during getDeployment()", e);
- }
-
- return fb.build();
- }
-
- @Override
- public void mutateMany(Map<String, Map<StaticBuffer, KCVMutation>> mutations, StoreTransaction txh) throws BackendException {
- logger.debug("Enter mutateMany");
- final MaskedTimestamp commitTime = new MaskedTimestamp(txh);
- // In case of an addition and deletion with identical timestamps, the
- // deletion tombstone wins.
- // http://hbase.apache.org/book/versions.html#d244e4250
- Map<StaticBuffer, Pair<Put, Delete>> commandsPerKey =
- convertToCommands(
- mutations,
- commitTime.getAdditionTime(times.getUnit()),
- commitTime.getDeletionTime(times.getUnit()));
-
- List<Row> batch = new ArrayList<Row>(commandsPerKey.size()); // actual batch operation
-
- // convert sorted commands into representation required for 'batch' operation
- for (Pair<Put, Delete> commands : commandsPerKey.values()) {
- if (commands.getFirst() != null)
- batch.add(commands.getFirst());
-
- if (commands.getSecond() != null)
- batch.add(commands.getSecond());
- }
-
- try {
- TableMask table = null;
-
- try {
- table = cnx.getTable(tableName);
- logger.debug("mutateMany : batch mutate started size {} ", batch.size());
- table.batch(batch, new Object[batch.size()]);
- logger.debug("mutateMany : batch mutate finished {} ", batch.size());
- } finally {
- IOUtils.closeQuietly(table);
- }
- } catch (IOException e) {
- throw new TemporaryBackendException(e);
- } catch (InterruptedException e) {
- throw new TemporaryBackendException(e);
- }
-
- sleepAfterWrite(txh, commitTime);
- }
-
- @Override
- public KeyColumnValueStore openDatabase(String longName) throws BackendException {
-
- return openDatabase(longName, -1);
- }
-
- @Override
- public KeyColumnValueStore openDatabase(final String longName, int ttlInSeconds) throws BackendException {
-
- HBaseKeyColumnValueStore store = openStores.get(longName);
-
- if (store == null) {
- final String cfName = shortCfNames ? shortenCfName(longName) : longName;
-
- final String llmPrefix = getName();
- llm = LocalLockMediators.INSTANCE.<StoreTransaction>get(llmPrefix, times);
- HBaseKeyColumnValueStore newStore = new HBaseKeyColumnValueStore(this, cnx, tableName, cfName, longName, llm);
-
- store = openStores.putIfAbsent(longName, newStore); // nothing bad happens if we loose to other thread
-
- if (store == null) {
- if (!skipSchemaCheck)
- ensureColumnFamilyExists(tableName, cfName, ttlInSeconds);
-
- store = newStore;
- }
- logger.info("Loaded 1.x Hbase Client Store Manager");
- }
-
- return store;
- }
-
-
- @Override
- public StoreTransaction beginTransaction(final BaseTransactionConfig config) throws BackendException {
- return new HBaseTransaction(config, llm);
- }
-
- @Override
- public String getName() {
- return tableName;
- }
-
- /**
- * Deletes the specified table with all its columns.
- * ATTENTION: Invoking this method will delete the table if it exists and therefore causes data loss.
- */
- @Override
- public void clearStorage() throws BackendException {
- try (AdminMask adm = getAdminInterface()) {
- adm.clearTable(tableName, times.getTime().getNativeTimestamp());
- } catch (IOException e)
- {
- throw new TemporaryBackendException(e);
- }
- }
-
- @Override
- public List<KeyRange> getLocalKeyPartition() throws BackendException {
-
- List<KeyRange> result = new LinkedList<KeyRange>();
-
- TableMask table = null;
- try {
- ensureTableExists(tableName, getCfNameForStoreName(GraphDatabaseConfiguration.SYSTEM_PROPERTIES_STORE_NAME), 0);
-
- table = cnx.getTable(tableName);
-
- HTable hTable = (HTable)table.getTableObject();
-
- Map<KeyRange, ServerName> normed =
- normalizeKeyBounds(hTable.getRegionLocations());
-
- for (Map.Entry<KeyRange, ServerName> e : normed.entrySet()) {
- if (NetworkUtil.isLocalConnection(e.getValue().getHostname())) {
- result.add(e.getKey());
- logger.debug("Found local key/row partition {} on host {}", e.getKey(), e.getValue());
- } else {
- logger.debug("Discarding remote {}", e.getValue());
- }
- }
- } catch (MasterNotRunningException e) {
- logger.warn("Unexpected MasterNotRunningException", e);
- } catch (ZooKeeperConnectionException e) {
- logger.warn("Unexpected ZooKeeperConnectionException", e);
- } catch (IOException e) {
- logger.warn("Unexpected IOException", e);
- } finally {
- IOUtils.closeQuietly(table);
- }
- return result;
- }
-
- /**
- * Given a map produced by {@link HTable#getRegionLocations()}, transform
- * each key from an {@link HRegionInfo} to a {@link KeyRange} expressing the
- * region's start and end key bounds using Titan-partitioning-friendly
- * conventions (start inclusive, end exclusive, zero bytes appended where
- * necessary to make all keys at least 4 bytes long).
- * <p/>
- * This method iterates over the entries in its map parameter and performs
- * the following conditional conversions on its keys. "Require" below means
- * either a {@link Preconditions} invocation or an assertion. HRegionInfo
- * sometimes returns start and end keys of zero length; this method replaces
- * zero length keys with null before doing any of the checks described
- * below. The parameter map and the values it contains are only read and
- * never modified.
- *
- * <ul>
- * <li>If an entry's HRegionInfo has null start and end keys, then first
- * require that the parameter map is a singleton, and then return a
- * single-entry map whose {@code KeyRange} has start and end buffers that
- * are both four bytes of zeros.</li>
- * <li>If the entry has a null end key (but non-null start key), put an
- * equivalent entry in the result map with a start key identical to the
- * input, except that zeros are appended to values less than 4 bytes long,
- * and an end key that is four bytes of zeros.
- * <li>If the entry has a null start key (but non-null end key), put an
- * equivalent entry in the result map where the start key is four bytes of
- * zeros, and the end key has zeros appended, if necessary, to make it at
- * least 4 bytes long, after which one is added to the padded value in
- * unsigned 32-bit arithmetic with overflow allowed.</li>
- * <li>Any entry which matches none of the above criteria results in an
- * equivalent entry in the returned map, except that zeros are appended to
- * both keys to make each at least 4 bytes long, and the end key is then
- * incremented as described in the last bullet point.</li>
- * </ul>
- *
- * After iterating over the parameter map, this method checks that it either
- * saw no entries with null keys, one entry with a null start key and a
- * different entry with a null end key, or one entry with both start and end
- * keys null. If any null keys are observed besides these three cases, the
- * method will die with a precondition failure.
- *
- * @param raw
- * A map of HRegionInfo and ServerName from HBase
- * @return Titan-friendly expression of each region's rowkey boundaries
- */
- private Map<KeyRange, ServerName> normalizeKeyBounds(NavigableMap<HRegionInfo, ServerName> raw) {
-
- Map.Entry<HRegionInfo, ServerName> nullStart = null;
- Map.Entry<HRegionInfo, ServerName> nullEnd = null;
-
- ImmutableMap.Builder<KeyRange, ServerName> b = ImmutableMap.builder();
-
- for (Map.Entry<HRegionInfo, ServerName> e : raw.entrySet()) {
- HRegionInfo regionInfo = e.getKey();
- byte startKey[] = regionInfo.getStartKey();
- byte endKey[] = regionInfo.getEndKey();
-
- if (0 == startKey.length) {
- startKey = null;
- logger.trace("Converted zero-length HBase startKey byte array to null");
- }
-
- if (0 == endKey.length) {
- endKey = null;
- logger.trace("Converted zero-length HBase endKey byte array to null");
- }
-
- if (null == startKey && null == endKey) {
- Preconditions.checkState(1 == raw.size());
- logger.debug("HBase table {} has a single region {}", tableName, regionInfo);
- // Choose arbitrary shared value = startKey = endKey
- return b.put(new KeyRange(FOUR_ZERO_BYTES, FOUR_ZERO_BYTES), e.getValue()).build();
- } else if (null == startKey) {
- logger.debug("Found HRegionInfo with null startKey on server {}: {}", e.getValue(), regionInfo);
- Preconditions.checkState(null == nullStart);
- nullStart = e;
- // I thought endBuf would be inclusive from the HBase javadoc, but in practice it is exclusive
- StaticBuffer endBuf = StaticArrayBuffer.of(zeroExtend(endKey));
- // Replace null start key with zeroes
- b.put(new KeyRange(FOUR_ZERO_BYTES, endBuf), e.getValue());
- } else if (null == endKey) {
- logger.debug("Found HRegionInfo with null endKey on server {}: {}", e.getValue(), regionInfo);
- Preconditions.checkState(null == nullEnd);
- nullEnd = e;
- // Replace null end key with zeroes
- b.put(new KeyRange(StaticArrayBuffer.of(zeroExtend(startKey)), FOUR_ZERO_BYTES), e.getValue());
- } else {
- Preconditions.checkState(null != startKey);
- Preconditions.checkState(null != endKey);
-
- // Convert HBase's inclusive end keys into exclusive Titan end keys
- StaticBuffer startBuf = StaticArrayBuffer.of(zeroExtend(startKey));
- StaticBuffer endBuf = StaticArrayBuffer.of(zeroExtend(endKey));
-
- KeyRange kr = new KeyRange(startBuf, endBuf);
- b.put(kr, e.getValue());
- logger.debug("Found HRegionInfo with non-null end and start keys on server {}: {}", e.getValue(), regionInfo);
- }
- }
-
- // Require either no null key bounds or a pair of them
- Preconditions.checkState(!(null == nullStart ^ null == nullEnd));
-
- // Check that every key in the result is at least 4 bytes long
- Map<KeyRange, ServerName> result = b.build();
- for (KeyRange kr : result.keySet()) {
- Preconditions.checkState(4 <= kr.getStart().length());
- Preconditions.checkState(4 <= kr.getEnd().length());
- }
-
- return result;
- }
-
- /**
- * If the parameter is shorter than 4 bytes, then create and return a new 4
- * byte array with the input array's bytes followed by zero bytes. Otherwise
- * return the parameter.
- *
- * @param dataToPad non-null but possibly zero-length byte array
- * @return either the parameter or a new array
- */
- private final byte[] zeroExtend(byte[] dataToPad) {
- assert null != dataToPad;
-
- final int targetLength = 4;
-
- if (targetLength <= dataToPad.length)
- return dataToPad;
-
- byte padded[] = new byte[targetLength];
-
- for (int i = 0; i < dataToPad.length; i++)
- padded[i] = dataToPad[i];
-
- for (int i = dataToPad.length; i < padded.length; i++)
- padded[i] = (byte)0;
-
- return padded;
- }
-
- public static String shortenCfName(String longName) throws PermanentBackendException {
- final String s;
- if (SHORT_CF_NAME_MAP.containsKey(longName)) {
- s = SHORT_CF_NAME_MAP.get(longName);
- Preconditions.checkNotNull(s);
- logger.debug("Substituted default CF name \"{}\" with short form \"{}\" to reduce HBase KeyValue size", longName, s);
- } else {
- if (SHORT_CF_NAME_MAP.containsValue(longName)) {
- String fmt = "Must use CF long-form name \"%s\" instead of the short-form name \"%s\" when configured with %s=true";
- String msg = String.format(fmt, SHORT_CF_NAME_MAP.inverse().get(longName), longName, SHORT_CF_NAMES.getName());
- throw new PermanentBackendException(msg);
- }
- s = longName;
- logger.debug("Kept default CF name \"{}\" because it has no associated short form", s);
- }
- return s;
- }
-
- private HTableDescriptor ensureTableExists(String tableName, String initialCFName, int ttlInSeconds) throws BackendException {
- AdminMask adm = null;
-
- HTableDescriptor desc;
-
- try { // Create our table, if necessary
- adm = getAdminInterface();
- /*
- * Some HBase versions/impls respond badly to attempts to create a
- * table without at least one CF. See #661. Creating a CF along with
- * the table avoids HBase carping.
- */
- if (adm.tableExists(tableName)) {
- desc = adm.getTableDescriptor(tableName);
- } else {
- desc = createTable(tableName, initialCFName, ttlInSeconds, adm);
- }
- } catch (IOException e) {
- throw new TemporaryBackendException(e);
- } finally {
- IOUtils.closeQuietly(adm);
- }
-
- return desc;
- }
-
- private HTableDescriptor createTable(String tableName, String cfName, int ttlInSeconds, AdminMask adm) throws IOException {
- HTableDescriptor desc = compat.newTableDescriptor(tableName);
-
- HColumnDescriptor cdesc = new HColumnDescriptor(cfName);
- setCFOptions(cdesc, ttlInSeconds);
-
- compat.addColumnFamilyToTableDescriptor(desc, cdesc);
-
- int count; // total regions to create
- String src;
-
- if (MIN_REGION_COUNT <= (count = regionCount)) {
- src = "region count configuration";
- } else if (0 < regionsPerServer &&
- MIN_REGION_COUNT <= (count = regionsPerServer * adm.getEstimatedRegionServerCount())) {
- src = "ClusterStatus server count";
- } else {
- count = -1;
- src = "default";
- }
-
- if (MIN_REGION_COUNT < count) {
- adm.createTable(desc, getStartKey(count), getEndKey(count), count);
- logger.debug("Created table {} with region count {} from {}", tableName, count, src);
- } else {
- adm.createTable(desc);
- logger.debug("Created table {} with default start key, end key, and region count", tableName);
- }
-
- return desc;
- }
-
- /**
- * This method generates the second argument to
- * {@link HBaseAdmin#createTable(HTableDescriptor, byte[], byte[], int)}.
- * <p/>
- * From the {@code createTable} javadoc:
- * "The start key specified will become the end key of the first region of
- * the table, and the end key specified will become the start key of the
- * last region of the table (the first region has a null start key and
- * the last region has a null end key)"
- * <p/>
- * To summarize, the {@code createTable} argument called "startKey" is
- * actually the end key of the first region.
- */
- private byte[] getStartKey(int regionCount) {
- ByteBuffer regionWidth = ByteBuffer.allocate(4);
- regionWidth.putInt((int)(((1L << 32) - 1L) / regionCount)).flip();
- return StaticArrayBuffer.of(regionWidth).getBytes(0, 4);
- }
-
- /**
- * Companion to {@link #getStartKey(int)}. See its javadoc for details.
- */
- private byte[] getEndKey(int regionCount) {
- ByteBuffer regionWidth = ByteBuffer.allocate(4);
- regionWidth.putInt((int)(((1L << 32) - 1L) / regionCount * (regionCount - 1))).flip();
- return StaticArrayBuffer.of(regionWidth).getBytes(0, 4);
- }
-
- private void ensureColumnFamilyExists(String tableName, String columnFamily, int ttlInSeconds) throws BackendException {
- AdminMask adm = null;
- try {
- adm = getAdminInterface();
- HTableDescriptor desc = ensureTableExists(tableName, columnFamily, ttlInSeconds);
-
- Preconditions.checkNotNull(desc);
-
- HColumnDescriptor cf = desc.getFamily(columnFamily.getBytes());
-
- // Create our column family, if necessary
- if (cf == null) {
- try {
- if (!adm.isTableDisabled(tableName)) {
- adm.disableTable(tableName);
- }
- } catch (TableNotEnabledException e) {
- logger.debug("Table {} already disabled", tableName);
- } catch (IOException e) {
- throw new TemporaryBackendException(e);
- }
-
- try {
- HColumnDescriptor cdesc = new HColumnDescriptor(columnFamily);
-
- setCFOptions(cdesc, ttlInSeconds);
-
- adm.addColumn(tableName, cdesc);
-
- logger.debug("Added HBase ColumnFamily {}, waiting for 1 sec. to propogate.", columnFamily);
-
- adm.enableTable(tableName);
- } catch (TableNotFoundException ee) {
- logger.error("TableNotFoundException", ee);
- throw new PermanentBackendException(ee);
- } catch (org.apache.hadoop.hbase.TableExistsException ee) {
- logger.debug("Swallowing exception {}", ee);
- } catch (IOException ee) {
- throw new TemporaryBackendException(ee);
- }
- }
- } finally {
- IOUtils.closeQuietly(adm);
- }
- }
-
- private void setCFOptions(HColumnDescriptor cdesc, int ttlInSeconds) {
- if (null != compression && !compression.equals(COMPRESSION_DEFAULT))
- compat.setCompression(cdesc, compression);
-
- if (ttlInSeconds > 0)
- cdesc.setTimeToLive(ttlInSeconds);
-
- cdesc.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF);
- }
-
- /**
- * Convert Titan internal Mutation representation into HBase native commands.
- *
- * @param mutations Mutations to convert into HBase commands.
- * @param putTimestamp The timestamp to use for Put commands.
- * @param delTimestamp The timestamp to use for Delete commands.
- * @return Commands sorted by key converted from Titan internal representation.
- * @throws com.thinkaurelius.titan.diskstorage.PermanentBackendException
- */
- private Map<StaticBuffer, Pair<Put, Delete>> convertToCommands(Map<String, Map<StaticBuffer, KCVMutation>> mutations,
- final long putTimestamp,
- final long delTimestamp) throws PermanentBackendException {
- Map<StaticBuffer, Pair<Put, Delete>> commandsPerKey = new HashMap<StaticBuffer, Pair<Put, Delete>>();
-
- for (Map.Entry<String, Map<StaticBuffer, KCVMutation>> entry : mutations.entrySet()) {
-
- String cfString = getCfNameForStoreName(entry.getKey());
- byte[] cfName = cfString.getBytes();
-
- for (Map.Entry<StaticBuffer, KCVMutation> m : entry.getValue().entrySet()) {
- byte[] key = m.getKey().as(StaticBuffer.ARRAY_FACTORY);
- KCVMutation mutation = m.getValue();
-
- Pair<Put, Delete> commands = commandsPerKey.get(m.getKey());
-
- if (commands == null) {
- commands = new Pair<Put, Delete>();
- commandsPerKey.put(m.getKey(), commands);
- }
-
- if (mutation.hasDeletions()) {
- if (commands.getSecond() == null) {
- Delete d = new Delete(key);
- compat.setTimestamp(d, delTimestamp);
- commands.setSecond(d);
- }
-
- for (StaticBuffer b : mutation.getDeletions()) {
- commands.getSecond().deleteColumns(cfName, b.as(StaticBuffer.ARRAY_FACTORY), delTimestamp);
- }
- }
-
- if (mutation.hasAdditions()) {
- if (commands.getFirst() == null) {
- Put p = new Put(key, putTimestamp);
- commands.setFirst(p);
- }
-
- for (Entry e : mutation.getAdditions()) {
- commands.getFirst().add(cfName,
- e.getColumnAs(StaticBuffer.ARRAY_FACTORY),
- putTimestamp,
- e.getValueAs(StaticBuffer.ARRAY_FACTORY));
- }
- }
- }
- }
-
- return commandsPerKey;
- }
-
- private String getCfNameForStoreName(String storeName) throws PermanentBackendException {
- return shortCfNames ? shortenCfName(storeName) : storeName;
- }
-
- private void checkConfigDeprecation(com.thinkaurelius.titan.diskstorage.configuration.Configuration config) {
- if (config.has(GraphDatabaseConfiguration.STORAGE_PORT)) {
- logger.warn("The configuration property {} is ignored for HBase. Set hbase.zookeeper.property.clientPort in hbase-site.xml or {}.hbase.zookeeper.property.clientPort in Titan's configuration file.",
- ConfigElement.getPath(GraphDatabaseConfiguration.STORAGE_PORT), ConfigElement.getPath(HBASE_CONFIGURATION_NAMESPACE));
- }
- }
-
- private AdminMask getAdminInterface() {
- try {
- return cnx.getAdmin();
- } catch (IOException e) {
- throw new TitanException(e);
- }
- }
-
- /**
- * Similar to {@link Function}, except that the {@code apply} method is allowed
- * to throw {@link BackendException}.
- */
- private static interface BackendFunction<F, T> {
-
- T apply(F input) throws BackendException;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseTransaction.java
----------------------------------------------------------------------
diff --git a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseTransaction.java b/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseTransaction.java
deleted file mode 100644
index e13593f..0000000
--- a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseTransaction.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Copyright 2012-2013 Aurelius LLC
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.thinkaurelius.titan.diskstorage.hbase;
-
-import com.thinkaurelius.titan.diskstorage.BackendException;
-import com.thinkaurelius.titan.diskstorage.BaseTransactionConfig;
-import com.thinkaurelius.titan.diskstorage.StaticBuffer;
-import com.thinkaurelius.titan.diskstorage.common.AbstractStoreTransaction;
-import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreTransaction;
-import com.thinkaurelius.titan.diskstorage.locking.LocalLockMediator;
-import com.thinkaurelius.titan.diskstorage.util.KeyColumn;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.LinkedHashSet;
-import java.util.Set;
-
-/**
- * This class overrides and adds nothing compared with
- * {@link com.thinkaurelius.titan.diskstorage.locking.consistentkey.ExpectedValueCheckingTransaction}; however, it creates a transaction type specific
- * to HBase, which lets us check for user errors like passing a Cassandra
- * transaction into a HBase method.
- *
- * @author Dan LaRocque <da...@hopcount.org>
- */
-public class HBaseTransaction extends AbstractStoreTransaction {
-
- private static final Logger log = LoggerFactory.getLogger(HBaseTransaction.class);
-
- LocalLockMediator<StoreTransaction> llm;
-
- Set<KeyColumn> keyColumnLocks = new LinkedHashSet<>();
-
- public HBaseTransaction(final BaseTransactionConfig config, LocalLockMediator<StoreTransaction> llm) {
- super(config);
- this.llm = llm;
- }
-
- @Override
- public synchronized void rollback() throws BackendException {
- super.rollback();
- log.debug("Rolled back transaction");
- deleteAllLocks();
- }
-
- @Override
- public synchronized void commit() throws BackendException {
- super.commit();
- log.debug("Committed transaction");
- deleteAllLocks();
- }
-
- public void updateLocks(KeyColumn lockID, StaticBuffer expectedValue) {
- keyColumnLocks.add(lockID);
- }
-
- private void deleteAllLocks() {
- for(KeyColumn kc : keyColumnLocks) {
- log.debug("Removed lock {} ", kc);
- llm.unlock(kc, this);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection0_98.java
----------------------------------------------------------------------
diff --git a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection0_98.java b/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection0_98.java
deleted file mode 100644
index 8660644..0000000
--- a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection0_98.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Copyright 2012-2013 Aurelius LLC
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.thinkaurelius.titan.diskstorage.hbase;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-
-public class HConnection0_98 implements ConnectionMask
-{
-
- private final HConnection cnx;
-
- public HConnection0_98(HConnection cnx)
- {
- this.cnx = cnx;
- }
-
- @Override
- public TableMask getTable(String name) throws IOException
- {
- return new HTable0_98(cnx.getTable(name));
- }
-
- @Override
- public AdminMask getAdmin() throws IOException
- {
- return new HBaseAdmin0_98(new HBaseAdmin(cnx));
- }
-
- @Override
- public void close() throws IOException
- {
- cnx.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection1_0.java
----------------------------------------------------------------------
diff --git a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection1_0.java b/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection1_0.java
deleted file mode 100644
index 91e5026..0000000
--- a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HConnection1_0.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Copyright 2012-2013 Aurelius LLC
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.thinkaurelius.titan.diskstorage.hbase;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-
-public class HConnection1_0 implements ConnectionMask
-{
-
- private final Connection cnx;
-
- public HConnection1_0(Connection cnx)
- {
- this.cnx = cnx;
- }
-
- @Override
- public TableMask getTable(String name) throws IOException
- {
- return new HTable1_0(cnx.getTable(TableName.valueOf(name)));
- }
-
- @Override
- public AdminMask getAdmin() throws IOException
- {
- return new HBaseAdmin1_0(new HBaseAdmin(cnx));
- }
-
- @Override
- public void close() throws IOException
- {
- cnx.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable0_98.java
----------------------------------------------------------------------
diff --git a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable0_98.java b/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable0_98.java
deleted file mode 100644
index b11532a..0000000
--- a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable0_98.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright 2012-2013 Aurelius LLC
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.thinkaurelius.titan.diskstorage.hbase;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Row;
-import org.apache.hadoop.hbase.client.Scan;
-
-public class HTable0_98 implements TableMask
-{
- private final HTableInterface table;
-
- public HTable0_98(HTableInterface table)
- {
- this.table = table;
- }
-
- @Override
- public ResultScanner getScanner(Scan filter) throws IOException
- {
- return table.getScanner(filter);
- }
-
- @Override
- public Result[] get(List<Get> gets) throws IOException
- {
- return table.get(gets);
- }
-
- @Override
- public void batch(List<Row> writes, Object[] results) throws IOException, InterruptedException
- {
- table.batch(writes, results);
- table.flushCommits();
- }
-
- @Override
- public void close() throws IOException
- {
- table.close();
- }
-
- @Override
- public Object getTableObject() {
- return table;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable1_0.java
----------------------------------------------------------------------
diff --git a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable1_0.java b/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable1_0.java
deleted file mode 100644
index 5c90617..0000000
--- a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HTable1_0.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright 2012-2013 Aurelius LLC
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.thinkaurelius.titan.diskstorage.hbase;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Row;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
-
-public class HTable1_0 implements TableMask
-{
- private final Table table;
-
- public HTable1_0(Table table)
- {
- this.table = table;
- }
-
- @Override
- public ResultScanner getScanner(Scan filter) throws IOException
- {
- return table.getScanner(filter);
- }
-
- @Override
- public Result[] get(List<Get> gets) throws IOException
- {
- return table.get(gets);
- }
-
- @Override
- public void batch(List<Row> writes, Object[] results) throws IOException, InterruptedException
- {
- table.batch(writes, results);
- /* table.flushCommits(); not needed anymore */
- }
-
- @Override
- public void close() throws IOException
- {
- table.close();
- }
-
- @Override
- public Object getTableObject() {
- return table;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/TableMask.java
----------------------------------------------------------------------
diff --git a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/TableMask.java b/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/TableMask.java
deleted file mode 100644
index 54f8743..0000000
--- a/titan/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/TableMask.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright 2012-2013 Aurelius LLC
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.thinkaurelius.titan.diskstorage.hbase;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Row;
-import org.apache.hadoop.hbase.client.Scan;
-
-/**
- * This interface hides ABI/API breaking changes that HBase has made to its Table/HTableInterface over the course
- * of development from 0.94 to 1.0 and beyond.
- */
-public interface TableMask extends Closeable
-{
-
- ResultScanner getScanner(Scan filter) throws IOException;
-
- Result[] get(List<Get> gets) throws IOException;
-
- void batch(List<Row> writes, Object[] results) throws IOException, InterruptedException;
-
- Object getTableObject();
-}
[8/9] incubator-atlas git commit: ATLAS-693 Titan 0.5.4
implementation of graph db abstraction. (jnhagelb via dkantor)
Posted by dk...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/pom.xml
----------------------------------------------------------------------
diff --git a/graphdb/titan0/pom.xml b/graphdb/titan0/pom.xml
new file mode 100644
index 0000000..f2dc9a8
--- /dev/null
+++ b/graphdb/titan0/pom.xml
@@ -0,0 +1,257 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+~ Licensed to the Apache Software Foundation (ASF) under one
+~ or more contributor license agreements. See the NOTICE file
+~ distributed with this work for additional information
+~ regarding copyright ownership. The ASF licenses this file
+~ to you under the Apache License, Version 2.0 (the
+~ "License"); you may not use this file except in compliance
+~ with the License. You may obtain a copy of the License at
+~
+~ http://www.apache.org/licenses/LICENSE-2.0
+~
+~ Unless required by applicable law or agreed to in writing, software
+~ distributed under the License is distributed on an "AS IS" BASIS,
+~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~ See the License for the specific language governing permissions and
+~ limitations under the License.
+-->
+
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns="http://maven.apache.org/POM/4.0.0"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>atlas-graphdb</artifactId>
+ <groupId>org.apache.atlas</groupId>
+ <version>0.8-incubating-SNAPSHOT</version>
+ </parent>
+ <artifactId>atlas-graphdb-titan0</artifactId>
+ <description>Apache Atlas Titan 0.5.4 Graph DB Impl</description>
+ <name>Apache Atlas Titan 0.5.4 Graph DB Impl</name>
+ <packaging>jar</packaging>
+
+ <properties>
+ <tinkerpop.version>2.6.0</tinkerpop.version>
+ <titan.version>0.5.4</titan.version>
+ </properties>
+
+ <dependencies>
+
+ <!-- for graphdb interface definitions -->
+ <dependency>
+ <groupId>org.apache.atlas</groupId>
+ <artifactId>atlas-graphdb-api</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.atlas</groupId>
+ <artifactId>atlas-graphdb-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.inject</groupId>
+ <artifactId>guice</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-configuration</groupId>
+ <artifactId>commons-configuration</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.thinkaurelius.titan</groupId>
+ <artifactId>titan-core</artifactId>
+ <version>${titan.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.tinkerpop.blueprints</groupId>
+ <artifactId>blueprints-core</artifactId>
+ <version>${tinkerpop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.tinkerpop.gremlin</groupId>
+ <artifactId>gremlin-java</artifactId>
+ <version>${tinkerpop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.vividsolutions</groupId>
+ <artifactId>jts</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-solrj</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.thinkaurelius.titan</groupId>
+ <artifactId>titan-es</artifactId>
+ <version>${titan.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.thinkaurelius.titan</groupId>
+ <artifactId>titan-berkeleyje</artifactId>
+ <version>${titan.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.thinkaurelius.titan</groupId>
+ <artifactId>titan-lucene</artifactId>
+ <version>${titan.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-collections</groupId>
+ <artifactId>commons-collections</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <plugins>
+ <!--
+ Create 'uber' jar that contains all of the dependencies (except those whose scope is provided)
+ Only Titan 0l5l4 and its dependencies are included. The other dependencies are bundled in the war file.
+ -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.4.3</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadedArtifactAttached>false</shadedArtifactAttached>
+ <artifactSet>
+ <excludes>
+ <!-- these are bundled with Atlas -->
+ <exclude>org.slf4j:*</exclude>
+ </excludes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>com.thinkaurelius.titan:titan-core</artifact>
+ <!-- force use of our custom LocalLockMediator implementation -->
+ <excludes>
+ <exclude>com/thinkaurelius/titan/diskstorage/locking/LocalLockMediator*</exclude>
+ </excludes>
+ </filter>
+
+ </filters>
+ <createSourcesJar>true</createSourcesJar>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>2.4</version>
+ <configuration>
+ <excludes>
+ <exclude>**/log4j.xml</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencyManagement>
+ <dependencies>
+ <!-- Graph DB -->
+ <dependency>
+ <groupId>com.tinkerpop.blueprints</groupId>
+ <artifactId>blueprints-core</artifactId>
+ <version>${tinkerpop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.thinkaurelius.titan</groupId>
+ <artifactId>titan-core</artifactId>
+ <version>${titan.version}</version>
+ <exclusions>
+ <!-- rexster does not work with servlet-api -->
+ <exclusion>
+ <groupId>com.tinkerpop.rexster</groupId>
+ <artifactId>rexster-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.tinkerpop.rexster</groupId>
+ <artifactId>rexster-server</artifactId>
+ </exclusion>
+ <!-- asm 4.0 does not work with jersey asm 3.1 -->
+ <exclusion>
+ <groupId>com.tinkerpop</groupId>
+ <artifactId>frames</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.esotericsoftware.reflectasm</groupId>
+ <artifactId>reflectasm</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.ow2.asm</groupId>
+ <artifactId>asm</artifactId>
+ </exclusion>
+ <exclusion> <!-- GPL license imported from ganglia -->
+ <groupId>org.acplt</groupId>
+ <artifactId>oncrpc</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>com.thinkaurelius.titan</groupId>
+ <artifactId>titan-berkeleyje</artifactId>
+ <version>${titan.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.thinkaurelius.titan</groupId>
+ <artifactId>titan-hbase</artifactId>
+ <version>${titan.version}</version>
+ </dependency>
+
+ </dependencies>
+ </dependencyManagement>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/AdminMask.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/AdminMask.java b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/AdminMask.java
new file mode 100644
index 0000000..e255f1b
--- /dev/null
+++ b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/AdminMask.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2012-2013 Aurelius LLC
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.thinkaurelius.titan.diskstorage.hbase;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.ClusterStatus;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+
+/**
+ * This interface hides ABI/API breaking changes that HBase has made to its Admin/HBaseAdmin over the course
+ * of development from 0.94 to 1.0 and beyond.
+ */
+public interface AdminMask extends Closeable
+{
+
+ void clearTable(String tableName, long timestamp) throws IOException;
+
+ HTableDescriptor getTableDescriptor(String tableName) throws TableNotFoundException, IOException;
+
+ boolean tableExists(String tableName) throws IOException;
+
+ void createTable(HTableDescriptor desc) throws IOException;
+
+ void createTable(HTableDescriptor desc, byte[] startKey, byte[] endKey, int numRegions) throws IOException;
+
+ /**
+ * Estimate the number of regionservers in the HBase cluster.
+ *
+ * This is usually implemented by calling
+ * {@link HBaseAdmin#getClusterStatus()} and then
+ * {@link ClusterStatus#getServers()} and finally {@code size()} on the
+ * returned server list.
+ *
+ * @return the number of servers in the cluster or -1 if it could not be determined
+ */
+ int getEstimatedRegionServerCount();
+
+ void disableTable(String tableName) throws IOException;
+
+ void enableTable(String tableName) throws IOException;
+
+ boolean isTableDisabled(String tableName) throws IOException;
+
+ void addColumn(String tableName, HColumnDescriptor columnDescriptor) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/ConnectionMask.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/ConnectionMask.java b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/ConnectionMask.java
new file mode 100644
index 0000000..feb578b
--- /dev/null
+++ b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/ConnectionMask.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2012-2013 Aurelius LLC
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.thinkaurelius.titan.diskstorage.hbase;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+/**
+ * This interface hides ABI/API breaking changes that HBase has made to its (H)Connection class over the course
+ * of development from 0.94 to 1.0 and beyond.
+ */
+public interface ConnectionMask extends Closeable
+{
+
+ TableMask getTable(String name) throws IOException;
+
+ AdminMask getAdmin() throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin0_98.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin0_98.java b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin0_98.java
new file mode 100644
index 0000000..0cd4795
--- /dev/null
+++ b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin0_98.java
@@ -0,0 +1,152 @@
+/*
+ * Copyright 2012-2013 Aurelius LLC
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.thinkaurelius.titan.diskstorage.hbase;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.thinkaurelius.titan.util.system.IOUtils;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+
+public class HBaseAdmin0_98 implements AdminMask
+{
+
+ private static final Logger log = LoggerFactory.getLogger(HBaseAdmin0_98.class);
+
+ private final HBaseAdmin adm;
+
+ public HBaseAdmin0_98(HBaseAdmin adm)
+ {
+ this.adm = adm;
+ }
+
+ @Override
+ public void clearTable(String tableName, long timestamp) throws IOException
+ {
+ if (!adm.tableExists(tableName)) {
+ log.debug("clearStorage() called before table {} was created, skipping.", tableName);
+ return;
+ }
+
+ // Unfortunately, linear scanning and deleting tables is faster in HBase < 1 when running integration tests than
+ // disabling and deleting tables.
+ HTable table = null;
+
+ try {
+ table = new HTable(adm.getConfiguration(), tableName);
+
+ Scan scan = new Scan();
+ scan.setBatch(100);
+ scan.setCacheBlocks(false);
+ scan.setCaching(2000);
+ scan.setTimeRange(0, Long.MAX_VALUE);
+ scan.setMaxVersions(1);
+
+ ResultScanner scanner = null;
+
+ try {
+ scanner = table.getScanner(scan);
+
+ for (Result res : scanner) {
+ Delete d = new Delete(res.getRow());
+
+ d.setTimestamp(timestamp);
+ table.delete(d);
+ }
+ } finally {
+ IOUtils.closeQuietly(scanner);
+ }
+ } finally {
+ IOUtils.closeQuietly(table);
+ }
+ }
+
+ @Override
+ public HTableDescriptor getTableDescriptor(String tableName) throws TableNotFoundException, IOException
+ {
+ return adm.getTableDescriptor(tableName.getBytes());
+ }
+
+ @Override
+ public boolean tableExists(String tableName) throws IOException
+ {
+ return adm.tableExists(tableName);
+ }
+
+ @Override
+ public void createTable(HTableDescriptor desc) throws IOException
+ {
+ adm.createTable(desc);
+ }
+
+ @Override
+ public void createTable(HTableDescriptor desc, byte[] startKey, byte[] endKey, int numRegions) throws IOException
+ {
+ adm.createTable(desc, startKey, endKey, numRegions);
+ }
+
+ @Override
+ public int getEstimatedRegionServerCount()
+ {
+ int serverCount = -1;
+ try {
+ serverCount = adm.getClusterStatus().getServers().size();
+ log.debug("Read {} servers from HBase ClusterStatus", serverCount);
+ } catch (IOException e) {
+ log.debug("Unable to retrieve HBase cluster status", e);
+ }
+ return serverCount;
+ }
+
+ @Override
+ public void disableTable(String tableName) throws IOException
+ {
+ adm.disableTable(tableName);
+ }
+
+ @Override
+ public void enableTable(String tableName) throws IOException
+ {
+ adm.enableTable(tableName);
+ }
+
+ @Override
+ public boolean isTableDisabled(String tableName) throws IOException
+ {
+ return adm.isTableDisabled(tableName);
+ }
+
+ @Override
+ public void addColumn(String tableName, HColumnDescriptor columnDescriptor) throws IOException
+ {
+ adm.addColumn(tableName, columnDescriptor);
+ }
+
+ @Override
+ public void close() throws IOException
+ {
+ adm.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin1_0.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin1_0.java b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin1_0.java
new file mode 100644
index 0000000..7e8f72d
--- /dev/null
+++ b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseAdmin1_0.java
@@ -0,0 +1,135 @@
+/*
+ * Copyright 2012-2013 Aurelius LLC
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.thinkaurelius.titan.diskstorage.hbase;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.TableNotDisabledException;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+
+public class HBaseAdmin1_0 implements AdminMask
+{
+
+ private static final Logger log = LoggerFactory.getLogger(HBaseAdmin1_0.class);
+
+ private final Admin adm;
+
+ public HBaseAdmin1_0(HBaseAdmin adm)
+ {
+ this.adm = adm;
+ }
+ @Override
+ public void clearTable(String tableString, long timestamp) throws IOException
+ {
+ TableName tableName = TableName.valueOf(tableString);
+
+ if (!adm.tableExists(tableName)) {
+ log.debug("Attempted to clear table {} before it exists (noop)", tableString);
+ return;
+ }
+
+ if (!adm.isTableDisabled(tableName))
+ adm.disableTable(tableName);
+
+ if (!adm.isTableDisabled(tableName))
+ throw new RuntimeException("Unable to disable table " + tableName);
+
+ // This API call appears to both truncate and reenable the table.
+ log.info("Truncating table {}", tableName);
+ adm.truncateTable(tableName, true /* preserve splits */);
+
+ try {
+ adm.enableTable(tableName);
+ } catch (TableNotDisabledException e) {
+ // This triggers seemingly every time in testing with 1.0.2.
+ log.debug("Table automatically reenabled by truncation: {}", tableName, e);
+ }
+ }
+
+ @Override
+ public HTableDescriptor getTableDescriptor(String tableString) throws TableNotFoundException, IOException
+ {
+ return adm.getTableDescriptor(TableName.valueOf(tableString));
+ }
+
+ @Override
+ public boolean tableExists(String tableString) throws IOException
+ {
+ return adm.tableExists(TableName.valueOf(tableString));
+ }
+
+ @Override
+ public void createTable(HTableDescriptor desc) throws IOException
+ {
+ adm.createTable(desc);
+ }
+
+ @Override
+ public void createTable(HTableDescriptor desc, byte[] startKey, byte[] endKey, int numRegions) throws IOException
+ {
+ adm.createTable(desc, startKey, endKey, numRegions);
+ }
+
+ @Override
+ public int getEstimatedRegionServerCount()
+ {
+ int serverCount = -1;
+ try {
+ serverCount = adm.getClusterStatus().getServers().size();
+ log.debug("Read {} servers from HBase ClusterStatus", serverCount);
+ } catch (IOException e) {
+ log.debug("Unable to retrieve HBase cluster status", e);
+ }
+ return serverCount;
+ }
+
+ @Override
+ public void disableTable(String tableString) throws IOException
+ {
+ adm.disableTable(TableName.valueOf(tableString));
+ }
+
+ @Override
+ public void enableTable(String tableString) throws IOException
+ {
+ adm.enableTable(TableName.valueOf(tableString));
+ }
+
+ @Override
+ public boolean isTableDisabled(String tableString) throws IOException
+ {
+ return adm.isTableDisabled(TableName.valueOf(tableString));
+ }
+
+ @Override
+ public void addColumn(String tableString, HColumnDescriptor columnDescriptor) throws IOException
+ {
+ adm.addColumn(TableName.valueOf(tableString), columnDescriptor);
+ }
+
+ @Override
+ public void close() throws IOException
+ {
+ adm.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat.java b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat.java
new file mode 100644
index 0000000..c9b03aa
--- /dev/null
+++ b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2012-2013 Aurelius LLC
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.thinkaurelius.titan.diskstorage.hbase;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.Delete;
+
+public interface HBaseCompat {
+
+ /**
+ * Configure the compression scheme {@code algo} on a column family
+ * descriptor {@code cd}. The {@code algo} parameter is a string value
+ * corresponding to one of the values of HBase's Compression enum. The
+ * Compression enum has moved between packages as HBase has evolved, which
+ * is why this method has a String argument in the signature instead of the
+ * enum itself.
+ *
+ * @param cd
+ * column family to configure
+ * @param algo
+ * compression type to use
+ */
+ public void setCompression(HColumnDescriptor cd, String algo);
+
+ /**
+ * Create and return a HTableDescriptor instance with the given name. The
+ * constructors on this method have remained stable over HBase development
+ * so far, but the old HTableDescriptor(String) constructor & byte[] friends
+ * are now marked deprecated and may eventually be removed in favor of the
+ * HTableDescriptor(TableName) constructor. That constructor (and the
+ * TableName type) only exists in newer HBase versions. Hence this method.
+ *
+ * @param tableName
+ * HBase table name
+ * @return a new table descriptor instance
+ */
+ public HTableDescriptor newTableDescriptor(String tableName);
+
+ ConnectionMask createConnection(Configuration conf) throws IOException;
+
+ void addColumnFamilyToTableDescriptor(HTableDescriptor tdesc, HColumnDescriptor cdesc);
+
+ void setTimestamp(Delete d, long timestamp);
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat0_98.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat0_98.java b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat0_98.java
new file mode 100644
index 0000000..2c0f3b4
--- /dev/null
+++ b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat0_98.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2012-2013 Aurelius LLC
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.thinkaurelius.titan.diskstorage.hbase;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.io.compress.Compression;
+
+public class HBaseCompat0_98 implements HBaseCompat {
+
+ @Override
+ public void setCompression(HColumnDescriptor cd, String algo) {
+ cd.setCompressionType(Compression.Algorithm.valueOf(algo));
+ }
+
+ @Override
+ public HTableDescriptor newTableDescriptor(String tableName) {
+ TableName tn = TableName.valueOf(tableName);
+ return new HTableDescriptor(tn);
+ }
+
+ @Override
+ public ConnectionMask createConnection(Configuration conf) throws IOException
+ {
+ return new HConnection0_98(HConnectionManager.createConnection(conf));
+ }
+
+ @Override
+ public void addColumnFamilyToTableDescriptor(HTableDescriptor tdesc, HColumnDescriptor cdesc)
+ {
+ tdesc.addFamily(cdesc);
+ }
+
+ @Override
+ public void setTimestamp(Delete d, long timestamp)
+ {
+ d.setTimestamp(timestamp);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_0.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_0.java b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_0.java
new file mode 100644
index 0000000..bb3fb3b
--- /dev/null
+++ b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_0.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2012-2013 Aurelius LLC
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.thinkaurelius.titan.diskstorage.hbase;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.io.compress.Compression;
+
+public class HBaseCompat1_0 implements HBaseCompat {
+
+ @Override
+ public void setCompression(HColumnDescriptor cd, String algo) {
+ cd.setCompressionType(Compression.Algorithm.valueOf(algo));
+ }
+
+ @Override
+ public HTableDescriptor newTableDescriptor(String tableName) {
+ TableName tn = TableName.valueOf(tableName);
+ return new HTableDescriptor(tn);
+ }
+
+ @Override
+ public ConnectionMask createConnection(Configuration conf) throws IOException
+ {
+ return new HConnection1_0(ConnectionFactory.createConnection(conf));
+ }
+
+ @Override
+ public void addColumnFamilyToTableDescriptor(HTableDescriptor tdesc, HColumnDescriptor cdesc)
+ {
+ tdesc.addFamily(cdesc);
+ }
+
+ @Override
+ public void setTimestamp(Delete d, long timestamp)
+ {
+ d.setTimestamp(timestamp);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_1.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_1.java b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_1.java
new file mode 100644
index 0000000..e5c3d31
--- /dev/null
+++ b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompat1_1.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2012-2013 Aurelius LLC
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.thinkaurelius.titan.diskstorage.hbase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.io.compress.Compression;
+
+import java.io.IOException;
+
+public class HBaseCompat1_1 implements HBaseCompat {
+
+ @Override
+ public void setCompression(HColumnDescriptor cd, String algo) {
+ cd.setCompressionType(Compression.Algorithm.valueOf(algo));
+ }
+
+ @Override
+ public HTableDescriptor newTableDescriptor(String tableName) {
+ TableName tn = TableName.valueOf(tableName);
+ return new HTableDescriptor(tn);
+ }
+
+ @Override
+ public ConnectionMask createConnection(Configuration conf) throws IOException
+ {
+ return new HConnection1_0(ConnectionFactory.createConnection(conf));
+ }
+
+ @Override
+ public void addColumnFamilyToTableDescriptor(HTableDescriptor tdesc, HColumnDescriptor cdesc)
+ {
+ tdesc.addFamily(cdesc);
+ }
+
+ @Override
+ public void setTimestamp(Delete d, long timestamp)
+ {
+ d.setTimestamp(timestamp);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompatLoader.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompatLoader.java b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompatLoader.java
new file mode 100644
index 0000000..2c0d6fe
--- /dev/null
+++ b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseCompatLoader.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2012-2013 Aurelius LLC
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.thinkaurelius.titan.diskstorage.hbase;
+
+import java.util.Arrays;
+
+import org.apache.hadoop.hbase.util.VersionInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HBaseCompatLoader {
+
+ private static final Logger log = LoggerFactory.getLogger(HBaseCompatLoader.class);
+
+ private static final String DEFAULT_HBASE_COMPAT_VERSION = "1.1";
+
+ private static final String DEFAULT_HBASE_CLASS_NAME = "com.thinkaurelius.titan.diskstorage.hbase.HBaseCompat1_1";
+
+ private static HBaseCompat cachedCompat;
+
+ public synchronized static HBaseCompat getCompat(String classOverride) {
+
+ if (null != cachedCompat) {
+ log.debug("Returning cached HBase compatibility layer: {}", cachedCompat);
+ return cachedCompat;
+ }
+
+ HBaseCompat compat;
+ String className = null;
+ String classNameSource = null;
+
+ if (null != classOverride) {
+ className = classOverride;
+ classNameSource = "from explicit configuration";
+ } else {
+ String hbaseVersion = VersionInfo.getVersion();
+ for (String supportedVersion : Arrays.asList("0.94", "0.96", "0.98", "1.0", "1.1")) {
+ if (hbaseVersion.startsWith(supportedVersion + ".")) {
+ className = "com.thinkaurelius.titan.diskstorage.hbase.HBaseCompat" + supportedVersion.replaceAll("\\.", "_");
+ classNameSource = "supporting runtime HBase version " + hbaseVersion;
+ break;
+ }
+ }
+ if (null == className) {
+ log.info("The HBase version {} is not explicitly supported by Titan. " +
+ "Loading Titan's compatibility layer for its most recent supported HBase version ({})",
+ hbaseVersion, DEFAULT_HBASE_COMPAT_VERSION);
+ className = DEFAULT_HBASE_CLASS_NAME;
+ classNameSource = " by default";
+ }
+ }
+
+ final String errTemplate = " when instantiating HBase compatibility class " + className;
+
+ try {
+ compat = (HBaseCompat)Class.forName(className).newInstance();
+ log.info("Instantiated HBase compatibility layer {}: {}", classNameSource, compat.getClass().getCanonicalName());
+ } catch (IllegalAccessException e) {
+ throw new RuntimeException(e.getClass().getSimpleName() + errTemplate, e);
+ } catch (InstantiationException e) {
+ throw new RuntimeException(e.getClass().getSimpleName() + errTemplate, e);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e.getClass().getSimpleName() + errTemplate, e);
+ }
+
+ return cachedCompat = compat;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueStore.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueStore.java b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueStore.java
new file mode 100644
index 0000000..c5f6e0d
--- /dev/null
+++ b/graphdb/titan0/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueStore.java
@@ -0,0 +1,425 @@
+/*
+ * Copyright 2012-2013 Aurelius LLC
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.thinkaurelius.titan.diskstorage.hbase;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Iterators;
+import com.thinkaurelius.titan.core.attribute.Duration;
+import com.thinkaurelius.titan.diskstorage.*;
+import com.thinkaurelius.titan.diskstorage.configuration.Configuration;
+import com.thinkaurelius.titan.diskstorage.keycolumnvalue.*;
+import com.thinkaurelius.titan.diskstorage.locking.LocalLockMediator;
+import com.thinkaurelius.titan.diskstorage.locking.PermanentLockingException;
+import com.thinkaurelius.titan.diskstorage.util.KeyColumn;
+import com.thinkaurelius.titan.diskstorage.util.RecordIterator;
+import com.thinkaurelius.titan.diskstorage.util.StaticArrayBuffer;
+import com.thinkaurelius.titan.diskstorage.util.StaticArrayEntry;
+import com.thinkaurelius.titan.diskstorage.util.StaticArrayEntryList;
+import com.thinkaurelius.titan.diskstorage.util.time.Timepoint;
+import com.thinkaurelius.titan.diskstorage.util.time.Timestamps;
+import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration;
+import com.thinkaurelius.titan.util.system.IOUtils;
+
+import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.filter.ColumnPaginationFilter;
+import org.apache.hadoop.hbase.filter.ColumnRangeFilter;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.*;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Here are some areas that might need work:
+ * <p/>
+ * - batching? (consider HTable#batch, HTable#setAutoFlush(false)
+ * - tuning HTable#setWriteBufferSize (?)
+ * - writing a server-side filter to replace ColumnCountGetFilter, which drops
+ * all columns on the row where it reaches its limit. This requires getSlice,
+ * currently, to impose its limit on the client side. That obviously won't
+ * scale.
+ * - RowMutations for combining Puts+Deletes (need a newer HBase than 0.92 for this)
+ * - (maybe) fiddle with HTable#setRegionCachePrefetch and/or #prewarmRegionCache
+ * <p/>
+ * There may be other problem areas. These are just the ones of which I'm aware.
+ */
+public class HBaseKeyColumnValueStore implements KeyColumnValueStore {
+
+ private static final Logger logger = LoggerFactory.getLogger(HBaseKeyColumnValueStore.class);
+
+ private final String tableName;
+ private final HBaseStoreManager storeManager;
+
+ // When using shortened CF names, columnFamily is the shortname and storeName is the longname
+ // When not using shortened CF names, they are the same
+ //private final String columnFamily;
+ private final String storeName;
+ // This is columnFamily.getBytes()
+ private final byte[] columnFamilyBytes;
+ private final HBaseGetter entryGetter;
+
+ private final ConnectionMask cnx;
+
+ private LocalLockMediator<StoreTransaction> localLockMediator;
+
+ private final Duration lockExpiryTimeMs;
+ private final Duration lockMaxWaitTimeMs;
+ private final Integer lockMaxRetries;
+
+ HBaseKeyColumnValueStore(HBaseStoreManager storeManager, ConnectionMask cnx, String tableName, String columnFamily, String storeName, LocalLockMediator<StoreTransaction> llm) {
+ this.storeManager = storeManager;
+ this.cnx = cnx;
+ this.tableName = tableName;
+ //this.columnFamily = columnFamily;
+ this.storeName = storeName;
+ this.columnFamilyBytes = columnFamily.getBytes();
+ this.entryGetter = new HBaseGetter(storeManager.getMetaDataSchema(storeName));
+ this.localLockMediator = llm;
+ Configuration storageConfig = storeManager.getStorageConfig();
+ this.lockExpiryTimeMs = storageConfig.get(GraphDatabaseConfiguration.LOCK_EXPIRE);
+ this.lockMaxWaitTimeMs = storageConfig.get(GraphDatabaseConfiguration.LOCK_WAIT);
+ this.lockMaxRetries = storageConfig.get(GraphDatabaseConfiguration.LOCK_RETRY);
+ }
+
+ @Override
+ public void close() throws BackendException {
+ }
+
+ @Override
+ public EntryList getSlice(KeySliceQuery query, StoreTransaction txh) throws BackendException {
+ Map<StaticBuffer, EntryList> result = getHelper(Arrays.asList(query.getKey()), getFilter(query));
+ return Iterables.getOnlyElement(result.values(), EntryList.EMPTY_LIST);
+ }
+
+ @Override
+ public Map<StaticBuffer,EntryList> getSlice(List<StaticBuffer> keys, SliceQuery query, StoreTransaction txh) throws BackendException {
+ return getHelper(keys, getFilter(query));
+ }
+
+ @Override
+ public void mutate(StaticBuffer key, List<Entry> additions, List<StaticBuffer> deletions, StoreTransaction txh) throws BackendException {
+ Map<StaticBuffer, KCVMutation> mutations = ImmutableMap.of(key, new KCVMutation(additions, deletions));
+ mutateMany(mutations, txh);
+ }
+
+ @Override
+ public void acquireLock(StaticBuffer key,
+ StaticBuffer column,
+ StaticBuffer expectedValue,
+ StoreTransaction txh) throws BackendException {
+
+ KeyColumn lockID = new KeyColumn(key, column);
+ logger.debug("Attempting to acquireLock on {} ", lockID);
+ int trialCount = 0;
+ boolean locked;
+ while (trialCount < lockMaxRetries) {
+ final Timepoint lockStartTime = Timestamps.MILLI.getTime(System.currentTimeMillis(), TimeUnit.MILLISECONDS);
+ locked = localLockMediator.lock(lockID, txh, lockStartTime.add(lockExpiryTimeMs));
+ trialCount++;
+ if (!locked) {
+ handleLockFailure(txh, lockID, trialCount);
+ } else {
+ logger.debug("Acquired lock on {}, {}", lockID, txh);
+ break;
+ }
+ }
+ ((HBaseTransaction) txh).updateLocks(lockID, expectedValue);
+ }
+
+ void handleLockFailure(StoreTransaction txh, KeyColumn lockID, int trialCount) throws PermanentLockingException {
+ if (trialCount < lockMaxRetries) {
+ try {
+ Thread.sleep(lockMaxWaitTimeMs.getLength(TimeUnit.DAYS.MILLISECONDS));
+ } catch (InterruptedException e) {
+ throw new PermanentLockingException(
+ "Interrupted while waiting for acquiring lock for transaction "
+ + txh + " lockID " + lockID + " on retry " + trialCount, e);
+ }
+ } else {
+ throw new PermanentLockingException("Could not lock the keyColumn " +
+ lockID + " on CF {} " + Bytes.toString(columnFamilyBytes));
+ }
+ }
+
+ @Override
+ public KeyIterator getKeys(KeyRangeQuery query, StoreTransaction txh) throws BackendException {
+ return executeKeySliceQuery(query.getKeyStart().as(StaticBuffer.ARRAY_FACTORY),
+ query.getKeyEnd().as(StaticBuffer.ARRAY_FACTORY),
+ new FilterList(FilterList.Operator.MUST_PASS_ALL),
+ query);
+ }
+
+ @Override
+ public String getName() {
+ return storeName;
+ }
+
+ @Override
+ public KeyIterator getKeys(SliceQuery query, StoreTransaction txh) throws BackendException {
+ return executeKeySliceQuery(new FilterList(FilterList.Operator.MUST_PASS_ALL), query);
+ }
+
+ public static Filter getFilter(SliceQuery query) {
+ byte[] colStartBytes = query.getSliceEnd().length() > 0 ? query.getSliceStart().as(StaticBuffer.ARRAY_FACTORY) : null;
+ byte[] colEndBytes = query.getSliceEnd().length() > 0 ? query.getSliceEnd().as(StaticBuffer.ARRAY_FACTORY) : null;
+
+ Filter filter = new ColumnRangeFilter(colStartBytes, true, colEndBytes, false);
+
+ if (query.hasLimit()) {
+ filter = new FilterList(FilterList.Operator.MUST_PASS_ALL,
+ filter,
+ new ColumnPaginationFilter(query.getLimit(), 0));
+ }
+
+ logger.debug("Generated HBase Filter {}", filter);
+
+ return filter;
+ }
+
+ private Map<StaticBuffer,EntryList> getHelper(List<StaticBuffer> keys, Filter getFilter) throws BackendException {
+ List<Get> requests = new ArrayList<Get>(keys.size());
+ {
+ for (StaticBuffer key : keys) {
+ Get g = new Get(key.as(StaticBuffer.ARRAY_FACTORY)).addFamily(columnFamilyBytes).setFilter(getFilter);
+ try {
+ g.setTimeRange(0, Long.MAX_VALUE);
+ } catch (IOException e) {
+ throw new PermanentBackendException(e);
+ }
+ requests.add(g);
+ }
+ }
+
+ Map<StaticBuffer,EntryList> resultMap = new HashMap<StaticBuffer,EntryList>(keys.size());
+
+ try {
+ TableMask table = null;
+ Result[] results = null;
+
+ try {
+ table = cnx.getTable(tableName);
+ logger.debug("Get requests {} {} ", Bytes.toString(columnFamilyBytes), requests.size());
+ results = table.get(requests);
+ logger.debug("Get requests finished {} {} ", Bytes.toString(columnFamilyBytes), requests.size());
+ } finally {
+ IOUtils.closeQuietly(table);
+ }
+
+ if (results == null)
+ return KCVSUtil.emptyResults(keys);
+
+ assert results.length==keys.size();
+
+ for (int i = 0; i < results.length; i++) {
+ Result result = results[i];
+ NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> f = result.getMap();
+
+ if (f == null) { // no result for this key
+ resultMap.put(keys.get(i), EntryList.EMPTY_LIST);
+ continue;
+ }
+
+ // actual key with <timestamp, value>
+ NavigableMap<byte[], NavigableMap<Long, byte[]>> r = f.get(columnFamilyBytes);
+ resultMap.put(keys.get(i), (r == null)
+ ? EntryList.EMPTY_LIST
+ : StaticArrayEntryList.ofBytes(r.entrySet(), entryGetter));
+ }
+
+ return resultMap;
+ } catch (IOException e) {
+ throw new TemporaryBackendException(e);
+ }
+ }
+
+ private void mutateMany(Map<StaticBuffer, KCVMutation> mutations, StoreTransaction txh) throws BackendException {
+ storeManager.mutateMany(ImmutableMap.of(storeName, mutations), txh);
+ }
+
+ private KeyIterator executeKeySliceQuery(FilterList filters, @Nullable SliceQuery columnSlice) throws BackendException {
+ return executeKeySliceQuery(null, null, filters, columnSlice);
+ }
+
+ private KeyIterator executeKeySliceQuery(@Nullable byte[] startKey,
+ @Nullable byte[] endKey,
+ FilterList filters,
+ @Nullable SliceQuery columnSlice) throws BackendException {
+ Scan scan = new Scan().addFamily(columnFamilyBytes);
+
+ try {
+ scan.setTimeRange(0, Long.MAX_VALUE);
+ } catch (IOException e) {
+ throw new PermanentBackendException(e);
+ }
+
+ if (startKey != null)
+ scan.setStartRow(startKey);
+
+ if (endKey != null)
+ scan.setStopRow(endKey);
+
+ if (columnSlice != null) {
+ filters.addFilter(getFilter(columnSlice));
+ }
+
+ TableMask table = null;
+
+ logger.debug("Scan for row keys {} {} ", Bytes.toString(startKey), Bytes.toString(endKey));
+ try {
+ table = cnx.getTable(tableName);
+ return new RowIterator(table, table.getScanner(scan.setFilter(filters)), columnFamilyBytes);
+ } catch (IOException e) {
+ IOUtils.closeQuietly(table);
+ throw new PermanentBackendException(e);
+ }
+ }
+
+ private class RowIterator implements KeyIterator {
+ private final Closeable table;
+ private final Iterator<Result> rows;
+ private final byte[] columnFamilyBytes;
+
+ private Result currentRow;
+ private boolean isClosed;
+
+ public RowIterator(Closeable table, ResultScanner rows, byte[] columnFamilyBytes) {
+ this.table = table;
+ this.columnFamilyBytes = Arrays.copyOf(columnFamilyBytes, columnFamilyBytes.length);
+ this.rows = Iterators.filter(rows.iterator(), new Predicate<Result>() {
+ @Override
+ public boolean apply(@Nullable Result result) {
+ if (result == null)
+ return false;
+
+ try {
+ StaticBuffer id = StaticArrayBuffer.of(result.getRow());
+ id.getLong(0);
+ } catch (NumberFormatException e) {
+ return false;
+ }
+
+ return true;
+ }
+ });
+ }
+
+ @Override
+ public RecordIterator<Entry> getEntries() {
+ ensureOpen();
+
+ return new RecordIterator<Entry>() {
+ private final NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> currentMap = currentRow.getMap();
+ private final Iterator<Map.Entry<byte[], NavigableMap<Long, byte[]>>> kv = currentMap == null ? null : currentMap.get(columnFamilyBytes).entrySet().iterator();
+
+ @Override
+ public boolean hasNext() {
+ ensureOpen();
+ return kv == null ? false : kv.hasNext();
+ }
+
+ @Override
+ public Entry next() {
+ ensureOpen();
+ return kv == null ? null : StaticArrayEntry.ofBytes(kv.next(), entryGetter);
+ }
+
+ @Override
+ public void close() {
+ isClosed = true;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+
+ @Override
+ public boolean hasNext() {
+ ensureOpen();
+ return rows.hasNext();
+ }
+
+ @Override
+ public StaticBuffer next() {
+ ensureOpen();
+
+ currentRow = rows.next();
+ return StaticArrayBuffer.of(currentRow.getRow());
+ }
+
+ @Override
+ public void close() {
+ IOUtils.closeQuietly(table);
+ isClosed = true;
+ logger.debug("RowIterator closed table {}", table);
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ private void ensureOpen() {
+ if (isClosed)
+ throw new IllegalStateException("Iterator has been closed.");
+ }
+ }
+
+ private static class HBaseGetter implements StaticArrayEntry.GetColVal<Map.Entry<byte[], NavigableMap<Long, byte[]>>, byte[]> {
+
+ private final EntryMetaData[] schema;
+
+ private HBaseGetter(EntryMetaData[] schema) {
+ this.schema = schema;
+ }
+
+ @Override
+ public byte[] getColumn(Map.Entry<byte[], NavigableMap<Long, byte[]>> element) {
+ return element.getKey();
+ }
+
+ @Override
+ public byte[] getValue(Map.Entry<byte[], NavigableMap<Long, byte[]>> element) {
+ return element.getValue().lastEntry().getValue();
+ }
+
+ @Override
+ public EntryMetaData[] getMetaSchema(Map.Entry<byte[], NavigableMap<Long, byte[]>> element) {
+ return schema;
+ }
+
+ @Override
+ public Object getMetaData(Map.Entry<byte[], NavigableMap<Long, byte[]>> element, EntryMetaData meta) {
+ switch(meta) {
+ case TIMESTAMP:
+ return element.getValue().lastEntry().getKey();
+ default:
+ throw new UnsupportedOperationException("Unsupported meta data: " + meta);
+ }
+ }
+ }
+}
[5/9] incubator-atlas git commit: ATLAS-693 Titan 0.5.4
implementation of graph db abstraction. (jnhagelb via dkantor)
Posted by dk...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Edge.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Edge.java b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Edge.java
new file mode 100644
index 0000000..1d5d409
--- /dev/null
+++ b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Edge.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.titan0;
+import org.apache.atlas.repository.graphdb.AtlasEdge;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+
+import com.tinkerpop.blueprints.Direction;
+import com.tinkerpop.blueprints.Edge;
+import com.tinkerpop.blueprints.Vertex;
+
+/**
+ * Titan 0.5.4 implementation of AtlasEdge.
+ */
+public class Titan0Edge extends Titan0Element<Edge> implements AtlasEdge<Titan0Vertex, Titan0Edge> {
+
+
+ public Titan0Edge(Titan0Graph graph, Edge edge) {
+ super(graph, edge);
+ }
+
+ @Override
+ public String getLabel() {
+ return wrappedElement.getLabel();
+ }
+
+ @Override
+ public Titan0Edge getE() {
+ return this;
+ }
+
+ @Override
+ public AtlasVertex<Titan0Vertex, Titan0Edge> getInVertex() {
+ Vertex v = wrappedElement.getVertex(Direction.IN);
+ return GraphDbObjectFactory.createVertex(graph, v);
+ }
+
+ @Override
+ public AtlasVertex<Titan0Vertex, Titan0Edge> getOutVertex() {
+ Vertex v = wrappedElement.getVertex(Direction.OUT);
+ return GraphDbObjectFactory.createVertex(graph, v);
+ }
+
+ @Override
+ public String toString() {
+ return "Titan0Edge [id=" + getId() + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Element.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Element.java b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Element.java
new file mode 100644
index 0000000..cacbaf8
--- /dev/null
+++ b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Element.java
@@ -0,0 +1,267 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.titan0;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.atlas.repository.graphdb.AtlasEdge;
+import org.apache.atlas.repository.graphdb.AtlasElement;
+import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+import com.thinkaurelius.titan.core.SchemaViolationException;
+import com.thinkaurelius.titan.core.TitanElement;
+import com.tinkerpop.blueprints.Element;
+import com.tinkerpop.blueprints.util.io.graphson.GraphSONMode;
+import com.tinkerpop.blueprints.util.io.graphson.GraphSONUtility;
+
+/**
+ * Titan 0.5.4 implementation of AtlasElement.
+ */
+public class Titan0Element<T extends Element> implements AtlasElement {
+
+ protected Titan0Graph graph;
+ protected T wrappedElement;
+
+ public Titan0Element(Titan0Graph graph, T element) {
+ wrappedElement = element;
+ this.graph = graph;
+ }
+
+ @Override
+ public Object getId() {
+ return wrappedElement.getId();
+ }
+
+ @Override
+ public Set<String> getPropertyKeys() {
+ return wrappedElement.getPropertyKeys();
+ }
+
+ @Override
+ public <U> void setProperty(String propertyName, U value) {
+ try {
+ wrappedElement.setProperty(propertyName, value);
+ } catch (SchemaViolationException e) {
+ throw new AtlasSchemaViolationException(e);
+ }
+ }
+
+ @Override
+ public <U> U getProperty(String propertyName, Class<U> clazz) {
+ return (U)convert(wrappedElement.getProperty(propertyName), clazz);
+ }
+
+ /**
+ * Gets all of the values of the given property.
+ * @param propertyName
+ * @return
+ */
+ @Override
+ public <T> Collection<T> getPropertyValues(String propertyName, Class<T> type) {
+ return Collections.singleton(getProperty(propertyName, type));
+ }
+
+ @Override
+ public void removeProperty(String propertyName) {
+ wrappedElement.removeProperty(propertyName);
+
+ }
+
+ @Override
+ public JSONObject toJson(Set<String> propertyKeys) throws JSONException {
+ return GraphSONUtility.jsonFromElement(wrappedElement, propertyKeys, GraphSONMode.NORMAL);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.atlas.repository.graphdb.AtlasElement#getListProperty(java.
+ * lang.String)
+ */
+ @Override
+ public List<String> getListProperty(String propertyName) {
+ return getProperty(propertyName, List.class);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.atlas.repository.graphdb.AtlasElement#setListProperty(java.
+ * lang.String, java.util.List)
+ */
+ @Override
+ public void setListProperty(String propertyName, List<String> values) {
+ setProperty(propertyName, values);
+
+ }
+
+ // not in interface
+ public T getWrappedElement() {
+ return wrappedElement;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = 37;
+ result = 17 * result + getClass().hashCode();
+ result = 17 * result + getWrappedElement().hashCode();
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if(other == null) {
+ return false;
+ }
+ if (other.getClass() != getClass()) {
+ return false;
+ }
+ Titan0Element otherElement = (Titan0Element) other;
+ return getWrappedElement().equals(otherElement.getWrappedElement());
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.atlas.repository.graphdb.AtlasElement#exists()
+ */
+ @Override
+ public boolean exists() {
+ try {
+ return ! ((TitanElement)wrappedElement).isRemoved();
+ }
+ catch(IllegalStateException e) {
+ return false;
+ }
+
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.atlas.repository.graphdb.AtlasElement#setJsonProperty(java.
+ * lang.String, java.lang.Object)
+ */
+ @Override
+ public <T> void setJsonProperty(String propertyName, T value) {
+ setProperty(propertyName, value);
+
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.atlas.repository.graphdb.AtlasElement#getJsonProperty(java.
+ * lang.String)
+ */
+ @Override
+ public <T> T getJsonProperty(String propertyName) {
+ return (T) getProperty(propertyName, String.class);
+ }
+
+ @Override
+ public String getIdForDisplay() {
+ return getId().toString();
+ }
+
+ private <T> T convert(Object propertyValue, Class<T> clazz) {
+ if(propertyValue == null) {
+ return null;
+ }
+ if(AtlasEdge.class.isAssignableFrom(clazz)) {
+ return (T)graph.getEdge(propertyValue.toString());
+ }
+ if(AtlasVertex.class.isAssignableFrom(clazz)) {
+ return (T)graph.getVertex(propertyValue.toString());
+ }
+ return (T)propertyValue;
+ }
+
+
+ @Override
+ public <V> List<V> getListProperty(String propertyName, Class<V> elementType) {
+
+ List<String> value = getListProperty(propertyName);
+
+ if(value == null) {
+ return null;
+ }
+
+ if(AtlasEdge.class.isAssignableFrom(elementType)) {
+
+
+ return (List<V>)Lists.transform(value, new Function<String,AtlasEdge>(){
+
+ @Override
+ public AtlasEdge apply(String input) {
+ return graph.getEdge(input);
+ }
+ });
+ }
+
+ if(AtlasVertex.class.isAssignableFrom(elementType)) {
+
+ return (List<V>)Lists.transform(value, new Function<String,AtlasVertex>(){
+
+ @Override
+ public AtlasVertex apply(String input) {
+ return graph.getVertex(input);
+ }
+ });
+ }
+
+ return (List<V>)value;
+ }
+
+
+ @Override
+ public void setPropertyFromElementsIds(String propertyName, List<AtlasElement> values) {
+ List<String> propertyValue = new ArrayList<>(values.size());
+ for(AtlasElement element: values) {
+ propertyValue.add(element.getId().toString());
+ }
+ setProperty(propertyName, propertyValue);
+ }
+
+
+ @Override
+ public void setPropertyFromElementId(String propertyName, AtlasElement value) {
+ setProperty(propertyName, value.getId().toString());
+
+ }
+
+
+ @Override
+ public boolean isIdAssigned() {
+
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Graph.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Graph.java b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Graph.java
new file mode 100644
index 0000000..51531ed
--- /dev/null
+++ b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Graph.java
@@ -0,0 +1,301 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.titan0;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import javax.script.Bindings;
+import javax.script.ScriptEngine;
+import javax.script.ScriptEngineManager;
+import javax.script.ScriptException;
+
+import org.apache.atlas.repository.graphdb.AtlasEdge;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
+import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
+import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
+import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.graphdb.GremlinVersion;
+import org.apache.atlas.repository.graphdb.titan0.query.Titan0GraphQuery;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.utils.IteratorToIterableAdapter;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import com.thinkaurelius.titan.core.SchemaViolationException;
+import com.thinkaurelius.titan.core.TitanGraph;
+import com.thinkaurelius.titan.core.TitanIndexQuery;
+import com.thinkaurelius.titan.core.util.TitanCleanup;
+import com.tinkerpop.blueprints.Edge;
+import com.tinkerpop.blueprints.Element;
+import com.tinkerpop.blueprints.Vertex;
+import com.tinkerpop.blueprints.util.io.graphson.GraphSONWriter;
+import com.tinkerpop.pipes.util.structures.Row;
+
+
+/**
+ * Titan 0.5.4 implementation of AtlasGraph.
+ */
+public class Titan0Graph implements AtlasGraph<Titan0Vertex, Titan0Edge> {
+
+ public Titan0Graph() {
+
+ }
+
+ @Override
+ public AtlasEdge<Titan0Vertex, Titan0Edge> addEdge(AtlasVertex<Titan0Vertex, Titan0Edge> outVertex,
+ AtlasVertex<Titan0Vertex, Titan0Edge> inVertex, String edgeLabel) {
+ try {
+ Edge edge = getGraph().addEdge(null, outVertex.getV().getWrappedElement(),
+ inVertex.getV().getWrappedElement(), edgeLabel);
+ return GraphDbObjectFactory.createEdge(this, edge);
+ } catch (SchemaViolationException e) {
+ throw new AtlasSchemaViolationException(e);
+ }
+ }
+
+ @Override
+ public AtlasGraphQuery<Titan0Vertex, Titan0Edge> query() {
+
+ return new Titan0GraphQuery(this);
+ }
+
+ @Override
+ public AtlasEdge<Titan0Vertex, Titan0Edge> getEdge(String edgeId) {
+ Edge edge = getGraph().getEdge(edgeId);
+ return GraphDbObjectFactory.createEdge(this, edge);
+ }
+
+ @Override
+ public void removeEdge(AtlasEdge<Titan0Vertex, Titan0Edge> edge) {
+ getGraph().removeEdge(edge.getE().getWrappedElement());
+
+ }
+
+ @Override
+ public void removeVertex(AtlasVertex<Titan0Vertex, Titan0Edge> vertex) {
+ getGraph().removeVertex(vertex.getV().getWrappedElement());
+
+ }
+
+ @Override
+ public Iterable<AtlasEdge<Titan0Vertex, Titan0Edge>> getEdges() {
+ Iterable<Edge> edges = getGraph().getEdges();
+ return wrapEdges(edges);
+ }
+
+ @Override
+ public Iterable<AtlasVertex<Titan0Vertex, Titan0Edge>> getVertices() {
+ Iterable<Vertex> vertices = getGraph().getVertices();
+ return wrapVertices(vertices);
+ }
+
+ @Override
+ public AtlasVertex<Titan0Vertex, Titan0Edge> addVertex() {
+ Vertex result = getGraph().addVertex(null);
+ return GraphDbObjectFactory.createVertex(this, result);
+ }
+
+ @Override
+ public void commit() {
+ getGraph().commit();
+ }
+
+ @Override
+ public void rollback() {
+ getGraph().rollback();
+ }
+
+ @Override
+ public AtlasIndexQuery<Titan0Vertex, Titan0Edge> indexQuery(String fulltextIndex, String graphQuery) {
+ TitanIndexQuery query = getGraph().indexQuery(fulltextIndex, graphQuery);
+ return new Titan0IndexQuery(this, query);
+ }
+
+ @Override
+ public AtlasGraphManagement getManagementSystem() {
+ return new Titan0DatabaseManager(getGraph().getManagementSystem());
+ }
+
+ @Override
+ public void shutdown() {
+ getGraph().shutdown();
+ }
+
+ @Override
+ public Set<String> getVertexIndexKeys() {
+ return getIndexKeys(Vertex.class);
+ }
+
+ @Override
+ public Set<String> getEdgeIndexKeys() {
+ return getIndexKeys(Edge.class);
+ }
+
+ private Set<String> getIndexKeys(Class<? extends Element> titanClass) {
+
+ return getGraph().getIndexedKeys(titanClass);
+ }
+
+ @Override
+ public AtlasVertex<Titan0Vertex, Titan0Edge> getVertex(String vertexId) {
+ Vertex v = getGraph().getVertex(vertexId);
+ return GraphDbObjectFactory.createVertex(this, v);
+ }
+
+ @Override
+ public Iterable<AtlasVertex<Titan0Vertex, Titan0Edge>> getVertices(String key, Object value) {
+
+ Iterable<Vertex> result = getGraph().getVertices(key, value);
+ return wrapVertices(result);
+ }
+
+ @Override
+ public Object getGremlinColumnValue(Object rowValue, String colName, int idx) {
+ Row<List> rV = (Row<List>) rowValue;
+ Object value = rV.getColumn(colName).get(idx);
+ return convertGremlinValue(value);
+ }
+
+ @Override
+ public Object convertGremlinValue(Object rawValue) {
+ if (rawValue instanceof Vertex) {
+ return GraphDbObjectFactory.createVertex(this, (Vertex) rawValue);
+ }
+ if (rawValue instanceof Edge) {
+ return GraphDbObjectFactory.createEdge(this, (Edge) rawValue);
+ }
+ return rawValue;
+ }
+
+ @Override
+ public GremlinVersion getSupportedGremlinVersion() {
+
+ return GremlinVersion.TWO;
+ }
+
+ @Override
+ public List<Object> convertPathQueryResultToList(Object rawValue) {
+ return (List<Object>) rawValue;
+ }
+
+ @Override
+ public void clear() {
+ TitanGraph graph = getGraph();
+ if (graph.isOpen()) {
+ // only a shut down graph can be cleared
+ graph.shutdown();
+ }
+ TitanCleanup.clear(graph);
+ }
+
+ private TitanGraph getGraph() {
+ // return the singleton instance of the graph in the plugin
+ return Titan0Database.getGraphInstance();
+ }
+
+ @Override
+ public void exportToGson(OutputStream os) throws IOException {
+ GraphSONWriter.outputGraph(getGraph(), os);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.atlas.repository.graphdb.AtlasGraph#executeGremlinScript(java.
+ * lang.String)
+ */
+ @Override
+ public Object executeGremlinScript(String gremlinQuery) throws ScriptException {
+
+ ScriptEngineManager manager = new ScriptEngineManager();
+ ScriptEngine engine = manager.getEngineByName("gremlin-groovy");
+ Bindings bindings = engine.createBindings();
+ bindings.put("g", getGraph());
+ Object result = engine.eval(gremlinQuery, bindings);
+ return result;
+ }
+
+ @Override
+ public String generatePersisentToLogicalConversionExpression(String expr, IDataType<?> type) {
+
+ //nothing special needed, value is stored in required type
+ return expr;
+ }
+
+ @Override
+ public boolean isPropertyValueConversionNeeded(IDataType<?> type) {
+
+ return false;
+ }
+
+ @Override
+ public boolean requiresInitialIndexedPredicate() {
+ return false;
+ }
+
+ @Override
+ public String getInitialIndexedPredicate() {
+ return "";
+ }
+
+ @Override
+ public String getOutputTransformationPredicate(boolean inSelect, boolean isPath) {
+ return "";
+ }
+
+ public Iterable<AtlasEdge<Titan0Vertex, Titan0Edge>> wrapEdges(Iterator<Edge> it) {
+
+ Iterable<Edge> iterable = new IteratorToIterableAdapter<Edge>(it);
+ return wrapEdges(iterable);
+ }
+
+ public Iterable<AtlasVertex<Titan0Vertex, Titan0Edge>> wrapVertices(Iterator<Vertex> it) {
+ Iterable<Vertex> iterable = new IteratorToIterableAdapter<Vertex>(it);
+ return wrapVertices(iterable);
+ }
+
+ public Iterable<AtlasVertex<Titan0Vertex, Titan0Edge>> wrapVertices(Iterable<Vertex> it) {
+
+ return Iterables.transform(it, new Function<Vertex, AtlasVertex<Titan0Vertex, Titan0Edge>>(){
+
+ @Override
+ public AtlasVertex<Titan0Vertex, Titan0Edge> apply(Vertex input) {
+ return GraphDbObjectFactory.createVertex(Titan0Graph.this, input);
+ }
+ });
+
+ }
+
+ public Iterable<AtlasEdge<Titan0Vertex, Titan0Edge>> wrapEdges(Iterable<Edge> it) {
+ Iterable<Edge> result = (Iterable<Edge>)it;
+ return Iterables.transform(result, new Function<Edge, AtlasEdge<Titan0Vertex, Titan0Edge>>(){
+
+ @Override
+ public AtlasEdge<Titan0Vertex, Titan0Edge> apply(Edge input) {
+ return GraphDbObjectFactory.createEdge(Titan0Graph.this, input);
+ }
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0GraphIndex.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0GraphIndex.java b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0GraphIndex.java
new file mode 100644
index 0000000..7beed78
--- /dev/null
+++ b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0GraphIndex.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graphdb.titan0;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.atlas.repository.graphdb.AtlasGraphIndex;
+import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
+
+import com.thinkaurelius.titan.core.PropertyKey;
+import com.thinkaurelius.titan.core.schema.TitanGraphIndex;
+import com.tinkerpop.blueprints.Edge;
+import com.tinkerpop.blueprints.Vertex;
+
+/**
+ * Titan 0.5.4 implementation of AtlasGraphIndex.
+ */
+public class Titan0GraphIndex implements AtlasGraphIndex {
+
+ private TitanGraphIndex wrappedIndex;
+
+ public Titan0GraphIndex(TitanGraphIndex toWrap) {
+ wrappedIndex = toWrap;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.atlas.repository.graphdb.AtlasGraphIndex#isMixedIndex()
+ */
+ @Override
+ public boolean isMixedIndex() {
+ return wrappedIndex.isMixedIndex();
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.atlas.repository.graphdb.AtlasGraphIndex#isEdgeIndex()
+ */
+ @Override
+ public boolean isEdgeIndex() {
+ return Edge.class.isAssignableFrom(wrappedIndex.getIndexedElement());
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.atlas.repository.graphdb.AtlasGraphIndex#isVertexIndex()
+ */
+ @Override
+ public boolean isVertexIndex() {
+ return Vertex.class.isAssignableFrom(wrappedIndex.getIndexedElement());
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.atlas.repository.graphdb.AtlasGraphIndex#isCompositeIndex()
+ */
+ @Override
+ public boolean isCompositeIndex() {
+ return wrappedIndex.isCompositeIndex();
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.atlas.repository.graphdb.AtlasGraphIndex#isUnique()
+ */
+ @Override
+ public boolean isUnique() {
+ return wrappedIndex.isUnique();
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.atlas.repository.graphdb.AtlasGraphIndex#getFieldKeys()
+ */
+ @Override
+ public Set<AtlasPropertyKey> getFieldKeys() {
+ PropertyKey[] keys = wrappedIndex.getFieldKeys();
+ Set<AtlasPropertyKey> result = new HashSet<AtlasPropertyKey>();
+ for(PropertyKey key : keys) {
+ result.add(GraphDbObjectFactory.createPropertyKey(key));
+ }
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0IndexQuery.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0IndexQuery.java b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0IndexQuery.java
new file mode 100644
index 0000000..18f0be5
--- /dev/null
+++ b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0IndexQuery.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.titan0;
+
+import java.util.Iterator;
+
+import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
+import com.thinkaurelius.titan.core.TitanIndexQuery;
+import com.tinkerpop.blueprints.Vertex;
+
+/**
+ * Titan 0.5.4 implementation of AtlasIndexQuery.
+ */
+public class Titan0IndexQuery implements AtlasIndexQuery<Titan0Vertex, Titan0Edge> {
+
+ private Titan0Graph graph;
+ private TitanIndexQuery wrappedIndexQuery;
+
+
+ public Titan0IndexQuery(Titan0Graph graph, TitanIndexQuery query) {
+ wrappedIndexQuery = query;
+ this.graph = graph;
+ }
+
+ @Override
+ public Iterator<AtlasIndexQuery.Result<Titan0Vertex, Titan0Edge>> vertices() {
+ Iterator<TitanIndexQuery.Result<Vertex>> results = wrappedIndexQuery.vertices().iterator();
+
+ Function<TitanIndexQuery.Result<Vertex>, AtlasIndexQuery.Result<Titan0Vertex, Titan0Edge>> function =
+ new Function<TitanIndexQuery.Result<Vertex>, AtlasIndexQuery.Result<Titan0Vertex, Titan0Edge>>() {
+
+ @Override
+ public AtlasIndexQuery.Result<Titan0Vertex, Titan0Edge> apply(TitanIndexQuery.Result<Vertex> source) {
+ return new ResultImpl(source);
+ }
+ };
+ return Iterators.transform(results, function);
+ }
+
+ private final class ResultImpl implements AtlasIndexQuery.Result<Titan0Vertex, Titan0Edge> {
+ private TitanIndexQuery.Result<Vertex> wrappedResult;
+
+ public ResultImpl(TitanIndexQuery.Result<Vertex> source) {
+ wrappedResult = source;
+ }
+
+ @Override
+ public AtlasVertex<Titan0Vertex, Titan0Edge> getVertex() {
+ return GraphDbObjectFactory.createVertex(graph, wrappedResult.getElement());
+ }
+
+ @Override
+ public double getScore() {
+ return wrappedResult.getScore();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0PropertyKey.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0PropertyKey.java b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0PropertyKey.java
new file mode 100644
index 0000000..1f9f6ef
--- /dev/null
+++ b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0PropertyKey.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graphdb.titan0;
+
+import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
+
+import com.thinkaurelius.titan.core.PropertyKey;
+
+/**
+ * Titan 0.5.4 implementaiton of AtlasPropertyKey.
+ */
+public class Titan0PropertyKey implements AtlasPropertyKey {
+
+ private PropertyKey wrappedPropertyKey;
+
+ public Titan0PropertyKey(PropertyKey toWrap) {
+ wrappedPropertyKey = toWrap;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.atlas.repository.graphdb.AtlasPropertyKey#getName()
+ */
+ @Override
+ public String getName() {
+ return wrappedPropertyKey.getName();
+ }
+
+ /**
+ * @return
+ */
+ public PropertyKey getWrappedPropertyKey() {
+ return wrappedPropertyKey;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof Titan0PropertyKey)) {
+ return false;
+ }
+ Titan0PropertyKey otherKey = (Titan0PropertyKey) other;
+ return wrappedPropertyKey.equals(otherKey.wrappedPropertyKey);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = 17;
+ result = 37 * result + wrappedPropertyKey.hashCode();
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Vertex.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Vertex.java b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Vertex.java
new file mode 100644
index 0000000..b26ff04
--- /dev/null
+++ b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Vertex.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.titan0;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+import org.apache.atlas.repository.graphdb.AtlasEdge;
+import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
+import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
+import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.graphdb.AtlasVertexQuery;
+
+import com.thinkaurelius.titan.core.SchemaViolationException;
+import com.thinkaurelius.titan.core.TitanProperty;
+import com.thinkaurelius.titan.core.TitanVertex;
+import com.tinkerpop.blueprints.Edge;
+import com.tinkerpop.blueprints.Vertex;
+
+/**
+ * Titan 0.5.4 implementation of AtlasVertex.
+ */
+public class Titan0Vertex extends Titan0Element<Vertex> implements AtlasVertex<Titan0Vertex, Titan0Edge> {
+
+ public Titan0Vertex(Titan0Graph graph, Vertex source) {
+ super(graph, source);
+ }
+
+ @Override
+ public Iterable<AtlasEdge<Titan0Vertex, Titan0Edge>> getEdges(AtlasEdgeDirection dir, String edgeLabel) {
+ Iterable<Edge> titanEdges = wrappedElement.getEdges(TitanObjectFactory.createDirection(dir), edgeLabel);
+ return graph.wrapEdges(titanEdges);
+ }
+
+ private TitanVertex getAsTitanVertex() {
+ return (TitanVertex) wrappedElement;
+ }
+
+ @Override
+ public Iterable<AtlasEdge<Titan0Vertex, Titan0Edge>> getEdges(AtlasEdgeDirection in) {
+ Iterable<Edge> titanResult = wrappedElement.getEdges(TitanObjectFactory.createDirection(in));
+ return graph.wrapEdges(titanResult);
+
+ }
+
+ @Override
+ public <T> T getProperty(String propertyName, Class<T> clazz) {
+
+ if (AtlasGraphManagement.MULTIPLICITY_MANY_PROPERTY_KEYS.contains(propertyName)) {
+ // throw exception in this case to be consistent with Titan 1.0.0
+ // behavior.
+ throw new IllegalStateException();
+ }
+ return super.getProperty(propertyName, clazz);
+ }
+
+ public <T> void setProperty(String propertyName, T value) {
+
+ try {
+ super.setProperty(propertyName, value);
+ } catch (UnsupportedOperationException e) {
+ // For consistency with Titan 1.0.0, treat sets of multiplicity many
+ // properties as adds. Handle this here since this is an uncommon
+ // occurrence.
+ if (AtlasGraphManagement.MULTIPLICITY_MANY_PROPERTY_KEYS.contains(propertyName)) {
+ addProperty(propertyName, value);
+ } else {
+ throw e;
+ }
+ }
+ }
+
+ @Override
+ public <T> void addProperty(String propertyName, T value) {
+ try {
+ getAsTitanVertex().addProperty(propertyName, value);
+ } catch (SchemaViolationException e) {
+ if (getPropertyValues(propertyName, value.getClass()).contains(value)) {
+ // follow java set semantics, don't throw an exception if
+ // value is already there.
+ return;
+ }
+ throw new AtlasSchemaViolationException(e);
+ }
+ }
+
+ @Override
+ public <T> Collection<T> getPropertyValues(String key, Class<T> clazz) {
+
+ TitanVertex tv = getAsTitanVertex();
+ Collection<T> result = new ArrayList<T>();
+ for (TitanProperty property : tv.getProperties(key)) {
+ result.add((T) property.getValue());
+ }
+ return result;
+ }
+
+ @Override
+ public AtlasVertexQuery<Titan0Vertex, Titan0Edge> query() {
+ return new Titan0VertexQuery(graph, wrappedElement.query());
+ }
+
+ @Override
+ public Titan0Vertex getV() {
+
+ return this;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.lang.Object#toString()
+ */
+ @Override
+ public String toString() {
+ return "Titan0Vertex [id=" + getId() + "]";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0VertexQuery.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0VertexQuery.java b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0VertexQuery.java
new file mode 100644
index 0000000..bd8b897
--- /dev/null
+++ b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0VertexQuery.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.titan0;
+
+import org.apache.atlas.repository.graphdb.AtlasEdge;
+import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.graphdb.AtlasVertexQuery;
+
+import com.tinkerpop.blueprints.Edge;
+import com.tinkerpop.blueprints.Vertex;
+import com.tinkerpop.blueprints.VertexQuery;
+
+/**
+ * Titan 0.5.4 implementation of AtlasVertexQuery.
+ */
+public class Titan0VertexQuery implements AtlasVertexQuery<Titan0Vertex, Titan0Edge> {
+
+ private Titan0Graph graph;
+ private VertexQuery vertexQuery;
+
+ public Titan0VertexQuery(Titan0Graph graph, VertexQuery vertexQuery) {
+ this.vertexQuery = vertexQuery;
+ this.graph = graph;
+ }
+
+ @Override
+ public AtlasVertexQuery<Titan0Vertex, Titan0Edge> direction(AtlasEdgeDirection queryDirection) {
+ vertexQuery.direction(TitanObjectFactory.createDirection(queryDirection));
+ return this;
+
+ }
+
+ @Override
+ public Iterable<AtlasVertex<Titan0Vertex, Titan0Edge>> vertices() {
+ Iterable<Vertex> vertices = vertexQuery.vertices();
+ return graph.wrapVertices(vertices);
+ }
+
+ @Override
+ public Iterable<AtlasEdge<Titan0Vertex, Titan0Edge>> edges() {
+ Iterable<Edge> edges = vertexQuery.edges();
+ return graph.wrapEdges(edges);
+ }
+
+ @Override
+ public long count() {
+ return vertexQuery.count();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/TitanObjectFactory.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/TitanObjectFactory.java b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/TitanObjectFactory.java
new file mode 100644
index 0000000..ab0e798
--- /dev/null
+++ b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/TitanObjectFactory.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.titan0;
+
+import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
+import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
+import org.apache.atlas.typesystem.types.Multiplicity;
+
+import com.thinkaurelius.titan.core.Cardinality;
+import com.thinkaurelius.titan.core.PropertyKey;
+import com.tinkerpop.blueprints.Direction;
+
+/**
+ * Factory that serves up instances of Titan/Tinkerpop classes that correspond to
+ * graph database abstraction layer/Atlas classes.
+ */
+public final class TitanObjectFactory {
+
+ private TitanObjectFactory() {
+
+ }
+
+ /**
+ * Retrieves the titan direction corresponding to the given
+ * AtlasEdgeDirection.
+ *
+ * @param dir
+ * @return
+ */
+ public static Direction createDirection(AtlasEdgeDirection dir) {
+
+ switch(dir) {
+ case IN:
+ return Direction.IN;
+ case OUT:
+ return Direction.OUT;
+ case BOTH:
+ return Direction.BOTH;
+ default:
+ throw new RuntimeException("Unrecognized direction: " + dir);
+ }
+ }
+
+
+ /**
+ * Converts a Multiplicity to a Cardinality.
+ *
+ * @param multiplicity
+ * @return
+ */
+ public static Cardinality createCardinality(Multiplicity multiplicity) {
+
+ if (multiplicity == Multiplicity.OPTIONAL || multiplicity == Multiplicity.REQUIRED) {
+ return Cardinality.SINGLE;
+ } else if (multiplicity == Multiplicity.COLLECTION) {
+ return Cardinality.LIST;
+ } else if (multiplicity == Multiplicity.SET) {
+ return Cardinality.SET;
+ }
+ // default to LIST as this is the most forgiving
+ return Cardinality.LIST;
+ }
+
+ public static PropertyKey createPropertyKey(AtlasPropertyKey key) {
+ return ((Titan0PropertyKey)key).getWrappedPropertyKey();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/query/NativeTitan0GraphQuery.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/query/NativeTitan0GraphQuery.java b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/query/NativeTitan0GraphQuery.java
new file mode 100644
index 0000000..cfd9905
--- /dev/null
+++ b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/query/NativeTitan0GraphQuery.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.titan0.query;
+
+import java.util.Collection;
+
+import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.graphdb.titan.query.NativeTitanGraphQuery;
+import org.apache.atlas.repository.graphdb.titan0.Titan0Database;
+import org.apache.atlas.repository.graphdb.titan0.Titan0Edge;
+import org.apache.atlas.repository.graphdb.titan0.Titan0Graph;
+import org.apache.atlas.repository.graphdb.titan0.Titan0Vertex;
+
+import com.thinkaurelius.titan.core.TitanGraphQuery;
+import com.thinkaurelius.titan.core.attribute.Contain;
+import com.thinkaurelius.titan.graphdb.query.TitanPredicate;
+import com.tinkerpop.blueprints.Compare;
+
+/**
+ * Titan 0.5.4 implementation of NativeTitanGraphQuery.
+ *
+ * @author jeff
+ *
+ */
+public class NativeTitan0GraphQuery implements NativeTitanGraphQuery<Titan0Vertex,Titan0Edge> {
+
+ private Titan0Graph graph_;
+ private TitanGraphQuery<?> query_;
+
+ public NativeTitan0GraphQuery(Titan0Graph graph) {
+ query_ = Titan0Database.getGraphInstance().query();
+ graph_ = graph;
+ }
+
+ @Override
+ public Iterable<AtlasVertex<Titan0Vertex,Titan0Edge>> vertices() {
+ Iterable it = query_.vertices();
+ return graph_.wrapVertices(it);
+ }
+
+
+ @Override
+ public void in(String propertyName, Collection<? extends Object> values) {
+ query_.has(propertyName, Contain.IN, values);
+
+ }
+
+ @Override
+ public void has(String propertyName, ComparisionOperator op, Object value) {
+
+ Compare c = getGremlinPredicate(op);
+ TitanPredicate pred = TitanPredicate.Converter.convert(c);
+ query_.has(propertyName, pred, value);
+ }
+
+ private Compare getGremlinPredicate(ComparisionOperator op) {
+ switch (op) {
+ case EQUAL:
+ return Compare.EQUAL;
+ case GREATER_THAN_EQUAL:
+ return Compare.GREATER_THAN_EQUAL;
+ case LESS_THAN_EQUAL:
+ return Compare.LESS_THAN_EQUAL;
+ case NOT_EQUAL:
+ return Compare.NOT_EQUAL;
+
+ default:
+ throw new RuntimeException("Unsupported comparison operator:" + op);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/query/Titan0GraphQuery.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/query/Titan0GraphQuery.java b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/query/Titan0GraphQuery.java
new file mode 100644
index 0000000..6c12ac2
--- /dev/null
+++ b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/query/Titan0GraphQuery.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.titan0.query;
+
+import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
+import org.apache.atlas.repository.graphdb.titan.query.TitanGraphQuery;
+import org.apache.atlas.repository.graphdb.titan.query.NativeTitanGraphQuery;
+import org.apache.atlas.repository.graphdb.titan.query.NativeTitanQueryFactory;
+import org.apache.atlas.repository.graphdb.titan0.Titan0Edge;
+import org.apache.atlas.repository.graphdb.titan0.Titan0Graph;
+import org.apache.atlas.repository.graphdb.titan0.Titan0Vertex;
+
+/**
+ * Titan 0.5.4 implementation of AtlasGraphQuery.
+ */
+public class Titan0GraphQuery extends TitanGraphQuery<Titan0Vertex,Titan0Edge> implements NativeTitanQueryFactory<Titan0Vertex,Titan0Edge> {
+
+ public Titan0GraphQuery(Titan0Graph graph, boolean isChildQuery) {
+ super(graph, isChildQuery);
+ }
+
+ public Titan0GraphQuery(Titan0Graph graph) {
+ super(graph);
+ }
+
+ @Override
+ public AtlasGraphQuery<Titan0Vertex, Titan0Edge> createChildQuery() {
+ return new Titan0GraphQuery((Titan0Graph)graph_, true);
+ }
+
+ @Override
+ protected NativeTitanQueryFactory<Titan0Vertex,Titan0Edge> getQueryFactory() {
+ return this;
+ }
+
+
+ @Override
+ public NativeTitanGraphQuery<Titan0Vertex, Titan0Edge> createNativeTitanQuery() {
+ return new NativeTitan0GraphQuery((Titan0Graph)graph_);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/main/java/org/apache/atlas/utils/IteratorToIterableAdapter.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/org/apache/atlas/utils/IteratorToIterableAdapter.java b/graphdb/titan0/src/main/java/org/apache/atlas/utils/IteratorToIterableAdapter.java
new file mode 100644
index 0000000..ed7c2a7
--- /dev/null
+++ b/graphdb/titan0/src/main/java/org/apache/atlas/utils/IteratorToIterableAdapter.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.utils;
+
+import java.util.Iterator;
+
+/**
+ * Adapter class that allows an Iterator to be presented as an instance of java.util.Iterable.
+ */
+public final class IteratorToIterableAdapter<T> implements Iterable<T> {
+
+ private final Iterator<T> wrapped;
+
+ public IteratorToIterableAdapter(Iterator<T> wrapped) {
+ this.wrapped = wrapped;
+ }
+
+ @Override
+ public Iterator<T> iterator() {
+ return wrapped;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueStoreTest.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueStoreTest.java b/graphdb/titan0/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueStoreTest.java
new file mode 100644
index 0000000..21087a5
--- /dev/null
+++ b/graphdb/titan0/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueStoreTest.java
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.thinkaurelius.titan.diskstorage.hbase;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.fail;
+
+import java.util.concurrent.TimeUnit;
+
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import com.thinkaurelius.titan.diskstorage.BackendException;
+import com.thinkaurelius.titan.diskstorage.EntryMetaData;
+import com.thinkaurelius.titan.diskstorage.StaticBuffer;
+import com.thinkaurelius.titan.diskstorage.configuration.Configuration;
+import com.thinkaurelius.titan.diskstorage.locking.LocalLockMediator;
+import com.thinkaurelius.titan.diskstorage.locking.PermanentLockingException;
+import com.thinkaurelius.titan.diskstorage.util.KeyColumn;
+import com.thinkaurelius.titan.diskstorage.util.time.StandardDuration;
+import com.thinkaurelius.titan.diskstorage.util.time.Timepoint;
+import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration;
+
+public class HBaseKeyColumnValueStoreTest {
+
+ @Mock
+ HBaseStoreManager storeManager;
+
+ @Mock
+ ConnectionMask connectionMask;
+
+ @Mock
+ LocalLockMediator localLockMediator;
+
+ @Mock
+ StaticBuffer key;
+
+ @Mock
+ StaticBuffer column;
+
+ @Mock
+ StaticBuffer expectedValue;
+
+ @Mock
+ HBaseTransaction transaction;
+
+ @Mock
+ Configuration storageConfig;
+
+ @BeforeMethod
+ public void setup() {
+ MockitoAnnotations.initMocks(this);
+ }
+
+ @Test
+ public void shouldSucceedInLockingIfLockMediatorSucceeds() throws BackendException {
+
+ when(storeManager.getMetaDataSchema("hbase")).thenReturn(new EntryMetaData[] {EntryMetaData.TIMESTAMP});
+ when(storeManager.getStorageConfig()).thenReturn(storageConfig);
+ when(storageConfig.get(GraphDatabaseConfiguration.LOCK_EXPIRE)).thenReturn(
+ new StandardDuration(300L, TimeUnit.MILLISECONDS));
+ when(storageConfig.get(GraphDatabaseConfiguration.LOCK_WAIT)).thenReturn(
+ new StandardDuration(10L, TimeUnit.MILLISECONDS));
+ when(storageConfig.get(GraphDatabaseConfiguration.LOCK_RETRY)).thenReturn(3);
+ KeyColumn lockID = new KeyColumn(key, column);
+ when(localLockMediator.lock(eq(lockID), eq(transaction), any(Timepoint.class))).
+ thenReturn(true);
+
+ HBaseKeyColumnValueStore hBaseKeyColumnValueStore =
+ new HBaseKeyColumnValueStore(storeManager, connectionMask, "titan", "e", "hbase", localLockMediator);
+ hBaseKeyColumnValueStore.acquireLock(key, column, expectedValue, transaction);
+
+ verify(transaction).updateLocks(lockID, expectedValue);
+ verify(localLockMediator, times(1)).lock(eq(lockID), eq(transaction), any(Timepoint.class));
+ }
+
+ @Test
+ public void shouldRetryRightNumberOfTimesIfLockMediationFails() throws BackendException {
+ when(storeManager.getMetaDataSchema("hbase")).thenReturn(new EntryMetaData[] {EntryMetaData.TIMESTAMP});
+ when(storeManager.getStorageConfig()).thenReturn(storageConfig);
+ when(storageConfig.get(GraphDatabaseConfiguration.LOCK_EXPIRE)).thenReturn(
+ new StandardDuration(300L, TimeUnit.MILLISECONDS));
+ when(storageConfig.get(GraphDatabaseConfiguration.LOCK_WAIT)).thenReturn(
+ new StandardDuration(10L, TimeUnit.MILLISECONDS));
+ when(storageConfig.get(GraphDatabaseConfiguration.LOCK_RETRY)).thenReturn(3);
+ KeyColumn lockID = new KeyColumn(key, column);
+ when(localLockMediator.lock(eq(lockID), eq(transaction), any(Timepoint.class))).
+ thenReturn(false).thenReturn(false).thenReturn(true);
+
+ HBaseKeyColumnValueStore hBaseKeyColumnValueStore =
+ new HBaseKeyColumnValueStore(storeManager, connectionMask, "titan", "e", "hbase", localLockMediator);
+ hBaseKeyColumnValueStore.acquireLock(key, column, expectedValue, transaction);
+
+ verify(transaction).updateLocks(lockID, expectedValue);
+ verify(localLockMediator, times(3)).lock(eq(lockID), eq(transaction), any(Timepoint.class));
+ }
+
+ @Test(expectedExceptions = PermanentLockingException.class)
+ public void shouldThrowExceptionAfterConfiguredRetriesIfLockMediationFails() throws BackendException {
+ when(storeManager.getMetaDataSchema("hbase")).thenReturn(new EntryMetaData[] {EntryMetaData.TIMESTAMP});
+ when(storeManager.getStorageConfig()).thenReturn(storageConfig);
+ when(storageConfig.get(GraphDatabaseConfiguration.LOCK_EXPIRE)).thenReturn(
+ new StandardDuration(300L, TimeUnit.MILLISECONDS));
+ when(storageConfig.get(GraphDatabaseConfiguration.LOCK_WAIT)).thenReturn(
+ new StandardDuration(10L, TimeUnit.MILLISECONDS));
+ when(storageConfig.get(GraphDatabaseConfiguration.LOCK_RETRY)).thenReturn(3);
+ KeyColumn lockID = new KeyColumn(key, column);
+ when(localLockMediator.lock(eq(lockID), eq(transaction), any(Timepoint.class))).
+ thenReturn(false).thenReturn(false).thenReturn(false);
+
+ HBaseKeyColumnValueStore hBaseKeyColumnValueStore =
+ new HBaseKeyColumnValueStore(storeManager, connectionMask, "titan", "e", "hbase", localLockMediator);
+ hBaseKeyColumnValueStore.acquireLock(key, column, expectedValue, transaction);
+
+ fail("Should fail as lock could not be acquired after 3 retries.");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/test/java/com/thinkaurelius/titan/diskstorage/locking/LocalLockMediatorTest.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/test/java/com/thinkaurelius/titan/diskstorage/locking/LocalLockMediatorTest.java b/graphdb/titan0/src/test/java/com/thinkaurelius/titan/diskstorage/locking/LocalLockMediatorTest.java
new file mode 100644
index 0000000..d0fd401
--- /dev/null
+++ b/graphdb/titan0/src/test/java/com/thinkaurelius/titan/diskstorage/locking/LocalLockMediatorTest.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2012-2013 Aurelius LLC
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.thinkaurelius.titan.diskstorage.locking;
+
+import com.thinkaurelius.titan.diskstorage.hbase.HBaseTransaction;
+import com.thinkaurelius.titan.diskstorage.util.time.TimestampProvider;
+import com.thinkaurelius.titan.diskstorage.util.time.Timestamps;
+import com.thinkaurelius.titan.diskstorage.StaticBuffer;
+import com.thinkaurelius.titan.diskstorage.util.KeyColumn;
+import com.thinkaurelius.titan.diskstorage.util.StaticArrayBuffer;
+import org.mockito.Mockito;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.util.concurrent.TimeUnit;
+
+public class LocalLockMediatorTest {
+
+ private static final String LOCK_NAMESPACE = "test";
+ private static final StaticBuffer LOCK_ROW = StaticArrayBuffer.of(new byte[]{1});
+ private static final StaticBuffer LOCK_COL = StaticArrayBuffer.of(new byte[]{1});
+ private static final KeyColumn kc = new KeyColumn(LOCK_ROW, LOCK_COL);
+ private static final HBaseTransaction mockTx1 = Mockito.mock(HBaseTransaction.class);
+ private static final HBaseTransaction mockTx2 = Mockito.mock(HBaseTransaction.class);
+
+ @Test
+ public void testLock() throws InterruptedException {
+ TimestampProvider times = Timestamps.MICRO;
+ LocalLockMediator<HBaseTransaction> llm =
+ new LocalLockMediator<HBaseTransaction>(LOCK_NAMESPACE, times);
+
+ //Expire immediately
+ Assert.assertTrue(llm.lock(kc, mockTx1, times.getTime(0, TimeUnit.NANOSECONDS)));
+ Assert.assertTrue(llm.lock(kc, mockTx2, times.getTime(Long.MAX_VALUE, TimeUnit.NANOSECONDS)));
+
+ llm = new LocalLockMediator<HBaseTransaction>(LOCK_NAMESPACE, times);
+
+ //Expire later
+ Assert.assertTrue(llm.lock(kc, mockTx1, times.getTime(Long.MAX_VALUE, TimeUnit.NANOSECONDS)));
+ //So second lock should fail on same keyCol
+ Assert.assertFalse(llm.lock(kc, mockTx2, times.getTime(Long.MAX_VALUE, TimeUnit.NANOSECONDS)));
+
+ //Unlock
+ Assert.assertTrue(llm.unlock(kc, mockTx1));
+ //Now locking should succeed
+ Assert.assertTrue(llm.lock(kc, mockTx2, times.getTime(Long.MAX_VALUE, TimeUnit.NANOSECONDS)));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/AbstractGraphDatabaseTest.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/AbstractGraphDatabaseTest.java b/graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/AbstractGraphDatabaseTest.java
new file mode 100644
index 0000000..35735e3
--- /dev/null
+++ b/graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/AbstractGraphDatabaseTest.java
@@ -0,0 +1,200 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graphdb.titan0;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
+import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeClass;
+
+/**
+ *
+ */
+public abstract class AbstractGraphDatabaseTest {
+
+ private static class RunnableWrapper implements Runnable {
+ private final Runnable r;
+ private Throwable exceptionThrown_ = null;
+
+ private RunnableWrapper(Runnable r) {
+ this.r = r;
+ }
+
+ @Override
+ public void run() {
+ try {
+ r.run();
+ }
+ catch(Throwable e) {
+ exceptionThrown_ = e;
+ }
+
+ }
+
+ public Throwable getExceptionThrown() {
+ return exceptionThrown_;
+ }
+ }
+
+ protected static final String WEIGHT_PROPERTY = "weight";
+ protected static final String TRAIT_NAMES = Constants.TRAIT_NAMES_PROPERTY_KEY;
+ protected static final String typeProperty = "__type";
+ protected static final String typeSystem = "typeSystem";
+
+ /**
+ *
+ */
+ private static final String BACKING_INDEX_NAME = "backing";
+
+ private AtlasGraph<?,?> graph = null;
+
+ @AfterMethod
+ public void commitGraph() {
+ //force any pending actions to be committed so we can be sure they don't cause errors.
+ pushChangesAndFlushCache();
+ graph.commit();
+ }
+ protected <V, E> void pushChangesAndFlushCache() {
+ AtlasGraph<V, E> graph = getGraph();
+ graph.commit();
+ }
+
+
+ @BeforeClass
+ public static void createIndices() {
+ Titan0Database db = new Titan0Database();
+ AtlasGraphManagement mgmt = db.getGraph().getManagementSystem();
+
+ if(mgmt.getGraphIndex(BACKING_INDEX_NAME) == null) {
+ mgmt.buildMixedVertexIndex(BACKING_INDEX_NAME, Constants.BACKING_INDEX);
+ }
+ mgmt.makePropertyKey("age13",Integer.class, Multiplicity.OPTIONAL);
+
+ createIndices(mgmt, "name", String.class, false, Multiplicity.REQUIRED);
+ createIndices(mgmt, WEIGHT_PROPERTY, Integer.class, false, Multiplicity.OPTIONAL);
+ createIndices(mgmt, "size15", String.class, false, Multiplicity.REQUIRED);
+ createIndices(mgmt, "typeName", String.class, false, Multiplicity.REQUIRED);
+ createIndices(mgmt, "__type", String.class, false, Multiplicity.REQUIRED);
+ createIndices(mgmt, Constants.GUID_PROPERTY_KEY, String.class, true, Multiplicity.REQUIRED);
+ createIndices(mgmt, Constants.TRAIT_NAMES_PROPERTY_KEY, String.class, false, Multiplicity.SET);
+ createIndices(mgmt, Constants.SUPER_TYPES_PROPERTY_KEY, String.class, false, Multiplicity.SET);
+ mgmt.commit();
+ }
+
+ @AfterClass
+ public static void cleanUp() {
+ Titan0Graph graph = new Titan0Graph();
+ graph.clear();
+
+ }
+
+ private static void createIndices(AtlasGraphManagement management, String propertyName, Class propertyClass,
+ boolean isUnique, Multiplicity cardinality) {
+
+
+
+ if(management.containsPropertyKey(propertyName)) {
+ //index was already created
+ return;
+ }
+
+ AtlasPropertyKey key = management.makePropertyKey(propertyName, propertyClass, cardinality);
+ try {
+ if(propertyClass != Integer.class) {
+ management.addIndexKey(BACKING_INDEX_NAME, key);
+ }
+ }
+ catch(Throwable t) {
+ //ok
+ t.printStackTrace();
+ }
+ try {
+ //if(propertyClass != Integer.class) {
+ management.createCompositeIndex(propertyName, key, isUnique);
+ //}
+ }
+ catch(Throwable t) {
+ //ok
+ t.printStackTrace();
+ }
+
+
+ }
+
+ /**
+ *
+ */
+ public AbstractGraphDatabaseTest() {
+ super();
+ }
+
+
+ protected final <V,E> AtlasGraph<V, E> getGraph() {
+ if(graph == null) {
+ graph = new Titan0Graph();
+ }
+ return (AtlasGraph<V,E>)graph;
+ }
+
+ protected Titan0Graph getTitan0Graph() {
+ AtlasGraph g = getGraph();
+ return (Titan0Graph)g;
+ }
+
+
+ protected List<AtlasVertex> newVertices_ = new ArrayList<>();
+
+ protected final <V, E> AtlasVertex<V, E> createVertex(AtlasGraph<V, E> graph) {
+ AtlasVertex<V,E> vertex = graph.addVertex();
+ newVertices_.add(vertex);
+ return vertex;
+ }
+
+ @AfterMethod
+ public void removeVertices() {
+ for(AtlasVertex vertex : newVertices_) {
+ if(vertex.exists()) {
+ getGraph().removeVertex(vertex);
+ }
+ }
+ getGraph().commit();
+ newVertices_.clear();
+ }
+ protected void runSynchronouslyInNewThread(final Runnable r) throws Throwable {
+
+ RunnableWrapper wrapper = new RunnableWrapper(r);
+ Thread th = new Thread(wrapper);
+ th.start();
+ th.join();
+ Throwable ex = wrapper.getExceptionThrown();
+ if(ex != null) {
+ throw ex;
+ }
+ }
+
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4fa10b6a/graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/GraphQueryTest.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/GraphQueryTest.java b/graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/GraphQueryTest.java
new file mode 100644
index 0000000..d02dce9
--- /dev/null
+++ b/graphdb/titan0/src/test/java/org/apache/atlas/repository/graphdb/titan0/GraphQueryTest.java
@@ -0,0 +1,447 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graphdb.titan0;
+
+
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
+import org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.testng.annotations.Test;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.Collections2;
+
+
+
+/**
+ * Tests for Titan0GraphQuery
+ */
+@Test
+public class GraphQueryTest extends AbstractGraphDatabaseTest {
+
+
+ @Test
+ public <V,E> void testQueryThatCannotRunInMemory() throws AtlasException {
+ AtlasGraph<V,E> graph = getGraph();
+ AtlasVertex<V,E> v1 = createVertex(graph);
+
+ v1.setProperty("name", "Fred");
+ v1.setProperty("size15", "15");
+
+ AtlasVertex<V,E> v2 = createVertex(graph);
+ v2.setProperty("name", "Fred");
+
+ AtlasVertex<V,E> v3 = createVertex(graph);
+ v3.setProperty("size15", "15");
+
+ graph.commit();
+
+ AtlasVertex<V,E> v4 = createVertex(graph);
+ v4.setProperty("name", "Fred");
+ v4.setProperty("size15", "15");
+
+ AtlasGraphQuery q = graph.query();
+ q.has("name", ComparisionOperator.NOT_EQUAL, "George");
+ q.has("size15","15");
+ graph.commit();
+ pause(); //pause to let the index get updated
+
+ assertQueryMatches(q, v1, v3, v4);
+
+ }
+
+ @Test
+ public void testCombinationOfAndsAndOrs() throws AtlasException {
+ Titan0Graph graph = getTitan0Graph();
+
+ AtlasVertex<Titan0Vertex,Titan0Edge> v1 = createVertex(graph);
+
+ v1.setProperty("name", "Fred");
+ v1.setProperty("size15", "15");
+ v1.setProperty("typeName", "Person");
+
+ AtlasVertex<Titan0Vertex,Titan0Edge> v2 = createVertex(graph);
+ v2.setProperty("name", "George");
+ v2.setProperty("size15", "16");
+ v2.setProperty("typeName", "Person");
+
+ AtlasVertex<Titan0Vertex,Titan0Edge> v3 = createVertex(graph);
+ v3.setProperty("name", "Jane");
+ v3.setProperty("size15", "17");
+ v3.setProperty("typeName", "Person");
+
+
+ AtlasVertex<Titan0Vertex,Titan0Edge> v4 = createVertex(graph);
+ v4.setProperty("name", "Bob");
+ v4.setProperty("size15", "18");
+ v4.setProperty("typeName", "Person");
+
+ AtlasVertex<Titan0Vertex,Titan0Edge> v5 = createVertex(graph);
+ v5.setProperty("name", "Julia");
+ v5.setProperty("size15", "19");
+ v5.setProperty("typeName", "Manager");
+
+
+ AtlasGraphQuery q = getGraphQuery();
+ q.has("typeName","Person");
+ //initially match
+ AtlasGraphQuery inner1a = q.createChildQuery();
+ AtlasGraphQuery inner1b = q.createChildQuery();
+ inner1a.has("name","Fred");
+ inner1b.has("name","Jane");
+ q.or(toList(inner1a, inner1b));
+
+
+ AtlasGraphQuery inner2a = q.createChildQuery();
+ AtlasGraphQuery inner2b = q.createChildQuery();
+ AtlasGraphQuery inner2c = q.createChildQuery();
+ inner2a.has("size15","18");
+ inner2b.has("size15","15");
+ inner2c.has("size15", "16");
+ q.or(toList(inner2a, inner2b, inner2c));
+
+ assertQueryMatches(q, v1);
+ graph.commit();
+ pause(); //let the index update
+ assertQueryMatches(q, v1);
+ }
+
+ @Test
+ public void testWithinStep() throws AtlasException {
+ Titan0Graph graph = getTitan0Graph();
+
+ AtlasVertex<Titan0Vertex,Titan0Edge> v1 = createVertex(graph);
+
+ v1.setProperty("name", "Fred");
+ v1.setProperty("size15", "15");
+ v1.setProperty("typeName", "Person");
+
+ AtlasVertex<Titan0Vertex,Titan0Edge> v2 = createVertex(graph);
+ v2.setProperty("name", "George");
+ v2.setProperty("size15", "16");
+ v2.setProperty("typeName", "Person");
+
+ AtlasVertex<Titan0Vertex,Titan0Edge> v3 = createVertex(graph);
+ v3.setProperty("name", "Jane");
+ v3.setProperty("size15", "17");
+ v3.setProperty("typeName", "Person");
+
+
+ AtlasVertex<Titan0Vertex,Titan0Edge> v4 = createVertex(graph);
+ v4.setProperty("name", "Bob");
+ v4.setProperty("size15", "18");
+ v4.setProperty("typeName", "Person");
+
+ AtlasVertex<Titan0Vertex,Titan0Edge> v5 = createVertex(graph);
+ v5.setProperty("name", "Julia");
+ v5.setProperty("size15", "19");
+ v5.setProperty("typeName", "Manager");
+
+
+ AtlasGraphQuery q = getGraphQuery();
+ q.has("typeName","Person");
+ //initially match
+ q.in("name", toList("Fred", "Jane"));
+ q.in("size15", toList("18", "15", "16"));
+
+ assertQueryMatches(q, v1);
+ graph.commit();
+ pause(); //let the index update
+ assertQueryMatches(q, v1);
+ }
+
+ @Test
+ public void testWithinStepWhereGraphIsStale() throws AtlasException {
+ Titan0Graph graph = getTitan0Graph();
+
+ AtlasVertex<Titan0Vertex,Titan0Edge> v1 = createVertex(graph);
+
+ v1.setProperty("name", "Fred");
+ v1.setProperty("size15", "15");
+ v1.setProperty("typeName", "Person");
+
+ AtlasVertex<Titan0Vertex,Titan0Edge> v2 = createVertex(graph);
+ v2.setProperty("name", "George");
+ v2.setProperty("size15", "16");
+ v2.setProperty("typeName", "Person");
+
+ AtlasVertex<Titan0Vertex,Titan0Edge> v3 = createVertex(graph);
+ v3.setProperty("name", "Jane");
+ v3.setProperty("size15", "17");
+ v3.setProperty("typeName", "Person");
+
+
+ AtlasVertex<Titan0Vertex,Titan0Edge> v4 = createVertex(graph);
+ v4.setProperty("name", "Bob");
+ v4.setProperty("size15", "18");
+ v4.setProperty("typeName", "Person");
+
+ AtlasVertex<Titan0Vertex,Titan0Edge> v5 = createVertex(graph);
+ v5.setProperty("name", "Julia");
+ v5.setProperty("size15", "19");
+ v5.setProperty("typeName", "Manager");
+
+
+ AtlasGraphQuery q = getGraphQuery();
+ q.has("typeName","Person");
+ //initially match
+ q.in("name", toList("Fred", "Jane"));
+
+ graph.commit();
+ pause(); //let the index update
+ assertQueryMatches(q, v1, v3);
+ v3.setProperty("name", "Janet"); //make v3 no longer match the query. Within step should filter out the vertex since it no longer matches.
+ assertQueryMatches(q, v1);
+ }
+
+ @Test
+ public void testSimpleOrQuery() throws AtlasException {
+ Titan0Graph graph = getTitan0Graph();
+
+
+ AtlasVertex<Titan0Vertex,Titan0Edge> v1 = createVertex(graph);
+
+ v1.setProperty("name", "Fred");
+ v1.setProperty("size15", "15");
+
+ AtlasVertex<Titan0Vertex,Titan0Edge> v2 = createVertex(graph);
+ v2.setProperty("name", "Fred");
+
+ AtlasVertex<Titan0Vertex,Titan0Edge> v3 = createVertex(graph);
+ v3.setProperty("size15", "15");
+
+ graph.commit();
+
+ AtlasVertex<Titan0Vertex,Titan0Edge> v4 = createVertex(graph);
+ v4.setProperty("name", "Fred");
+ v4.setProperty("size15", "15");
+
+ AtlasVertex<Titan0Vertex,Titan0Edge> v5 = createVertex(graph);
+ v5.setProperty("name", "George");
+ v5.setProperty("size15", "16");
+
+ AtlasGraphQuery q = graph.query();
+ AtlasGraphQuery inner1 = q.createChildQuery().has("name", "Fred");
+ AtlasGraphQuery inner2 = q.createChildQuery().has("size15", "15");
+ q.or(toList(inner1, inner2));
+ assertQueryMatches(q, v1, v2, v3, v4);
+ graph.commit();
+ pause(); //pause to let the indexer get updated (this fails frequently without a pause)
+ assertQueryMatches(q, v1, v2, v3, v4);
+ }
+
+
+
+
+ @Test
+ public <V,E> void testQueryMatchesAddedVertices() throws AtlasException {
+ AtlasGraph<V,E> graph = getGraph();
+
+ AtlasVertex<V,E> v1 = createVertex(graph);
+
+ v1.setProperty("name", "Fred");
+ v1.setProperty("size15", "15");
+
+ AtlasVertex<V,E> v2 = createVertex(graph);
+ v2.setProperty("name", "Fred");
+
+ AtlasVertex<V,E> v3 = createVertex(graph);
+ v3.setProperty("size15", "15");
+
+ graph.commit();
+
+ AtlasVertex<V,E> v4 = createVertex(graph);
+ v4.setProperty("name", "Fred");
+ v4.setProperty("size15", "15");
+
+ AtlasGraphQuery q = getGraphQuery();
+ q.has("name", "Fred");
+ q.has("size15","15");
+
+ assertQueryMatches(q, v1, v4);
+ graph.commit();
+ assertQueryMatches(q, v1, v4);
+
+ }
+
+
+ @Test
+ public <V,E> void testQueryDoesNotMatchRemovedVertices() throws AtlasException {
+ AtlasGraph<V,E> graph = getGraph();
+
+ AtlasVertex<V,E> v1 = createVertex(graph);
+
+ v1.setProperty("name", "Fred");
+ v1.setProperty("size15", "15");
+
+ AtlasVertex<V,E> v2 = createVertex(graph);
+ v2.setProperty("name", "Fred");
+
+ AtlasVertex<V,E> v3 = createVertex(graph);
+ v3.setProperty("size15", "15");
+
+ AtlasVertex<V,E> v4 = createVertex(graph);
+ v4.setProperty("name", "Fred");
+ v4.setProperty("size15", "15");
+
+ graph.commit();
+
+ graph.removeVertex(v1);
+
+ AtlasGraphQuery q = getGraphQuery();
+ q.has("name", "Fred");
+ q.has("size15","15");
+
+ assertQueryMatches(q, v4);
+ graph.commit();
+
+ assertQueryMatches(q, v4);
+ }
+
+ @Test
+ public <V,E> void testQueryDoesNotMatchUncommittedAddedAndRemovedVertices() throws AtlasException {
+ AtlasGraph<V,E> graph = getGraph();
+
+ AtlasVertex<V,E> v1 = createVertex(graph);
+
+ v1.setProperty("name", "Fred");
+ v1.setProperty("size15", "15");
+
+ AtlasVertex<V,E> v2 = createVertex(graph);
+ v2.setProperty("name", "Fred");
+
+ AtlasVertex<V,E> v3 = createVertex(graph);
+ v3.setProperty("size15", "15");
+
+ AtlasVertex<V,E> v4 = createVertex(graph);
+ v4.setProperty("name", "Fred");
+ v4.setProperty("size15", "15");
+
+
+ AtlasGraphQuery q = getGraphQuery();
+ q.has("name", "Fred");
+ q.has("size15","15");
+
+ assertQueryMatches(q, v1, v4);
+
+ graph.removeVertex(v1);
+
+
+ assertQueryMatches(q, v4);
+ graph.commit();
+
+ assertQueryMatches(q, v4);
+ }
+
+
+ @Test
+ public <V,E> void testQueryResultsReflectPropertyAdd() throws AtlasException {
+ AtlasGraph<V,E> graph = getGraph();
+
+ AtlasVertex<V,E> v1 = createVertex(graph);
+ v1.setProperty("name", "Fred");
+ v1.setProperty("size15", "15");
+ v1.addProperty(TRAIT_NAMES, "trait1");
+ v1.addProperty(TRAIT_NAMES, "trait2");
+
+ AtlasVertex<V,E> v2 = createVertex(graph);
+ v2.setProperty("name", "Fred");
+ v2.addProperty(TRAIT_NAMES, "trait1");
+
+ AtlasVertex<V,E> v3 = createVertex(graph);
+ v3.setProperty("size15", "15");
+ v3.addProperty(TRAIT_NAMES, "trait2");
+
+ AtlasGraphQuery query = getGraphQuery();
+ query.has("name", "Fred");
+ query.has(TRAIT_NAMES, "trait1");
+ query.has("size15", "15");
+
+ assertQueryMatches(query, v1);
+ //make v3 match the query
+ v3.setProperty("name", "Fred");
+ v3.addProperty(TRAIT_NAMES, "trait1");
+ assertQueryMatches(query, v1, v3);
+ v3.removeProperty(TRAIT_NAMES);
+ assertQueryMatches(query, v1);
+ v3.addProperty(TRAIT_NAMES, "trait2");
+ assertQueryMatches(query, v1);
+ v1.removeProperty(TRAIT_NAMES);
+ assertQueryMatches(query);
+ graph.commit();
+ assertQueryMatches(query);
+
+ }
+
+ private static <T> List<T> toList(Iterable<T> itr) {
+ List<T> result = new ArrayList<T>();
+ for(T object : itr) {
+ result.add(object);
+ }
+ return result;
+
+ }
+
+ private <V,E >void assertQueryMatches(AtlasGraphQuery expr, AtlasVertex... expectedResults) throws AtlasException {
+
+ //getGraph().commit();
+ Collection<AtlasVertex<Titan0Vertex, Titan0Edge>> temp = toList(expr.vertices());
+ //filter out vertices from previous test executions
+ Collection<AtlasVertex<Titan0Vertex, Titan0Edge>> result = Collections2.filter(temp, new Predicate<AtlasVertex<Titan0Vertex, Titan0Edge>>() {
+
+ @Override
+ public boolean apply(AtlasVertex<Titan0Vertex, Titan0Edge> input) {
+ return newVertices_.contains(input);
+ }
+
+ });
+ assertEquals("Expected/found result sizes differ. Expected: " + Arrays.asList(expectedResults).toString() +", found: " + result, expectedResults.length, result.size());
+
+ for(AtlasVertex<V,E> v : expectedResults) {
+ assertTrue(result.contains(v));
+ }
+ }
+
+ private static List<Object> toList(Object...objects) {
+ return Arrays.asList(objects);
+ }
+
+ private AtlasGraphQuery<Titan0Vertex, Titan0Edge> getGraphQuery() {
+ return getTitan0Graph().query();
+ }
+
+ private void pause() {
+ try {
+ Thread.sleep(5000);
+ }
+ catch(InterruptedException e)
+ {}
+ }
+}