You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gora.apache.org by kt...@apache.org on 2015/11/17 17:20:11 UTC
gora git commit: GORA-414 Switch to Accumulo 1.6.4 and switched a
test to MiniAccumulo
Repository: gora
Updated Branches:
refs/heads/master a40f72f98 -> 356c1f8a8
GORA-414 Switch to Accumulo 1.6.4 and switched a test to MiniAccumulo
Project: http://git-wip-us.apache.org/repos/asf/gora/repo
Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/356c1f8a
Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/356c1f8a
Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/356c1f8a
Branch: refs/heads/master
Commit: 356c1f8a825f883c8de43a3e25019ff230a26457
Parents: a40f72f
Author: Keith Turner <kt...@apache.org>
Authored: Mon Nov 16 18:43:46 2015 -0500
Committer: Keith Turner <ke...@deenlo.com>
Committed: Mon Nov 16 21:14:08 2015 -0500
----------------------------------------------------------------------
gora-accumulo/pom.xml | 10 ++++-
.../gora/accumulo/store/AccumuloStore.java | 39 ++++++++++++--------
.../gora/accumulo/GoraAccumuloTestDriver.java | 8 +++-
.../src/test/resources/gora.properties | 8 ++--
4 files changed, 43 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/gora/blob/356c1f8a/gora-accumulo/pom.xml
----------------------------------------------------------------------
diff --git a/gora-accumulo/pom.xml b/gora-accumulo/pom.xml
index 8ab5143..bc131fa 100644
--- a/gora-accumulo/pom.xml
+++ b/gora-accumulo/pom.xml
@@ -50,7 +50,7 @@
</ciManagement>
<properties>
- <accumulo.version>1.5.1</accumulo.version>
+ <accumulo.version>1.6.4</accumulo.version>
<osgi.import>*</osgi.import>
<osgi.export>org.apache.gora.accumulo*;version="${project.version}";-noimport:=true</osgi.export>
</properties>
@@ -127,6 +127,14 @@
</dependency>
<dependency>
+ <!-- Accumulo mini was trying to load the Shell, which failed with the version of jline in the parent pom -->
+ <groupId>jline</groupId>
+ <artifactId>jline</artifactId>
+ <version>2.11</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/gora/blob/356c1f8a/gora-accumulo/src/main/java/org/apache/gora/accumulo/store/AccumuloStore.java
----------------------------------------------------------------------
diff --git a/gora-accumulo/src/main/java/org/apache/gora/accumulo/store/AccumuloStore.java b/gora-accumulo/src/main/java/org/apache/gora/accumulo/store/AccumuloStore.java
index 31141c3..e6b6e65 100644
--- a/gora-accumulo/src/main/java/org/apache/gora/accumulo/store/AccumuloStore.java
+++ b/gora-accumulo/src/main/java/org/apache/gora/accumulo/store/AccumuloStore.java
@@ -35,7 +35,6 @@ import java.util.concurrent.TimeUnit;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
-import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchWriter;
@@ -67,11 +66,10 @@ import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.SortedKeyIterator;
import org.apache.accumulo.core.iterators.user.TimestampFilter;
import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.accumulo.core.security.CredentialHelper;
-import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.security.Credentials;
import org.apache.accumulo.core.util.Pair;
-import org.apache.accumulo.core.util.TextUtil;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.avro.Schema;
import org.apache.avro.Schema.Field;
@@ -121,7 +119,7 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T
private Connector conn;
private BatchWriter batchWriter;
private AccumuloMapping mapping;
- private TCredentials credentials;
+ private Credentials credentials;
private Encoder encoder;
public static final Logger LOG = LoggerFactory.getLogger(AccumuloStore.class);
@@ -188,6 +186,15 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T
}
+ private static byte[] getBytes(Text text) {
+ byte[] bytes = text.getBytes();
+ if (bytes.length != text.getLength()) {
+ bytes = new byte[text.getLength()];
+ System.arraycopy(text.getBytes(), 0, bytes, 0, bytes.length);
+ }
+ return bytes;
+ }
+
public K fromBytes(Class<K> clazz, byte[] val) {
return fromBytes(encoder, clazz, val);
}
@@ -363,7 +370,7 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T
} else {
conn = new MockInstance().getConnector(user, token);
}
- credentials = CredentialHelper.create(user, token, conn.getInstance().getInstanceID());
+ credentials = new Credentials(user, token);
if (autoCreateSchema && !schemaExists())
createSchema();
@@ -452,8 +459,10 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T
conn.tableOperations().setProperty(mapping.tableName, entry.getKey(), entry.getValue());
}
- } catch (AccumuloException | AccumuloSecurityException | TableExistsException e) {
+ } catch (AccumuloException | AccumuloSecurityException e) {
LOG.error(e.getMessage(), e);
+ } catch (TableExistsException e) {
+ LOG.debug(e.getMessage(), e);
}
}
@@ -614,7 +623,7 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T
public T get(K key, String[] fields) {
try {
// TODO make isolated scanner optional?
- Scanner scanner = new IsolatedScanner(conn.createScanner(mapping.tableName, Constants.NO_AUTHS));
+ Scanner scanner = new IsolatedScanner(conn.createScanner(mapping.tableName, Authorizations.EMPTY));
Range rowRange = new Range(new Text(toBytes(key)));
scanner.setRange(rowRange);
@@ -818,7 +827,7 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T
private Scanner createScanner(Query<K,T> query) throws TableNotFoundException {
// TODO make isolated scanner optional?
- Scanner scanner = new IsolatedScanner(conn.createScanner(mapping.tableName, Constants.NO_AUTHS));
+ Scanner scanner = new IsolatedScanner(conn.createScanner(mapping.tableName, Authorizations.EMPTY));
setFetchColumns(scanner, query.getFields());
scanner.setRange(createRange(query));
@@ -871,12 +880,12 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T
if (conn instanceof MockConnector)
tl = new MockTabletLocator();
else
- tl = TabletLocator.getInstance(conn.getInstance(), new Text(Tables.getTableId(conn.getInstance(), mapping.tableName)));
+ tl = TabletLocator.getLocator(conn.getInstance(), new Text(Tables.getTableId(conn.getInstance(), mapping.tableName)));
Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<>();
tl.invalidateCache();
- while (tl.binRanges(Collections.singletonList(createRange(query)), binnedRanges, credentials).size() > 0) {
+ while (tl.binRanges(credentials, Collections.singletonList(createRange(query)), binnedRanges).size() > 0) {
// TODO log?
if (!Tables.exists(conn.getInstance(), Tables.getTableId(conn.getInstance(), mapping.tableName)))
throw new TableDeletedException(Tables.getTableId(conn.getInstance(), mapping.tableName));
@@ -913,18 +922,18 @@ public class AccumuloStore<K,T extends PersistentBase> extends DataStoreBase<K,T
K startKey = null;
if (startRow == null || !ke.contains(startRow)) {
if (ke.getPrevEndRow() != null) {
- startKey = followingKey(encoder, getKeyClass(), TextUtil.getBytes(ke.getPrevEndRow()));
+ startKey = followingKey(encoder, getKeyClass(), getBytes(ke.getPrevEndRow()));
}
} else {
- startKey = fromBytes(getKeyClass(), TextUtil.getBytes(startRow));
+ startKey = fromBytes(getKeyClass(), getBytes(startRow));
}
K endKey = null;
if (endRow == null || !ke.contains(endRow)) {
if (ke.getEndRow() != null)
- endKey = lastPossibleKey(encoder, getKeyClass(), TextUtil.getBytes(ke.getEndRow()));
+ endKey = lastPossibleKey(encoder, getKeyClass(), getBytes(ke.getEndRow()));
} else {
- endKey = fromBytes(getKeyClass(), TextUtil.getBytes(endRow));
+ endKey = fromBytes(getKeyClass(), getBytes(endRow));
}
PartitionQueryImpl<K, T> pqi = new PartitionQueryImpl<>(query, startKey, endKey, location);
http://git-wip-us.apache.org/repos/asf/gora/blob/356c1f8a/gora-accumulo/src/test/java/org/apache/gora/accumulo/GoraAccumuloTestDriver.java
----------------------------------------------------------------------
diff --git a/gora-accumulo/src/test/java/org/apache/gora/accumulo/GoraAccumuloTestDriver.java b/gora-accumulo/src/test/java/org/apache/gora/accumulo/GoraAccumuloTestDriver.java
index 0d33444..bd3b502 100644
--- a/gora-accumulo/src/test/java/org/apache/gora/accumulo/GoraAccumuloTestDriver.java
+++ b/gora-accumulo/src/test/java/org/apache/gora/accumulo/GoraAccumuloTestDriver.java
@@ -18,6 +18,7 @@
package org.apache.gora.accumulo;
import org.apache.accumulo.minicluster.MiniAccumuloCluster;
+import org.apache.accumulo.minicluster.MiniAccumuloConfig;
import org.apache.gora.GoraTestDriver;
import org.apache.gora.accumulo.store.AccumuloStore;
import org.junit.Rule;
@@ -33,7 +34,7 @@ public class GoraAccumuloTestDriver extends GoraTestDriver {
private static final Logger LOG = LoggerFactory.getLogger(GoraAccumuloTestDriver.class);
private static MiniAccumuloCluster cluster = null;
- private static final String PASSWORD = "password";
+ private static final String PASSWORD = "drowssap";
@Rule
public TemporaryFolder tmpDir = new TemporaryFolder();
@@ -48,7 +49,10 @@ public class GoraAccumuloTestDriver extends GoraTestDriver {
log.info("Starting Accumulo MiniAccumuloCluster...");
try {
tmpDir.create();
- cluster = new MiniAccumuloCluster(tmpDir.getRoot(), PASSWORD);
+ MiniAccumuloConfig miniCfg = new MiniAccumuloConfig(tmpDir.getRoot(), PASSWORD);
+ miniCfg.setInstanceName("goraTest");
+ miniCfg.setZooKeeperPort(56321);
+ cluster = new MiniAccumuloCluster(miniCfg);
cluster.start();
} catch (Exception e) {
LOG.error("Error starting Accumulo MiniAccumuloCluster: {}", e.getMessage());
http://git-wip-us.apache.org/repos/asf/gora/blob/356c1f8a/gora-accumulo/src/test/resources/gora.properties
----------------------------------------------------------------------
diff --git a/gora-accumulo/src/test/resources/gora.properties b/gora-accumulo/src/test/resources/gora.properties
index f89a360..8d1c4f5 100644
--- a/gora-accumulo/src/test/resources/gora.properties
+++ b/gora-accumulo/src/test/resources/gora.properties
@@ -14,8 +14,8 @@
# limitations under the License.
gora.datastore.default=org.apache.gora.accumulo.store.AccumuloStore
-gora.datastore.accumulo.mock=true
-gora.datastore.accumulo.instance=a14
-gora.datastore.accumulo.zookeepers=localhost
+gora.datastore.accumulo.mock=false
+gora.datastore.accumulo.instance=goraTest
+gora.datastore.accumulo.zookeepers=localhost:56321
gora.datastore.accumulo.user=root
-gora.datastore.accumulo.password=
\ No newline at end of file
+gora.datastore.accumulo.password=drowssap
\ No newline at end of file