You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2017/06/12 22:07:57 UTC
[1/8] accumulo git commit: ACCUMULO-4640 Fix broken
SiteConfigurationTest
Repository: accumulo
Updated Branches:
refs/heads/1.7 4effd0e9a -> de83e61c7
refs/heads/1.8 da695f604 -> d99158ff0
refs/heads/master e4a381de9 -> 15b9c3eec
ACCUMULO-4640 Fix broken SiteConfigurationTest
Avoid using non-existent constructor in mock object creation
Also fix a trivial javadoc warning
Also fix a trivial test failure with Java 8 and Powermock
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/de83e61c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/de83e61c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/de83e61c
Branch: refs/heads/1.7
Commit: de83e61c7f965b6f83482c6070b483bd1835b5a6
Parents: 4effd0e
Author: Christopher Tubbs <ct...@apache.org>
Authored: Mon Jun 12 14:23:39 2017 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Mon Jun 12 14:23:39 2017 -0400
----------------------------------------------------------------------
.../main/java/org/apache/accumulo/core/file/rfile/RFile.java | 2 +-
.../org/apache/accumulo/core/conf/SiteConfigurationTest.java | 5 ++---
.../java/org/apache/accumulo/shell/ShellSetInstanceTest.java | 2 ++
3 files changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/de83e61c/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
index bab2266..d5779ce 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
@@ -1022,7 +1022,7 @@ public class RFile {
* @return map of locality group names to column families. The default locality group will have {@code null} for a name. RFile will only track up to
* {@value Writer#MAX_CF_IN_DLG} families for the default locality group. After this it will stop tracking. For the case where the default group has
* more thn {@value Writer#MAX_CF_IN_DLG} families an empty list of families is returned.
- * @see LocalityGroupUtil#seek(Reader, Range, String, Map)
+ * @see LocalityGroupUtil#seek(RFile.Reader, Range, String, Map)
*/
public Map<String,ArrayList<ByteSequence>> getLocalityGroupCF() {
Map<String,ArrayList<ByteSequence>> cf = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/de83e61c/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java b/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java
index f89dbfa..29304c8 100644
--- a/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java
@@ -44,13 +44,12 @@ public class SiteConfigurationTest {
}
@Test
- public void testOnlySensitivePropertiesExtractedFromCredetialProvider() throws SecurityException, NoSuchMethodException {
+ public void testOnlySensitivePropertiesExtractedFromCredentialProvider() throws SecurityException, NoSuchMethodException {
if (!isCredentialProviderAvailable) {
return;
}
- SiteConfiguration siteCfg = EasyMock.createMockBuilder(SiteConfiguration.class).addMockedMethod("getHadoopConfiguration")
- .withConstructor(AccumuloConfiguration.class).withArgs(DefaultConfiguration.getInstance()).createMock();
+ SiteConfiguration siteCfg = EasyMock.createMockBuilder(SiteConfiguration.class).addMockedMethod("getHadoopConfiguration").createMock();
siteCfg.set(Property.INSTANCE_SECRET, "ignored");
http://git-wip-us.apache.org/repos/asf/accumulo/blob/de83e61c/shell/src/test/java/org/apache/accumulo/shell/ShellSetInstanceTest.java
----------------------------------------------------------------------
diff --git a/shell/src/test/java/org/apache/accumulo/shell/ShellSetInstanceTest.java b/shell/src/test/java/org/apache/accumulo/shell/ShellSetInstanceTest.java
index 4041164..40f3442 100644
--- a/shell/src/test/java/org/apache/accumulo/shell/ShellSetInstanceTest.java
+++ b/shell/src/test/java/org/apache/accumulo/shell/ShellSetInstanceTest.java
@@ -56,10 +56,12 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("javax.security.*")
@PrepareForTest({Shell.class, ZooUtil.class, ConfigSanityCheck.class})
public class ShellSetInstanceTest {
public static class TestOutputStream extends OutputStream {
[4/8] accumulo git commit: Merge branch '1.7' into 1.8
Posted by ct...@apache.org.
Merge branch '1.7' into 1.8
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d99158ff
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d99158ff
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d99158ff
Branch: refs/heads/master
Commit: d99158ff056ed9cccb3fc8be7a9c52ef1bafd385
Parents: da695f6 de83e61
Author: Christopher Tubbs <ct...@apache.org>
Authored: Mon Jun 12 15:02:03 2017 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Mon Jun 12 15:02:03 2017 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/client/rfile/RFileWriter.java | 6 +++---
.../org/apache/accumulo/core/conf/SiteConfigurationTest.java | 5 ++---
2 files changed, 5 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d99158ff/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriter.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriter.java
index 9995888,0000000..a69f83e
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriter.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriter.java
@@@ -1,234 -1,0 +1,234 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.core.client.rfile;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.data.ArrayByteSequence;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileSKVWriter;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.commons.collections.map.LRUMap;
+
+import com.google.common.base.Preconditions;
+
+//formatter was adding spaces that checkstyle did not like, so turned off formatter
+//@formatter:off
+/**
+ * This class provides an API for writing RFiles. It can be used to create file for bulk import into Accumulo using
+ * {@link TableOperations#importDirectory(String, String, String, boolean)}
+ *
+ * <p>
+ * A RFileWriter has the following constraints. Violating these constraints will result in runtime exceptions.
+ *
+ * <ul>
+ * <li>Keys must be appended in sorted order within a locality group.</li>
+ * <li>Locality groups must have a mutually exclusive set of column families.</li>
+ * <li>The default locality group must be started last.</li>
+ * </ul>
+ *
+ * <p>
+ * Below is an example of using RFileWriter
+ *
+ * <pre>
+ * <code>
- * {@code Iterable<Entry<Key, Value>>} localityGroup1Data = ...
- * {@code Iterable<Entry<Key, Value>>} localityGroup2Data = ...
- * {@code Iterable<Entry<Key, Value>>} defaultGroupData = ...
++ * Iterable<Entry<Key, Value>> localityGroup1Data = ...
++ * Iterable<Entry<Key, Value>> localityGroup2Data = ...
++ * Iterable<Entry<Key, Value>> defaultGroupData = ...
+ *
+ * try(RFileWriter writer = RFile.newWriter().to(file).build()) {
+ *
+ * // Start a locality group before appending data.
+ * writer.startNewLocalityGroup("groupA", "columnFam1", "columnFam2");
+ * // Append data to the locality group that was started above. Must append in sorted order.
+ * writer.append(localityGroup1Data);
+ *
+ * // Add another locality group.
+ * writer.startNewLocalityGroup("groupB", "columnFam3", "columnFam4");
+ * writer.append(localityGroup2Data);
+ *
+ * // The default locality group must be started last. The column families for the default group do not need to be specified.
+ * writer.startDefaultLocalityGroup();
+ * // Data appended here can not contain any column families specified in previous locality groups.
+ * writer.append(defaultGroupData);
+ *
+ * // This is a try-with-resources so the writer is closed here at the end of the code block.
+ * }
+ * </code>
+ * </pre>
+ *
+ * <p>
+ * Create instances by calling {@link RFile#newWriter()}
+ *
+ * @since 1.8.0
+ */
+// @formatter:on
+public class RFileWriter implements AutoCloseable {
+
+ private FileSKVWriter writer;
+ private final LRUMap validVisibilities;
+ private boolean startedLG;
+ private boolean startedDefaultLG;
+
+ RFileWriter(FileSKVWriter fileSKVWriter, int visCacheSize) {
+ this.writer = fileSKVWriter;
+ this.validVisibilities = new LRUMap(visCacheSize);
+ }
+
+ private void _startNewLocalityGroup(String name, Set<ByteSequence> columnFamilies) throws IOException {
+ Preconditions.checkState(!startedDefaultLG, "Cannont start a locality group after starting the default locality group");
+ writer.startNewLocalityGroup(name, columnFamilies);
+ startedLG = true;
+ }
+
+ /**
+ * Before appending any data, a locality group must be started. The default locality group must be started last.
+ *
+ * @param name
+ * locality group name, used for informational purposes
+ * @param families
+ * the column families the locality group can contain
+ *
+ * @throws IllegalStateException
+ * When default locality group already started.
+ */
+ public void startNewLocalityGroup(String name, List<byte[]> families) throws IOException {
+ HashSet<ByteSequence> fams = new HashSet<>();
+ for (byte[] family : families) {
+ fams.add(new ArrayByteSequence(family));
+ }
+ _startNewLocalityGroup(name, fams);
+ }
+
+ /**
+ * See javadoc for {@link #startNewLocalityGroup(String, List)}
+ *
+ * @throws IllegalStateException
+ * When default locality group already started.
+ */
+ public void startNewLocalityGroup(String name, byte[]... families) throws IOException {
+ startNewLocalityGroup(name, Arrays.asList(families));
+ }
+
+ /**
+ * See javadoc for {@link #startNewLocalityGroup(String, List)}.
+ *
+ * @param families
+ * will be encoded using UTF-8
+ *
+ * @throws IllegalStateException
+ * When default locality group already started.
+ */
+ public void startNewLocalityGroup(String name, Set<String> families) throws IOException {
+ HashSet<ByteSequence> fams = new HashSet<>();
+ for (String family : families) {
+ fams.add(new ArrayByteSequence(family));
+ }
+ _startNewLocalityGroup(name, fams);
+ }
+
+ /**
+ * See javadoc for {@link #startNewLocalityGroup(String, List)}.
+ *
+ * @param families
+ * will be encoded using UTF-8
+ *
+ * @throws IllegalStateException
+ * When default locality group already started.
+ */
+ public void startNewLocalityGroup(String name, String... families) throws IOException {
+ HashSet<ByteSequence> fams = new HashSet<>();
+ for (String family : families) {
+ fams.add(new ArrayByteSequence(family));
+ }
+ _startNewLocalityGroup(name, fams);
+ }
+
+ /**
+ * A locality group in which the column families do not need to specified. The locality group must be started after all other locality groups. Can not append
+ * column families that were in a previous locality group. If no locality groups were started, then the first append will start the default locality group.
+ *
+ * @throws IllegalStateException
+ * When default locality group already started.
+ */
+
+ public void startDefaultLocalityGroup() throws IOException {
+ Preconditions.checkState(!startedDefaultLG);
+ writer.startDefaultLocalityGroup();
+ startedDefaultLG = true;
+ startedLG = true;
+ }
+
+ /**
+ * Append the key and value to the last locality group that was started. If no locality group was started, then the default group will automatically be
+ * started.
+ *
+ * @param key
+ * This key must be greater than or equal to the last key appended. For non-default locality groups, the keys column family must be one of the column
+ * families specified when calling startNewLocalityGroup(). Must be non-null.
+ * @param val
+ * value to append, must be non-null.
+ *
+ * @throws IllegalArgumentException
+ * This is thrown when data is appended out of order OR when the key contains a invalid visibility OR when a column family is not valid for a
+ * locality group.
+ */
+ public void append(Key key, Value val) throws IOException {
+ if (!startedLG) {
+ startDefaultLocalityGroup();
+ }
+ Boolean wasChecked = (Boolean) validVisibilities.get(key.getColumnVisibilityData());
+ if (wasChecked == null) {
+ byte[] cv = key.getColumnVisibilityData().toArray();
+ new ColumnVisibility(cv);
+ validVisibilities.put(new ArrayByteSequence(Arrays.copyOf(cv, cv.length)), Boolean.TRUE);
+ }
+ writer.append(key, val);
+ }
+
+ /**
+ * Append the keys and values to the last locality group that was started.
+ *
+ * @param keyValues
+ * The keys must be in sorted order. The first key returned by the iterable must be greater than or equal to the last key appended. For non-default
+ * locality groups, the keys column family must be one of the column families specified when calling startNewLocalityGroup(). Must be non-null. If no
+ * locality group was started, then the default group will automatically be started.
+ *
+ * @throws IllegalArgumentException
+ * This is thrown when data is appended out of order OR when the key contains a invalid visibility OR when a column family is not valid for a
+ * locality group.
+ */
+ public void append(Iterable<Entry<Key,Value>> keyValues) throws IOException {
+ for (Entry<Key,Value> entry : keyValues) {
+ append(entry.getKey(), entry.getValue());
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ writer.close();
+ }
+}
[2/8] accumulo git commit: ACCUMULO-4640 Fix broken
SiteConfigurationTest
Posted by ct...@apache.org.
ACCUMULO-4640 Fix broken SiteConfigurationTest
Avoid using non-existent constructor in mock object creation
Also fix a trivial javadoc warning
Also fix a trivial test failure with Java 8 and Powermock
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/de83e61c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/de83e61c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/de83e61c
Branch: refs/heads/1.8
Commit: de83e61c7f965b6f83482c6070b483bd1835b5a6
Parents: 4effd0e
Author: Christopher Tubbs <ct...@apache.org>
Authored: Mon Jun 12 14:23:39 2017 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Mon Jun 12 14:23:39 2017 -0400
----------------------------------------------------------------------
.../main/java/org/apache/accumulo/core/file/rfile/RFile.java | 2 +-
.../org/apache/accumulo/core/conf/SiteConfigurationTest.java | 5 ++---
.../java/org/apache/accumulo/shell/ShellSetInstanceTest.java | 2 ++
3 files changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/de83e61c/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
index bab2266..d5779ce 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
@@ -1022,7 +1022,7 @@ public class RFile {
* @return map of locality group names to column families. The default locality group will have {@code null} for a name. RFile will only track up to
* {@value Writer#MAX_CF_IN_DLG} families for the default locality group. After this it will stop tracking. For the case where the default group has
* more thn {@value Writer#MAX_CF_IN_DLG} families an empty list of families is returned.
- * @see LocalityGroupUtil#seek(Reader, Range, String, Map)
+ * @see LocalityGroupUtil#seek(RFile.Reader, Range, String, Map)
*/
public Map<String,ArrayList<ByteSequence>> getLocalityGroupCF() {
Map<String,ArrayList<ByteSequence>> cf = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/de83e61c/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java b/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java
index f89dbfa..29304c8 100644
--- a/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java
@@ -44,13 +44,12 @@ public class SiteConfigurationTest {
}
@Test
- public void testOnlySensitivePropertiesExtractedFromCredetialProvider() throws SecurityException, NoSuchMethodException {
+ public void testOnlySensitivePropertiesExtractedFromCredentialProvider() throws SecurityException, NoSuchMethodException {
if (!isCredentialProviderAvailable) {
return;
}
- SiteConfiguration siteCfg = EasyMock.createMockBuilder(SiteConfiguration.class).addMockedMethod("getHadoopConfiguration")
- .withConstructor(AccumuloConfiguration.class).withArgs(DefaultConfiguration.getInstance()).createMock();
+ SiteConfiguration siteCfg = EasyMock.createMockBuilder(SiteConfiguration.class).addMockedMethod("getHadoopConfiguration").createMock();
siteCfg.set(Property.INSTANCE_SECRET, "ignored");
http://git-wip-us.apache.org/repos/asf/accumulo/blob/de83e61c/shell/src/test/java/org/apache/accumulo/shell/ShellSetInstanceTest.java
----------------------------------------------------------------------
diff --git a/shell/src/test/java/org/apache/accumulo/shell/ShellSetInstanceTest.java b/shell/src/test/java/org/apache/accumulo/shell/ShellSetInstanceTest.java
index 4041164..40f3442 100644
--- a/shell/src/test/java/org/apache/accumulo/shell/ShellSetInstanceTest.java
+++ b/shell/src/test/java/org/apache/accumulo/shell/ShellSetInstanceTest.java
@@ -56,10 +56,12 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("javax.security.*")
@PrepareForTest({Shell.class, ZooUtil.class, ConfigSanityCheck.class})
public class ShellSetInstanceTest {
public static class TestOutputStream extends OutputStream {
[8/8] accumulo git commit: ACCUMULO-4652 Fix SLF4J error in build
Posted by ct...@apache.org.
ACCUMULO-4652 Fix SLF4J error in build
Ensure log4j-slf4j is on the classpath for the exec-maven-plugin
execution of the ConfigurationDocGen during the build.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/15b9c3ee
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/15b9c3ee
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/15b9c3ee
Branch: refs/heads/master
Commit: 15b9c3eec1160ea3a121f157532151df0d2135f2
Parents: 20ba43a
Author: Christopher Tubbs <ct...@apache.org>
Authored: Mon Jun 12 18:05:37 2017 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Mon Jun 12 18:05:37 2017 -0400
----------------------------------------------------------------------
core/pom.xml | 6 +++---
pom.xml | 2 +-
2 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/15b9c3ee/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 96510f5..8b32bef 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -268,7 +268,7 @@
<goals>
<goal>exec</goal>
</goals>
- <phase>compile</phase>
+ <phase>prepare-package</phase>
<configuration>
<executable>mkdir</executable>
<arguments>
@@ -282,10 +282,10 @@
<goals>
<goal>java</goal>
</goals>
- <phase>compile</phase>
+ <phase>package</phase>
<configuration>
<mainClass>org.apache.accumulo.core.conf.ConfigurationDocGen</mainClass>
- <classpathScope>compile</classpathScope>
+ <classpathScope>test</classpathScope>
<arguments>
<argument>--generate-markdown</argument>
<argument>${project.build.directory}/generated-docs/configuration-properties.md</argument>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/15b9c3ee/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 69b4e77..89f6b26 100644
--- a/pom.xml
+++ b/pom.xml
@@ -758,7 +758,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
- <version>1.4.0</version>
+ <version>1.5.0</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
[7/8] accumulo git commit: ACCUMULO-4642 Remove warning and unneeded
objects
Posted by ct...@apache.org.
ACCUMULO-4642 Remove warning and unneeded objects
Remove deprecation warning for TabletBalancer init methods and
unnecessary creation of new instances of AccumuloServerContext.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/20ba43a1
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/20ba43a1
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/20ba43a1
Branch: refs/heads/master
Commit: 20ba43a17455d3b793edeb071d3792617b9acd89
Parents: f5371f9
Author: Christopher Tubbs <ct...@apache.org>
Authored: Mon Jun 12 15:27:14 2017 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Mon Jun 12 15:27:14 2017 -0400
----------------------------------------------------------------------
.../accumulo/server/master/balancer/ChaoticLoadBalancer.java | 8 --------
.../src/main/java/org/apache/accumulo/master/Master.java | 2 +-
.../apache/accumulo/master/MasterClientServiceHandler.java | 7 ++-----
3 files changed, 3 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/20ba43a1/server/base/src/main/java/org/apache/accumulo/server/master/balancer/ChaoticLoadBalancer.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/ChaoticLoadBalancer.java b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/ChaoticLoadBalancer.java
index c93c4b9..ad03d85 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/ChaoticLoadBalancer.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/ChaoticLoadBalancer.java
@@ -31,8 +31,6 @@ import org.apache.accumulo.core.master.thrift.TableInfo;
import org.apache.accumulo.core.master.thrift.TabletServerStatus;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
-import org.apache.accumulo.server.AccumuloServerContext;
-import org.apache.accumulo.server.conf.ServerConfigurationFactory;
import org.apache.accumulo.server.master.state.TServerInstance;
import org.apache.accumulo.server.master.state.TabletMigration;
import org.apache.thrift.TException;
@@ -163,10 +161,4 @@ public class ChaoticLoadBalancer extends TabletBalancer {
return 100;
}
- @Override
- public void init(ServerConfigurationFactory conf) {}
-
- @Override
- public void init(AccumuloServerContext context) {}
-
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/20ba43a1/server/master/src/main/java/org/apache/accumulo/master/Master.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java b/server/master/src/main/java/org/apache/accumulo/master/Master.java
index 51d0748..379b955 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@ -611,7 +611,7 @@ public class Master extends AccumuloServerContext implements LiveTServerSet.List
ThriftTransportPool.getInstance().setIdleTime(aconf.getTimeInMillis(Property.GENERAL_RPC_TIMEOUT));
tserverSet = new LiveTServerSet(this, this);
this.tabletBalancer = aconf.instantiateClassProperty(Property.MASTER_TABLET_BALANCER, TabletBalancer.class, new DefaultLoadBalancer());
- this.tabletBalancer.init(new AccumuloServerContext(instance, serverConfig));
+ this.tabletBalancer.init(this);
try {
AccumuloVFSClassLoader.getContextManager().setContextConfig(new ContextManager.DefaultContextsConfig(new Iterable<Entry<String,String>>() {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/20ba43a1/server/master/src/main/java/org/apache/accumulo/master/MasterClientServiceHandler.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/MasterClientServiceHandler.java b/server/master/src/main/java/org/apache/accumulo/master/MasterClientServiceHandler.java
index 0f2e232..2497a44 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/MasterClientServiceHandler.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/MasterClientServiceHandler.java
@@ -78,7 +78,6 @@ import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
import org.apache.accumulo.fate.zookeeper.IZooReaderWriter.Mutator;
import org.apache.accumulo.master.tableOps.TraceRepo;
import org.apache.accumulo.master.tserverOps.ShutdownTServer;
-import org.apache.accumulo.server.AccumuloServerContext;
import org.apache.accumulo.server.client.ClientServiceHandler;
import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection;
import org.apache.accumulo.server.master.balancer.DefaultLoadBalancer;
@@ -106,11 +105,9 @@ public class MasterClientServiceHandler extends FateServiceHandler implements Ma
private static final Logger log = Master.log;
private static final Logger drainLog = LoggerFactory.getLogger("org.apache.accumulo.master.MasterDrainImpl");
- private Instance instance;
protected MasterClientServiceHandler(Master master) {
super(master);
- this.instance = master.getInstance();
}
@Override
@@ -260,7 +257,7 @@ public class MasterClientServiceHandler extends FateServiceHandler implements Ma
private String getNamespaceIdFromTableId(TableOperation tableOp, String tableId) throws ThriftTableOperationException {
String namespaceId;
try {
- namespaceId = Tables.getNamespaceId(instance, tableId);
+ namespaceId = Tables.getNamespaceId(master.getInstance(), tableId);
} catch (TableNotFoundException e) {
throw new ThriftTableOperationException(tableId, null, tableOp, TableOperationExceptionType.NOTFOUND, e.getMessage());
}
@@ -466,7 +463,7 @@ public class MasterClientServiceHandler extends FateServiceHandler implements Ma
if (property.equals(Property.MASTER_TABLET_BALANCER.getKey())) {
TabletBalancer balancer = master.getConfiguration().instantiateClassProperty(Property.MASTER_TABLET_BALANCER, TabletBalancer.class,
new DefaultLoadBalancer());
- balancer.init(new AccumuloServerContext(instance, master.getConfigurationFactory()));
+ balancer.init(master);
master.tabletBalancer = balancer;
log.info("tablet balancer changed to " + master.tabletBalancer.getClass().getName());
}
[6/8] accumulo git commit: Merge branch '1.8'
Posted by ct...@apache.org.
Merge branch '1.8'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f5371f99
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f5371f99
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f5371f99
Branch: refs/heads/master
Commit: f5371f99d8473bb47a30025c0b214ba1f4044cd0
Parents: e4a381d d99158f
Author: Christopher Tubbs <ct...@apache.org>
Authored: Mon Jun 12 15:06:46 2017 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Mon Jun 12 15:06:46 2017 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/conf/SiteConfigurationTest.java | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f5371f99/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java
----------------------------------------------------------------------
[3/8] accumulo git commit: ACCUMULO-4640 Fix broken
SiteConfigurationTest
Posted by ct...@apache.org.
ACCUMULO-4640 Fix broken SiteConfigurationTest
Avoid using non-existent constructor in mock object creation
Also fix a trivial javadoc warning
Also fix a trivial test failure with Java 8 and Powermock
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/de83e61c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/de83e61c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/de83e61c
Branch: refs/heads/master
Commit: de83e61c7f965b6f83482c6070b483bd1835b5a6
Parents: 4effd0e
Author: Christopher Tubbs <ct...@apache.org>
Authored: Mon Jun 12 14:23:39 2017 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Mon Jun 12 14:23:39 2017 -0400
----------------------------------------------------------------------
.../main/java/org/apache/accumulo/core/file/rfile/RFile.java | 2 +-
.../org/apache/accumulo/core/conf/SiteConfigurationTest.java | 5 ++---
.../java/org/apache/accumulo/shell/ShellSetInstanceTest.java | 2 ++
3 files changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/de83e61c/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
index bab2266..d5779ce 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
@@ -1022,7 +1022,7 @@ public class RFile {
* @return map of locality group names to column families. The default locality group will have {@code null} for a name. RFile will only track up to
* {@value Writer#MAX_CF_IN_DLG} families for the default locality group. After this it will stop tracking. For the case where the default group has
* more thn {@value Writer#MAX_CF_IN_DLG} families an empty list of families is returned.
- * @see LocalityGroupUtil#seek(Reader, Range, String, Map)
+ * @see LocalityGroupUtil#seek(RFile.Reader, Range, String, Map)
*/
public Map<String,ArrayList<ByteSequence>> getLocalityGroupCF() {
Map<String,ArrayList<ByteSequence>> cf = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/de83e61c/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java b/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java
index f89dbfa..29304c8 100644
--- a/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/conf/SiteConfigurationTest.java
@@ -44,13 +44,12 @@ public class SiteConfigurationTest {
}
@Test
- public void testOnlySensitivePropertiesExtractedFromCredetialProvider() throws SecurityException, NoSuchMethodException {
+ public void testOnlySensitivePropertiesExtractedFromCredentialProvider() throws SecurityException, NoSuchMethodException {
if (!isCredentialProviderAvailable) {
return;
}
- SiteConfiguration siteCfg = EasyMock.createMockBuilder(SiteConfiguration.class).addMockedMethod("getHadoopConfiguration")
- .withConstructor(AccumuloConfiguration.class).withArgs(DefaultConfiguration.getInstance()).createMock();
+ SiteConfiguration siteCfg = EasyMock.createMockBuilder(SiteConfiguration.class).addMockedMethod("getHadoopConfiguration").createMock();
siteCfg.set(Property.INSTANCE_SECRET, "ignored");
http://git-wip-us.apache.org/repos/asf/accumulo/blob/de83e61c/shell/src/test/java/org/apache/accumulo/shell/ShellSetInstanceTest.java
----------------------------------------------------------------------
diff --git a/shell/src/test/java/org/apache/accumulo/shell/ShellSetInstanceTest.java b/shell/src/test/java/org/apache/accumulo/shell/ShellSetInstanceTest.java
index 4041164..40f3442 100644
--- a/shell/src/test/java/org/apache/accumulo/shell/ShellSetInstanceTest.java
+++ b/shell/src/test/java/org/apache/accumulo/shell/ShellSetInstanceTest.java
@@ -56,10 +56,12 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("javax.security.*")
@PrepareForTest({Shell.class, ZooUtil.class, ConfigSanityCheck.class})
public class ShellSetInstanceTest {
public static class TestOutputStream extends OutputStream {
[5/8] accumulo git commit: Merge branch '1.7' into 1.8
Posted by ct...@apache.org.
Merge branch '1.7' into 1.8
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d99158ff
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d99158ff
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d99158ff
Branch: refs/heads/1.8
Commit: d99158ff056ed9cccb3fc8be7a9c52ef1bafd385
Parents: da695f6 de83e61
Author: Christopher Tubbs <ct...@apache.org>
Authored: Mon Jun 12 15:02:03 2017 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Mon Jun 12 15:02:03 2017 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/client/rfile/RFileWriter.java | 6 +++---
.../org/apache/accumulo/core/conf/SiteConfigurationTest.java | 5 ++---
2 files changed, 5 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d99158ff/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriter.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriter.java
index 9995888,0000000..a69f83e
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriter.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriter.java
@@@ -1,234 -1,0 +1,234 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.core.client.rfile;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.data.ArrayByteSequence;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileSKVWriter;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.commons.collections.map.LRUMap;
+
+import com.google.common.base.Preconditions;
+
+//formatter was adding spaces that checkstyle did not like, so turned off formatter
+//@formatter:off
+/**
+ * This class provides an API for writing RFiles. It can be used to create file for bulk import into Accumulo using
+ * {@link TableOperations#importDirectory(String, String, String, boolean)}
+ *
+ * <p>
+ * A RFileWriter has the following constraints. Violating these constraints will result in runtime exceptions.
+ *
+ * <ul>
+ * <li>Keys must be appended in sorted order within a locality group.</li>
+ * <li>Locality groups must have a mutually exclusive set of column families.</li>
+ * <li>The default locality group must be started last.</li>
+ * </ul>
+ *
+ * <p>
+ * Below is an example of using RFileWriter
+ *
+ * <pre>
+ * <code>
- * {@code Iterable<Entry<Key, Value>>} localityGroup1Data = ...
- * {@code Iterable<Entry<Key, Value>>} localityGroup2Data = ...
- * {@code Iterable<Entry<Key, Value>>} defaultGroupData = ...
++ * Iterable<Entry<Key, Value>> localityGroup1Data = ...
++ * Iterable<Entry<Key, Value>> localityGroup2Data = ...
++ * Iterable<Entry<Key, Value>> defaultGroupData = ...
+ *
+ * try(RFileWriter writer = RFile.newWriter().to(file).build()) {
+ *
+ * // Start a locality group before appending data.
+ * writer.startNewLocalityGroup("groupA", "columnFam1", "columnFam2");
+ * // Append data to the locality group that was started above. Must append in sorted order.
+ * writer.append(localityGroup1Data);
+ *
+ * // Add another locality group.
+ * writer.startNewLocalityGroup("groupB", "columnFam3", "columnFam4");
+ * writer.append(localityGroup2Data);
+ *
+ * // The default locality group must be started last. The column families for the default group do not need to be specified.
+ * writer.startDefaultLocalityGroup();
+ * // Data appended here can not contain any column families specified in previous locality groups.
+ * writer.append(defaultGroupData);
+ *
+ * // This is a try-with-resources so the writer is closed here at the end of the code block.
+ * }
+ * </code>
+ * </pre>
+ *
+ * <p>
+ * Create instances by calling {@link RFile#newWriter()}
+ *
+ * @since 1.8.0
+ */
+// @formatter:on
+public class RFileWriter implements AutoCloseable {
+
+ private FileSKVWriter writer;
+ private final LRUMap validVisibilities;
+ private boolean startedLG;
+ private boolean startedDefaultLG;
+
+ RFileWriter(FileSKVWriter fileSKVWriter, int visCacheSize) {
+ this.writer = fileSKVWriter;
+ this.validVisibilities = new LRUMap(visCacheSize);
+ }
+
+ private void _startNewLocalityGroup(String name, Set<ByteSequence> columnFamilies) throws IOException {
+ Preconditions.checkState(!startedDefaultLG, "Cannont start a locality group after starting the default locality group");
+ writer.startNewLocalityGroup(name, columnFamilies);
+ startedLG = true;
+ }
+
+ /**
+ * Before appending any data, a locality group must be started. The default locality group must be started last.
+ *
+ * @param name
+ * locality group name, used for informational purposes
+ * @param families
+ * the column families the locality group can contain
+ *
+ * @throws IllegalStateException
+ * When default locality group already started.
+ */
+ public void startNewLocalityGroup(String name, List<byte[]> families) throws IOException {
+ HashSet<ByteSequence> fams = new HashSet<>();
+ for (byte[] family : families) {
+ fams.add(new ArrayByteSequence(family));
+ }
+ _startNewLocalityGroup(name, fams);
+ }
+
+ /**
+ * See javadoc for {@link #startNewLocalityGroup(String, List)}
+ *
+ * @throws IllegalStateException
+ * When default locality group already started.
+ */
+ public void startNewLocalityGroup(String name, byte[]... families) throws IOException {
+ startNewLocalityGroup(name, Arrays.asList(families));
+ }
+
+ /**
+ * See javadoc for {@link #startNewLocalityGroup(String, List)}.
+ *
+ * @param families
+ * will be encoded using UTF-8
+ *
+ * @throws IllegalStateException
+ * When default locality group already started.
+ */
+ public void startNewLocalityGroup(String name, Set<String> families) throws IOException {
+ HashSet<ByteSequence> fams = new HashSet<>();
+ for (String family : families) {
+ fams.add(new ArrayByteSequence(family));
+ }
+ _startNewLocalityGroup(name, fams);
+ }
+
+ /**
+ * See javadoc for {@link #startNewLocalityGroup(String, List)}.
+ *
+ * @param families
+ * will be encoded using UTF-8
+ *
+ * @throws IllegalStateException
+ * When default locality group already started.
+ */
+ public void startNewLocalityGroup(String name, String... families) throws IOException {
+ HashSet<ByteSequence> fams = new HashSet<>();
+ for (String family : families) {
+ fams.add(new ArrayByteSequence(family));
+ }
+ _startNewLocalityGroup(name, fams);
+ }
+
+ /**
+ * A locality group in which the column families do not need to specified. The locality group must be started after all other locality groups. Can not append
+ * column families that were in a previous locality group. If no locality groups were started, then the first append will start the default locality group.
+ *
+ * @throws IllegalStateException
+ * When default locality group already started.
+ */
+
+ public void startDefaultLocalityGroup() throws IOException {
+ Preconditions.checkState(!startedDefaultLG);
+ writer.startDefaultLocalityGroup();
+ startedDefaultLG = true;
+ startedLG = true;
+ }
+
+ /**
+ * Append the key and value to the last locality group that was started. If no locality group was started, then the default group will automatically be
+ * started.
+ *
+ * @param key
+ * This key must be greater than or equal to the last key appended. For non-default locality groups, the keys column family must be one of the column
+ * families specified when calling startNewLocalityGroup(). Must be non-null.
+ * @param val
+ * value to append, must be non-null.
+ *
+ * @throws IllegalArgumentException
+ * This is thrown when data is appended out of order OR when the key contains a invalid visibility OR when a column family is not valid for a
+ * locality group.
+ */
+ public void append(Key key, Value val) throws IOException {
+ if (!startedLG) {
+ startDefaultLocalityGroup();
+ }
+ Boolean wasChecked = (Boolean) validVisibilities.get(key.getColumnVisibilityData());
+ if (wasChecked == null) {
+ byte[] cv = key.getColumnVisibilityData().toArray();
+ new ColumnVisibility(cv);
+ validVisibilities.put(new ArrayByteSequence(Arrays.copyOf(cv, cv.length)), Boolean.TRUE);
+ }
+ writer.append(key, val);
+ }
+
+ /**
+ * Append the keys and values to the last locality group that was started.
+ *
+ * @param keyValues
+ * The keys must be in sorted order. The first key returned by the iterable must be greater than or equal to the last key appended. For non-default
+ * locality groups, the keys column family must be one of the column families specified when calling startNewLocalityGroup(). Must be non-null. If no
+ * locality group was started, then the default group will automatically be started.
+ *
+ * @throws IllegalArgumentException
+ * This is thrown when data is appended out of order OR when the key contains a invalid visibility OR when a column family is not valid for a
+ * locality group.
+ */
+ public void append(Iterable<Entry<Key,Value>> keyValues) throws IOException {
+ for (Entry<Key,Value> entry : keyValues) {
+ append(entry.getKey(), entry.getValue());
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ writer.close();
+ }
+}