You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by cj...@apache.org on 2015/01/21 02:00:36 UTC
accumulo git commit: ACCUMULO-2793 Adding documentation to
administration section of user manual for migrating from a non-HA volume to
an HA volume. Also adding log error when the second pair item in an
instance.volumes.replacements entry appears in the
Repository: accumulo
Updated Branches:
refs/heads/1.6 c5153331c -> 21ab05578
ACCUMULO-2793 Adding documentation to administration section of user manual for migrating from a non-HA volume to an HA volume. Also adding log error when the second pair item in an instance.volumes.replacements entry appears in the instance.volumes property during the "add volumes" initialize command.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/21ab0557
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/21ab0557
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/21ab0557
Branch: refs/heads/1.6
Commit: 21ab0557828394891d574567bc529867d11012e3
Parents: c515333
Author: Corey J. Nolet <cj...@gmail.com>
Authored: Thu Jan 15 22:12:27 2015 -0500
Committer: Corey J. Nolet <cj...@gmail.com>
Committed: Tue Jan 20 20:00:15 2015 -0500
----------------------------------------------------------------------
.../chapters/administration.tex | 65 ++++++++++++++++++++
.../apache/accumulo/server/ServerConstants.java | 1 +
.../apache/accumulo/server/init/Initialize.java | 17 +++--
3 files changed, 79 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/21ab0557/docs/src/main/latex/accumulo_user_manual/chapters/administration.tex
----------------------------------------------------------------------
diff --git a/docs/src/main/latex/accumulo_user_manual/chapters/administration.tex b/docs/src/main/latex/accumulo_user_manual/chapters/administration.tex
index 4b917d1..120e88b 100644
--- a/docs/src/main/latex/accumulo_user_manual/chapters/administration.tex
+++ b/docs/src/main/latex/accumulo_user_manual/chapters/administration.tex
@@ -539,3 +539,68 @@ The Accumulo client library is configured to retry failed mutations and in many
cases clients will be able to continue processing after the recovery process without
throwing an exception.
+\section{Migrating Accumulo from non-HA Namenode to HA Namenode}
+
+The following steps will allow a non-HA instance to be migrated to an HA instance. Consider an HDFS URL
+\texttt{hdfs://namenode.example.com:8020} which is going to be moved to \texttt{hdfs://nameservice1}.
+
+Before moving HDFS over to the HA namenode, use \texttt{\$ACCUMULO\_HOME/bin/accumulo admin volumes} to confirm
+that the only volume displayed is the volume from the current namenode's HDFS URL.
+
+\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
+Listing volumes referenced in zookeeper
+ Volume : hdfs://namenode.example.com:8020/accumulo
+
+Listing volumes referenced in accumulo.root tablets section
+ Volume : hdfs://namenode.example.com:8020/accumulo
+Listing volumes referenced in accumulo.root deletes section (volume replacement occurrs at deletion time)
+
+Listing volumes referenced in accumulo.metadata tablets section
+ Volume : hdfs://namenode.example.com:8020/accumulo
+
+Listing volumes referenced in accumulo.metadata deletes section (volume replacement occurrs at deletion time)
+\end{verbatim}
+
+After verifying the current volume is correct, shut down the cluster and transition HDFS to the HA nameservice.
+
+Edit \texttt{\$ACCUMULO\_HOME/conf/accumulo-site.xml} to notify accumulo that a volume is being replaced. First,
+add the new nameservice volume to the \texttt{instance.volumes} property. Next, add the
+\texttt{instance.volumes.replacements} property in the form of \texttt{old new}. It's important to not include
+the volume that's being replaced in \texttt{instance.volumes}, otherwise it's possible accumulo could continue
+to write to the volume.
+
+\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
+<!-- instance.dfs.uri and instance.dfs.dir should not be set-->
+<property>
+ <name>instance.volumes</name>
+ <value>hdfs://nameservice1/accumulo</value>
+</property>
+<property>
+ <name>instance.volumes.replacements</name>
+ <value>hdfs://namenode.example.com:8020/accumulo hdfs://nameservice1/accumulo</value>
+</property>
+\end{verbatim}
+
+Run \texttt{\$ACCUMULO\_HOME/bin/accumulo init --add-volumes} and start up the accumulo cluster. Verify that the
+new nameservice volume shows up with \texttt{\$ACCUMULO\_HOME/bin/accumulo admin volumes}.
+
+\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
+Listing volumes referenced in zookeeper
+ Volume : hdfs://namenode.example.com:8020/accumulo
+ Volume : hdfs://nameservice1/accumulo
+
+Listing volumes referenced in accumulo.root tablets section
+ Volume : hdfs://namenode.example.com:8020/accumulo
+ Volume : hdfs://nameservice1/accumulo
+Listing volumes referenced in accumulo.root deletes section (volume replacement occurrs at deletion time)
+
+Listing volumes referenced in accumulo.metadata tablets section
+ Volume : hdfs://namenode.example.com:8020/accumulo
+ Volume : hdfs://nameservice1/accumulo
+Listing volumes referenced in accumulo.metadata deletes section (volume replacement occurrs at deletion time)
+\end{verbatim}
+
+Some erroneous GarbageCollector messages may still be seen for a small period while data is transitioning to
+the new volumes. This is expected and can usually be ignored.
+
+
http://git-wip-us.apache.org/repos/asf/accumulo/blob/21ab0557/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
index 51fa47e..2b654ca 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
@@ -158,6 +158,7 @@ public class ServerConstants {
List<Pair<Path,Path>> ret = new ArrayList<Pair<Path,Path>>();
for (String pair : pairs) {
+
String uris[] = pair.split("\\s+");
if (uris.length != 2)
throw new IllegalArgumentException(Property.INSTANCE_VOLUMES_REPLACEMENTS.getKey() + " contains malformed pair " + pair);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/21ab0557/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
index e0a3797..fd46ab7 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
@@ -20,7 +20,6 @@ import static com.google.common.base.Charsets.UTF_8;
import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN;
import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN;
-
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Arrays;
@@ -31,7 +30,6 @@ import java.util.Map.Entry;
import java.util.UUID;
import jline.console.ConsoleReader;
-
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.cli.Help;
import org.apache.accumulo.core.client.AccumuloSecurityException;
@@ -53,6 +51,7 @@ import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.ColumnFQ;
+import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
@@ -122,6 +121,7 @@ public class Initialize {
}
private static HashMap<String,String> initialMetadataConf = new HashMap<String,String>();
+
static {
initialMetadataConf.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "32K");
initialMetadataConf.put(Property.TABLE_FILE_REPLICATION.getKey(), "5");
@@ -518,17 +518,26 @@ public class Initialize {
}
private static void addVolumes(VolumeManager fs) throws IOException {
+
+ String[] volumeURIs = VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration());
+
HashSet<String> initializedDirs = new HashSet<String>();
- initializedDirs.addAll(Arrays.asList(ServerConstants.checkBaseUris(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration()), true)));
+ initializedDirs.addAll(Arrays.asList(ServerConstants.checkBaseUris(volumeURIs, true)));
HashSet<String> uinitializedDirs = new HashSet<String>();
- uinitializedDirs.addAll(Arrays.asList(VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration())));
+ uinitializedDirs.addAll(Arrays.asList(volumeURIs));
uinitializedDirs.removeAll(initializedDirs);
Path aBasePath = new Path(initializedDirs.iterator().next());
Path iidPath = new Path(aBasePath, ServerConstants.INSTANCE_ID_DIR);
Path versionPath = new Path(aBasePath, ServerConstants.VERSION_DIR);
+ for (Pair<Path,Path> replacementVolume : ServerConstants.getVolumeReplacements()) {
+ if (aBasePath.equals(replacementVolume.getSecond()))
+ log.error(aBasePath + " is set to be replaced in " + Property.INSTANCE_VOLUMES_REPLACEMENTS + " and should not appear in " +
+ Property.INSTANCE_VOLUMES + ". It is highly recommended that this property be removed as data could still be written to this volume.");
+ }
+
UUID uuid = UUID.fromString(ZooUtil.getInstanceIDFromHdfs(iidPath, ServerConfiguration.getSiteConfiguration()));
if (ServerConstants.DATA_VERSION != Accumulo.getAccumuloPersistentVersion(versionPath.getFileSystem(CachedConfiguration.getInstance()), versionPath)) {