You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@accumulo.apache.org by "EdColeman (via GitHub)" <gi...@apache.org> on 2023/10/20 00:47:28 UTC
[PR] 3.1 upgrade process with no chop changes [accumulo]
EdColeman opened a new pull request, #3876:
URL: https://github.com/apache/accumulo/pull/3876
Partially resolves #3768 (removal of `ExternalCompactionColumnFamily` needs to be implemented. Posting as draft as remaining work is completed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "EdColeman (via GitHub)" <gi...@apache.org>.
EdColeman commented on code in PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#discussion_r1370970030
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants.ChoppedColumnFamily;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsConversion(json)) {
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper because of ZooKeeper exception", ex);
+ }
+ }
+
+ @Override
+ public void upgradeRoot(@NonNull ServerContext context) {
+ log.debug("Upgrade root: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootName = Ample.DataLevel.METADATA.metaTable();
+ processReferences(context, rootName);
+ }
+
+ @Override
+ public void upgradeMetadata(@NonNull ServerContext context) {
+ log.debug("Upgrade metadata: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var metaName = Ample.DataLevel.USER.metaTable();
+ processReferences(context, metaName);
+ }
+
+ private void processReferences(ServerContext context, String tableName) {
+ // not using ample to avoid StoredTabletFile because old file ref is incompatible
+ try (AccumuloClient c = Accumulo.newClient().from(context.getProperties()).build();
+ BatchWriter batchWriter = c.createBatchWriter(tableName);
+ Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+
+ scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ scanner.fetchColumnFamily(ChoppedColumnFamily.NAME);
+ scanner.fetchColumnFamily(ExternalCompactionColumnFamily.NAME);
+ scanner.forEach((k, v) -> {
+ var family = k.getColumnFamily();
+ if (family.equals(DataFileColumnFamily.NAME)) {
+ upgradeDataFileCF(k, v, batchWriter, tableName);
+ } else if (family.equals(ChoppedColumnFamily.NAME)) {
+ removeChoppedCF(k, batchWriter, tableName);
+ } else if (family.equals(ExternalCompactionColumnFamily.NAME)) {
+ removeExternalCompactionCF(k, batchWriter, tableName);
+ } else {
+ log.warn("Received unexpected column family processing references: " + family);
+ }
+ });
+ } catch (MutationsRejectedException mex) {
+ log.warn("Failed to update reference for table: " + tableName);
+ log.warn("Constraint violations: {}", mex.getConstraintViolationSummaries());
+ throw new IllegalStateException("Failed to process table: " + tableName, mex);
+ } catch (Exception ex) {
+ throw new IllegalStateException("Failed to process table: " + tableName, ex);
+ }
+ }
+
+ @VisibleForTesting
+ void upgradeDataFileCF(final Key key, final Value value, final BatchWriter batchWriter,
+ final String tableName) {
+ String file = key.getColumnQualifier().toString();
+ // filter out references if they are in the correct format already.
+ if (fileNeedsConversion(file)) {
+ var fileJson = StoredTabletFile.of(new Path(file)).getMetadataText();
+ try {
+ Mutation update = new Mutation(key.getRow());
+ update.at().family(DataFileColumnFamily.STR_NAME).qualifier(fileJson).put(value);
+ log.trace("table: {}, adding: {}", tableName, update.prettyPrint());
+ batchWriter.addMutation(update);
+
+ Mutation delete = new Mutation(key.getRow());
+ delete.at().family(DataFileColumnFamily.STR_NAME).qualifier(file).delete();
+ log.trace("table {}: deleting: {}", tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ // include constraint violation info in log - but stop upgrade
+ log.warn(
+ "Failed to update file reference for table: " + tableName + ". row: " + key.getRow());
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ throw new IllegalStateException("File conversion failed. Aborting upgrade", ex);
+ }
+ }
+ }
+
+ @VisibleForTesting
+ void removeChoppedCF(final Key key, final BatchWriter batchWriter, final String tableName) {
+ Mutation delete = null;
+ try {
+ delete = new Mutation(key.getRow()).at().family(ChoppedColumnFamily.STR_NAME)
+ .qualifier(ChoppedColumnFamily.STR_NAME).delete();
+ log.warn(
+ "Deleting chopped reference from:{}. Previous split or delete may not have completed cleanly. Ref: {}",
+ tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ log.warn("Failed to delete obsolete chopped CF reference for table: " + tableName + ". Ref: "
+ + delete.prettyPrint() + ". Will try to continue. Ref may need to be manually removed");
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ }
+ }
+
+ @VisibleForTesting
+ void removeExternalCompactionCF(final Key key, final BatchWriter batchWriter,
+ final String tableName) {
+ Mutation delete = null;
+ try {
+ delete = new Mutation(key.getRow()).at().family(ExternalCompactionColumnFamily.NAME)
+ .qualifier(key.getColumnQualifier()).delete();
+ log.warn(
+ "Deleting external compaction reference from:{}. Previous compaction may not have completed. Ref: {}",
+ tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ log.warn("Failed to delete obsolete external compaction CF reference for table: " + tableName
Review Comment:
Added in bceac27d1c
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "EdColeman (via GitHub)" <gi...@apache.org>.
EdColeman commented on code in PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#discussion_r1384186872
##########
core/src/main/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadata.java:
##########
@@ -59,11 +60,14 @@ public class RootTabletMetadata {
};
// JSON Mapping Version 1. Released with Accumulo version 2.1.0
- private static final int VERSION = 1;
+ private static final int VERSION_1 = 1;
+ // JSON Mapping Version 2. Released with Accumulo version 3,1
+ private static final int VERSION_2 = 2;
+ private static final int VERSION = VERSION_2;
// This class is used to serialize and deserialize root tablet metadata using GSon. Any changes to
// this class must consider persisted data.
- private static class Data {
+ public static class Data {
Review Comment:
Addressed in cc6b6e25801c
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "cshannon (via GitHub)" <gi...@apache.org>.
cshannon commented on code in PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#discussion_r1367909313
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.ComparablePair;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.io.Text;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsConversion(json)) {
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper because of ZooKeeper exception", ex);
+ }
+ }
+
+ @Override
+ public void upgradeRoot(@NonNull ServerContext context) {
+ log.debug("Upgrade root: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+
+ var rootName = Ample.DataLevel.METADATA.metaTable();
+ var filesToConvert = getFileReferences(context, rootName);
+ convertFileReferences(context, rootName, filesToConvert);
+ deleteObsoleteReferences(context, rootName);
+ }
+
+ @Override
+ public void upgradeMetadata(@NonNull ServerContext context) {
+ log.debug("Upgrade metadata: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+
+ var metaName = Ample.DataLevel.USER.metaTable();
+ var filesToConvert = getFileReferences(context, metaName);
+ convertFileReferences(context, metaName, filesToConvert);
+ deleteObsoleteReferences(context, metaName);
+ }
+
+ private Map<ComparablePair<KeyExtent,String>,Value>
+ getFileReferences(@NonNull final ServerContext context, @NonNull final String tableName) {
+
+ log.trace("gather file references for table: {}", tableName);
+
+ Map<ComparablePair<KeyExtent,String>,Value> filesToConvert = new TreeMap<>();
+
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.STR_NAME);
+ scanner.forEach((k, v) -> {
+ KeyExtent ke = KeyExtent.fromMetaRow(k.getRow());
+ String file = k.getColumnQualifier().toString();
+ // filter out references that are in the correct format already.
+ if (fileNeedsConversion(file)) {
+ var prev = filesToConvert.put(new ComparablePair<>(ke, file), v);
+ if (prev != null) {
+ throw new IllegalStateException(
+ "upgrade for table: " + tableName + " aborted, would have missed: " + prev);
+ }
+ }
+ });
+ } catch (TableNotFoundException ex) {
+ throw new IllegalStateException("failed to read metadata table for upgrade", ex);
+ }
+ log.debug("Number of files to convert for table: {}, number of files: {}", tableName,
+ filesToConvert.size());
+ return filesToConvert;
+ }
+
+ private void convertFileReferences(final ServerContext context, final String tableName,
+ Map<ComparablePair<KeyExtent,String>,Value> filesToConvert) {
+
+ // not using ample to avoid StoredTabletFile because old file ref is incompatible
+ try (AccumuloClient c = Accumulo.newClient().from(context.getProperties()).build();
Review Comment:
I like these changes a lot, this will be much more efficient in terms of having to make a single pass and also with memory usage as the batch writer will flush as needed.
The main thing left would be is if we need to use an `IsolatedScanner` or not to buffer the entire row into memory so the changes in place don't cause issues. The code already handles things if already upgraded so the main thing I'd want to avoid is processing the same columns in the row more than once and I am not sure if that could happen or not.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "EdColeman (via GitHub)" <gi...@apache.org>.
EdColeman commented on code in PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#discussion_r1387263925
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants.ChoppedColumnFamily;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsUpgrade(json)) {
+ log.info("Root metadata in ZooKeeper before upgrade: {}", json);
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ log.info("Root metadata in ZooKeeper after upgrade: {}", rtm.toJson());
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper because of ZooKeeper exception", ex);
+ }
+ }
+
+ @Override
+ public void upgradeRoot(@NonNull ServerContext context) {
+ log.debug("Upgrade root: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootName = Ample.DataLevel.METADATA.metaTable();
+ processReferences(context, rootName);
+ }
+
+ @Override
+ public void upgradeMetadata(@NonNull ServerContext context) {
+ log.debug("Upgrade metadata: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var metaName = Ample.DataLevel.USER.metaTable();
+ processReferences(context, metaName);
+ }
+
+ private void processReferences(ServerContext context, String tableName) {
+ // not using ample to avoid StoredTabletFile because old file ref is incompatible
+ try (AccumuloClient c = Accumulo.newClient().from(context.getProperties()).build();
+ BatchWriter batchWriter = c.createBatchWriter(tableName); Scanner scanner =
+ new IsolatedScanner(context.createScanner(tableName, Authorizations.EMPTY))) {
+
+ scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ scanner.fetchColumnFamily(ChoppedColumnFamily.NAME);
+ scanner.fetchColumnFamily(ExternalCompactionColumnFamily.NAME);
+ scanner.forEach((k, v) -> {
+ var family = k.getColumnFamily();
+ if (family.equals(DataFileColumnFamily.NAME)) {
+ upgradeDataFileCF(k, v, batchWriter, tableName);
+ } else if (family.equals(ChoppedColumnFamily.NAME)) {
+ removeChoppedCF(k, batchWriter, tableName);
+ } else if (family.equals(ExternalCompactionColumnFamily.NAME)) {
+ removeExternalCompactionCF(k, batchWriter, tableName);
+ } else {
+ throw new IllegalStateException("Processing: " + tableName
+ + " Received unexpected column family processing references: " + family);
+ }
+ });
+ } catch (MutationsRejectedException mex) {
+ log.warn("Failed to update reference for table: " + tableName);
+ log.warn("Constraint violations: {}", mex.getConstraintViolationSummaries());
+ throw new IllegalStateException("Failed to process table: " + tableName, mex);
+ } catch (Exception ex) {
+ throw new IllegalStateException("Failed to process table: " + tableName, ex);
+ }
+ }
+
+ @VisibleForTesting
+ void upgradeDataFileCF(final Key key, final Value value, final BatchWriter batchWriter,
+ final String tableName) {
+ String file = key.getColumnQualifier().toString();
+ // filter out references if they are in the correct format already.
+ if (fileNeedsConversion(file)) {
+ var fileJson = StoredTabletFile.of(new Path(file)).getMetadataText();
+ try {
+ Mutation update = new Mutation(key.getRow());
Review Comment:
For the upgrade process, there is no difference if the upgrade code were to fail on mutation updates if it was for a mutation within a tablet or for a whole tablet mutation - that is, there should be no impact to the upgrade code being re-run with a partial upgrade that had previously completed at any stage.
Being that the upgrade code is running before things are brought online, any inconsistency in file representations needs to be resolved before the next level can be brought online. For example, until all root table file references pointing to the metadata tablet files are resolved, the upgrade will not progress to the point of processing the metadata file references. This is independent if the inconsistencies were within a tablet of for any tablet in general.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "ctubbsii (via GitHub)" <gi...@apache.org>.
ctubbsii commented on PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#issuecomment-1772039761
> removal of `ExternalCompactionColumnFamily` needs to be implemented
I don't think that's related to the no-chop merge change, is it? Or is that a separate thing that was removed in 3.1?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "EdColeman (via GitHub)" <gi...@apache.org>.
EdColeman commented on code in PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#discussion_r1367822306
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.ComparablePair;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.io.Text;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsConversion(json)) {
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper because of ZooKeeper exception", ex);
+ }
+ }
+
+ @Override
+ public void upgradeRoot(@NonNull ServerContext context) {
+ log.debug("Upgrade root: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+
+ var rootName = Ample.DataLevel.METADATA.metaTable();
+ var filesToConvert = getFileReferences(context, rootName);
+ convertFileReferences(context, rootName, filesToConvert);
+ deleteObsoleteReferences(context, rootName);
+ }
+
+ @Override
+ public void upgradeMetadata(@NonNull ServerContext context) {
+ log.debug("Upgrade metadata: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+
+ var metaName = Ample.DataLevel.USER.metaTable();
+ var filesToConvert = getFileReferences(context, metaName);
+ convertFileReferences(context, metaName, filesToConvert);
+ deleteObsoleteReferences(context, metaName);
+ }
+
+ private Map<ComparablePair<KeyExtent,String>,Value>
+ getFileReferences(@NonNull final ServerContext context, @NonNull final String tableName) {
+
+ log.trace("gather file references for table: {}", tableName);
+
+ Map<ComparablePair<KeyExtent,String>,Value> filesToConvert = new TreeMap<>();
+
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.STR_NAME);
+ scanner.forEach((k, v) -> {
+ KeyExtent ke = KeyExtent.fromMetaRow(k.getRow());
+ String file = k.getColumnQualifier().toString();
+ // filter out references that are in the correct format already.
+ if (fileNeedsConversion(file)) {
+ var prev = filesToConvert.put(new ComparablePair<>(ke, file), v);
+ if (prev != null) {
+ throw new IllegalStateException(
+ "upgrade for table: " + tableName + " aborted, would have missed: " + prev);
+ }
+ }
+ });
+ } catch (TableNotFoundException ex) {
+ throw new IllegalStateException("failed to read metadata table for upgrade", ex);
+ }
+ log.debug("Number of files to convert for table: {}, number of files: {}", tableName,
+ filesToConvert.size());
+ return filesToConvert;
+ }
+
+ private void convertFileReferences(final ServerContext context, final String tableName,
+ Map<ComparablePair<KeyExtent,String>,Value> filesToConvert) {
+
+ // not using ample to avoid StoredTabletFile because old file ref is incompatible
+ try (AccumuloClient c = Accumulo.newClient().from(context.getProperties()).build();
+ BatchWriter batchWriter = c.createBatchWriter(tableName)) {
+ filesToConvert.forEach((refPair, value) -> {
+ try {
+ log.trace("update file reference for table: {}. row: {} to: {} value: {}", tableName,
+ refPair.getFirst().toMetaRow(), refPair.getSecond(), value);
+
+ var row = refPair.getFirst().toMetaRow();
+ var fileJson = new Text(StoredTabletFile.serialize(refPair.getSecond(), new Range()));
Review Comment:
Changed in e8ae63227647 (along with other changes)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "EdColeman (via GitHub)" <gi...@apache.org>.
EdColeman commented on code in PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#discussion_r1384187250
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants.ChoppedColumnFamily;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsUpgrade(json)) {
+ log.info("Root metadata in ZooKeeper before upgrade: {}", json);
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ log.info("Root metadata in ZooKeeper after upgrade: {}", rtm.toJson());
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper because of ZooKeeper exception", ex);
+ }
+ }
+
+ @Override
+ public void upgradeRoot(@NonNull ServerContext context) {
+ log.debug("Upgrade root: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootName = Ample.DataLevel.METADATA.metaTable();
+ processReferences(context, rootName);
+ }
+
+ @Override
+ public void upgradeMetadata(@NonNull ServerContext context) {
+ log.debug("Upgrade metadata: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var metaName = Ample.DataLevel.USER.metaTable();
+ processReferences(context, metaName);
+ }
+
+ private void processReferences(ServerContext context, String tableName) {
+ // not using ample to avoid StoredTabletFile because old file ref is incompatible
+ try (AccumuloClient c = Accumulo.newClient().from(context.getProperties()).build();
+ BatchWriter batchWriter = c.createBatchWriter(tableName); Scanner scanner =
+ new IsolatedScanner(context.createScanner(tableName, Authorizations.EMPTY))) {
+
+ scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ scanner.fetchColumnFamily(ChoppedColumnFamily.NAME);
+ scanner.fetchColumnFamily(ExternalCompactionColumnFamily.NAME);
+ scanner.forEach((k, v) -> {
+ var family = k.getColumnFamily();
+ if (family.equals(DataFileColumnFamily.NAME)) {
+ upgradeDataFileCF(k, v, batchWriter, tableName);
+ } else if (family.equals(ChoppedColumnFamily.NAME)) {
+ removeChoppedCF(k, batchWriter, tableName);
+ } else if (family.equals(ExternalCompactionColumnFamily.NAME)) {
+ removeExternalCompactionCF(k, batchWriter, tableName);
+ } else {
+ throw new IllegalStateException("Processing: " + tableName
+ + " Received unexpected column family processing references: " + family);
+ }
+ });
+ } catch (MutationsRejectedException mex) {
+ log.warn("Failed to update reference for table: " + tableName);
+ log.warn("Constraint violations: {}", mex.getConstraintViolationSummaries());
+ throw new IllegalStateException("Failed to process table: " + tableName, mex);
+ } catch (Exception ex) {
+ throw new IllegalStateException("Failed to process table: " + tableName, ex);
+ }
+ }
+
+ @VisibleForTesting
+ void upgradeDataFileCF(final Key key, final Value value, final BatchWriter batchWriter,
+ final String tableName) {
+ String file = key.getColumnQualifier().toString();
+ // filter out references if they are in the correct format already.
+ if (fileNeedsConversion(file)) {
+ var fileJson = StoredTabletFile.of(new Path(file)).getMetadataText();
+ try {
+ Mutation update = new Mutation(key.getRow());
+ update.at().family(DataFileColumnFamily.STR_NAME).qualifier(fileJson).put(value);
+ log.trace("table: {}, adding: {}", tableName, update.prettyPrint());
+ batchWriter.addMutation(update);
+
+ Mutation delete = new Mutation(key.getRow());
+ delete.at().family(DataFileColumnFamily.STR_NAME).qualifier(file).delete();
+ log.trace("table {}: deleting: {}", tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ // include constraint violation info in log - but stop upgrade
+ log.warn(
+ "Failed to update file reference for table: " + tableName + ". row: " + key.getRow());
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ throw new IllegalStateException("File conversion failed. Aborting upgrade", ex);
+ }
+ }
+ }
+
+ @VisibleForTesting
+ void removeChoppedCF(final Key key, final BatchWriter batchWriter, final String tableName) {
+ Mutation delete = null;
+ try {
+ delete = new Mutation(key.getRow()).at().family(ChoppedColumnFamily.STR_NAME)
+ .qualifier(ChoppedColumnFamily.STR_NAME).delete();
+ log.warn(
+ "Deleting chopped reference from:{}. Previous split or delete may not have completed cleanly. Ref: {}",
+ tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ log.warn("Failed to delete obsolete chopped CF reference for table: " + tableName + ". Ref: "
+ + delete.prettyPrint() + ". Will try to continue. Ref may need to be manually removed");
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ throw new IllegalStateException(
+ "Failed to delete obsolete chopped CF reference for table: " + tableName, ex);
+ }
+ }
+
+ @VisibleForTesting
+ void removeExternalCompactionCF(final Key key, final BatchWriter batchWriter,
+ final String tableName) {
+ Mutation delete = null;
+ try {
+ delete = new Mutation(key.getRow()).at().family(ExternalCompactionColumnFamily.NAME)
+ .qualifier(key.getColumnQualifier()).delete();
+ log.debug(
+ "Deleting external compaction reference from:{}. Previous compaction may not have completed. Ref: {}",
+ tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ log.warn("Failed to delete obsolete external compaction CF reference for table: " + tableName
+ + ". Ref: " + delete.prettyPrint()
+ + ". Will try to continue. Ref may need to be manually removed");
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ throw new IllegalStateException(
+ "Failed to delete obsolete external compaction CF reference for table: " + tableName, ex);
+ }
+ }
+
+ /**
+ * Quick sanity check to see if value has been converted by checking the candidate can be parsed
Review Comment:
Fixed in cc6b6e25801c
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "cshannon (via GitHub)" <gi...@apache.org>.
cshannon commented on code in PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#discussion_r1367721118
##########
core/src/main/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadata.java:
##########
@@ -59,11 +60,14 @@ public class RootTabletMetadata {
};
// JSON Mapping Version 1. Released with Accumulo version 2.1.0
- private static final int VERSION = 1;
+ private static final int VERSION_1 = 1;
+ // JSON Mapping Version 1. Released with Accumulo version 3,1
Review Comment:
```suggestion
// JSON Mapping Version 2. Released with Accumulo version 3,1
```
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.ComparablePair;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.io.Text;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsConversion(json)) {
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper because of ZooKeeper exception", ex);
+ }
+ }
+
+ @Override
+ public void upgradeRoot(@NonNull ServerContext context) {
+ log.debug("Upgrade root: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+
+ var rootName = Ample.DataLevel.METADATA.metaTable();
+ var filesToConvert = getFileReferences(context, rootName);
+ convertFileReferences(context, rootName, filesToConvert);
+ deleteObsoleteReferences(context, rootName);
+ }
+
+ @Override
+ public void upgradeMetadata(@NonNull ServerContext context) {
+ log.debug("Upgrade metadata: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+
+ var metaName = Ample.DataLevel.USER.metaTable();
+ var filesToConvert = getFileReferences(context, metaName);
+ convertFileReferences(context, metaName, filesToConvert);
+ deleteObsoleteReferences(context, metaName);
+ }
+
+ private Map<ComparablePair<KeyExtent,String>,Value>
+ getFileReferences(@NonNull final ServerContext context, @NonNull final String tableName) {
+
+ log.trace("gather file references for table: {}", tableName);
+
+ Map<ComparablePair<KeyExtent,String>,Value> filesToConvert = new TreeMap<>();
+
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.STR_NAME);
+ scanner.forEach((k, v) -> {
+ KeyExtent ke = KeyExtent.fromMetaRow(k.getRow());
+ String file = k.getColumnQualifier().toString();
+ // filter out references that are in the correct format already.
+ if (fileNeedsConversion(file)) {
+ var prev = filesToConvert.put(new ComparablePair<>(ke, file), v);
+ if (prev != null) {
+ throw new IllegalStateException(
+ "upgrade for table: " + tableName + " aborted, would have missed: " + prev);
+ }
+ }
+ });
+ } catch (TableNotFoundException ex) {
+ throw new IllegalStateException("failed to read metadata table for upgrade", ex);
+ }
+ log.debug("Number of files to convert for table: {}, number of files: {}", tableName,
+ filesToConvert.size());
+ return filesToConvert;
+ }
+
+ private void convertFileReferences(final ServerContext context, final String tableName,
+ Map<ComparablePair<KeyExtent,String>,Value> filesToConvert) {
+
+ // not using ample to avoid StoredTabletFile because old file ref is incompatible
+ try (AccumuloClient c = Accumulo.newClient().from(context.getProperties()).build();
Review Comment:
Right now the conversion is happening in two steps where the metadata is first scanned to load the file references into a map and then that is iterated over and updated. One thing I am wondering is if this is a problem if there's a lot of files in the system. I'm not sure what a practical upper bounds would be since depends on a lot of factors but if it's say hundreds of thousands or millions that would be a lot to load at once. We could try and break it up maybe into batches (maybe a tablet at a time or some sort of limit on files at one time and keep track of where you left off).
I haven't really tried this before also thinking maybe we just modify the files while scanning in the same loop to avoid having to load into a map. The main thing I am not sure of is whether or not that would cause issues with the updates and we'd need to use an isolated scanner but that also buffers the entire row into memory as well.
@keith-turner any thoughts on this?
##########
core/src/test/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadataTest.java:
##########
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.metadata.schema;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class RootTabletMetadataTest {
+ private static final Logger LOG = LoggerFactory.getLogger(RootTabletMetadataTest.class);
+
+ @Test
+ public void convertRoot1File() {
+ String root21ZkData =
+ "{\"version\":1,\"columnValues\":{\"file\":{\"hdfs://localhost:8020/accumulo/tables/+r/root_tablet/A000000v.rf\":\"1368,61\"},\"last\":{\"100025091780006\":\"localhost:9997\"},\"loc\":{\"100025091780006\":\"localhost:9997\"},\"srv\":{\"dir\":\"root_tablet\",\"flush\":\"3\",\"lock\":\"tservers/localhost:9997/zlock#9db8961a-4ee9-400e-8e80-3353148baadd#0000000000$100025091780006\",\"time\":\"L53\"},\"~tab\":{\"~pr\":\"\\u0000\"}}}";
+
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(root21ZkData);
+ LOG.debug("converted column values: {}", rtm.toTabletMetadata().getFiles());
+ assertEquals(1, rtm.toTabletMetadata().getFiles().size());
+
+ LOG.info("FILES: {}", rtm.toTabletMetadata().getFilesMap());
+ }
+
+ @Test
+ public void convertRoot2Files() {
+ String root212ZkData2Files =
+ "{\"version\":1,\"columnValues\":{\"file\":{\"hdfs://localhost:8020/accumulo/tables/+r/root_tablet/00000_00000.rf\":\"0,0\",\"hdfs://localhost:8020/accumulo/tables/+r/root_tablet/F000000c.rf\":\"926,18\"},\"last\":{\"10001a84d7d0005\":\"localhost:9997\"},\"loc\":{\"10001a84d7d0005\":\"localhost:9997\"},\"srv\":{\"dir\":\"root_tablet\",\"flush\":\"2\",\"lock\":\"tservers/localhost:9997/zlock#d21adaa4-0f97-4004-9ff8-cce9dbb6687f#0000000000$10001a84d7d0005\",\"time\":\"L6\"},\"~tab\":{\"~pr\":\"\\u0000\"}}}\n";
+
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(root212ZkData2Files);
+ LOG.debug("converted column values: {}", rtm.toTabletMetadata());
+ assertEquals(2, rtm.toTabletMetadata().getFiles().size());
Review Comment:
Besides just checking size we should also verify correctness. It would be good to verify the files match the previous path after upgrade and also the ranges are infinite, etc.
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.ComparablePair;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.io.Text;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsConversion(json)) {
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper because of ZooKeeper exception", ex);
+ }
+ }
+
+ @Override
+ public void upgradeRoot(@NonNull ServerContext context) {
+ log.debug("Upgrade root: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+
+ var rootName = Ample.DataLevel.METADATA.metaTable();
+ var filesToConvert = getFileReferences(context, rootName);
+ convertFileReferences(context, rootName, filesToConvert);
+ deleteObsoleteReferences(context, rootName);
+ }
+
+ @Override
+ public void upgradeMetadata(@NonNull ServerContext context) {
+ log.debug("Upgrade metadata: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+
+ var metaName = Ample.DataLevel.USER.metaTable();
+ var filesToConvert = getFileReferences(context, metaName);
+ convertFileReferences(context, metaName, filesToConvert);
+ deleteObsoleteReferences(context, metaName);
+ }
+
+ private Map<ComparablePair<KeyExtent,String>,Value>
+ getFileReferences(@NonNull final ServerContext context, @NonNull final String tableName) {
+
+ log.trace("gather file references for table: {}", tableName);
+
+ Map<ComparablePair<KeyExtent,String>,Value> filesToConvert = new TreeMap<>();
+
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.STR_NAME);
+ scanner.forEach((k, v) -> {
+ KeyExtent ke = KeyExtent.fromMetaRow(k.getRow());
+ String file = k.getColumnQualifier().toString();
+ // filter out references that are in the correct format already.
+ if (fileNeedsConversion(file)) {
+ var prev = filesToConvert.put(new ComparablePair<>(ke, file), v);
+ if (prev != null) {
+ throw new IllegalStateException(
+ "upgrade for table: " + tableName + " aborted, would have missed: " + prev);
+ }
+ }
+ });
+ } catch (TableNotFoundException ex) {
+ throw new IllegalStateException("failed to read metadata table for upgrade", ex);
+ }
+ log.debug("Number of files to convert for table: {}, number of files: {}", tableName,
+ filesToConvert.size());
+ return filesToConvert;
+ }
+
+ private void convertFileReferences(final ServerContext context, final String tableName,
+ Map<ComparablePair<KeyExtent,String>,Value> filesToConvert) {
+
+ // not using ample to avoid StoredTabletFile because old file ref is incompatible
+ try (AccumuloClient c = Accumulo.newClient().from(context.getProperties()).build();
+ BatchWriter batchWriter = c.createBatchWriter(tableName)) {
+ filesToConvert.forEach((refPair, value) -> {
+ try {
+ log.trace("update file reference for table: {}. row: {} to: {} value: {}", tableName,
+ refPair.getFirst().toMetaRow(), refPair.getSecond(), value);
+
+ var row = refPair.getFirst().toMetaRow();
+ var fileJson = new Text(StoredTabletFile.serialize(refPair.getSecond(), new Range()));
Review Comment:
```suggestion
var fileJson = StoredTabletFile.of(new Path(refPair.getSecond())).getMetadataText();
```
Both ways are equivalent so this is optional but this is a little nicer not having to create the empty range.
##########
core/src/main/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadata.java:
##########
@@ -73,10 +77,27 @@ private static class Data {
*/
private final TreeMap<String,TreeMap<String,String>> columnValues;
- public Data(int version, TreeMap<String,TreeMap<String,String>> columnValues) {
+ private Data(int version, TreeMap<String,TreeMap<String,String>> columnValues) {
this.version = version;
this.columnValues = columnValues;
}
+
+ public int getVersion() {
+ return version;
+ }
+
+ /**
+ * For external use, return a copy so the original remains immutable.
+ */
+ public TreeMap<String,TreeMap<String,String>> getColumnValues() {
+ return new TreeMap<>(columnValues);
Review Comment:
Instead of making a copy and using TreeMap I would recommend using an Immutable map and then we don't need to worry about copying data (assuming the values are supposed to be immutable). Since it's a SortedMap you could look at using Guava's `ImmutableSortedMap`. It might be a little tricky because the map contains another map so you may need to make them both immutable depending on how it's used. You also could just wrap the map(s) using `UnmodifiableMap`
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.ComparablePair;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.io.Text;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsConversion(json)) {
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper because of ZooKeeper exception", ex);
+ }
+ }
+
+ @Override
+ public void upgradeRoot(@NonNull ServerContext context) {
+ log.debug("Upgrade root: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+
+ var rootName = Ample.DataLevel.METADATA.metaTable();
+ var filesToConvert = getFileReferences(context, rootName);
+ convertFileReferences(context, rootName, filesToConvert);
+ deleteObsoleteReferences(context, rootName);
+ }
+
+ @Override
+ public void upgradeMetadata(@NonNull ServerContext context) {
+ log.debug("Upgrade metadata: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+
+ var metaName = Ample.DataLevel.USER.metaTable();
+ var filesToConvert = getFileReferences(context, metaName);
+ convertFileReferences(context, metaName, filesToConvert);
+ deleteObsoleteReferences(context, metaName);
+ }
+
+ private Map<ComparablePair<KeyExtent,String>,Value>
+ getFileReferences(@NonNull final ServerContext context, @NonNull final String tableName) {
+
+ log.trace("gather file references for table: {}", tableName);
+
+ Map<ComparablePair<KeyExtent,String>,Value> filesToConvert = new TreeMap<>();
+
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.STR_NAME);
+ scanner.forEach((k, v) -> {
+ KeyExtent ke = KeyExtent.fromMetaRow(k.getRow());
+ String file = k.getColumnQualifier().toString();
+ // filter out references that are in the correct format already.
+ if (fileNeedsConversion(file)) {
+ var prev = filesToConvert.put(new ComparablePair<>(ke, file), v);
+ if (prev != null) {
+ throw new IllegalStateException(
+ "upgrade for table: " + tableName + " aborted, would have missed: " + prev);
+ }
+ }
+ });
+ } catch (TableNotFoundException ex) {
+ throw new IllegalStateException("failed to read metadata table for upgrade", ex);
+ }
+ log.debug("Number of files to convert for table: {}, number of files: {}", tableName,
+ filesToConvert.size());
+ return filesToConvert;
+ }
+
+ private void convertFileReferences(final ServerContext context, final String tableName,
+ Map<ComparablePair<KeyExtent,String>,Value> filesToConvert) {
+
+ // not using ample to avoid StoredTabletFile because old file ref is incompatible
+ try (AccumuloClient c = Accumulo.newClient().from(context.getProperties()).build();
+ BatchWriter batchWriter = c.createBatchWriter(tableName)) {
+ filesToConvert.forEach((refPair, value) -> {
+ try {
+ log.trace("update file reference for table: {}. row: {} to: {} value: {}", tableName,
+ refPair.getFirst().toMetaRow(), refPair.getSecond(), value);
+
+ var row = refPair.getFirst().toMetaRow();
+ var fileJson = new Text(StoredTabletFile.serialize(refPair.getSecond(), new Range()));
+
+ Mutation m = new Mutation(row);
+ m.at().family(MetadataSchema.TabletsSection.DataFileColumnFamily.STR_NAME)
+ .qualifier(fileJson).put(value);
+
+ log.trace("table: {}, adding: {}", tableName, m.prettyPrint());
+ batchWriter.addMutation(m);
+
+ Mutation delete = new Mutation(row);
+ delete.at().family(MetadataSchema.TabletsSection.DataFileColumnFamily.STR_NAME)
+ .qualifier(refPair.getSecond()).delete();
+
+ log.trace("table {}: deleting: {}", tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ throw new IllegalStateException("Failed to update file entries for table: " + tableName,
+ ex);
+ }
+ });
+ batchWriter.flush();
+ } catch (Exception ex) {
+ throw new IllegalStateException(ex);
+ }
+ }
+
+ /**
+ * Removes chopped and external compaction references that have obsolete encoding that is
+ * incompatible with StoredTabletFile json encoding. These references should be rare. If they are
+ * present, some operations likely terminated abnormally under the old version before shutdown.
+ * The deletions are logged so those operations may be re-run if desired.
+ */
+ private void deleteObsoleteReferences(ServerContext context, String tableName) {
+ log.debug("processing obsolete references for table: {}", tableName);
+ try (AccumuloClient c = Accumulo.newClient().from(context.getProperties()).build();
+ BatchWriter batchWriter = c.createBatchWriter(tableName)) {
+
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.fetchColumnFamily(UpgraderDeprecatedConstants.ChoppedColumnFamily.STR_NAME);
+ scanner
+ .fetchColumnFamily(MetadataSchema.TabletsSection.ExternalCompactionColumnFamily.NAME);
+ scanner.forEach((k, v) -> {
+ Mutation delete;
+ var family = k.getColumnFamily();
+ if (family.equals(UpgraderDeprecatedConstants.ChoppedColumnFamily.NAME)) {
+ delete = buildChoppedDeleteMutation(k, tableName);
+ } else if (family
+ .equals(MetadataSchema.TabletsSection.ExternalCompactionColumnFamily.NAME)) {
+ delete = buildExternalCompactionDelete(k, tableName);
+ } else {
+ throw new IllegalStateException(
+ "unexpected column Family: '{}' seen processing obsolete references");
+ }
+ try {
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ log.warn("Failed to delete obsolete reference for table: " + tableName + ". Ref: "
+ + delete.prettyPrint()
+ + ". Will try to continue. Ref may need to be manually removed");
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ }
+ });
+ }
+ } catch (MutationsRejectedException ex) {
+ log.warn("Failed to delete obsolete reference for table: " + tableName + " on close");
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ throw new IllegalStateException(ex);
+ } catch (Exception ex) {
+ throw new IllegalStateException(
+ "Processing obsolete referenced for table: " + tableName + " failed. Upgrade aborting",
+ ex);
+ }
+ }
+
+ private Mutation buildChoppedDeleteMutation(final Key k, final String tableName) {
+ Mutation delete = new Mutation(k.getRow()).at()
+ .family(UpgraderDeprecatedConstants.ChoppedColumnFamily.STR_NAME)
+ .qualifier(UpgraderDeprecatedConstants.ChoppedColumnFamily.STR_NAME).delete();
+ log.warn(
+ "Deleting chopped reference from:{}. Previous split or delete may not have completed cleanly. Ref: {}",
+ tableName, delete.prettyPrint());
+ return delete;
+ }
+
+ private Mutation buildExternalCompactionDelete(Key k, String tableName) {
+ Mutation delete = new Mutation(k.getRow()).at()
+ .family(MetadataSchema.TabletsSection.ExternalCompactionColumnFamily.NAME)
+ .qualifier(k.getColumnQualifier()).delete();
+ log.warn(
+ "Deleting external compaction reference from:{}. Previous compaction may not have completed. Ref: {}",
+ tableName, delete.prettyPrint());
+ return delete;
+ }
+
+ /**
+ * Quick sanity check to see if value has been converted by checking the candidate can be parsed
+ * into a StoredTabletFile. If parsing does not throw a parsing exception. then assume it has been
+ * converted. If validate cannot parse the candidate a JSON, then it is assumed that it has not
+ * been converted. Other parsing errors will propagate as IllegalArgumentExceptions.
+ *
+ * @param candidate a possible file: reference.
+ * @return false if a valid StoredTabletFile, true if it cannot be parsed as JSON. Otherwise,
+ * propagate an IllegalArgumentException
+ */
+ private boolean fileNeedsConversion(@NonNull final String candidate) {
+ try {
+ StoredTabletFile.validate(candidate);
Review Comment:
Some trace logging might be good here when validating just in case we need to troubleshoot something weird going on with the metadata not converting, etc.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "cshannon (via GitHub)" <gi...@apache.org>.
cshannon commented on PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#issuecomment-1773363619
@EdColeman - I have started to review this but probably won't finish today however I plan to work this weekend so should finish then. I am going to spend some time testing it myself with Uno, etc to make sure it all looks good.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "EdColeman (via GitHub)" <gi...@apache.org>.
EdColeman commented on code in PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#discussion_r1367822283
##########
core/src/test/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadataTest.java:
##########
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.metadata.schema;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class RootTabletMetadataTest {
+ private static final Logger LOG = LoggerFactory.getLogger(RootTabletMetadataTest.class);
+
+ @Test
+ public void convertRoot1File() {
+ String root21ZkData =
+ "{\"version\":1,\"columnValues\":{\"file\":{\"hdfs://localhost:8020/accumulo/tables/+r/root_tablet/A000000v.rf\":\"1368,61\"},\"last\":{\"100025091780006\":\"localhost:9997\"},\"loc\":{\"100025091780006\":\"localhost:9997\"},\"srv\":{\"dir\":\"root_tablet\",\"flush\":\"3\",\"lock\":\"tservers/localhost:9997/zlock#9db8961a-4ee9-400e-8e80-3353148baadd#0000000000$100025091780006\",\"time\":\"L53\"},\"~tab\":{\"~pr\":\"\\u0000\"}}}";
+
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(root21ZkData);
+ LOG.debug("converted column values: {}", rtm.toTabletMetadata().getFiles());
+ assertEquals(1, rtm.toTabletMetadata().getFiles().size());
+
+ LOG.info("FILES: {}", rtm.toTabletMetadata().getFilesMap());
+ }
+
+ @Test
+ public void convertRoot2Files() {
+ String root212ZkData2Files =
+ "{\"version\":1,\"columnValues\":{\"file\":{\"hdfs://localhost:8020/accumulo/tables/+r/root_tablet/00000_00000.rf\":\"0,0\",\"hdfs://localhost:8020/accumulo/tables/+r/root_tablet/F000000c.rf\":\"926,18\"},\"last\":{\"10001a84d7d0005\":\"localhost:9997\"},\"loc\":{\"10001a84d7d0005\":\"localhost:9997\"},\"srv\":{\"dir\":\"root_tablet\",\"flush\":\"2\",\"lock\":\"tservers/localhost:9997/zlock#d21adaa4-0f97-4004-9ff8-cce9dbb6687f#0000000000$10001a84d7d0005\",\"time\":\"L6\"},\"~tab\":{\"~pr\":\"\\u0000\"}}}\n";
+
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(root212ZkData2Files);
+ LOG.debug("converted column values: {}", rtm.toTabletMetadata());
+ assertEquals(2, rtm.toTabletMetadata().getFiles().size());
Review Comment:
Added contains checks in e8ae63227647
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "EdColeman (via GitHub)" <gi...@apache.org>.
EdColeman commented on code in PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#discussion_r1370920750
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants.ChoppedColumnFamily;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsConversion(json)) {
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
Review Comment:
Added in aa2c468a5b. Use IsolatedScanner and ZooKeeper root dump before / after.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "EdColeman (via GitHub)" <gi...@apache.org>.
EdColeman commented on code in PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#discussion_r1370969239
##########
core/src/main/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadata.java:
##########
@@ -73,10 +77,20 @@ private static class Data {
*/
private final TreeMap<String,TreeMap<String,String>> columnValues;
- public Data(int version, TreeMap<String,TreeMap<String,String>> columnValues) {
+ private Data(int version, TreeMap<String,TreeMap<String,String>> columnValues) {
this.version = version;
this.columnValues = columnValues;
}
+
+ public int getVersion() {
+ return version;
+ }
+
+ public static boolean needsConversion(final String json) {
Review Comment:
Changed in bceac27d1c
##########
core/src/test/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadataTest.java:
##########
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.metadata.schema;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.hadoop.fs.Path;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class RootTabletMetadataTest {
+ private static final Logger LOG = LoggerFactory.getLogger(RootTabletMetadataTest.class);
+
+ @Test
+ public void convertRoot1File() {
+ String root21ZkData =
+ "{\"version\":1,\"columnValues\":{\"file\":{\"hdfs://localhost:8020/accumulo/tables/+r/root_tablet/A000000v.rf\":\"1368,61\"},\"last\":{\"100025091780006\":\"localhost:9997\"},\"loc\":{\"100025091780006\":\"localhost:9997\"},\"srv\":{\"dir\":\"root_tablet\",\"flush\":\"3\",\"lock\":\"tservers/localhost:9997/zlock#9db8961a-4ee9-400e-8e80-3353148baadd#0000000000$100025091780006\",\"time\":\"L53\"},\"~tab\":{\"~pr\":\"\\u0000\"}}}";
+
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(root21ZkData);
+ LOG.debug("converted column values: {}", rtm.toTabletMetadata().getFiles());
+
+ var files = rtm.toTabletMetadata().getFiles();
+ LOG.info("FILES: {}", rtm.toTabletMetadata().getFilesMap());
+
+ assertEquals(1, files.size());
+ assertTrue(files.contains(StoredTabletFile
+ .of(new Path("hdfs://localhost:8020/accumulo/tables/+r/root_tablet/A000000v.rf"))));
+ }
+
+ @Test
+ public void convertRoot2Files() {
+ String root212ZkData2Files =
+ "{\"version\":1,\"columnValues\":{\"file\":{\"hdfs://localhost:8020/accumulo/tables/+r/root_tablet/00000_00000.rf\":\"0,0\",\"hdfs://localhost:8020/accumulo/tables/+r/root_tablet/F000000c.rf\":\"926,18\"},\"last\":{\"10001a84d7d0005\":\"localhost:9997\"},\"loc\":{\"10001a84d7d0005\":\"localhost:9997\"},\"srv\":{\"dir\":\"root_tablet\",\"flush\":\"2\",\"lock\":\"tservers/localhost:9997/zlock#d21adaa4-0f97-4004-9ff8-cce9dbb6687f#0000000000$10001a84d7d0005\",\"time\":\"L6\"},\"~tab\":{\"~pr\":\"\\u0000\"}}}\n";
+
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(root212ZkData2Files);
+ LOG.debug("converted column values: {}", rtm.toTabletMetadata());
+
Review Comment:
Added tests in bceac27d1c
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "EdColeman (via GitHub)" <gi...@apache.org>.
EdColeman commented on code in PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#discussion_r1385193521
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants.ChoppedColumnFamily;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsUpgrade(json)) {
+ log.info("Root metadata in ZooKeeper before upgrade: {}", json);
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ log.info("Root metadata in ZooKeeper after upgrade: {}", rtm.toJson());
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper because of ZooKeeper exception", ex);
+ }
+ }
+
+ @Override
+ public void upgradeRoot(@NonNull ServerContext context) {
+ log.debug("Upgrade root: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootName = Ample.DataLevel.METADATA.metaTable();
+ processReferences(context, rootName);
+ }
+
+ @Override
+ public void upgradeMetadata(@NonNull ServerContext context) {
+ log.debug("Upgrade metadata: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var metaName = Ample.DataLevel.USER.metaTable();
+ processReferences(context, metaName);
+ }
+
+ private void processReferences(ServerContext context, String tableName) {
+ // not using ample to avoid StoredTabletFile because old file ref is incompatible
+ try (AccumuloClient c = Accumulo.newClient().from(context.getProperties()).build();
+ BatchWriter batchWriter = c.createBatchWriter(tableName); Scanner scanner =
+ new IsolatedScanner(context.createScanner(tableName, Authorizations.EMPTY))) {
+
+ scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ scanner.fetchColumnFamily(ChoppedColumnFamily.NAME);
+ scanner.fetchColumnFamily(ExternalCompactionColumnFamily.NAME);
+ scanner.forEach((k, v) -> {
+ var family = k.getColumnFamily();
+ if (family.equals(DataFileColumnFamily.NAME)) {
+ upgradeDataFileCF(k, v, batchWriter, tableName);
+ } else if (family.equals(ChoppedColumnFamily.NAME)) {
+ removeChoppedCF(k, batchWriter, tableName);
+ } else if (family.equals(ExternalCompactionColumnFamily.NAME)) {
+ removeExternalCompactionCF(k, batchWriter, tableName);
+ } else {
+ throw new IllegalStateException("Processing: " + tableName
+ + " Received unexpected column family processing references: " + family);
+ }
+ });
+ } catch (MutationsRejectedException mex) {
+ log.warn("Failed to update reference for table: " + tableName);
+ log.warn("Constraint violations: {}", mex.getConstraintViolationSummaries());
+ throw new IllegalStateException("Failed to process table: " + tableName, mex);
+ } catch (Exception ex) {
+ throw new IllegalStateException("Failed to process table: " + tableName, ex);
+ }
+ }
+
+ @VisibleForTesting
+ void upgradeDataFileCF(final Key key, final Value value, final BatchWriter batchWriter,
+ final String tableName) {
+ String file = key.getColumnQualifier().toString();
+ // filter out references if they are in the correct format already.
+ if (fileNeedsConversion(file)) {
+ var fileJson = StoredTabletFile.of(new Path(file)).getMetadataText();
+ try {
+ Mutation update = new Mutation(key.getRow());
+ update.at().family(DataFileColumnFamily.STR_NAME).qualifier(fileJson).put(value);
+ log.trace("table: {}, adding: {}", tableName, update.prettyPrint());
+ batchWriter.addMutation(update);
+
+ Mutation delete = new Mutation(key.getRow());
+ delete.at().family(DataFileColumnFamily.STR_NAME).qualifier(file).delete();
+ log.trace("table {}: deleting: {}", tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ // include constraint violation info in log - but stop upgrade
+ log.warn(
+ "Failed to update file reference for table: " + tableName + ". row: " + key.getRow());
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ throw new IllegalStateException("File conversion failed. Aborting upgrade", ex);
+ }
+ }
+ }
+
+ @VisibleForTesting
+ void removeChoppedCF(final Key key, final BatchWriter batchWriter, final String tableName) {
+ Mutation delete = null;
+ try {
+ delete = new Mutation(key.getRow()).at().family(ChoppedColumnFamily.STR_NAME)
+ .qualifier(ChoppedColumnFamily.STR_NAME).delete();
+ log.warn(
+ "Deleting chopped reference from:{}. Previous split or delete may not have completed cleanly. Ref: {}",
+ tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ log.warn("Failed to delete obsolete chopped CF reference for table: " + tableName + ". Ref: "
+ + delete.prettyPrint() + ". Will try to continue. Ref may need to be manually removed");
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ throw new IllegalStateException(
+ "Failed to delete obsolete chopped CF reference for table: " + tableName, ex);
+ }
+ }
+
+ @VisibleForTesting
+ void removeExternalCompactionCF(final Key key, final BatchWriter batchWriter,
+ final String tableName) {
+ Mutation delete = null;
+ try {
+ delete = new Mutation(key.getRow()).at().family(ExternalCompactionColumnFamily.NAME)
+ .qualifier(key.getColumnQualifier()).delete();
+ log.debug(
+ "Deleting external compaction reference from:{}. Previous compaction may not have completed. Ref: {}",
+ tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ log.warn("Failed to delete obsolete external compaction CF reference for table: " + tableName
+ + ". Ref: " + delete.prettyPrint()
+ + ". Will try to continue. Ref may need to be manually removed");
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ throw new IllegalStateException(
+ "Failed to delete obsolete external compaction CF reference for table: " + tableName, ex);
+ }
+ }
+
+ /**
+ * Quick sanity check to see if value has been converted by checking the candidate can be parsed
+ * into a StoredTabletFile. If parsing does not throw a parsing exception. then assume it has been
+ * converted. If validate cannot parse the candidate a JSON, then it is assumed that it has not
+ * been converted. Other parsing errors will propagate as IllegalArgumentExceptions.
+ *
+ * @param candidate a possible file: reference.
+ * @return false if a valid StoredTabletFile, true if it cannot be parsed as JSON. Otherwise,
+ * propagate an IllegalArgumentException
+ */
+ private boolean fileNeedsConversion(@NonNull final String candidate) {
+ try {
+ StoredTabletFile.validate(candidate);
+ log.trace("file reference does not need conversion: {}", candidate);
+ return false;
+ } catch (JsonSyntaxException ex) {
Review Comment:
Modified the needs conversion check in 882be1e05f8
Changed the check to look for a starting `{` and an ending `}`. The data written to the metadata is well controlled, so it may be enough that it looks "json like" instead of trying fully to parse. If the column qualifier data is a plain file reference (2.1) or something that looks like a json object starting with 3.1 and the no chop fence encoding then that is likely sufficient for what this is trying to do in a upgrade.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "EdColeman (via GitHub)" <gi...@apache.org>.
EdColeman commented on code in PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#discussion_r1367822234
##########
core/src/main/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadata.java:
##########
@@ -59,11 +60,14 @@ public class RootTabletMetadata {
};
// JSON Mapping Version 1. Released with Accumulo version 2.1.0
- private static final int VERSION = 1;
+ private static final int VERSION_1 = 1;
+ // JSON Mapping Version 1. Released with Accumulo version 3,1
Review Comment:
Addressed in e8ae63227647
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "EdColeman (via GitHub)" <gi...@apache.org>.
EdColeman commented on code in PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#discussion_r1367822318
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.ComparablePair;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.io.Text;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsConversion(json)) {
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper because of ZooKeeper exception", ex);
+ }
+ }
+
+ @Override
+ public void upgradeRoot(@NonNull ServerContext context) {
+ log.debug("Upgrade root: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+
+ var rootName = Ample.DataLevel.METADATA.metaTable();
+ var filesToConvert = getFileReferences(context, rootName);
+ convertFileReferences(context, rootName, filesToConvert);
+ deleteObsoleteReferences(context, rootName);
+ }
+
+ @Override
+ public void upgradeMetadata(@NonNull ServerContext context) {
+ log.debug("Upgrade metadata: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+
+ var metaName = Ample.DataLevel.USER.metaTable();
+ var filesToConvert = getFileReferences(context, metaName);
+ convertFileReferences(context, metaName, filesToConvert);
+ deleteObsoleteReferences(context, metaName);
+ }
+
+ private Map<ComparablePair<KeyExtent,String>,Value>
+ getFileReferences(@NonNull final ServerContext context, @NonNull final String tableName) {
+
+ log.trace("gather file references for table: {}", tableName);
+
+ Map<ComparablePair<KeyExtent,String>,Value> filesToConvert = new TreeMap<>();
+
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.STR_NAME);
+ scanner.forEach((k, v) -> {
+ KeyExtent ke = KeyExtent.fromMetaRow(k.getRow());
+ String file = k.getColumnQualifier().toString();
+ // filter out references that are in the correct format already.
+ if (fileNeedsConversion(file)) {
+ var prev = filesToConvert.put(new ComparablePair<>(ke, file), v);
+ if (prev != null) {
+ throw new IllegalStateException(
+ "upgrade for table: " + tableName + " aborted, would have missed: " + prev);
+ }
+ }
+ });
+ } catch (TableNotFoundException ex) {
+ throw new IllegalStateException("failed to read metadata table for upgrade", ex);
+ }
+ log.debug("Number of files to convert for table: {}, number of files: {}", tableName,
+ filesToConvert.size());
+ return filesToConvert;
+ }
+
+ private void convertFileReferences(final ServerContext context, final String tableName,
+ Map<ComparablePair<KeyExtent,String>,Value> filesToConvert) {
+
+ // not using ample to avoid StoredTabletFile because old file ref is incompatible
+ try (AccumuloClient c = Accumulo.newClient().from(context.getProperties()).build();
+ BatchWriter batchWriter = c.createBatchWriter(tableName)) {
+ filesToConvert.forEach((refPair, value) -> {
+ try {
+ log.trace("update file reference for table: {}. row: {} to: {} value: {}", tableName,
+ refPair.getFirst().toMetaRow(), refPair.getSecond(), value);
+
+ var row = refPair.getFirst().toMetaRow();
+ var fileJson = new Text(StoredTabletFile.serialize(refPair.getSecond(), new Range()));
+
+ Mutation m = new Mutation(row);
+ m.at().family(MetadataSchema.TabletsSection.DataFileColumnFamily.STR_NAME)
+ .qualifier(fileJson).put(value);
+
+ log.trace("table: {}, adding: {}", tableName, m.prettyPrint());
+ batchWriter.addMutation(m);
+
+ Mutation delete = new Mutation(row);
+ delete.at().family(MetadataSchema.TabletsSection.DataFileColumnFamily.STR_NAME)
+ .qualifier(refPair.getSecond()).delete();
+
+ log.trace("table {}: deleting: {}", tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ throw new IllegalStateException("Failed to update file entries for table: " + tableName,
+ ex);
+ }
+ });
+ batchWriter.flush();
+ } catch (Exception ex) {
+ throw new IllegalStateException(ex);
+ }
+ }
+
+ /**
+ * Removes chopped and external compaction references that have obsolete encoding that is
+ * incompatible with StoredTabletFile json encoding. These references should be rare. If they are
+ * present, some operations likely terminated abnormally under the old version before shutdown.
+ * The deletions are logged so those operations may be re-run if desired.
+ */
+ private void deleteObsoleteReferences(ServerContext context, String tableName) {
+ log.debug("processing obsolete references for table: {}", tableName);
+ try (AccumuloClient c = Accumulo.newClient().from(context.getProperties()).build();
+ BatchWriter batchWriter = c.createBatchWriter(tableName)) {
+
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.fetchColumnFamily(UpgraderDeprecatedConstants.ChoppedColumnFamily.STR_NAME);
+ scanner
+ .fetchColumnFamily(MetadataSchema.TabletsSection.ExternalCompactionColumnFamily.NAME);
+ scanner.forEach((k, v) -> {
+ Mutation delete;
+ var family = k.getColumnFamily();
+ if (family.equals(UpgraderDeprecatedConstants.ChoppedColumnFamily.NAME)) {
+ delete = buildChoppedDeleteMutation(k, tableName);
+ } else if (family
+ .equals(MetadataSchema.TabletsSection.ExternalCompactionColumnFamily.NAME)) {
+ delete = buildExternalCompactionDelete(k, tableName);
+ } else {
+ throw new IllegalStateException(
+ "unexpected column Family: '{}' seen processing obsolete references");
+ }
+ try {
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ log.warn("Failed to delete obsolete reference for table: " + tableName + ". Ref: "
+ + delete.prettyPrint()
+ + ". Will try to continue. Ref may need to be manually removed");
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ }
+ });
+ }
+ } catch (MutationsRejectedException ex) {
+ log.warn("Failed to delete obsolete reference for table: " + tableName + " on close");
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ throw new IllegalStateException(ex);
+ } catch (Exception ex) {
+ throw new IllegalStateException(
+ "Processing obsolete referenced for table: " + tableName + " failed. Upgrade aborting",
+ ex);
+ }
+ }
+
+ private Mutation buildChoppedDeleteMutation(final Key k, final String tableName) {
+ Mutation delete = new Mutation(k.getRow()).at()
+ .family(UpgraderDeprecatedConstants.ChoppedColumnFamily.STR_NAME)
+ .qualifier(UpgraderDeprecatedConstants.ChoppedColumnFamily.STR_NAME).delete();
+ log.warn(
+ "Deleting chopped reference from:{}. Previous split or delete may not have completed cleanly. Ref: {}",
+ tableName, delete.prettyPrint());
+ return delete;
+ }
+
+ private Mutation buildExternalCompactionDelete(Key k, String tableName) {
+ Mutation delete = new Mutation(k.getRow()).at()
+ .family(MetadataSchema.TabletsSection.ExternalCompactionColumnFamily.NAME)
+ .qualifier(k.getColumnQualifier()).delete();
+ log.warn(
+ "Deleting external compaction reference from:{}. Previous compaction may not have completed. Ref: {}",
+ tableName, delete.prettyPrint());
+ return delete;
+ }
+
+ /**
+ * Quick sanity check to see if value has been converted by checking the candidate can be parsed
+ * into a StoredTabletFile. If parsing does not throw a parsing exception. then assume it has been
+ * converted. If validate cannot parse the candidate a JSON, then it is assumed that it has not
+ * been converted. Other parsing errors will propagate as IllegalArgumentExceptions.
+ *
+ * @param candidate a possible file: reference.
+ * @return false if a valid StoredTabletFile, true if it cannot be parsed as JSON. Otherwise,
+ * propagate an IllegalArgumentException
+ */
+ private boolean fileNeedsConversion(@NonNull final String candidate) {
+ try {
+ StoredTabletFile.validate(candidate);
Review Comment:
Added in e8ae63227647
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "EdColeman (via GitHub)" <gi...@apache.org>.
EdColeman commented on code in PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#discussion_r1367822837
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.ComparablePair;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.io.Text;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsConversion(json)) {
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper because of ZooKeeper exception", ex);
+ }
+ }
+
+ @Override
+ public void upgradeRoot(@NonNull ServerContext context) {
+ log.debug("Upgrade root: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+
+ var rootName = Ample.DataLevel.METADATA.metaTable();
+ var filesToConvert = getFileReferences(context, rootName);
+ convertFileReferences(context, rootName, filesToConvert);
+ deleteObsoleteReferences(context, rootName);
+ }
+
+ @Override
+ public void upgradeMetadata(@NonNull ServerContext context) {
+ log.debug("Upgrade metadata: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+
+ var metaName = Ample.DataLevel.USER.metaTable();
+ var filesToConvert = getFileReferences(context, metaName);
+ convertFileReferences(context, metaName, filesToConvert);
+ deleteObsoleteReferences(context, metaName);
+ }
+
+ private Map<ComparablePair<KeyExtent,String>,Value>
+ getFileReferences(@NonNull final ServerContext context, @NonNull final String tableName) {
+
+ log.trace("gather file references for table: {}", tableName);
+
+ Map<ComparablePair<KeyExtent,String>,Value> filesToConvert = new TreeMap<>();
+
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.STR_NAME);
+ scanner.forEach((k, v) -> {
+ KeyExtent ke = KeyExtent.fromMetaRow(k.getRow());
+ String file = k.getColumnQualifier().toString();
+ // filter out references that are in the correct format already.
+ if (fileNeedsConversion(file)) {
+ var prev = filesToConvert.put(new ComparablePair<>(ke, file), v);
+ if (prev != null) {
+ throw new IllegalStateException(
+ "upgrade for table: " + tableName + " aborted, would have missed: " + prev);
+ }
+ }
+ });
+ } catch (TableNotFoundException ex) {
+ throw new IllegalStateException("failed to read metadata table for upgrade", ex);
+ }
+ log.debug("Number of files to convert for table: {}, number of files: {}", tableName,
+ filesToConvert.size());
+ return filesToConvert;
+ }
+
+ private void convertFileReferences(final ServerContext context, final String tableName,
+ Map<ComparablePair<KeyExtent,String>,Value> filesToConvert) {
+
+ // not using ample to avoid StoredTabletFile because old file ref is incompatible
+ try (AccumuloClient c = Accumulo.newClient().from(context.getProperties()).build();
Review Comment:
Changed so that the upgrade file conversion, chop and external compaction removals are done in a single pass through the metadata without creating a map of the file references in e8ae63227647. This reduced the three passes through the metadata that was previous done on first implementation to a single pass and no intermediate storage is used.
Did not use an IsolatedScanner - not sure if that is necessary or not.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#discussion_r1370482709
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants.ChoppedColumnFamily;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsConversion(json)) {
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper because of ZooKeeper exception", ex);
+ }
+ }
+
+ @Override
+ public void upgradeRoot(@NonNull ServerContext context) {
+ log.debug("Upgrade root: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootName = Ample.DataLevel.METADATA.metaTable();
+ processReferences(context, rootName);
+ }
+
+ @Override
+ public void upgradeMetadata(@NonNull ServerContext context) {
+ log.debug("Upgrade metadata: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var metaName = Ample.DataLevel.USER.metaTable();
+ processReferences(context, metaName);
+ }
+
+ private void processReferences(ServerContext context, String tableName) {
+ // not using ample to avoid StoredTabletFile because old file ref is incompatible
+ try (AccumuloClient c = Accumulo.newClient().from(context.getProperties()).build();
+ BatchWriter batchWriter = c.createBatchWriter(tableName);
+ Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+
+ scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ scanner.fetchColumnFamily(ChoppedColumnFamily.NAME);
+ scanner.fetchColumnFamily(ExternalCompactionColumnFamily.NAME);
+ scanner.forEach((k, v) -> {
+ var family = k.getColumnFamily();
+ if (family.equals(DataFileColumnFamily.NAME)) {
+ upgradeDataFileCF(k, v, batchWriter, tableName);
+ } else if (family.equals(ChoppedColumnFamily.NAME)) {
+ removeChoppedCF(k, batchWriter, tableName);
+ } else if (family.equals(ExternalCompactionColumnFamily.NAME)) {
+ removeExternalCompactionCF(k, batchWriter, tableName);
+ } else {
+ log.warn("Received unexpected column family processing references: " + family);
+ }
+ });
+ } catch (MutationsRejectedException mex) {
+ log.warn("Failed to update reference for table: " + tableName);
+ log.warn("Constraint violations: {}", mex.getConstraintViolationSummaries());
+ throw new IllegalStateException("Failed to process table: " + tableName, mex);
+ } catch (Exception ex) {
+ throw new IllegalStateException("Failed to process table: " + tableName, ex);
+ }
+ }
+
+ @VisibleForTesting
+ void upgradeDataFileCF(final Key key, final Value value, final BatchWriter batchWriter,
+ final String tableName) {
+ String file = key.getColumnQualifier().toString();
+ // filter out references if they are in the correct format already.
+ if (fileNeedsConversion(file)) {
+ var fileJson = StoredTabletFile.of(new Path(file)).getMetadataText();
+ try {
+ Mutation update = new Mutation(key.getRow());
+ update.at().family(DataFileColumnFamily.STR_NAME).qualifier(fileJson).put(value);
+ log.trace("table: {}, adding: {}", tableName, update.prettyPrint());
+ batchWriter.addMutation(update);
+
+ Mutation delete = new Mutation(key.getRow());
+ delete.at().family(DataFileColumnFamily.STR_NAME).qualifier(file).delete();
+ log.trace("table {}: deleting: {}", tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ // include constraint violation info in log - but stop upgrade
+ log.warn(
+ "Failed to update file reference for table: " + tableName + ". row: " + key.getRow());
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ throw new IllegalStateException("File conversion failed. Aborting upgrade", ex);
+ }
+ }
+ }
+
+ @VisibleForTesting
+ void removeChoppedCF(final Key key, final BatchWriter batchWriter, final String tableName) {
+ Mutation delete = null;
+ try {
+ delete = new Mutation(key.getRow()).at().family(ChoppedColumnFamily.STR_NAME)
+ .qualifier(ChoppedColumnFamily.STR_NAME).delete();
+ log.warn(
+ "Deleting chopped reference from:{}. Previous split or delete may not have completed cleanly. Ref: {}",
+ tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ log.warn("Failed to delete obsolete chopped CF reference for table: " + tableName + ". Ref: "
+ + delete.prettyPrint() + ". Will try to continue. Ref may need to be manually removed");
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ }
+ }
+
+ @VisibleForTesting
+ void removeExternalCompactionCF(final Key key, final BatchWriter batchWriter,
+ final String tableName) {
+ Mutation delete = null;
+ try {
+ delete = new Mutation(key.getRow()).at().family(ExternalCompactionColumnFamily.NAME)
+ .qualifier(key.getColumnQualifier()).delete();
+ log.warn(
Review Comment:
Its not unexpected rf problematic to have these external compactions, so the warn level does not seem necessary.
```suggestion
log.debug(
```
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants.ChoppedColumnFamily;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsConversion(json)) {
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
Review Comment:
Since the root tablet metadata is so important, may be good to log the root tablet metadata before and after.
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants.ChoppedColumnFamily;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsConversion(json)) {
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper because of ZooKeeper exception", ex);
+ }
+ }
+
+ @Override
+ public void upgradeRoot(@NonNull ServerContext context) {
+ log.debug("Upgrade root: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootName = Ample.DataLevel.METADATA.metaTable();
+ processReferences(context, rootName);
+ }
+
+ @Override
+ public void upgradeMetadata(@NonNull ServerContext context) {
+ log.debug("Upgrade metadata: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var metaName = Ample.DataLevel.USER.metaTable();
+ processReferences(context, metaName);
+ }
+
+ private void processReferences(ServerContext context, String tableName) {
+ // not using ample to avoid StoredTabletFile because old file ref is incompatible
+ try (AccumuloClient c = Accumulo.newClient().from(context.getProperties()).build();
+ BatchWriter batchWriter = c.createBatchWriter(tableName);
+ Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+
+ scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ scanner.fetchColumnFamily(ChoppedColumnFamily.NAME);
+ scanner.fetchColumnFamily(ExternalCompactionColumnFamily.NAME);
+ scanner.forEach((k, v) -> {
+ var family = k.getColumnFamily();
+ if (family.equals(DataFileColumnFamily.NAME)) {
+ upgradeDataFileCF(k, v, batchWriter, tableName);
+ } else if (family.equals(ChoppedColumnFamily.NAME)) {
+ removeChoppedCF(k, batchWriter, tableName);
+ } else if (family.equals(ExternalCompactionColumnFamily.NAME)) {
+ removeExternalCompactionCF(k, batchWriter, tableName);
+ } else {
+ log.warn("Received unexpected column family processing references: " + family);
Review Comment:
I would throw an exception here instead of logging a warning because its completely unexpected.
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants.ChoppedColumnFamily;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsConversion(json)) {
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper because of ZooKeeper exception", ex);
+ }
+ }
+
+ @Override
+ public void upgradeRoot(@NonNull ServerContext context) {
+ log.debug("Upgrade root: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootName = Ample.DataLevel.METADATA.metaTable();
+ processReferences(context, rootName);
+ }
+
+ @Override
+ public void upgradeMetadata(@NonNull ServerContext context) {
+ log.debug("Upgrade metadata: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var metaName = Ample.DataLevel.USER.metaTable();
+ processReferences(context, metaName);
+ }
+
+ private void processReferences(ServerContext context, String tableName) {
+ // not using ample to avoid StoredTabletFile because old file ref is incompatible
+ try (AccumuloClient c = Accumulo.newClient().from(context.getProperties()).build();
+ BatchWriter batchWriter = c.createBatchWriter(tableName);
+ Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+
+ scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ scanner.fetchColumnFamily(ChoppedColumnFamily.NAME);
+ scanner.fetchColumnFamily(ExternalCompactionColumnFamily.NAME);
+ scanner.forEach((k, v) -> {
+ var family = k.getColumnFamily();
+ if (family.equals(DataFileColumnFamily.NAME)) {
+ upgradeDataFileCF(k, v, batchWriter, tableName);
+ } else if (family.equals(ChoppedColumnFamily.NAME)) {
+ removeChoppedCF(k, batchWriter, tableName);
+ } else if (family.equals(ExternalCompactionColumnFamily.NAME)) {
+ removeExternalCompactionCF(k, batchWriter, tableName);
+ } else {
+ log.warn("Received unexpected column family processing references: " + family);
+ }
+ });
+ } catch (MutationsRejectedException mex) {
+ log.warn("Failed to update reference for table: " + tableName);
+ log.warn("Constraint violations: {}", mex.getConstraintViolationSummaries());
+ throw new IllegalStateException("Failed to process table: " + tableName, mex);
+ } catch (Exception ex) {
+ throw new IllegalStateException("Failed to process table: " + tableName, ex);
+ }
+ }
+
+ @VisibleForTesting
+ void upgradeDataFileCF(final Key key, final Value value, final BatchWriter batchWriter,
+ final String tableName) {
+ String file = key.getColumnQualifier().toString();
+ // filter out references if they are in the correct format already.
+ if (fileNeedsConversion(file)) {
+ var fileJson = StoredTabletFile.of(new Path(file)).getMetadataText();
+ try {
+ Mutation update = new Mutation(key.getRow());
+ update.at().family(DataFileColumnFamily.STR_NAME).qualifier(fileJson).put(value);
+ log.trace("table: {}, adding: {}", tableName, update.prettyPrint());
+ batchWriter.addMutation(update);
+
+ Mutation delete = new Mutation(key.getRow());
+ delete.at().family(DataFileColumnFamily.STR_NAME).qualifier(file).delete();
+ log.trace("table {}: deleting: {}", tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ // include constraint violation info in log - but stop upgrade
+ log.warn(
+ "Failed to update file reference for table: " + tableName + ". row: " + key.getRow());
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ throw new IllegalStateException("File conversion failed. Aborting upgrade", ex);
+ }
+ }
+ }
+
+ @VisibleForTesting
+ void removeChoppedCF(final Key key, final BatchWriter batchWriter, final String tableName) {
+ Mutation delete = null;
+ try {
+ delete = new Mutation(key.getRow()).at().family(ChoppedColumnFamily.STR_NAME)
+ .qualifier(ChoppedColumnFamily.STR_NAME).delete();
+ log.warn(
+ "Deleting chopped reference from:{}. Previous split or delete may not have completed cleanly. Ref: {}",
+ tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ log.warn("Failed to delete obsolete chopped CF reference for table: " + tableName + ". Ref: "
+ + delete.prettyPrint() + ". Will try to continue. Ref may need to be manually removed");
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ }
+ }
+
+ @VisibleForTesting
+ void removeExternalCompactionCF(final Key key, final BatchWriter batchWriter,
+ final String tableName) {
+ Mutation delete = null;
+ try {
+ delete = new Mutation(key.getRow()).at().family(ExternalCompactionColumnFamily.NAME)
+ .qualifier(key.getColumnQualifier()).delete();
+ log.warn(
+ "Deleting external compaction reference from:{}. Previous compaction may not have completed. Ref: {}",
+ tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ log.warn("Failed to delete obsolete external compaction CF reference for table: " + tableName
Review Comment:
Also need to throw an exception here.
##########
core/src/main/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadata.java:
##########
@@ -73,10 +77,20 @@ private static class Data {
*/
private final TreeMap<String,TreeMap<String,String>> columnValues;
- public Data(int version, TreeMap<String,TreeMap<String,String>> columnValues) {
+ private Data(int version, TreeMap<String,TreeMap<String,String>> columnValues) {
this.version = version;
this.columnValues = columnValues;
}
+
+ public int getVersion() {
+ return version;
+ }
+
+ public static boolean needsConversion(final String json) {
Review Comment:
```suggestion
public static boolean needsUpgrade(final String json) {
```
##########
core/src/test/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadataTest.java:
##########
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.metadata.schema;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.hadoop.fs.Path;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class RootTabletMetadataTest {
Review Comment:
Could add the following test also
* Pass version 5 data and ensure an exception is thrown
* Pass version 2 data to upgrade and ensure it behaves as expected
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.ComparablePair;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.io.Text;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsConversion(json)) {
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper because of ZooKeeper exception", ex);
+ }
+ }
+
+ @Override
+ public void upgradeRoot(@NonNull ServerContext context) {
+ log.debug("Upgrade root: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+
+ var rootName = Ample.DataLevel.METADATA.metaTable();
+ var filesToConvert = getFileReferences(context, rootName);
+ convertFileReferences(context, rootName, filesToConvert);
+ deleteObsoleteReferences(context, rootName);
+ }
+
+ @Override
+ public void upgradeMetadata(@NonNull ServerContext context) {
+ log.debug("Upgrade metadata: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+
+ var metaName = Ample.DataLevel.USER.metaTable();
+ var filesToConvert = getFileReferences(context, metaName);
+ convertFileReferences(context, metaName, filesToConvert);
+ deleteObsoleteReferences(context, metaName);
+ }
+
+ private Map<ComparablePair<KeyExtent,String>,Value>
+ getFileReferences(@NonNull final ServerContext context, @NonNull final String tableName) {
+
+ log.trace("gather file references for table: {}", tableName);
+
+ Map<ComparablePair<KeyExtent,String>,Value> filesToConvert = new TreeMap<>();
+
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.STR_NAME);
+ scanner.forEach((k, v) -> {
+ KeyExtent ke = KeyExtent.fromMetaRow(k.getRow());
+ String file = k.getColumnQualifier().toString();
+ // filter out references that are in the correct format already.
+ if (fileNeedsConversion(file)) {
+ var prev = filesToConvert.put(new ComparablePair<>(ke, file), v);
+ if (prev != null) {
+ throw new IllegalStateException(
+ "upgrade for table: " + tableName + " aborted, would have missed: " + prev);
+ }
+ }
+ });
+ } catch (TableNotFoundException ex) {
+ throw new IllegalStateException("failed to read metadata table for upgrade", ex);
+ }
+ log.debug("Number of files to convert for table: {}, number of files: {}", tableName,
+ filesToConvert.size());
+ return filesToConvert;
+ }
+
+ private void convertFileReferences(final ServerContext context, final String tableName,
+ Map<ComparablePair<KeyExtent,String>,Value> filesToConvert) {
+
+ // not using ample to avoid StoredTabletFile because old file ref is incompatible
+ try (AccumuloClient c = Accumulo.newClient().from(context.getProperties()).build();
Review Comment:
> Did not use an IsolatedScanner - not sure if that is necessary or not.
Its good to use an isolated scanner for metadata as it ensure that partial changes from a mutation are not seen.
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants.ChoppedColumnFamily;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsConversion(json)) {
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper because of ZooKeeper exception", ex);
+ }
+ }
+
+ @Override
+ public void upgradeRoot(@NonNull ServerContext context) {
+ log.debug("Upgrade root: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootName = Ample.DataLevel.METADATA.metaTable();
+ processReferences(context, rootName);
+ }
+
+ @Override
+ public void upgradeMetadata(@NonNull ServerContext context) {
+ log.debug("Upgrade metadata: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var metaName = Ample.DataLevel.USER.metaTable();
+ processReferences(context, metaName);
+ }
+
+ private void processReferences(ServerContext context, String tableName) {
+ // not using ample to avoid StoredTabletFile because old file ref is incompatible
+ try (AccumuloClient c = Accumulo.newClient().from(context.getProperties()).build();
+ BatchWriter batchWriter = c.createBatchWriter(tableName);
+ Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+
+ scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ scanner.fetchColumnFamily(ChoppedColumnFamily.NAME);
+ scanner.fetchColumnFamily(ExternalCompactionColumnFamily.NAME);
+ scanner.forEach((k, v) -> {
+ var family = k.getColumnFamily();
+ if (family.equals(DataFileColumnFamily.NAME)) {
+ upgradeDataFileCF(k, v, batchWriter, tableName);
+ } else if (family.equals(ChoppedColumnFamily.NAME)) {
+ removeChoppedCF(k, batchWriter, tableName);
+ } else if (family.equals(ExternalCompactionColumnFamily.NAME)) {
+ removeExternalCompactionCF(k, batchWriter, tableName);
+ } else {
+ log.warn("Received unexpected column family processing references: " + family);
+ }
+ });
+ } catch (MutationsRejectedException mex) {
+ log.warn("Failed to update reference for table: " + tableName);
+ log.warn("Constraint violations: {}", mex.getConstraintViolationSummaries());
+ throw new IllegalStateException("Failed to process table: " + tableName, mex);
+ } catch (Exception ex) {
+ throw new IllegalStateException("Failed to process table: " + tableName, ex);
+ }
+ }
+
+ @VisibleForTesting
+ void upgradeDataFileCF(final Key key, final Value value, final BatchWriter batchWriter,
+ final String tableName) {
+ String file = key.getColumnQualifier().toString();
+ // filter out references if they are in the correct format already.
+ if (fileNeedsConversion(file)) {
+ var fileJson = StoredTabletFile.of(new Path(file)).getMetadataText();
+ try {
+ Mutation update = new Mutation(key.getRow());
+ update.at().family(DataFileColumnFamily.STR_NAME).qualifier(fileJson).put(value);
+ log.trace("table: {}, adding: {}", tableName, update.prettyPrint());
+ batchWriter.addMutation(update);
+
+ Mutation delete = new Mutation(key.getRow());
+ delete.at().family(DataFileColumnFamily.STR_NAME).qualifier(file).delete();
+ log.trace("table {}: deleting: {}", tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ // include constraint violation info in log - but stop upgrade
+ log.warn(
+ "Failed to update file reference for table: " + tableName + ". row: " + key.getRow());
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ throw new IllegalStateException("File conversion failed. Aborting upgrade", ex);
+ }
+ }
+ }
+
+ @VisibleForTesting
+ void removeChoppedCF(final Key key, final BatchWriter batchWriter, final String tableName) {
+ Mutation delete = null;
+ try {
+ delete = new Mutation(key.getRow()).at().family(ChoppedColumnFamily.STR_NAME)
+ .qualifier(ChoppedColumnFamily.STR_NAME).delete();
+ log.warn(
+ "Deleting chopped reference from:{}. Previous split or delete may not have completed cleanly. Ref: {}",
+ tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ log.warn("Failed to delete obsolete chopped CF reference for table: " + tableName + ". Ref: "
Review Comment:
Need to rethrow this exception. The batch writer queues up mutations and does not send them immediately. When addMuation throws an exception it may not have been caused by the most recently added mutation. Could be from a mutation for changing a file.
##########
core/src/test/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadataTest.java:
##########
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.metadata.schema;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.hadoop.fs.Path;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class RootTabletMetadataTest {
+ private static final Logger LOG = LoggerFactory.getLogger(RootTabletMetadataTest.class);
+
+ @Test
+ public void convertRoot1File() {
+ String root21ZkData =
+ "{\"version\":1,\"columnValues\":{\"file\":{\"hdfs://localhost:8020/accumulo/tables/+r/root_tablet/A000000v.rf\":\"1368,61\"},\"last\":{\"100025091780006\":\"localhost:9997\"},\"loc\":{\"100025091780006\":\"localhost:9997\"},\"srv\":{\"dir\":\"root_tablet\",\"flush\":\"3\",\"lock\":\"tservers/localhost:9997/zlock#9db8961a-4ee9-400e-8e80-3353148baadd#0000000000$100025091780006\",\"time\":\"L53\"},\"~tab\":{\"~pr\":\"\\u0000\"}}}";
+
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(root21ZkData);
+ LOG.debug("converted column values: {}", rtm.toTabletMetadata().getFiles());
+
+ var files = rtm.toTabletMetadata().getFiles();
+ LOG.info("FILES: {}", rtm.toTabletMetadata().getFilesMap());
+
+ assertEquals(1, files.size());
+ assertTrue(files.contains(StoredTabletFile
+ .of(new Path("hdfs://localhost:8020/accumulo/tables/+r/root_tablet/A000000v.rf"))));
+ }
+
+ @Test
+ public void convertRoot2Files() {
+ String root212ZkData2Files =
+ "{\"version\":1,\"columnValues\":{\"file\":{\"hdfs://localhost:8020/accumulo/tables/+r/root_tablet/00000_00000.rf\":\"0,0\",\"hdfs://localhost:8020/accumulo/tables/+r/root_tablet/F000000c.rf\":\"926,18\"},\"last\":{\"10001a84d7d0005\":\"localhost:9997\"},\"loc\":{\"10001a84d7d0005\":\"localhost:9997\"},\"srv\":{\"dir\":\"root_tablet\",\"flush\":\"2\",\"lock\":\"tservers/localhost:9997/zlock#d21adaa4-0f97-4004-9ff8-cce9dbb6687f#0000000000$10001a84d7d0005\",\"time\":\"L6\"},\"~tab\":{\"~pr\":\"\\u0000\"}}}\n";
+
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(root212ZkData2Files);
+ LOG.debug("converted column values: {}", rtm.toTabletMetadata());
+
Review Comment:
Can test the needsConversion function.
```suggestion
assertTrue(RootTabletMetadata.needsConversion(root212ZkData2Files));
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "EdColeman (via GitHub)" <gi...@apache.org>.
EdColeman commented on code in PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#discussion_r1370969619
##########
core/src/test/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadataTest.java:
##########
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.metadata.schema;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.hadoop.fs.Path;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class RootTabletMetadataTest {
Review Comment:
Added test in bceac27d1c (used -1 as version so it should always be invalid)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "EdColeman (via GitHub)" <gi...@apache.org>.
EdColeman commented on code in PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#discussion_r1384195638
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants.ChoppedColumnFamily;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsUpgrade(json)) {
+ log.info("Root metadata in ZooKeeper before upgrade: {}", json);
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ log.info("Root metadata in ZooKeeper after upgrade: {}", rtm.toJson());
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper because of ZooKeeper exception", ex);
+ }
+ }
+
+ @Override
+ public void upgradeRoot(@NonNull ServerContext context) {
+ log.debug("Upgrade root: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootName = Ample.DataLevel.METADATA.metaTable();
+ processReferences(context, rootName);
+ }
+
+ @Override
+ public void upgradeMetadata(@NonNull ServerContext context) {
+ log.debug("Upgrade metadata: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var metaName = Ample.DataLevel.USER.metaTable();
+ processReferences(context, metaName);
+ }
+
+ private void processReferences(ServerContext context, String tableName) {
+ // not using ample to avoid StoredTabletFile because old file ref is incompatible
+ try (AccumuloClient c = Accumulo.newClient().from(context.getProperties()).build();
+ BatchWriter batchWriter = c.createBatchWriter(tableName); Scanner scanner =
+ new IsolatedScanner(context.createScanner(tableName, Authorizations.EMPTY))) {
+
+ scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ scanner.fetchColumnFamily(ChoppedColumnFamily.NAME);
+ scanner.fetchColumnFamily(ExternalCompactionColumnFamily.NAME);
+ scanner.forEach((k, v) -> {
+ var family = k.getColumnFamily();
+ if (family.equals(DataFileColumnFamily.NAME)) {
+ upgradeDataFileCF(k, v, batchWriter, tableName);
+ } else if (family.equals(ChoppedColumnFamily.NAME)) {
+ removeChoppedCF(k, batchWriter, tableName);
+ } else if (family.equals(ExternalCompactionColumnFamily.NAME)) {
+ removeExternalCompactionCF(k, batchWriter, tableName);
+ } else {
+ throw new IllegalStateException("Processing: " + tableName
+ + " Received unexpected column family processing references: " + family);
+ }
+ });
+ } catch (MutationsRejectedException mex) {
+ log.warn("Failed to update reference for table: " + tableName);
+ log.warn("Constraint violations: {}", mex.getConstraintViolationSummaries());
+ throw new IllegalStateException("Failed to process table: " + tableName, mex);
+ } catch (Exception ex) {
+ throw new IllegalStateException("Failed to process table: " + tableName, ex);
+ }
+ }
+
+ @VisibleForTesting
+ void upgradeDataFileCF(final Key key, final Value value, final BatchWriter batchWriter,
+ final String tableName) {
+ String file = key.getColumnQualifier().toString();
+ // filter out references if they are in the correct format already.
+ if (fileNeedsConversion(file)) {
+ var fileJson = StoredTabletFile.of(new Path(file)).getMetadataText();
+ try {
+ Mutation update = new Mutation(key.getRow());
Review Comment:
I worried about memory usage for a large number of files as a single mutation - but this would also eliminate the need to check for the file needs conversion issue comment on above.
The `fileNeedsConversion` check was added to handle partial upgrades where some files had been converted and then an error would halt the upgrade leaving some files converted, some not. The halt was trigger by errors during development, but seemed that it could allow an admin to perform corrective action and the "restart" the upgrade if it happened in general.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "EdColeman (via GitHub)" <gi...@apache.org>.
EdColeman commented on code in PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#discussion_r1384187129
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants.ChoppedColumnFamily;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsUpgrade(json)) {
+ log.info("Root metadata in ZooKeeper before upgrade: {}", json);
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ log.info("Root metadata in ZooKeeper after upgrade: {}", rtm.toJson());
Review Comment:
Changed in cc6b6e25801c
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "EdColeman (via GitHub)" <gi...@apache.org>.
EdColeman commented on PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#issuecomment-1772985051
Also verified that a 3.0 --> 3.1 upgrade with an uno instance works.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "EdColeman (via GitHub)" <gi...@apache.org>.
EdColeman commented on code in PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#discussion_r1371659241
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants.ChoppedColumnFamily;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsConversion(json)) {
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper because of ZooKeeper exception", ex);
+ }
+ }
+
+ @Override
+ public void upgradeRoot(@NonNull ServerContext context) {
+ log.debug("Upgrade root: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootName = Ample.DataLevel.METADATA.metaTable();
+ processReferences(context, rootName);
+ }
+
+ @Override
+ public void upgradeMetadata(@NonNull ServerContext context) {
+ log.debug("Upgrade metadata: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var metaName = Ample.DataLevel.USER.metaTable();
+ processReferences(context, metaName);
+ }
+
+ private void processReferences(ServerContext context, String tableName) {
+ // not using ample to avoid StoredTabletFile because old file ref is incompatible
+ try (AccumuloClient c = Accumulo.newClient().from(context.getProperties()).build();
+ BatchWriter batchWriter = c.createBatchWriter(tableName);
+ Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+
+ scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ scanner.fetchColumnFamily(ChoppedColumnFamily.NAME);
+ scanner.fetchColumnFamily(ExternalCompactionColumnFamily.NAME);
+ scanner.forEach((k, v) -> {
+ var family = k.getColumnFamily();
+ if (family.equals(DataFileColumnFamily.NAME)) {
+ upgradeDataFileCF(k, v, batchWriter, tableName);
+ } else if (family.equals(ChoppedColumnFamily.NAME)) {
+ removeChoppedCF(k, batchWriter, tableName);
+ } else if (family.equals(ExternalCompactionColumnFamily.NAME)) {
+ removeExternalCompactionCF(k, batchWriter, tableName);
+ } else {
+ log.warn("Received unexpected column family processing references: " + family);
+ }
+ });
+ } catch (MutationsRejectedException mex) {
+ log.warn("Failed to update reference for table: " + tableName);
+ log.warn("Constraint violations: {}", mex.getConstraintViolationSummaries());
+ throw new IllegalStateException("Failed to process table: " + tableName, mex);
+ } catch (Exception ex) {
+ throw new IllegalStateException("Failed to process table: " + tableName, ex);
+ }
+ }
+
+ @VisibleForTesting
+ void upgradeDataFileCF(final Key key, final Value value, final BatchWriter batchWriter,
+ final String tableName) {
+ String file = key.getColumnQualifier().toString();
+ // filter out references if they are in the correct format already.
+ if (fileNeedsConversion(file)) {
+ var fileJson = StoredTabletFile.of(new Path(file)).getMetadataText();
+ try {
+ Mutation update = new Mutation(key.getRow());
+ update.at().family(DataFileColumnFamily.STR_NAME).qualifier(fileJson).put(value);
+ log.trace("table: {}, adding: {}", tableName, update.prettyPrint());
+ batchWriter.addMutation(update);
+
+ Mutation delete = new Mutation(key.getRow());
+ delete.at().family(DataFileColumnFamily.STR_NAME).qualifier(file).delete();
+ log.trace("table {}: deleting: {}", tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ // include constraint violation info in log - but stop upgrade
+ log.warn(
+ "Failed to update file reference for table: " + tableName + ". row: " + key.getRow());
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ throw new IllegalStateException("File conversion failed. Aborting upgrade", ex);
+ }
+ }
+ }
+
+ @VisibleForTesting
+ void removeChoppedCF(final Key key, final BatchWriter batchWriter, final String tableName) {
+ Mutation delete = null;
+ try {
+ delete = new Mutation(key.getRow()).at().family(ChoppedColumnFamily.STR_NAME)
+ .qualifier(ChoppedColumnFamily.STR_NAME).delete();
+ log.warn(
+ "Deleting chopped reference from:{}. Previous split or delete may not have completed cleanly. Ref: {}",
+ tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ log.warn("Failed to delete obsolete chopped CF reference for table: " + tableName + ". Ref: "
+ + delete.prettyPrint() + ". Will try to continue. Ref may need to be manually removed");
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ }
+ }
+
+ @VisibleForTesting
+ void removeExternalCompactionCF(final Key key, final BatchWriter batchWriter,
+ final String tableName) {
+ Mutation delete = null;
+ try {
+ delete = new Mutation(key.getRow()).at().family(ExternalCompactionColumnFamily.NAME)
+ .qualifier(key.getColumnQualifier()).delete();
+ log.warn(
Review Comment:
Currently, we block the upgrade from even starting if there are outstanding FATE transactions - do we need something equivalent to stop / block external compactions?
Also, during the upgrade only the root and the metadata tablets would be online and I don't think we would want concurrent operations occurring on those tables while the upgrade is executing.
My assumption was that because no FATEs are present, any flags present would be from operations that terminated and did not get a chance to clean-up. I may be important to know which things did not stop cleanly - so it can be determined if post-upgrade clean-up operations, such as re-queuing the interrupted compaction is necessary.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "EdColeman (via GitHub)" <gi...@apache.org>.
EdColeman merged PR #3876:
URL: https://github.com/apache/accumulo/pull/3876
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#discussion_r1384090341
##########
core/src/test/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadataTest.java:
##########
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.metadata.schema;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.hadoop.fs.Path;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class RootTabletMetadataTest {
+ private static final Logger LOG = LoggerFactory.getLogger(RootTabletMetadataTest.class);
+
+ @Test
+ public void convertRoot1File() {
+ String root21ZkData =
+ "{\"version\":1,\"columnValues\":{\"file\":{\"hdfs://localhost:8020/accumulo/tables/+r/root_tablet/A000000v.rf\":\"1368,61\"},\"last\":{\"100025091780006\":\"localhost:9997\"},\"loc\":{\"100025091780006\":\"localhost:9997\"},\"srv\":{\"dir\":\"root_tablet\",\"flush\":\"3\",\"lock\":\"tservers/localhost:9997/zlock#9db8961a-4ee9-400e-8e80-3353148baadd#0000000000$100025091780006\",\"time\":\"L53\"},\"~tab\":{\"~pr\":\"\\u0000\"}}}";
Review Comment:
If only we could move to JDK 17, then we could use [text blocks](https://docs.oracle.com/en/java/javase/15/text-blocks/index.html) for this and it would be a lot nicer.
##########
core/src/main/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadata.java:
##########
@@ -59,11 +60,14 @@ public class RootTabletMetadata {
};
// JSON Mapping Version 1. Released with Accumulo version 2.1.0
- private static final int VERSION = 1;
+ private static final int VERSION_1 = 1;
+ // JSON Mapping Version 2. Released with Accumulo version 3,1
+ private static final int VERSION_2 = 2;
+ private static final int VERSION = VERSION_2;
// This class is used to serialize and deserialize root tablet metadata using GSon. Any changes to
// this class must consider persisted data.
- private static class Data {
+ public static class Data {
Review Comment:
Feel free to ignore this comment. I would keep this class private
```suggestion
private static class Data {
```
and add the the following method to `RootTabletMetadata` and have everything call it.
```java
public static boolean needsUpgrade(final String json) {
return Data.needsUpgrade(json);
}
```
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants.ChoppedColumnFamily;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsUpgrade(json)) {
+ log.info("Root metadata in ZooKeeper before upgrade: {}", json);
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ log.info("Root metadata in ZooKeeper after upgrade: {}", rtm.toJson());
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper because of ZooKeeper exception", ex);
Review Comment:
```suggestion
"Could not read or write root metadata in ZooKeeper because of ZooKeeper exception", ex);
```
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants.ChoppedColumnFamily;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsUpgrade(json)) {
+ log.info("Root metadata in ZooKeeper before upgrade: {}", json);
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ log.info("Root metadata in ZooKeeper after upgrade: {}", rtm.toJson());
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper because of ZooKeeper exception", ex);
+ }
+ }
+
+ @Override
+ public void upgradeRoot(@NonNull ServerContext context) {
+ log.debug("Upgrade root: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootName = Ample.DataLevel.METADATA.metaTable();
+ processReferences(context, rootName);
+ }
+
+ @Override
+ public void upgradeMetadata(@NonNull ServerContext context) {
+ log.debug("Upgrade metadata: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var metaName = Ample.DataLevel.USER.metaTable();
+ processReferences(context, metaName);
+ }
+
+ private void processReferences(ServerContext context, String tableName) {
+ // not using ample to avoid StoredTabletFile because old file ref is incompatible
+ try (AccumuloClient c = Accumulo.newClient().from(context.getProperties()).build();
+ BatchWriter batchWriter = c.createBatchWriter(tableName); Scanner scanner =
+ new IsolatedScanner(context.createScanner(tableName, Authorizations.EMPTY))) {
+
+ scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ scanner.fetchColumnFamily(ChoppedColumnFamily.NAME);
+ scanner.fetchColumnFamily(ExternalCompactionColumnFamily.NAME);
+ scanner.forEach((k, v) -> {
+ var family = k.getColumnFamily();
+ if (family.equals(DataFileColumnFamily.NAME)) {
+ upgradeDataFileCF(k, v, batchWriter, tableName);
+ } else if (family.equals(ChoppedColumnFamily.NAME)) {
+ removeChoppedCF(k, batchWriter, tableName);
+ } else if (family.equals(ExternalCompactionColumnFamily.NAME)) {
+ removeExternalCompactionCF(k, batchWriter, tableName);
+ } else {
+ throw new IllegalStateException("Processing: " + tableName
+ + " Received unexpected column family processing references: " + family);
+ }
+ });
+ } catch (MutationsRejectedException mex) {
+ log.warn("Failed to update reference for table: " + tableName);
+ log.warn("Constraint violations: {}", mex.getConstraintViolationSummaries());
+ throw new IllegalStateException("Failed to process table: " + tableName, mex);
+ } catch (Exception ex) {
+ throw new IllegalStateException("Failed to process table: " + tableName, ex);
+ }
+ }
+
+ @VisibleForTesting
+ void upgradeDataFileCF(final Key key, final Value value, final BatchWriter batchWriter,
+ final String tableName) {
+ String file = key.getColumnQualifier().toString();
+ // filter out references if they are in the correct format already.
+ if (fileNeedsConversion(file)) {
+ var fileJson = StoredTabletFile.of(new Path(file)).getMetadataText();
+ try {
+ Mutation update = new Mutation(key.getRow());
+ update.at().family(DataFileColumnFamily.STR_NAME).qualifier(fileJson).put(value);
+ log.trace("table: {}, adding: {}", tableName, update.prettyPrint());
+ batchWriter.addMutation(update);
+
+ Mutation delete = new Mutation(key.getRow());
+ delete.at().family(DataFileColumnFamily.STR_NAME).qualifier(file).delete();
+ log.trace("table {}: deleting: {}", tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ // include constraint violation info in log - but stop upgrade
+ log.warn(
+ "Failed to update file reference for table: " + tableName + ". row: " + key.getRow());
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ throw new IllegalStateException("File conversion failed. Aborting upgrade", ex);
+ }
+ }
+ }
+
+ @VisibleForTesting
+ void removeChoppedCF(final Key key, final BatchWriter batchWriter, final String tableName) {
+ Mutation delete = null;
+ try {
+ delete = new Mutation(key.getRow()).at().family(ChoppedColumnFamily.STR_NAME)
+ .qualifier(ChoppedColumnFamily.STR_NAME).delete();
+ log.warn(
+ "Deleting chopped reference from:{}. Previous split or delete may not have completed cleanly. Ref: {}",
+ tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ log.warn("Failed to delete obsolete chopped CF reference for table: " + tableName + ". Ref: "
+ + delete.prettyPrint() + ". Will try to continue. Ref may need to be manually removed");
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ throw new IllegalStateException(
+ "Failed to delete obsolete chopped CF reference for table: " + tableName, ex);
+ }
+ }
+
+ @VisibleForTesting
+ void removeExternalCompactionCF(final Key key, final BatchWriter batchWriter,
+ final String tableName) {
+ Mutation delete = null;
+ try {
+ delete = new Mutation(key.getRow()).at().family(ExternalCompactionColumnFamily.NAME)
+ .qualifier(key.getColumnQualifier()).delete();
+ log.debug(
+ "Deleting external compaction reference from:{}. Previous compaction may not have completed. Ref: {}",
+ tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ log.warn("Failed to delete obsolete external compaction CF reference for table: " + tableName
+ + ". Ref: " + delete.prettyPrint()
+ + ". Will try to continue. Ref may need to be manually removed");
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ throw new IllegalStateException(
+ "Failed to delete obsolete external compaction CF reference for table: " + tableName, ex);
+ }
+ }
+
+ /**
+ * Quick sanity check to see if value has been converted by checking the candidate can be parsed
+ * into a StoredTabletFile. If parsing does not throw a parsing exception. then assume it has been
+ * converted. If validate cannot parse the candidate a JSON, then it is assumed that it has not
+ * been converted. Other parsing errors will propagate as IllegalArgumentExceptions.
+ *
+ * @param candidate a possible file: reference.
+ * @return false if a valid StoredTabletFile, true if it cannot be parsed as JSON. Otherwise,
+ * propagate an IllegalArgumentException
+ */
+ private boolean fileNeedsConversion(@NonNull final String candidate) {
+ try {
+ StoredTabletFile.validate(candidate);
+ log.trace("file reference does not need conversion: {}", candidate);
+ return false;
+ } catch (JsonSyntaxException ex) {
Review Comment:
Using exceptions for expected situations may be slow, needs more research. In the quick reading I did on the subject the expensive part of throwing an exception is creating the stack trace, which makes sense. If this is happening for a large number of files and its slow, then it could make upgrade slow. I think we need to a follow on issue for this, it may not be a problem but would be good to investigate. Could either try to benchmark it or just avoid throwing and catching exceptions as part of normal processing.
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants.ChoppedColumnFamily;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsUpgrade(json)) {
+ log.info("Root metadata in ZooKeeper before upgrade: {}", json);
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ log.info("Root metadata in ZooKeeper after upgrade: {}", rtm.toJson());
Review Comment:
Wondering if this log message should go after the ZK write.
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants.ChoppedColumnFamily;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsUpgrade(json)) {
+ log.info("Root metadata in ZooKeeper before upgrade: {}", json);
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ log.info("Root metadata in ZooKeeper after upgrade: {}", rtm.toJson());
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper because of ZooKeeper exception", ex);
+ }
+ }
+
+ @Override
+ public void upgradeRoot(@NonNull ServerContext context) {
+ log.debug("Upgrade root: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootName = Ample.DataLevel.METADATA.metaTable();
+ processReferences(context, rootName);
+ }
+
+ @Override
+ public void upgradeMetadata(@NonNull ServerContext context) {
+ log.debug("Upgrade metadata: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var metaName = Ample.DataLevel.USER.metaTable();
+ processReferences(context, metaName);
+ }
+
+ private void processReferences(ServerContext context, String tableName) {
+ // not using ample to avoid StoredTabletFile because old file ref is incompatible
+ try (AccumuloClient c = Accumulo.newClient().from(context.getProperties()).build();
+ BatchWriter batchWriter = c.createBatchWriter(tableName); Scanner scanner =
+ new IsolatedScanner(context.createScanner(tableName, Authorizations.EMPTY))) {
+
+ scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ scanner.fetchColumnFamily(ChoppedColumnFamily.NAME);
+ scanner.fetchColumnFamily(ExternalCompactionColumnFamily.NAME);
+ scanner.forEach((k, v) -> {
+ var family = k.getColumnFamily();
+ if (family.equals(DataFileColumnFamily.NAME)) {
+ upgradeDataFileCF(k, v, batchWriter, tableName);
+ } else if (family.equals(ChoppedColumnFamily.NAME)) {
+ removeChoppedCF(k, batchWriter, tableName);
+ } else if (family.equals(ExternalCompactionColumnFamily.NAME)) {
+ removeExternalCompactionCF(k, batchWriter, tableName);
+ } else {
+ throw new IllegalStateException("Processing: " + tableName
+ + " Received unexpected column family processing references: " + family);
+ }
+ });
+ } catch (MutationsRejectedException mex) {
+ log.warn("Failed to update reference for table: " + tableName);
+ log.warn("Constraint violations: {}", mex.getConstraintViolationSummaries());
+ throw new IllegalStateException("Failed to process table: " + tableName, mex);
+ } catch (Exception ex) {
+ throw new IllegalStateException("Failed to process table: " + tableName, ex);
+ }
+ }
+
+ @VisibleForTesting
+ void upgradeDataFileCF(final Key key, final Value value, final BatchWriter batchWriter,
+ final String tableName) {
+ String file = key.getColumnQualifier().toString();
+ // filter out references if they are in the correct format already.
+ if (fileNeedsConversion(file)) {
+ var fileJson = StoredTabletFile.of(new Path(file)).getMetadataText();
+ try {
+ Mutation update = new Mutation(key.getRow());
Review Comment:
It would be nice to generate a single mutation for all the tablets files. That may be faster and would result in atomically updating the tablets files. However I don't think this has to be done. If it were done, it could be in a follow on PR. What is here seems correct and is probably fast enough for the one time upgrade case.
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants.ChoppedColumnFamily;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsUpgrade(json)) {
+ log.info("Root metadata in ZooKeeper before upgrade: {}", json);
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ log.info("Root metadata in ZooKeeper after upgrade: {}", rtm.toJson());
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper because of ZooKeeper exception", ex);
+ }
+ }
+
+ @Override
+ public void upgradeRoot(@NonNull ServerContext context) {
+ log.debug("Upgrade root: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootName = Ample.DataLevel.METADATA.metaTable();
+ processReferences(context, rootName);
+ }
+
+ @Override
+ public void upgradeMetadata(@NonNull ServerContext context) {
+ log.debug("Upgrade metadata: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var metaName = Ample.DataLevel.USER.metaTable();
+ processReferences(context, metaName);
+ }
+
+ private void processReferences(ServerContext context, String tableName) {
+ // not using ample to avoid StoredTabletFile because old file ref is incompatible
+ try (AccumuloClient c = Accumulo.newClient().from(context.getProperties()).build();
+ BatchWriter batchWriter = c.createBatchWriter(tableName); Scanner scanner =
+ new IsolatedScanner(context.createScanner(tableName, Authorizations.EMPTY))) {
+
+ scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ scanner.fetchColumnFamily(ChoppedColumnFamily.NAME);
+ scanner.fetchColumnFamily(ExternalCompactionColumnFamily.NAME);
+ scanner.forEach((k, v) -> {
+ var family = k.getColumnFamily();
+ if (family.equals(DataFileColumnFamily.NAME)) {
+ upgradeDataFileCF(k, v, batchWriter, tableName);
+ } else if (family.equals(ChoppedColumnFamily.NAME)) {
+ removeChoppedCF(k, batchWriter, tableName);
+ } else if (family.equals(ExternalCompactionColumnFamily.NAME)) {
+ removeExternalCompactionCF(k, batchWriter, tableName);
+ } else {
+ throw new IllegalStateException("Processing: " + tableName
+ + " Received unexpected column family processing references: " + family);
+ }
+ });
+ } catch (MutationsRejectedException mex) {
+ log.warn("Failed to update reference for table: " + tableName);
+ log.warn("Constraint violations: {}", mex.getConstraintViolationSummaries());
+ throw new IllegalStateException("Failed to process table: " + tableName, mex);
+ } catch (Exception ex) {
+ throw new IllegalStateException("Failed to process table: " + tableName, ex);
+ }
+ }
+
+ @VisibleForTesting
+ void upgradeDataFileCF(final Key key, final Value value, final BatchWriter batchWriter,
+ final String tableName) {
+ String file = key.getColumnQualifier().toString();
+ // filter out references if they are in the correct format already.
+ if (fileNeedsConversion(file)) {
+ var fileJson = StoredTabletFile.of(new Path(file)).getMetadataText();
+ try {
+ Mutation update = new Mutation(key.getRow());
+ update.at().family(DataFileColumnFamily.STR_NAME).qualifier(fileJson).put(value);
+ log.trace("table: {}, adding: {}", tableName, update.prettyPrint());
+ batchWriter.addMutation(update);
+
+ Mutation delete = new Mutation(key.getRow());
+ delete.at().family(DataFileColumnFamily.STR_NAME).qualifier(file).delete();
+ log.trace("table {}: deleting: {}", tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ // include constraint violation info in log - but stop upgrade
+ log.warn(
+ "Failed to update file reference for table: " + tableName + ". row: " + key.getRow());
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ throw new IllegalStateException("File conversion failed. Aborting upgrade", ex);
+ }
+ }
+ }
+
+ @VisibleForTesting
+ void removeChoppedCF(final Key key, final BatchWriter batchWriter, final String tableName) {
+ Mutation delete = null;
+ try {
+ delete = new Mutation(key.getRow()).at().family(ChoppedColumnFamily.STR_NAME)
+ .qualifier(ChoppedColumnFamily.STR_NAME).delete();
+ log.warn(
+ "Deleting chopped reference from:{}. Previous split or delete may not have completed cleanly. Ref: {}",
+ tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ log.warn("Failed to delete obsolete chopped CF reference for table: " + tableName + ". Ref: "
+ + delete.prettyPrint() + ". Will try to continue. Ref may need to be manually removed");
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ throw new IllegalStateException(
+ "Failed to delete obsolete chopped CF reference for table: " + tableName, ex);
+ }
+ }
+
+ @VisibleForTesting
+ void removeExternalCompactionCF(final Key key, final BatchWriter batchWriter,
+ final String tableName) {
+ Mutation delete = null;
+ try {
+ delete = new Mutation(key.getRow()).at().family(ExternalCompactionColumnFamily.NAME)
+ .qualifier(key.getColumnQualifier()).delete();
+ log.debug(
+ "Deleting external compaction reference from:{}. Previous compaction may not have completed. Ref: {}",
+ tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ log.warn("Failed to delete obsolete external compaction CF reference for table: " + tableName
+ + ". Ref: " + delete.prettyPrint()
+ + ". Will try to continue. Ref may need to be manually removed");
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ throw new IllegalStateException(
+ "Failed to delete obsolete external compaction CF reference for table: " + tableName, ex);
+ }
+ }
+
+ /**
+ * Quick sanity check to see if value has been converted by checking the candidate can be parsed
Review Comment:
Could use language that is more inclusive here.
```suggestion
* Quick validation to see if value has been converted by checking the candidate can be parsed
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "EdColeman (via GitHub)" <gi...@apache.org>.
EdColeman commented on PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#issuecomment-1772782205
This has been tested with uno going from 2.1 directly to 3.1 with chopped and external compaction references manually inserted into the metadata table in 2.1 before shutdown. On upgrade, the removal of those references are reported as:
```
2023-10-20T13:40:17,350 [upgrade.Upgrader11to12] DEBUG: processing obsolete references for table: accumulo.metadata
2023-10-20T13:40:17,356 [upgrade.Upgrader11to12] WARN : Deleting chopped reference from:accumulo.metadata. Previous split or delete may not have completed cleanly. Ref: mutation: 1;0020
update: chopped:chopped value [delete]
2023-10-20T13:40:17,356 [upgrade.Upgrader11to12] WARN : Deleting external compaction reference from:accumulo.metadata. Previous compaction may not have completed. Ref: mutation: 1;0020
update: ecomp:ECID:0000001 value [delete]
2023-10-20T13:40:17,356 [upgrade.Upgrader11to12] WARN : Deleting chopped reference from:accumulo.metadata. Previous split or delete may not have completed cleanly. Ref: mutation: 1;0040
update: chopped:chopped value [delete]
2023-10-20T13:40:17,356 [upgrade.Upgrader11to12] WARN : Deleting external compaction reference from:accumulo.metadata. Previous compaction may not have completed. Ref: mutation: 1;0040
update: ecomp:ECID:0000001 value [delete]
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "EdColeman (via GitHub)" <gi...@apache.org>.
EdColeman commented on code in PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#discussion_r1370969883
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants.ChoppedColumnFamily;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsConversion(json)) {
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper because of ZooKeeper exception", ex);
+ }
+ }
+
+ @Override
+ public void upgradeRoot(@NonNull ServerContext context) {
+ log.debug("Upgrade root: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootName = Ample.DataLevel.METADATA.metaTable();
+ processReferences(context, rootName);
+ }
+
+ @Override
+ public void upgradeMetadata(@NonNull ServerContext context) {
+ log.debug("Upgrade metadata: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var metaName = Ample.DataLevel.USER.metaTable();
+ processReferences(context, metaName);
+ }
+
+ private void processReferences(ServerContext context, String tableName) {
+ // not using ample to avoid StoredTabletFile because old file ref is incompatible
+ try (AccumuloClient c = Accumulo.newClient().from(context.getProperties()).build();
+ BatchWriter batchWriter = c.createBatchWriter(tableName);
+ Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+
+ scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ scanner.fetchColumnFamily(ChoppedColumnFamily.NAME);
+ scanner.fetchColumnFamily(ExternalCompactionColumnFamily.NAME);
+ scanner.forEach((k, v) -> {
+ var family = k.getColumnFamily();
+ if (family.equals(DataFileColumnFamily.NAME)) {
+ upgradeDataFileCF(k, v, batchWriter, tableName);
+ } else if (family.equals(ChoppedColumnFamily.NAME)) {
+ removeChoppedCF(k, batchWriter, tableName);
+ } else if (family.equals(ExternalCompactionColumnFamily.NAME)) {
+ removeExternalCompactionCF(k, batchWriter, tableName);
+ } else {
+ log.warn("Received unexpected column family processing references: " + family);
Review Comment:
Added in bceac27d1c.
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants.ChoppedColumnFamily;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsConversion(json)) {
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper because of ZooKeeper exception", ex);
+ }
+ }
+
+ @Override
+ public void upgradeRoot(@NonNull ServerContext context) {
+ log.debug("Upgrade root: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootName = Ample.DataLevel.METADATA.metaTable();
+ processReferences(context, rootName);
+ }
+
+ @Override
+ public void upgradeMetadata(@NonNull ServerContext context) {
+ log.debug("Upgrade metadata: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var metaName = Ample.DataLevel.USER.metaTable();
+ processReferences(context, metaName);
+ }
+
+ private void processReferences(ServerContext context, String tableName) {
+ // not using ample to avoid StoredTabletFile because old file ref is incompatible
+ try (AccumuloClient c = Accumulo.newClient().from(context.getProperties()).build();
+ BatchWriter batchWriter = c.createBatchWriter(tableName);
+ Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+
+ scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ scanner.fetchColumnFamily(ChoppedColumnFamily.NAME);
+ scanner.fetchColumnFamily(ExternalCompactionColumnFamily.NAME);
+ scanner.forEach((k, v) -> {
+ var family = k.getColumnFamily();
+ if (family.equals(DataFileColumnFamily.NAME)) {
+ upgradeDataFileCF(k, v, batchWriter, tableName);
+ } else if (family.equals(ChoppedColumnFamily.NAME)) {
+ removeChoppedCF(k, batchWriter, tableName);
+ } else if (family.equals(ExternalCompactionColumnFamily.NAME)) {
+ removeExternalCompactionCF(k, batchWriter, tableName);
+ } else {
+ log.warn("Received unexpected column family processing references: " + family);
+ }
+ });
+ } catch (MutationsRejectedException mex) {
+ log.warn("Failed to update reference for table: " + tableName);
+ log.warn("Constraint violations: {}", mex.getConstraintViolationSummaries());
+ throw new IllegalStateException("Failed to process table: " + tableName, mex);
+ } catch (Exception ex) {
+ throw new IllegalStateException("Failed to process table: " + tableName, ex);
+ }
+ }
+
+ @VisibleForTesting
+ void upgradeDataFileCF(final Key key, final Value value, final BatchWriter batchWriter,
+ final String tableName) {
+ String file = key.getColumnQualifier().toString();
+ // filter out references if they are in the correct format already.
+ if (fileNeedsConversion(file)) {
+ var fileJson = StoredTabletFile.of(new Path(file)).getMetadataText();
+ try {
+ Mutation update = new Mutation(key.getRow());
+ update.at().family(DataFileColumnFamily.STR_NAME).qualifier(fileJson).put(value);
+ log.trace("table: {}, adding: {}", tableName, update.prettyPrint());
+ batchWriter.addMutation(update);
+
+ Mutation delete = new Mutation(key.getRow());
+ delete.at().family(DataFileColumnFamily.STR_NAME).qualifier(file).delete();
+ log.trace("table {}: deleting: {}", tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ // include constraint violation info in log - but stop upgrade
+ log.warn(
+ "Failed to update file reference for table: " + tableName + ". row: " + key.getRow());
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ throw new IllegalStateException("File conversion failed. Aborting upgrade", ex);
+ }
+ }
+ }
+
+ @VisibleForTesting
+ void removeChoppedCF(final Key key, final BatchWriter batchWriter, final String tableName) {
+ Mutation delete = null;
+ try {
+ delete = new Mutation(key.getRow()).at().family(ChoppedColumnFamily.STR_NAME)
+ .qualifier(ChoppedColumnFamily.STR_NAME).delete();
+ log.warn(
+ "Deleting chopped reference from:{}. Previous split or delete may not have completed cleanly. Ref: {}",
+ tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ log.warn("Failed to delete obsolete chopped CF reference for table: " + tableName + ". Ref: "
Review Comment:
Added in bceac27d1c
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#discussion_r1370980320
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants.ChoppedColumnFamily;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsConversion(json)) {
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper because of ZooKeeper exception", ex);
+ }
+ }
+
+ @Override
+ public void upgradeRoot(@NonNull ServerContext context) {
+ log.debug("Upgrade root: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootName = Ample.DataLevel.METADATA.metaTable();
+ processReferences(context, rootName);
+ }
+
+ @Override
+ public void upgradeMetadata(@NonNull ServerContext context) {
+ log.debug("Upgrade metadata: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var metaName = Ample.DataLevel.USER.metaTable();
+ processReferences(context, metaName);
+ }
+
+ private void processReferences(ServerContext context, String tableName) {
+ // not using ample to avoid StoredTabletFile because old file ref is incompatible
+ try (AccumuloClient c = Accumulo.newClient().from(context.getProperties()).build();
+ BatchWriter batchWriter = c.createBatchWriter(tableName);
+ Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+
+ scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ scanner.fetchColumnFamily(ChoppedColumnFamily.NAME);
+ scanner.fetchColumnFamily(ExternalCompactionColumnFamily.NAME);
+ scanner.forEach((k, v) -> {
+ var family = k.getColumnFamily();
+ if (family.equals(DataFileColumnFamily.NAME)) {
+ upgradeDataFileCF(k, v, batchWriter, tableName);
+ } else if (family.equals(ChoppedColumnFamily.NAME)) {
+ removeChoppedCF(k, batchWriter, tableName);
+ } else if (family.equals(ExternalCompactionColumnFamily.NAME)) {
+ removeExternalCompactionCF(k, batchWriter, tableName);
+ } else {
+ log.warn("Received unexpected column family processing references: " + family);
+ }
+ });
+ } catch (MutationsRejectedException mex) {
+ log.warn("Failed to update reference for table: " + tableName);
+ log.warn("Constraint violations: {}", mex.getConstraintViolationSummaries());
+ throw new IllegalStateException("Failed to process table: " + tableName, mex);
+ } catch (Exception ex) {
+ throw new IllegalStateException("Failed to process table: " + tableName, ex);
+ }
+ }
+
+ @VisibleForTesting
+ void upgradeDataFileCF(final Key key, final Value value, final BatchWriter batchWriter,
+ final String tableName) {
+ String file = key.getColumnQualifier().toString();
+ // filter out references if they are in the correct format already.
+ if (fileNeedsConversion(file)) {
+ var fileJson = StoredTabletFile.of(new Path(file)).getMetadataText();
+ try {
+ Mutation update = new Mutation(key.getRow());
+ update.at().family(DataFileColumnFamily.STR_NAME).qualifier(fileJson).put(value);
+ log.trace("table: {}, adding: {}", tableName, update.prettyPrint());
+ batchWriter.addMutation(update);
+
+ Mutation delete = new Mutation(key.getRow());
+ delete.at().family(DataFileColumnFamily.STR_NAME).qualifier(file).delete();
+ log.trace("table {}: deleting: {}", tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ // include constraint violation info in log - but stop upgrade
+ log.warn(
+ "Failed to update file reference for table: " + tableName + ". row: " + key.getRow());
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ throw new IllegalStateException("File conversion failed. Aborting upgrade", ex);
+ }
+ }
+ }
+
+ @VisibleForTesting
+ void removeChoppedCF(final Key key, final BatchWriter batchWriter, final String tableName) {
+ Mutation delete = null;
+ try {
+ delete = new Mutation(key.getRow()).at().family(ChoppedColumnFamily.STR_NAME)
+ .qualifier(ChoppedColumnFamily.STR_NAME).delete();
+ log.warn(
+ "Deleting chopped reference from:{}. Previous split or delete may not have completed cleanly. Ref: {}",
+ tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ log.warn("Failed to delete obsolete chopped CF reference for table: " + tableName + ". Ref: "
+ + delete.prettyPrint() + ". Will try to continue. Ref may need to be manually removed");
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ }
+ }
+
+ @VisibleForTesting
+ void removeExternalCompactionCF(final Key key, final BatchWriter batchWriter,
+ final String tableName) {
+ Mutation delete = null;
+ try {
+ delete = new Mutation(key.getRow()).at().family(ExternalCompactionColumnFamily.NAME)
+ .qualifier(key.getColumnQualifier()).delete();
+ log.warn(
Review Comment:
Its not expected that an external compaction would run across an upgrade and removing entries related to any that were running is not very interesting. It could possibly log a lot of warns if there are a lot of ecomp entries and that would be a lot of noise that is not very useful IMO.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "EdColeman (via GitHub)" <gi...@apache.org>.
EdColeman commented on code in PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#discussion_r1374855491
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants.ChoppedColumnFamily;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsConversion(json)) {
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper because of ZooKeeper exception", ex);
+ }
+ }
+
+ @Override
+ public void upgradeRoot(@NonNull ServerContext context) {
+ log.debug("Upgrade root: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootName = Ample.DataLevel.METADATA.metaTable();
+ processReferences(context, rootName);
+ }
+
+ @Override
+ public void upgradeMetadata(@NonNull ServerContext context) {
+ log.debug("Upgrade metadata: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var metaName = Ample.DataLevel.USER.metaTable();
+ processReferences(context, metaName);
+ }
+
+ private void processReferences(ServerContext context, String tableName) {
+ // not using ample to avoid StoredTabletFile because old file ref is incompatible
+ try (AccumuloClient c = Accumulo.newClient().from(context.getProperties()).build();
+ BatchWriter batchWriter = c.createBatchWriter(tableName);
+ Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+
+ scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ scanner.fetchColumnFamily(ChoppedColumnFamily.NAME);
+ scanner.fetchColumnFamily(ExternalCompactionColumnFamily.NAME);
+ scanner.forEach((k, v) -> {
+ var family = k.getColumnFamily();
+ if (family.equals(DataFileColumnFamily.NAME)) {
+ upgradeDataFileCF(k, v, batchWriter, tableName);
+ } else if (family.equals(ChoppedColumnFamily.NAME)) {
+ removeChoppedCF(k, batchWriter, tableName);
+ } else if (family.equals(ExternalCompactionColumnFamily.NAME)) {
+ removeExternalCompactionCF(k, batchWriter, tableName);
+ } else {
+ log.warn("Received unexpected column family processing references: " + family);
+ }
+ });
+ } catch (MutationsRejectedException mex) {
+ log.warn("Failed to update reference for table: " + tableName);
+ log.warn("Constraint violations: {}", mex.getConstraintViolationSummaries());
+ throw new IllegalStateException("Failed to process table: " + tableName, mex);
+ } catch (Exception ex) {
+ throw new IllegalStateException("Failed to process table: " + tableName, ex);
+ }
+ }
+
+ @VisibleForTesting
+ void upgradeDataFileCF(final Key key, final Value value, final BatchWriter batchWriter,
+ final String tableName) {
+ String file = key.getColumnQualifier().toString();
+ // filter out references if they are in the correct format already.
+ if (fileNeedsConversion(file)) {
+ var fileJson = StoredTabletFile.of(new Path(file)).getMetadataText();
+ try {
+ Mutation update = new Mutation(key.getRow());
+ update.at().family(DataFileColumnFamily.STR_NAME).qualifier(fileJson).put(value);
+ log.trace("table: {}, adding: {}", tableName, update.prettyPrint());
+ batchWriter.addMutation(update);
+
+ Mutation delete = new Mutation(key.getRow());
+ delete.at().family(DataFileColumnFamily.STR_NAME).qualifier(file).delete();
+ log.trace("table {}: deleting: {}", tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ // include constraint violation info in log - but stop upgrade
+ log.warn(
+ "Failed to update file reference for table: " + tableName + ". row: " + key.getRow());
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ throw new IllegalStateException("File conversion failed. Aborting upgrade", ex);
+ }
+ }
+ }
+
+ @VisibleForTesting
+ void removeChoppedCF(final Key key, final BatchWriter batchWriter, final String tableName) {
+ Mutation delete = null;
+ try {
+ delete = new Mutation(key.getRow()).at().family(ChoppedColumnFamily.STR_NAME)
+ .qualifier(ChoppedColumnFamily.STR_NAME).delete();
+ log.warn(
+ "Deleting chopped reference from:{}. Previous split or delete may not have completed cleanly. Ref: {}",
+ tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ log.warn("Failed to delete obsolete chopped CF reference for table: " + tableName + ". Ref: "
+ + delete.prettyPrint() + ". Will try to continue. Ref may need to be manually removed");
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ }
+ }
+
+ @VisibleForTesting
+ void removeExternalCompactionCF(final Key key, final BatchWriter batchWriter,
+ final String tableName) {
+ Mutation delete = null;
+ try {
+ delete = new Mutation(key.getRow()).at().family(ExternalCompactionColumnFamily.NAME)
+ .qualifier(key.getColumnQualifier()).delete();
+ log.warn(
Review Comment:
Changed to debug in 77dd24939a
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "EdColeman (via GitHub)" <gi...@apache.org>.
EdColeman commented on code in PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#discussion_r1367822257
##########
core/src/main/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadata.java:
##########
@@ -73,10 +77,27 @@ private static class Data {
*/
private final TreeMap<String,TreeMap<String,String>> columnValues;
- public Data(int version, TreeMap<String,TreeMap<String,String>> columnValues) {
+ private Data(int version, TreeMap<String,TreeMap<String,String>> columnValues) {
this.version = version;
this.columnValues = columnValues;
}
+
+ public int getVersion() {
+ return version;
+ }
+
+ /**
+ * For external use, return a copy so the original remains immutable.
+ */
+ public TreeMap<String,TreeMap<String,String>> getColumnValues() {
+ return new TreeMap<>(columnValues);
Review Comment:
With refactoring, this method was not needed outside of the class and the method was removed in e8ae63227647
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "EdColeman (via GitHub)" <gi...@apache.org>.
EdColeman commented on PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#issuecomment-1776261977
Added tests for Upgrader11to12 with 3ff4d1655b432a
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "EdColeman (via GitHub)" <gi...@apache.org>.
EdColeman commented on code in PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#discussion_r1370897218
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants.ChoppedColumnFamily;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsConversion(json)) {
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper because of ZooKeeper exception", ex);
+ }
+ }
+
+ @Override
+ public void upgradeRoot(@NonNull ServerContext context) {
+ log.debug("Upgrade root: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootName = Ample.DataLevel.METADATA.metaTable();
+ processReferences(context, rootName);
+ }
+
+ @Override
+ public void upgradeMetadata(@NonNull ServerContext context) {
+ log.debug("Upgrade metadata: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var metaName = Ample.DataLevel.USER.metaTable();
+ processReferences(context, metaName);
+ }
+
+ private void processReferences(ServerContext context, String tableName) {
+ // not using ample to avoid StoredTabletFile because old file ref is incompatible
+ try (AccumuloClient c = Accumulo.newClient().from(context.getProperties()).build();
+ BatchWriter batchWriter = c.createBatchWriter(tableName);
+ Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+
+ scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ scanner.fetchColumnFamily(ChoppedColumnFamily.NAME);
+ scanner.fetchColumnFamily(ExternalCompactionColumnFamily.NAME);
+ scanner.forEach((k, v) -> {
+ var family = k.getColumnFamily();
+ if (family.equals(DataFileColumnFamily.NAME)) {
+ upgradeDataFileCF(k, v, batchWriter, tableName);
+ } else if (family.equals(ChoppedColumnFamily.NAME)) {
+ removeChoppedCF(k, batchWriter, tableName);
+ } else if (family.equals(ExternalCompactionColumnFamily.NAME)) {
+ removeExternalCompactionCF(k, batchWriter, tableName);
+ } else {
+ log.warn("Received unexpected column family processing references: " + family);
+ }
+ });
+ } catch (MutationsRejectedException mex) {
+ log.warn("Failed to update reference for table: " + tableName);
+ log.warn("Constraint violations: {}", mex.getConstraintViolationSummaries());
+ throw new IllegalStateException("Failed to process table: " + tableName, mex);
+ } catch (Exception ex) {
+ throw new IllegalStateException("Failed to process table: " + tableName, ex);
+ }
+ }
+
+ @VisibleForTesting
+ void upgradeDataFileCF(final Key key, final Value value, final BatchWriter batchWriter,
+ final String tableName) {
+ String file = key.getColumnQualifier().toString();
+ // filter out references if they are in the correct format already.
+ if (fileNeedsConversion(file)) {
+ var fileJson = StoredTabletFile.of(new Path(file)).getMetadataText();
+ try {
+ Mutation update = new Mutation(key.getRow());
+ update.at().family(DataFileColumnFamily.STR_NAME).qualifier(fileJson).put(value);
+ log.trace("table: {}, adding: {}", tableName, update.prettyPrint());
+ batchWriter.addMutation(update);
+
+ Mutation delete = new Mutation(key.getRow());
+ delete.at().family(DataFileColumnFamily.STR_NAME).qualifier(file).delete();
+ log.trace("table {}: deleting: {}", tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ // include constraint violation info in log - but stop upgrade
+ log.warn(
+ "Failed to update file reference for table: " + tableName + ". row: " + key.getRow());
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ throw new IllegalStateException("File conversion failed. Aborting upgrade", ex);
+ }
+ }
+ }
+
+ @VisibleForTesting
+ void removeChoppedCF(final Key key, final BatchWriter batchWriter, final String tableName) {
+ Mutation delete = null;
+ try {
+ delete = new Mutation(key.getRow()).at().family(ChoppedColumnFamily.STR_NAME)
+ .qualifier(ChoppedColumnFamily.STR_NAME).delete();
+ log.warn(
+ "Deleting chopped reference from:{}. Previous split or delete may not have completed cleanly. Ref: {}",
+ tableName, delete.prettyPrint());
+ batchWriter.addMutation(delete);
+ } catch (MutationsRejectedException ex) {
+ log.warn("Failed to delete obsolete chopped CF reference for table: " + tableName + ". Ref: "
+ + delete.prettyPrint() + ". Will try to continue. Ref may need to be manually removed");
+ log.warn("Constraint violations: {}", ex.getConstraintViolationSummaries());
+ }
+ }
+
+ @VisibleForTesting
+ void removeExternalCompactionCF(final Key key, final BatchWriter batchWriter,
+ final String tableName) {
+ Mutation delete = null;
+ try {
+ delete = new Mutation(key.getRow()).at().family(ExternalCompactionColumnFamily.NAME)
+ .qualifier(key.getColumnQualifier()).delete();
+ log.warn(
Review Comment:
The warn was so that that would stand out in the logs. This will run once, and debug statements could be easier lost / ignored. The removal of external compaction markers is because they would likely cause problems because they do not include the fence parameters.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] 3.1 upgrade process with no chop changes [accumulo]
Posted by "EdColeman (via GitHub)" <gi...@apache.org>.
EdColeman commented on code in PR #3876:
URL: https://github.com/apache/accumulo/pull/3876#discussion_r1384187005
##########
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java:
##########
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.upgrade;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
+import static org.apache.accumulo.core.metadata.schema.UpgraderDeprecatedConstants.ChoppedColumnFamily;
+import static org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.JsonSyntaxException;
+
+public class Upgrader11to12 implements Upgrader {
+
+ private static final Logger log = LoggerFactory.getLogger(Upgrader11to12.class);
+
+ @Override
+ public void upgradeZookeeper(@NonNull ServerContext context) {
+ log.debug("Upgrade ZooKeeper: upgrading to data version {}", METADATA_FILE_JSON_ENCODING);
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+ if (RootTabletMetadata.Data.needsUpgrade(json)) {
+ log.info("Root metadata in ZooKeeper before upgrade: {}", json);
+ RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ log.info("Root metadata in ZooKeeper after upgrade: {}", rtm.toJson());
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8), stat.getVersion());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper because of ZooKeeper exception", ex);
Review Comment:
Fixed in cc6b6e25801c
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org