You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by to...@apache.org on 2016/09/08 11:16:28 UTC
svn commit: r1759778 - in /jackrabbit/oak/trunk/oak-upgrade/src:
main/java/org/apache/jackrabbit/oak/upgrade/cli/
main/java/org/apache/jackrabbit/oak/upgrade/cli/blob/
main/java/org/apache/jackrabbit/oak/upgrade/cli/node/
main/java/org/apache/jackrabbi...
Author: tomekr
Date: Thu Sep 8 11:16:28 2016
New Revision: 1759778
URL: http://svn.apache.org/viewvc?rev=1759778&view=rev
Log:
OAK-4639: Enable --missingblobstore by default for the sidegrade
Added:
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/blob/ConstantBlobStoreFactory.java
- copied, changed from r1759760, jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/NodeStoreFactory.java
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/DatastoreArguments.java
jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/blob/CopyBinariesTest.java
Removed:
jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/blob/CopyReferencesTest.java
jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/blob/MissingBlobStoreTest.java
Modified:
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/MigrationFactory.java
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/JdbcFactory.java
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/MongoFactory.java
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/NodeStoreFactory.java
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentFactory.java
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentTarFactory.java
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/StoreFactory.java
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationOptions.java
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/OptionParserFactory.java
jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/StoreArguments.java
jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/AbstractOak2OakTest.java
jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/JdbcNodeStoreContainer.java
jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentNodeStoreContainer.java
jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentTarNodeStoreContainer.java
Modified: jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/MigrationFactory.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/MigrationFactory.java?rev=1759778&r1=1759777&r2=1759778&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/MigrationFactory.java (original)
+++ jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/MigrationFactory.java Thu Sep 8 11:16:28 2016
@@ -29,6 +29,7 @@ import org.apache.jackrabbit.oak.spi.com
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.apache.jackrabbit.oak.upgrade.RepositorySidegrade;
import org.apache.jackrabbit.oak.upgrade.RepositoryUpgrade;
+import org.apache.jackrabbit.oak.upgrade.cli.parser.DatastoreArguments;
import org.apache.jackrabbit.oak.upgrade.cli.parser.MigrationOptions;
import org.apache.jackrabbit.oak.upgrade.cli.parser.StoreArguments;
@@ -57,28 +58,21 @@ public class MigrationFactory {
}
public RepositorySidegrade createSidegrade() throws IOException {
- BlobStore srcBlobStore = stores.getSrcBlobStore().create(closer);
+ BlobStore srcBlobStore = stores.getDatastores().getSrcBlobStore().create(closer);
NodeStore srcStore = stores.getSrcStore().create(srcBlobStore, closer);
NodeStore dstStore = createTarget(closer, srcBlobStore);
return createSidegrade(srcStore, dstStore);
}
protected NodeStore createTarget(Closer closer, BlobStore srcBlobStore) throws IOException {
- BlobStore dstBlobStore;
- if (options.isCopyBinariesByReference()) {
- dstBlobStore = srcBlobStore;
- } else {
- dstBlobStore = stores.getDstBlobStore().create(closer);
- }
+ BlobStore dstBlobStore = stores.getDatastores().getDstBlobStore(srcBlobStore).create(closer);
NodeStore dstStore = stores.getDstStore().create(dstBlobStore, closer);
return dstStore;
}
protected RepositoryUpgrade createUpgrade(RepositoryContext source, NodeStore dstStore) {
RepositoryUpgrade upgrade = new RepositoryUpgrade(source, dstStore);
- if (source.getDataStore() != null && options.isCopyBinariesByReference()) {
- upgrade.setCopyBinariesByReference(true);
- }
+ upgrade.setCopyBinariesByReference(stores.getDatastores().getBlobMigrationCase() == DatastoreArguments.BlobMigrationCase.COPY_REFERENCES);
upgrade.setCopyVersions(options.getCopyVersions());
upgrade.setCopyOrphanedVersions(options.getCopyOrphanedVersions());
if (options.getIncludePaths() != null) {
Copied: jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/blob/ConstantBlobStoreFactory.java (from r1759760, jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/NodeStoreFactory.java)
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/blob/ConstantBlobStoreFactory.java?p2=jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/blob/ConstantBlobStoreFactory.java&p1=jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/NodeStoreFactory.java&r1=1759760&r2=1759778&rev=1759778&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/NodeStoreFactory.java (original)
+++ jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/blob/ConstantBlobStoreFactory.java Thu Sep 8 11:16:28 2016
@@ -14,16 +14,23 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.jackrabbit.oak.upgrade.cli.node;
+package org.apache.jackrabbit.oak.upgrade.cli.blob;
+
+import com.google.common.io.Closer;
+import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import java.io.IOException;
-import org.apache.jackrabbit.oak.spi.blob.BlobStore;
-import org.apache.jackrabbit.oak.spi.state.NodeStore;
+public class ConstantBlobStoreFactory implements BlobStoreFactory {
-import com.google.common.io.Closer;
+ private final BlobStore blobStore;
-public interface NodeStoreFactory {
+ public ConstantBlobStoreFactory(BlobStore blobStore) {
+ this.blobStore = blobStore;
+ }
- NodeStore create(BlobStore blobStore, Closer closer) throws IOException;
+ @Override
+ public BlobStore create(Closer closer) throws IOException {
+ return blobStore;
+ }
}
Modified: jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/JdbcFactory.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/JdbcFactory.java?rev=1759778&r1=1759777&r2=1759778&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/JdbcFactory.java (original)
+++ jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/JdbcFactory.java Thu Sep 8 11:16:28 2016
@@ -18,6 +18,7 @@ package org.apache.jackrabbit.oak.upgrad
import org.apache.jackrabbit.oak.plugins.document.DocumentMK;
import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
+import org.apache.jackrabbit.oak.plugins.document.rdb.RDBBlobStore;
import org.apache.jackrabbit.oak.plugins.document.rdb.RDBDataSourceFactory;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
@@ -28,6 +29,7 @@ import com.google.common.io.Closer;
import javax.sql.DataSource;
import java.io.Closeable;
+import java.io.IOException;
public class JdbcFactory implements NodeStoreFactory {
@@ -53,15 +55,11 @@ public class JdbcFactory implements Node
@Override
public NodeStore create(BlobStore blobStore, Closer closer) {
- DataSource ds = RDBDataSourceFactory.forJdbcUrl(jdbcUri, user, password);
- if (ds instanceof Closeable) {
- closer.register((Closeable)ds);
- }
DocumentMK.Builder builder = MongoFactory.getBuilder(cacheSize);
if (blobStore != null) {
builder.setBlobStore(blobStore);
}
- builder.setRDBConnection(ds);
+ builder.setRDBConnection(getDataSource(closer));
log.info("Initialized DocumentNodeStore on RDB with Cache size : {} MB, Fast migration : {}", cacheSize,
builder.isDisableBranches());
DocumentNodeStore documentNodeStore = builder.getNodeStore();
@@ -69,6 +67,28 @@ public class JdbcFactory implements Node
return documentNodeStore;
}
+ private DataSource getDataSource(Closer closer) {
+ DataSource ds = RDBDataSourceFactory.forJdbcUrl(jdbcUri, user, password);
+ if (ds instanceof Closeable) {
+ closer.register((Closeable)ds);
+ }
+ return ds;
+ }
+
+ @Override
+ public boolean hasExternalBlobReferences() throws IOException {
+ Closer closer = Closer.create();
+ try {
+ DataSource ds = getDataSource(closer);
+ RDBBlobStore blobStore = new RDBBlobStore(ds);
+ return !blobStore.getAllChunkIds(0).hasNext();
+ } catch(Throwable e) {
+ throw closer.rethrow(e);
+ } finally {
+ closer.close();
+ }
+ }
+
@Override
public String toString() {
return String.format("DocumentNodeStore[%s]", jdbcUri);
Modified: jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/MongoFactory.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/MongoFactory.java?rev=1759778&r1=1759777&r2=1759778&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/MongoFactory.java (original)
+++ jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/MongoFactory.java Thu Sep 8 11:16:28 2016
@@ -16,8 +16,10 @@
*/
package org.apache.jackrabbit.oak.upgrade.cli.node;
+import com.mongodb.DB;
import org.apache.jackrabbit.oak.plugins.document.DocumentMK;
import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
+import org.apache.jackrabbit.oak.plugins.document.mongo.MongoBlobStore;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
@@ -44,22 +46,39 @@ public class MongoFactory implements Nod
@Override
public NodeStore create(BlobStore blobStore, Closer closer) throws UnknownHostException {
+ DocumentMK.Builder builder = getBuilder(cacheSize);
+ builder.setMongoDB(getDB(closer));
+ if (blobStore != null) {
+ builder.setBlobStore(blobStore);
+ }
+ DocumentNodeStore documentNodeStore = builder.getNodeStore();
+ closer.register(asCloseable(documentNodeStore));
+ return documentNodeStore;
+ }
+
+ private DB getDB(Closer closer) throws UnknownHostException {
String db;
if (uri.getDatabase() == null) {
db = "aem-author"; // assume an author instance
} else {
db = uri.getDatabase();
}
- DocumentMK.Builder builder = getBuilder(cacheSize);
MongoClient client = new MongoClient(uri);
closer.register(asCloseable(client));
- builder.setMongoDB(client.getDB(db));
- if (blobStore != null) {
- builder.setBlobStore(blobStore);
+ return client.getDB(db);
+ }
+
+ @Override
+ public boolean hasExternalBlobReferences() throws IOException {
+ Closer closer = Closer.create();
+ try {
+ MongoBlobStore mongoBlobStore = new MongoBlobStore(getDB(closer));
+ return !mongoBlobStore.getAllChunkIds(0).hasNext();
+ } catch(Throwable e) {
+ throw closer.rethrow(e);
+ } finally {
+ closer.close();
}
- DocumentNodeStore documentNodeStore = builder.getNodeStore();
- closer.register(asCloseable(documentNodeStore));
- return documentNodeStore;
}
static Closeable asCloseable(final DocumentNodeStore documentNodeStore) {
Modified: jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/NodeStoreFactory.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/NodeStoreFactory.java?rev=1759778&r1=1759777&r2=1759778&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/NodeStoreFactory.java (original)
+++ jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/NodeStoreFactory.java Thu Sep 8 11:16:28 2016
@@ -26,4 +26,6 @@ import com.google.common.io.Closer;
public interface NodeStoreFactory {
NodeStore create(BlobStore blobStore, Closer closer) throws IOException;
+
+ boolean hasExternalBlobReferences() throws IOException;
}
Modified: jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentFactory.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentFactory.java?rev=1759778&r1=1759777&r2=1759778&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentFactory.java (original)
+++ jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentFactory.java Thu Sep 8 11:16:28 2016
@@ -20,6 +20,7 @@ import java.io.Closeable;
import java.io.File;
import java.io.IOException;
+import org.apache.jackrabbit.oak.plugins.blob.ReferenceCollector;
import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeBuilder;
import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeState;
import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore;
@@ -33,6 +34,8 @@ import org.apache.jackrabbit.oak.spi.sta
import com.google.common.io.Closer;
+import javax.annotation.Nullable;
+
import static com.google.common.base.Preconditions.checkArgument;
public class SegmentFactory implements NodeStoreFactory {
@@ -98,6 +101,38 @@ public class SegmentFactory implements N
});
}
+ @Override
+ public boolean hasExternalBlobReferences() throws IOException {
+ Builder builder = FileStore.builder(new File(dir, "segmentstore"));
+ builder.withMaxFileSize(256);
+ if (disableMmap) {
+ builder.withMemoryMapping(false);
+ } else {
+ builder.withDefaultMemoryMapping();
+ }
+ FileStore fs;
+ try {
+ fs = builder.build();
+ } catch (InvalidFileStoreVersionException e) {
+ throw new IOException(e);
+ }
+ try {
+ fs.getTracker().collectBlobReferences(new ReferenceCollector() {
+ @Override
+ public void addReference(String reference, @Nullable String nodeId) {
+ // FIXME the collector should allow to stop processing
+ // see java.nio.file.FileVisitor
+ throw new ExternalBlobFound();
+ }
+ });
+ return false;
+ } catch (ExternalBlobFound e) {
+ return true;
+ } finally {
+ fs.close();
+ }
+ }
+
public File getRepositoryDir() {
return dir;
}
@@ -115,4 +150,7 @@ public class SegmentFactory implements N
public String toString() {
return String.format("SegmentNodeStore[%s]", dir);
}
+
+ private static class ExternalBlobFound extends RuntimeException {
+ }
}
Modified: jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentTarFactory.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentTarFactory.java?rev=1759778&r1=1759777&r2=1759778&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentTarFactory.java (original)
+++ jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentTarFactory.java Thu Sep 8 11:16:28 2016
@@ -24,6 +24,7 @@ import java.io.File;
import java.io.IOException;
import com.google.common.io.Closer;
+import org.apache.jackrabbit.oak.plugins.blob.ReferenceCollector;
import org.apache.jackrabbit.oak.segment.SegmentNodeBuilder;
import org.apache.jackrabbit.oak.segment.SegmentNodeState;
import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
@@ -35,6 +36,8 @@ import org.apache.jackrabbit.oak.spi.sta
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import javax.annotation.Nullable;
+
public class SegmentTarFactory implements NodeStoreFactory {
private final File dir;
@@ -97,6 +100,39 @@ public class SegmentTarFactory implement
});
}
+
+ @Override
+ public boolean hasExternalBlobReferences() throws IOException {
+ final FileStoreBuilder builder = fileStoreBuilder(new File(dir, "segmentstore"));
+ builder.withMaxFileSize(256);
+ if (disableMmap) {
+ builder.withMemoryMapping(false);
+ } else {
+ builder.withDefaultMemoryMapping();
+ }
+ final FileStore fs;
+ try {
+ fs = builder.build();
+ } catch (InvalidFileStoreVersionException e) {
+ throw new IOException(e);
+ }
+ try {
+ fs.collectBlobReferences(new ReferenceCollector() {
+ @Override
+ public void addReference(String reference, @Nullable String nodeId) {
+ // FIXME the collector should allow to stop processing
+ // see java.nio.file.FileVisitor
+ throw new ExternalBlobFound();
+ }
+ });
+ return false;
+ } catch (ExternalBlobFound e) {
+ return true;
+ } finally {
+ fs.close();
+ }
+ }
+
public File getRepositoryDir() {
return dir;
}
@@ -114,4 +150,7 @@ public class SegmentTarFactory implement
public String toString() {
return String.format("SegmentTarNodeStore[%s]", dir);
}
+
+ private static class ExternalBlobFound extends RuntimeException {
+ }
}
Modified: jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/StoreFactory.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/StoreFactory.java?rev=1759778&r1=1759777&r2=1759778&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/StoreFactory.java (original)
+++ jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/StoreFactory.java Thu Sep 8 11:16:28 2016
@@ -59,4 +59,12 @@ public class StoreFactory {
public boolean isJcr2() {
return jcr2Factory != null;
}
+
+ public boolean hasExternalBlobReferences() throws IOException {
+ if (isJcr2()) {
+ return true;
+ } else {
+ return nodeStoreFactory.hasExternalBlobReferences();
+ }
+ }
}
Added: jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/DatastoreArguments.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/DatastoreArguments.java?rev=1759778&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/DatastoreArguments.java (added)
+++ jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/DatastoreArguments.java Thu Sep 8 11:16:28 2016
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.upgrade.cli.parser;
+
+import org.apache.commons.lang.text.StrSubstitutor;
+import org.apache.jackrabbit.oak.spi.blob.BlobStore;
+import org.apache.jackrabbit.oak.upgrade.cli.blob.BlobStoreFactory;
+import org.apache.jackrabbit.oak.upgrade.cli.blob.ConstantBlobStoreFactory;
+import org.apache.jackrabbit.oak.upgrade.cli.blob.DummyBlobStoreFactory;
+import org.apache.jackrabbit.oak.upgrade.cli.blob.FileBlobStoreFactory;
+import org.apache.jackrabbit.oak.upgrade.cli.blob.FileDataStoreFactory;
+import org.apache.jackrabbit.oak.upgrade.cli.blob.MissingBlobStoreFactory;
+import org.apache.jackrabbit.oak.upgrade.cli.blob.S3DataStoreFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static com.google.common.collect.Maps.newHashMap;
+import static org.apache.jackrabbit.oak.upgrade.cli.parser.OptionParserFactory.COPY_BINARIES;
+import static org.apache.jackrabbit.oak.upgrade.cli.parser.OptionParserFactory.DST_FBS;
+import static org.apache.jackrabbit.oak.upgrade.cli.parser.OptionParserFactory.DST_FDS;
+import static org.apache.jackrabbit.oak.upgrade.cli.parser.OptionParserFactory.DST_S3;
+import static org.apache.jackrabbit.oak.upgrade.cli.parser.OptionParserFactory.DST_S3_CONFIG;
+import static org.apache.jackrabbit.oak.upgrade.cli.parser.OptionParserFactory.IGNORE_MISSING_BINARIES;
+import static org.apache.jackrabbit.oak.upgrade.cli.parser.OptionParserFactory.SRC_FBS;
+import static org.apache.jackrabbit.oak.upgrade.cli.parser.OptionParserFactory.SRC_FDS;
+import static org.apache.jackrabbit.oak.upgrade.cli.parser.OptionParserFactory.SRC_S3;
+import static org.apache.jackrabbit.oak.upgrade.cli.parser.OptionParserFactory.SRC_S3_CONFIG;
+import static org.apache.jackrabbit.oak.upgrade.cli.parser.StoreType.JCR2_DIR_XML;
+
+/**
+ * This class parses the input provided by the user and analyses the given node stores
+ * in order to find out which datastore combination should be used for the migration.
+ *
+ * The desired outcome for the combinations of user input can be found in the table below.
+ * The table is a kind of heuristics that tries to match the user intentions.
+ * <pre>
+ * For sidegrade:
+ || src blobstore defined || src blobs embedded || dst blobstore defined || --copy-binaries || outcome src blobstore || outcome action
+ | - | - | - | - | missing | copy references¹
+ | - | - | - | + | missing | (x) not supported
+ | - | - | + | * | missing | (x) not supported
+ | - | + | - | * | embedded | copy to embedded
+ | - | + | + | * | embedded | copy to defined blobstore
+ | + | * | - | - | as in src | copy references
+ | + | * | - | + | as in src | copy to embedded
+ | + | * | + | * | as in src | copy to defined blobstore
+
+ ¹ - (x) not supported for SegmentMK -> MongoMK migration
+
+ For upgrade:
+
+ || dst blobstore defined || --copy-binaries || outcome src blobstore || outcome action
+ | - | - | defined by JCR2 | copy references
+ | - | + | defined by JCR2 | copy to embedded
+ | + | * | defined by JCR2 | copy to defined blobstore
+ * </pre>
+ */
+public class DatastoreArguments {
+
+ private static final Logger log = LoggerFactory.getLogger(DatastoreArguments.class);
+
+ private final BlobStoreFactory definedSrcBlob;
+
+ private final BlobStoreFactory definedDstBlob;
+
+ private final StoreArguments storeArguments;
+
+ private final BlobMigrationCase blobMigrationCase;
+
+ private final MigrationCliArguments parser;
+
+ public DatastoreArguments(MigrationCliArguments parser, StoreArguments storeArguments) throws CliArgumentException {
+ this.storeArguments = storeArguments;
+ this.parser = parser;
+
+ try {
+ blobMigrationCase = discoverBlobMigrationCase();
+ } catch (IOException e) {
+ log.error("Can't figure out the right blob migration path", e);
+ throw new CliArgumentException(1);
+ }
+
+ if (blobMigrationCase == BlobMigrationCase.UNSUPPORTED) {
+ throw new CliArgumentException("This combination of data- and node-stores is not supported", 1);
+ }
+
+ try {
+ definedSrcBlob = isSrcBlobStoreDefined() ? getDefinedSrcBlobStore() : null;
+ definedDstBlob = isDstBlobStoreDefined() ? getDefinedDstBlobStore() : null;
+ } catch(IOException e) {
+ log.error("Can't read the blob configuration", e);
+ throw new CliArgumentException(1);
+ }
+
+ log.info(blobMigrationCase.getDescription(this));
+ }
+
+ public BlobStoreFactory getSrcBlobStore() throws IOException {
+ BlobStoreFactory result;
+ if (isSrcBlobStoreDefined()) {
+ result = definedSrcBlob;
+ } else if (blobMigrationCase == BlobMigrationCase.COPY_REFERENCES) {
+ result = new MissingBlobStoreFactory();
+ } else {
+ result = new DummyBlobStoreFactory(); // embedded
+ }
+ log.info("Source blob store: {}", result);
+ return result;
+ }
+
+ public BlobStoreFactory getDstBlobStore(BlobStore srcBlobStore) throws IOException {
+ BlobStoreFactory result;
+ if (isDstBlobStoreDefined()) {
+ result = definedDstBlob;
+ } else if (blobMigrationCase == BlobMigrationCase.COPY_REFERENCES && (isSrcBlobStoreDefined() || storeArguments.getSrcType() == JCR2_DIR_XML)) {
+ result = new ConstantBlobStoreFactory(srcBlobStore);
+ } else if (blobMigrationCase == BlobMigrationCase.COPY_REFERENCES) {
+ result = new MissingBlobStoreFactory();
+ } else {
+ result = new DummyBlobStoreFactory(); // embedded
+ }
+
+ log.info("Destination blob store: {}", result);
+ return result;
+ }
+
+ public boolean isSrcBlobStoreDefined() {
+ return parser.hasOption(SRC_FBS) || (parser.hasOption(SRC_S3_CONFIG) && parser.hasOption(SRC_S3)) || (parser.hasOption(SRC_FDS));
+ }
+
+ public boolean isDstBlobStoreDefined() {
+ return parser.hasOption(DST_FBS) || (parser.hasOption(DST_S3_CONFIG) && parser.hasOption(DST_S3)) || parser.hasOption(DST_FDS);
+ }
+
+ private BlobStoreFactory getDefinedSrcBlobStore() throws IOException {
+ boolean ignoreMissingBinaries = parser.hasOption(IGNORE_MISSING_BINARIES);
+ if (parser.hasOption(SRC_FBS)) {
+ return new FileBlobStoreFactory(parser.getOption(SRC_FBS));
+ } else if (parser.hasOption(SRC_S3_CONFIG) && parser.hasOption(SRC_S3)) {
+ return new S3DataStoreFactory(parser.getOption(SRC_S3_CONFIG), parser.getOption(SRC_S3), ignoreMissingBinaries);
+ } else if (parser.hasOption(SRC_FDS)) {
+ return new FileDataStoreFactory(parser.getOption(SRC_FDS), ignoreMissingBinaries);
+ } else {
+ return null;
+ }
+ }
+
+ private BlobStoreFactory getDefinedDstBlobStore() throws IOException {
+ if (parser.hasOption(DST_FBS)) {
+ return new FileBlobStoreFactory(parser.getOption(DST_FBS));
+ } else if (parser.hasOption(DST_S3_CONFIG) && parser.hasOption(DST_S3)) {
+ return new S3DataStoreFactory(parser.getOption(DST_S3_CONFIG), parser.getOption(DST_S3), false);
+ } else if (parser.hasOption(DST_FDS)) {
+ return new FileDataStoreFactory(parser.getOption(DST_FDS), false);
+ } else {
+ return null;
+ }
+ }
+
+ public enum BlobMigrationCase {
+ COPY_REFERENCES("Only blob references will be copied"),
+ EMBEDDED_TO_EMBEDDED("Blobs embedded in ${srcnode} will be embedded in ${dstnode}"),
+ EMBEDDED_TO_EXTERNAL("Blobs embedded in ${srcnode} will be copied to ${dstblob}"),
+ EXTERNAL_TO_EMBEDDED("Blobs stored in ${srcblob} will be embedded in ${dstnode}"),
+ EXTERNAL_TO_EXTERNAL("Blobs stored in ${srcblob} will be copied to ${dstblob}"),
+ UNSUPPORTED("Unsupported case");
+
+ private final String description;
+
+ BlobMigrationCase(String description) {
+ this.description = description;
+ }
+
+ private String getDescription(DatastoreArguments datastoreArguments) {
+ Map<String, String> map = newHashMap();
+ map.put("srcnode", datastoreArguments.storeArguments.getSrcDescriptor());
+ map.put("dstnode", datastoreArguments.storeArguments.getDstDescriptor());
+
+ if (datastoreArguments.storeArguments.getSrcType() == JCR2_DIR_XML) {
+ map.put("srcblob", "CRX2 datastore");
+ } else {
+ map.put("srcblob", datastoreArguments.definedSrcBlob == null ? "?" : datastoreArguments.definedSrcBlob.toString());
+ }
+ map.put("dstblob", datastoreArguments.definedDstBlob == null ? "?" : datastoreArguments.definedDstBlob.toString());
+
+ StrSubstitutor subst = new StrSubstitutor(map);
+ return subst.replace(description);
+ }
+
+ }
+
+ public BlobMigrationCase getBlobMigrationCase() {
+ return blobMigrationCase;
+ }
+
+ private BlobMigrationCase discoverBlobMigrationCase() throws IOException {
+ boolean srcDefined = isSrcBlobStoreDefined() || storeArguments.getSrcType() == JCR2_DIR_XML;
+ boolean dstDefined = isDstBlobStoreDefined();
+ boolean srcEmbedded = !storeArguments.srcHasExternalBlobReferences();
+ boolean copyBinaries = parser.hasOption(COPY_BINARIES);
+
+ boolean srcSegment = storeArguments.getSrcType().isSegment();
+ boolean dstSegment = storeArguments.getDstType().isSegment();
+
+ // default case, no datastore-related arguments given, but blobs are stored externally
+ if (!srcDefined && !dstDefined && !srcEmbedded && !copyBinaries) {
+ if (srcSegment && !dstSegment) { // segment -> document is not supported for this case
+ return BlobMigrationCase.UNSUPPORTED;
+ } else { // we try to copy references using MissingBlobStore
+ return BlobMigrationCase.COPY_REFERENCES;
+ }
+ // can't copy binaries if they are stored externally and we don't know where
+ } else if (!srcDefined && !dstDefined && !srcEmbedded && copyBinaries) {
+ return BlobMigrationCase.UNSUPPORTED;
+ // can't copy binaries if they are stored externally and we don't know where
+ // (even if the destination datastore is defined)
+ } else if (!srcDefined && !srcEmbedded && dstDefined) {
+ return BlobMigrationCase.UNSUPPORTED;
+ // source is embedded and no destination given
+ } else if (!srcDefined && srcEmbedded && !dstDefined) {
+ return BlobMigrationCase.EMBEDDED_TO_EMBEDDED;
+ // source is embedded and the destination is given
+ } else if (!srcDefined && srcEmbedded && dstDefined) {
+ return BlobMigrationCase.EMBEDDED_TO_EXTERNAL;
+ // source is given, no destination, but also no --copy-binaries -> copy references
+ } else if (srcDefined && !dstDefined && !copyBinaries) {
+ return BlobMigrationCase.COPY_REFERENCES;
+ // source is given, no destination, but --copy-binaries -> copy to embedded
+ } else if (srcDefined && !dstDefined && copyBinaries) {
+ return BlobMigrationCase.EXTERNAL_TO_EMBEDDED;
+ // source and destination is given
+ } else if (srcDefined && dstDefined) {
+ return BlobMigrationCase.EXTERNAL_TO_EXTERNAL;
+ }
+ return BlobMigrationCase.UNSUPPORTED;
+ }
+}
Modified: jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationOptions.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationOptions.java?rev=1759778&r1=1759777&r2=1759778&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationOptions.java (original)
+++ jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationOptions.java Thu Sep 8 11:16:28 2016
@@ -30,7 +30,7 @@ public class MigrationOptions {
private static final DateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd");
- private final boolean copyBinariesByReference;
+ private final boolean copyBinaries;
private final boolean disableMmap;
@@ -59,8 +59,8 @@ public class MigrationOptions {
private final boolean ignoreMissingBinaries;
public MigrationOptions(MigrationCliArguments args) {
- this.copyBinariesByReference = !args.hasOption(OptionParserFactory.COPY_BINARIES);
this.disableMmap = args.hasOption(OptionParserFactory.DISABLE_MMAP);
+ this.copyBinaries = args.hasOption(OptionParserFactory.COPY_BINARIES);
if (args.hasOption(OptionParserFactory.CACHE_SIZE)) {
this.cacheSizeInMB = args.getIntOption(OptionParserFactory.CACHE_SIZE);
} else {
@@ -91,8 +91,8 @@ public class MigrationOptions {
logOptions();
}
- public boolean isCopyBinariesByReference() {
- return copyBinariesByReference;
+ public boolean isCopyBinaries() {
+ return copyBinaries;
}
public boolean isDisableMmap() {
@@ -148,12 +148,6 @@ public class MigrationOptions {
}
private void logOptions() {
- if (copyBinariesByReference) {
- log.info("DataStore needs to be shared with new repository");
- } else {
- log.info("Binary content would be copied to the NodeStore.");
- }
-
if (disableMmap) {
log.info("Disabling memory mapped file access for Segment Store");
}
@@ -233,4 +227,4 @@ public class MigrationOptions {
return calendar;
}
-}
\ No newline at end of file
+}
Modified: jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/OptionParserFactory.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/OptionParserFactory.java?rev=1759778&r1=1759777&r2=1759778&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/OptionParserFactory.java (original)
+++ jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/OptionParserFactory.java Thu Sep 8 11:16:28 2016
@@ -59,8 +59,6 @@ public class OptionParserFactory {
public static final String DST_S3 = "s3datastore";
public static final String DST_S3_CONFIG = "s3config";
-
- public static final String MISSING_BLOBSTORE = "missingblobstore";
public static final String COPY_VERSIONS = "copy-versions";
@@ -109,7 +107,6 @@ public class OptionParserFactory {
op.accepts(DST_S3, "Datastore directory to be used for the target S3").withRequiredArg().ofType(String.class);
op.accepts(DST_S3_CONFIG, "Configuration file for the target S3DataStore").withRequiredArg()
.ofType(String.class);
- op.accepts(MISSING_BLOBSTORE, "Try to upgrade the NodeStore without access to the external Datastore");
op.accepts(IGNORE_MISSING_BINARIES, "Don't break the migration if some binaries are missing");
}
Modified: jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/StoreArguments.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/StoreArguments.java?rev=1759778&r1=1759777&r2=1759778&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/StoreArguments.java (original)
+++ jackrabbit/oak/trunk/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/StoreArguments.java Thu Sep 8 11:16:28 2016
@@ -23,32 +23,13 @@ import java.util.Iterator;
import java.util.List;
import org.apache.jackrabbit.oak.plugins.segment.SegmentVersion;
-import org.apache.jackrabbit.oak.upgrade.cli.blob.BlobStoreFactory;
-import org.apache.jackrabbit.oak.upgrade.cli.blob.DummyBlobStoreFactory;
-import org.apache.jackrabbit.oak.upgrade.cli.blob.FileBlobStoreFactory;
-import org.apache.jackrabbit.oak.upgrade.cli.blob.FileDataStoreFactory;
-import org.apache.jackrabbit.oak.upgrade.cli.blob.MissingBlobStoreFactory;
-import org.apache.jackrabbit.oak.upgrade.cli.blob.S3DataStoreFactory;
import org.apache.jackrabbit.oak.upgrade.cli.node.StoreFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.jackrabbit.oak.upgrade.cli.parser.OptionParserFactory.IGNORE_MISSING_BINARIES;
-import static org.apache.jackrabbit.oak.upgrade.cli.parser.OptionParserFactory.SRC_FBS;
-import static org.apache.jackrabbit.oak.upgrade.cli.parser.OptionParserFactory.SRC_FDS;
-import static org.apache.jackrabbit.oak.upgrade.cli.parser.OptionParserFactory.SRC_S3;
-import static org.apache.jackrabbit.oak.upgrade.cli.parser.OptionParserFactory.SRC_S3_CONFIG;
-import static org.apache.jackrabbit.oak.upgrade.cli.parser.OptionParserFactory.DST_FBS;
-import static org.apache.jackrabbit.oak.upgrade.cli.parser.OptionParserFactory.DST_FDS;
-import static org.apache.jackrabbit.oak.upgrade.cli.parser.OptionParserFactory.DST_S3;
-import static org.apache.jackrabbit.oak.upgrade.cli.parser.OptionParserFactory.DST_S3_CONFIG;
-import static org.apache.jackrabbit.oak.upgrade.cli.parser.OptionParserFactory.MISSING_BLOBSTORE;
-
import static org.apache.jackrabbit.oak.upgrade.cli.parser.StoreType.JCR2_DIR;
import static org.apache.jackrabbit.oak.upgrade.cli.parser.StoreType.JCR2_DIR_XML;
import static org.apache.jackrabbit.oak.upgrade.cli.parser.StoreType.JCR2_XML;
-import static org.apache.jackrabbit.oak.upgrade.cli.parser.StoreType.JDBC;
-import static org.apache.jackrabbit.oak.upgrade.cli.parser.StoreType.MONGO;
import static org.apache.jackrabbit.oak.upgrade.cli.parser.StoreType.SEGMENT;
import static org.apache.jackrabbit.oak.upgrade.cli.parser.StoreType.SEGMENT_TAR;
import static org.apache.jackrabbit.oak.upgrade.cli.parser.StoreType.getMatchingType;
@@ -69,6 +50,10 @@ public class StoreArguments {
private final StoreDescriptor dst;
+ private final DatastoreArguments datastores;
+
+ private Boolean srcHasExternalBlobRefs;
+
public StoreArguments(MigrationCliArguments parser) throws CliArgumentException {
this.parser = parser;
@@ -84,9 +69,7 @@ public class StoreArguments {
logSegmentVersion();
}
- if (parser.hasOption(MISSING_BLOBSTORE) && !nodeStoresSupportMissingBlobStore()) {
- throw new CliArgumentException("This combination of node stores is not supported by the --" + MISSING_BLOBSTORE, 1);
- }
+ datastores = new DatastoreArguments(parser, this);
}
public StoreFactory getSrcStore() {
@@ -97,6 +80,10 @@ public class StoreArguments {
return dst.getFactory(MigrationDirection.DST, parser);
}
+ public DatastoreArguments getDatastores() {
+ return datastores;
+ }
+
public StoreType getSrcType() {
return src.getType();
}
@@ -105,39 +92,12 @@ public class StoreArguments {
return dst.getType();
}
- public BlobStoreFactory getSrcBlobStore() throws IOException {
- BlobStoreFactory factory;
- boolean ignoreMissingBinaries = parser.hasOption(IGNORE_MISSING_BINARIES);
- if (parser.hasOption(SRC_FBS)) {
- factory = new FileBlobStoreFactory(parser.getOption(SRC_FBS));
- } else if (parser.hasOption(SRC_S3_CONFIG) && parser.hasOption(SRC_S3)) {
- factory = new S3DataStoreFactory(parser.getOption(SRC_S3_CONFIG), parser.getOption(SRC_S3), ignoreMissingBinaries);
- } else if (parser.hasOption(SRC_FDS)) {
- factory = new FileDataStoreFactory(parser.getOption(SRC_FDS), ignoreMissingBinaries);
- } else if (parser.hasOption(MISSING_BLOBSTORE)) {
- factory = new MissingBlobStoreFactory();
- } else {
- factory = new DummyBlobStoreFactory();
- }
- log.info("Source blob store: {}", factory);
- return factory;
- }
-
- public BlobStoreFactory getDstBlobStore() throws IOException {
- BlobStoreFactory factory;
- if (parser.hasOption(DST_FBS)) {
- factory = new FileBlobStoreFactory(parser.getOption(DST_FBS));
- } else if (parser.hasOption(DST_S3_CONFIG) && parser.hasOption(DST_S3)) {
- factory = new S3DataStoreFactory(parser.getOption(DST_S3_CONFIG), parser.getOption(DST_S3), false);
- } else if (parser.hasOption(DST_FDS)) {
- factory = new FileDataStoreFactory(parser.getOption(DST_FDS), false);
- } else if (parser.hasOption(MISSING_BLOBSTORE)) {
- factory = new MissingBlobStoreFactory();
- } else {
- factory = new DummyBlobStoreFactory();
- }
- log.info("Destination blob store: {}", factory);
- return factory;
+ String getSrcDescriptor() {
+ return src.toString();
+ }
+
+ String getDstDescriptor() {
+ return dst.toString();
}
public boolean isInPlaceUpgrade() {
@@ -151,6 +111,13 @@ public class StoreArguments {
return src.getPaths();
}
+ boolean srcHasExternalBlobReferences() throws IOException {
+ if (srcHasExternalBlobRefs == null) {
+ srcHasExternalBlobRefs = src.getFactory(StoreArguments.MigrationDirection.SRC, parser).hasExternalBlobReferences();
+ }
+ return srcHasExternalBlobRefs;
+ }
+
private static List<StoreDescriptor> createStoreDescriptors(List<String> arguments) throws CliArgumentException {
List<StoreDescriptor> descriptors = mapToStoreDescriptors(arguments);
mergeCrx2Descriptors(descriptors);
@@ -256,21 +223,6 @@ public class StoreArguments {
}
}
- private boolean nodeStoresSupportMissingBlobStore() {
- StoreType srcType = src.getType();
- StoreType dstType = dst.getType();
-
- if (srcType.isSegment() && dstType.isSegment()) {
- return true;
- } else if (srcType == MONGO && (dstType.isSegment() || dstType == MONGO)) {
- return true;
- } else if (srcType == JDBC && (dstType.isSegment() || dstType == JDBC)) {
- return true;
- } else {
- return false;
- }
- }
-
enum MigrationDirection {
SRC, DST
}
@@ -310,5 +262,6 @@ public class StoreArguments {
return String.format("%s%s", type, Arrays.toString(getPaths()));
}
}
+
}
}
Modified: jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/AbstractOak2OakTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/AbstractOak2OakTest.java?rev=1759778&r1=1759777&r2=1759778&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/AbstractOak2OakTest.java (original)
+++ jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/AbstractOak2OakTest.java Thu Sep 8 11:16:28 2016
@@ -56,6 +56,7 @@ import org.apache.jackrabbit.oak.upgrade
import org.apache.jackrabbit.oak.upgrade.cli.container.NodeStoreContainer;
import org.apache.jackrabbit.oak.upgrade.cli.container.SegmentNodeStoreContainer;
import org.apache.jackrabbit.oak.upgrade.cli.container.SegmentTarNodeStoreContainer;
+import org.apache.jackrabbit.oak.upgrade.cli.parser.CliArgumentException;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -118,8 +119,12 @@ public abstract class AbstractOak2OakTes
@After
public void clean() throws IOException {
try {
- session.logout();
- repository.shutdown();
+ if (session != null) {
+ session.logout();
+ }
+ if (repository != null) {
+ repository.shutdown();
+ }
} finally {
IOUtils.closeQuietly(getDestinationContainer());
getDestinationContainer().clean();
@@ -147,7 +152,7 @@ public abstract class AbstractOak2OakTes
}
@Test
- public void validateMigration() throws RepositoryException, IOException {
+ public void validateMigration() throws RepositoryException, IOException, CliArgumentException {
verifyContent(session);
verifyBlob(session);
if (supportsCheckpointMigration()) {
Added: jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/blob/CopyBinariesTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/blob/CopyBinariesTest.java?rev=1759778&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/blob/CopyBinariesTest.java (added)
+++ jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/blob/CopyBinariesTest.java Thu Sep 8 11:16:28 2016
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.upgrade.cli.blob;
+
+import static java.util.Arrays.asList;
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import com.google.common.base.Joiner;
+import joptsimple.OptionSet;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.upgrade.cli.AbstractOak2OakTest;
+import org.apache.jackrabbit.oak.upgrade.cli.OakUpgrade;
+import org.apache.jackrabbit.oak.upgrade.cli.container.BlobStoreContainer;
+import org.apache.jackrabbit.oak.upgrade.cli.container.FileDataStoreContainer;
+import org.apache.jackrabbit.oak.upgrade.cli.container.JdbcNodeStoreContainer;
+import org.apache.jackrabbit.oak.upgrade.cli.container.NodeStoreContainer;
+import org.apache.jackrabbit.oak.upgrade.cli.container.SegmentNodeStoreContainer;
+import org.apache.jackrabbit.oak.upgrade.cli.container.SegmentTarNodeStoreContainer;
+import org.apache.jackrabbit.oak.upgrade.cli.parser.CliArgumentException;
+import org.apache.jackrabbit.oak.upgrade.cli.parser.DatastoreArguments;
+import org.apache.jackrabbit.oak.upgrade.cli.parser.MigrationCliArguments;
+import org.apache.jackrabbit.oak.upgrade.cli.parser.OptionParserFactory;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.jcr.RepositoryException;
+
+@RunWith(Parameterized.class)
+public class CopyBinariesTest extends AbstractOak2OakTest {
+
+ private static final Logger log = LoggerFactory.getLogger(CopyBinariesTest.class);
+
+ @Parameterized.Parameters(name = "{0}")
+ public static Collection<Object[]> data() throws IOException {
+ List<Object[]> params = new ArrayList<Object[]>();
+
+ BlobStoreContainer blob = new FileDataStoreContainer();
+ BlobStoreContainer blob2 = new FileDataStoreContainer();
+ params.add(new Object[]{
+ "Copy references, no blobstores defined, segment -> segment",
+ new SegmentNodeStoreContainer(blob),
+ new SegmentNodeStoreContainer(blob),
+ asList(),
+ DatastoreArguments.BlobMigrationCase.COPY_REFERENCES
+ });
+ params.add(new Object[]{
+ "Copy references, no blobstores defined, segment-tar -> segment-tar",
+ new SegmentTarNodeStoreContainer(blob),
+ new SegmentTarNodeStoreContainer(blob),
+ asList(),
+ DatastoreArguments.BlobMigrationCase.COPY_REFERENCES
+ });
+ params.add(new Object[]{
+ "Copy references, no blobstores defined, document -> segment-tar",
+ new JdbcNodeStoreContainer(blob),
+ new SegmentNodeStoreContainer(blob),
+ asList("--src-user=sa", "--src-password=sa"),
+ DatastoreArguments.BlobMigrationCase.COPY_REFERENCES
+ });
+ params.add(new Object[]{
+ "Copy references, no blobstores defined, segment-tar -> document",
+ new SegmentTarNodeStoreContainer(blob),
+ new JdbcNodeStoreContainer(blob),
+ asList("--user=sa", "--password=sa"),
+ DatastoreArguments.BlobMigrationCase.UNSUPPORTED
+ });
+ params.add(new Object[]{
+ "Missing source, external destination",
+ new SegmentTarNodeStoreContainer(blob),
+ new SegmentTarNodeStoreContainer(blob),
+ asList("--datastore=" + blob.getDescription()),
+ DatastoreArguments.BlobMigrationCase.UNSUPPORTED
+ });
+ params.add(new Object[]{
+ "Copy embedded to embedded, no blobstores defined",
+ new SegmentTarNodeStoreContainer(),
+ new SegmentTarNodeStoreContainer(),
+ asList(),
+ DatastoreArguments.BlobMigrationCase.EMBEDDED_TO_EMBEDDED
+ });
+ params.add(new Object[]{
+ "Copy embedded to external, no blobstores defined",
+ new SegmentTarNodeStoreContainer(),
+ new SegmentTarNodeStoreContainer(blob),
+ asList("--datastore=" + blob.getDescription()),
+ DatastoreArguments.BlobMigrationCase.EMBEDDED_TO_EXTERNAL
+ });
+ params.add(new Object[]{
+ "Copy references, src blobstore defined",
+ new SegmentTarNodeStoreContainer(blob),
+ new SegmentTarNodeStoreContainer(blob),
+ asList("--src-datastore=" + blob.getDescription()),
+ DatastoreArguments.BlobMigrationCase.COPY_REFERENCES
+ });
+ params.add(new Object[]{
+ "Copy external to embedded, src blobstore defined",
+ new SegmentTarNodeStoreContainer(blob),
+ new SegmentTarNodeStoreContainer(),
+ asList("--copy-binaries", "--src-datastore=" + blob.getDescription()),
+ DatastoreArguments.BlobMigrationCase.EXTERNAL_TO_EMBEDDED
+ });
+ params.add(new Object[]{
+ "Copy external to external, src blobstore defined",
+ new SegmentTarNodeStoreContainer(blob),
+ new SegmentTarNodeStoreContainer(blob2),
+ asList("--copy-binaries", "--src-datastore=" + blob.getDescription(), "--datastore=" + blob2.getDescription()),
+ DatastoreArguments.BlobMigrationCase.EXTERNAL_TO_EXTERNAL
+ });
+ return params;
+ }
+
+ private final NodeStoreContainer source;
+
+ private final NodeStoreContainer destination;
+
+ private final List<String> args;
+
+ private final DatastoreArguments.BlobMigrationCase blobMigrationCase;
+
+ public CopyBinariesTest(String name, NodeStoreContainer source, NodeStoreContainer destination, List<String> args, DatastoreArguments.BlobMigrationCase blobMigrationCase) throws IOException, CliArgumentException {
+ this.source = source;
+ this.destination = destination;
+ this.args = args;
+ this.blobMigrationCase = blobMigrationCase;
+
+ this.source.clean();
+ this.destination.clean();
+ }
+
+ @Override
+ protected NodeStoreContainer getSourceContainer() {
+ return source;
+ }
+
+ @Override
+ protected NodeStoreContainer getDestinationContainer() {
+ return destination;
+ }
+
+ @Override
+ protected String[] getArgs() {
+ List<String> result = new ArrayList<>(args);
+ result.addAll(asList("--disable-mmap", source.getDescription(), destination.getDescription()));
+ return result.toArray(new String[result.size()]);
+ }
+
+ @Before
+ @Override
+ public void prepare() throws Exception {
+ NodeStore source = getSourceContainer().open();
+ try {
+ initContent(source);
+ } finally {
+ getSourceContainer().close();
+ }
+
+ String[] args = getArgs();
+ log.info("oak2oak {}", Joiner.on(' ').join(args));
+ try {
+ OptionSet options = OptionParserFactory.create().parse(args);
+ MigrationCliArguments cliArgs = new MigrationCliArguments(options);
+ OakUpgrade.migrate(cliArgs);
+ assertEquals(blobMigrationCase, cliArgs.getStoreArguments().getDatastores().getBlobMigrationCase());
+ } catch(CliArgumentException e) {
+ if (blobMigrationCase == DatastoreArguments.BlobMigrationCase.UNSUPPORTED) {
+ return;
+ } else {
+ throw e;
+ }
+ }
+ createSession();
+ }
+
+ @Test
+ @Override
+ public void validateMigration() throws RepositoryException, IOException, CliArgumentException {
+ if (blobMigrationCase == DatastoreArguments.BlobMigrationCase.UNSUPPORTED) {
+ return;
+ }
+ verifyContent(session);
+ verifyBlob(session);
+ }
+}
\ No newline at end of file
Modified: jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/JdbcNodeStoreContainer.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/JdbcNodeStoreContainer.java?rev=1759778&r1=1759777&r2=1759778&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/JdbcNodeStoreContainer.java (original)
+++ jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/JdbcNodeStoreContainer.java Thu Sep 8 11:16:28 2016
@@ -62,7 +62,10 @@ public class JdbcNodeStoreContainer impl
@Override
public void close() {
try {
- closer.close();
+ if (closer != null) {
+ closer.close();
+ closer = null;
+ }
} catch (IOException e) {
LOG.error("Can't close document node store", e);
}
Modified: jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentNodeStoreContainer.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentNodeStoreContainer.java?rev=1759778&r1=1759777&r2=1759778&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentNodeStoreContainer.java (original)
+++ jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentNodeStoreContainer.java Thu Sep 8 11:16:28 2016
@@ -53,6 +53,7 @@ public class SegmentNodeStoreContainer i
@Override
public NodeStore open() throws IOException {
+ directory.mkdirs();
FileStore.Builder builder = FileStore.builder(new File(directory, "segmentstore"));
if (blob != null) {
builder.withBlobStore(blob.open());
Modified: jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentTarNodeStoreContainer.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentTarNodeStoreContainer.java?rev=1759778&r1=1759777&r2=1759778&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentTarNodeStoreContainer.java (original)
+++ jackrabbit/oak/trunk/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentTarNodeStoreContainer.java Thu Sep 8 11:16:28 2016
@@ -53,6 +53,7 @@ public class SegmentTarNodeStoreContaine
@Override
public NodeStore open() throws IOException {
+ directory.mkdirs();
FileStoreBuilder builder = fileStoreBuilder(new File(directory, "segmentstore"));
if (blob != null) {
builder.withBlobStore(blob.open());
@@ -67,7 +68,10 @@ public class SegmentTarNodeStoreContaine
@Override
public void close() {
- fs.close();
+ if (fs != null) {
+ fs.close();
+ fs = null;
+ }
}
@Override