You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by to...@apache.org on 2017/10/12 09:27:32 UTC
svn commit: r1811927 - in /jackrabbit/oak/branches/1.6/oak-upgrade: pom.xml
src/main/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3DataStoreFactory.java
src/test/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3ToInlinedTest.java
Author: tomekr
Date: Thu Oct 12 09:27:32 2017
New Revision: 1811927
URL: http://svn.apache.org/viewvc?rev=1811927&view=rev
Log:
OAK-6611: Many S3DataStore errors during migration with oak-upgrade
Backported from 1.7
Added:
jackrabbit/oak/branches/1.6/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3ToInlinedTest.java
Modified:
jackrabbit/oak/branches/1.6/oak-upgrade/pom.xml
jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3DataStoreFactory.java
Modified: jackrabbit/oak/branches/1.6/oak-upgrade/pom.xml
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-upgrade/pom.xml?rev=1811927&r1=1811926&r2=1811927&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-upgrade/pom.xml (original)
+++ jackrabbit/oak/branches/1.6/oak-upgrade/pom.xml Thu Oct 12 09:27:32 2017
@@ -139,6 +139,14 @@
<!-- Test Dependencies -->
<dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-commons</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-all</artifactId>
<version>1.3</version>
Modified: jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3DataStoreFactory.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3DataStoreFactory.java?rev=1811927&r1=1811926&r2=1811927&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3DataStoreFactory.java (original)
+++ jackrabbit/oak/branches/1.6/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3DataStoreFactory.java Thu Oct 12 09:27:32 2017
@@ -22,6 +22,7 @@ import java.io.FileReader;
import java.io.IOException;
import java.util.HashSet;
import java.util.Properties;
+import java.util.concurrent.Executors;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -32,16 +33,14 @@ import org.apache.jackrabbit.core.data.D
import org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStore;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import com.google.common.io.Closer;
import com.google.common.io.Files;
+import org.apache.jackrabbit.oak.stats.DefaultStatisticsProvider;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
public class S3DataStoreFactory implements BlobStoreFactory {
- private static final Logger log = LoggerFactory.getLogger(S3DataStoreFactory.class);
-
private static final Pattern STRIP_VALUE_PATTERN = Pattern.compile("^[TILFDXSCB]?\"(.*)\"\\W*$");
private final Properties props;
@@ -76,6 +75,13 @@ public class S3DataStoreFactory implemen
S3DataStore delegate = new S3DataStore();
delegate.setProperties(props);
delegate.setPath(directory);
+
+ // Initialize a default stats provider
+ StatisticsProvider statsProvider = new DefaultStatisticsProvider(Executors.newSingleThreadScheduledExecutor());
+ delegate.setStatisticsProvider(statsProvider);
+ // Reduce staging purge interval to 60 seconds
+ delegate.setStagingPurgeInterval(60);
+
try {
delegate.init(tempHomeDir.getPath());
} catch (RepositoryException e) {
@@ -94,9 +100,17 @@ public class S3DataStoreFactory implemen
@Override
public void close() throws IOException {
try {
- store.close();
- } catch (DataStoreException e) {
+ while (store.getStats().get(1).getElementCount() > 0) {
+ Thread.sleep(100);
+ }
+ } catch (InterruptedException e) {
throw new IOException(e);
+ } finally {
+ try {
+ store.close();
+ } catch (DataStoreException e) {
+ throw new IOException(e);
+ }
}
}
};
Added: jackrabbit/oak/branches/1.6/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3ToInlinedTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3ToInlinedTest.java?rev=1811927&view=auto
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3ToInlinedTest.java (added)
+++ jackrabbit/oak/branches/1.6/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/blob/S3ToInlinedTest.java Thu Oct 12 09:27:32 2017
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.upgrade.cli.blob;
+
+import java.io.IOException;
+
+import ch.qos.logback.classic.Level;
+import org.apache.jackrabbit.oak.commons.junit.LogCustomizer;
+import org.apache.jackrabbit.oak.plugins.blob.UploadStagingCache;
+import org.apache.jackrabbit.oak.upgrade.cli.AbstractOak2OakTest;
+import org.apache.jackrabbit.oak.upgrade.cli.container.BlobStoreContainer;
+import org.apache.jackrabbit.oak.upgrade.cli.container.NodeStoreContainer;
+import org.apache.jackrabbit.oak.upgrade.cli.container.S3DataStoreContainer;
+import org.apache.jackrabbit.oak.upgrade.cli.container.SegmentTarNodeStoreContainer;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class S3ToInlinedTest extends AbstractOak2OakTest {
+ private static final String S3_PROPERTIES = System.getProperty("s3.properties");
+
+ private final NodeStoreContainer source;
+
+ private final NodeStoreContainer destination;
+
+ private final BlobStoreContainer sourceBlob;
+
+ private LogCustomizer customLogs;
+
+ public S3ToInlinedTest() throws IOException {
+ Assume.assumeTrue(S3_PROPERTIES != null && !S3_PROPERTIES.isEmpty());
+ sourceBlob = new S3DataStoreContainer(S3_PROPERTIES);
+ source = new SegmentTarNodeStoreContainer(sourceBlob);
+ destination = new SegmentTarNodeStoreContainer();
+ }
+
+ @Override
+ protected NodeStoreContainer getSourceContainer() {
+ return source;
+ }
+
+ @Override
+ protected NodeStoreContainer getDestinationContainer() {
+ return destination;
+ }
+
+ @Override
+ protected String[] getArgs() {
+ return new String[] { "--copy-binaries", "--src-s3datastore", sourceBlob.getDescription(), "--src-s3config",
+ S3_PROPERTIES, source.getDescription(), destination.getDescription() };
+ }
+
+ @Override
+ protected boolean supportsCheckpointMigration() {
+ return true;
+ }
+
+ @Before
+ public void prepare() throws Exception {
+ // Capture logs
+ customLogs = LogCustomizer
+ .forLogger(UploadStagingCache.class.getName())
+ .enable(Level.INFO)
+ .filter(Level.INFO)
+ .contains("Uploads in progress on close [0]")
+ .create();
+ customLogs.starting();
+ super.prepare();
+ }
+
+ /**
+ * Tests whether all the s3 uploads finished
+ */
+ @Test
+ public void testAsyncUploadFinished() {
+ assertEquals(1, customLogs.getLogs().size());
+ customLogs.finished();
+ }
+
+}