You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2017/05/18 10:00:08 UTC
svn commit: r1795497 - in /jackrabbit/oak/branches/1.6: ./ oak-run/pom.xml
oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java
Author: amitj
Date: Thu May 18 10:00:08 2017
New Revision: 1795497
URL: http://svn.apache.org/viewvc?rev=1795497&view=rev
Log:
OAK-6229: NPE when running datastorecheck command with S3
- Merge r1795475 from trunk
- Added explicit dependency to httpcore-4.4.4 and httpclient-4.5.2 as stale versions not supported by the latest aws-java-sdk were being pulled in by solr
- Added explicit dependency to jackson core, annotations and databind as stale versions not supported by the latest aws-java-sdk, were being pulled in by oak-http
Modified:
jackrabbit/oak/branches/1.6/ (props changed)
jackrabbit/oak/branches/1.6/oak-run/pom.xml
jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
jackrabbit/oak/branches/1.6/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java
Propchange: jackrabbit/oak/branches/1.6/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu May 18 10:00:08 2017
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1781068,1781075,1781248,1781386,1781846,1781907,1782000,1782029,1782196,1782447,1782476,1782770,1782945,1782966,1782973,1782990,1783061,1783066,1783089,1783104-1783105,1783110,1783619,1783720,1783731,1783733,1783738,1783742,1783773,1783855,1783891,1784023,1784130,1784162,1784251,1784401,1784551,1784574,1784689,1785095,1785108,1785283,1785838,1785919,1785946,1787074,1787217,1787425,1788378,1788387-1788389,1788850,1789056,1790382,1792463,1792742,1792746,1793088,1793618,1793627,1793644,1795314,1795330
+/jackrabbit/oak/trunk:1781068,1781075,1781248,1781386,1781846,1781907,1782000,1782029,1782196,1782447,1782476,1782770,1782945,1782966,1782973,1782990,1783061,1783066,1783089,1783104-1783105,1783110,1783619,1783720,1783731,1783733,1783738,1783742,1783773,1783855,1783891,1784023,1784130,1784162,1784251,1784401,1784551,1784574,1784689,1785095,1785108,1785283,1785838,1785919,1785946,1787074,1787217,1787425,1788378,1788387-1788389,1788850,1789056,1790382,1792463,1792742,1792746,1793088,1793618,1793627,1793644,1795314,1795330,1795475
/jackrabbit/trunk:1345480
Modified: jackrabbit/oak/branches/1.6/oak-run/pom.xml
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-run/pom.xml?rev=1795497&r1=1795496&r2=1795497&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-run/pom.xml (original)
+++ jackrabbit/oak/branches/1.6/oak-run/pom.xml Thu May 18 10:00:08 2017
@@ -411,14 +411,62 @@
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</dependency>
-
+
+ <!-- httpcore & httpclient -->
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ <version>4.4.4</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.5.2</version>
+ </dependency>
+
+ <!-- jackson libs -->
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>2.6.6</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>2.6.6</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>2.6.6</version>
+ </dependency>
<!-- Test dependencies -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
-
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-blob-cloud</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-blob-cloud-azure</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<profiles>
Modified: jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java?rev=1795497&r1=1795496&r2=1795497&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java (original)
+++ jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java Thu May 18 10:00:08 2017
@@ -151,7 +151,9 @@ class Utils {
String cfgPath = s3dsConfig.value(options);
Properties props = loadAndTransformProps(cfgPath);
s3ds.setProperties(props);
- s3ds.init(null);
+ File homeDir = Files.createTempDir();
+ closer.register(asCloseable(homeDir));
+ s3ds.init(homeDir.getAbsolutePath());
delegate = s3ds;
} else if (options.has(azureBlobDSConfig)) {
AzureDataStore azureds = new AzureDataStore();
Modified: jackrabbit/oak/branches/1.6/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java?rev=1795497&r1=1795496&r2=1795497&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java (original)
+++ jackrabbit/oak/branches/1.6/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java Thu May 18 10:00:08 2017
@@ -21,7 +21,6 @@ package org.apache.jackrabbit.oak.run;
import static com.google.common.base.Charsets.UTF_8;
import static org.junit.Assert.assertEquals;
-import java.io.BufferedWriter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
@@ -34,6 +33,7 @@ import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import java.util.Properties;
import java.util.Random;
import java.util.Set;
@@ -41,9 +41,12 @@ import com.google.common.collect.Immutab
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
-import com.google.common.io.Files;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.FileFilterUtils;
-import org.apache.commons.lang3.StringEscapeUtils;
+import org.apache.felix.cm.file.ConfigurationHandler;
+import org.apache.jackrabbit.core.data.DataStore;
+import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureDataStoreUtils;
+import org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils;
import org.apache.jackrabbit.oak.commons.FileIOUtils;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
import org.apache.jackrabbit.oak.plugins.blob.datastore.OakFileDataStore;
@@ -81,18 +84,48 @@ public class DataStoreCheckTest {
private String dsPath;
+ private DataStoreBlobStore setupDataStore;
+
+ private String dsOption;
+
@Before
public void setup() throws Exception {
- OakFileDataStore delegate = new OakFileDataStore();
- dsPath = temporaryFolder.newFolder().getAbsolutePath();
- delegate.setPath(dsPath);
- delegate.init(null);
- DataStoreBlobStore blobStore = new DataStoreBlobStore(delegate);
+ if (S3DataStoreUtils.isS3Configured()) {
+ Properties props = S3DataStoreUtils.getS3Config();
+ props.setProperty("cacheSize", "0");
+ DataStore ds = S3DataStoreUtils.getS3DataStore(S3DataStoreUtils.getFixtures().get(0),
+ props,
+ temporaryFolder.newFolder().getAbsolutePath());
+ setupDataStore = new DataStoreBlobStore(ds);
+ cfgFilePath = createTempConfig(temporaryFolder.newFile(), props);
+ dsOption = "s3ds";
+ } else if (AzureDataStoreUtils.isAzureConfigured()) {
+ Properties props = AzureDataStoreUtils.getAzureConfig();
+ props.setProperty("cacheSize", "0");
+ DataStore ds = AzureDataStoreUtils.getAzureDataStore(props,
+ temporaryFolder.newFolder().getAbsolutePath());
+ setupDataStore = new DataStoreBlobStore(ds);
+ cfgFilePath = createTempConfig(temporaryFolder.newFile(), props);
+ dsOption = "azureblobds";
+ }
+ else {
+ OakFileDataStore delegate = new OakFileDataStore();
+ dsPath = temporaryFolder.newFolder().getAbsolutePath();
+ delegate.setPath(dsPath);
+ delegate.init(null);
+ setupDataStore = new DataStoreBlobStore(delegate);
+
+ File cfgFile = temporaryFolder.newFile();
+ Properties props = new Properties();
+ props.put("path", dsPath);
+ cfgFilePath = createTempConfig(cfgFile, props);
+ dsOption = "fds";
+ }
File storeFile = temporaryFolder.newFolder();
storePath = storeFile.getAbsolutePath();
FileStore fileStore = FileStoreBuilder.fileStoreBuilder(storeFile)
- .withBlobStore(blobStore)
+ .withBlobStore(setupDataStore)
.withMaxFileSize(256)
.withSegmentCacheSize(64)
.build();
@@ -104,7 +137,7 @@ public class DataStoreCheckTest {
blobsAdded = Sets.newHashSet();
for (int i = 0; i < numBlobs; i++) {
SegmentBlob b = (SegmentBlob) store.createBlob(randomStream(i, 18342));
- Iterator<String> idIter = blobStore.resolveChunks(b.getBlobId());
+ Iterator<String> idIter = setupDataStore.resolveChunks(b.getBlobId());
while (idIter.hasNext()) {
String chunk = idIter.next();
blobsAdded.add(chunk);
@@ -115,14 +148,7 @@ public class DataStoreCheckTest {
store.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY);
log.info("Created blobs : {}", blobsAdded);
- File cfgFile = temporaryFolder.newFile();
- BufferedWriter writer = Files.newWriter(cfgFile, UTF_8);
- FileIOUtils.writeAsLine(writer, "path=\"" + StringEscapeUtils.escapeJava(dsPath) + "\"",false);
- writer.close();
- cfgFilePath = cfgFile.getAbsolutePath();
-
fileStore.close();
- blobStore.close();
}
@After
@@ -133,6 +159,7 @@ public class DataStoreCheckTest {
@Test
public void testCorrect() throws Exception {
File dump = temporaryFolder.newFolder();
+ setupDataStore.close();
testAllParams(dump);
}
@@ -140,17 +167,12 @@ public class DataStoreCheckTest {
public void testConsistency() throws Exception {
File dump = temporaryFolder.newFolder();
- // Delete a random blob from datastore
- OakFileDataStore delegate = new OakFileDataStore();
- delegate.setPath(dsPath);
- delegate.init(null);
- DataStoreBlobStore blobStore = new DataStoreBlobStore(delegate);
-
Random rand = new Random();
String deletedBlobId = Iterables.get(blobsAdded, rand.nextInt(blobsAdded.size()));
blobsAdded.remove(deletedBlobId);
- long count = blobStore.countDeleteChunks(ImmutableList.of(deletedBlobId), 0);
+ long count = setupDataStore.countDeleteChunks(ImmutableList.of(deletedBlobId), 0);
assertEquals(1, count);
+ setupDataStore.close();
testAllParams(dump);
@@ -159,10 +181,10 @@ public class DataStoreCheckTest {
assertFileEquals(dump, "[consistency]", Sets.newHashSet(deletedBlobId));
}
- public void testAllParams(File dump) throws Exception {
+ private void testAllParams(File dump) throws Exception {
DataStoreCheckCommand checkCommand = new DataStoreCheckCommand();
List<String> argsList = Lists
- .newArrayList("--id", "--ref", "--consistency", "--fds", cfgFilePath, "--store", storePath,
+ .newArrayList("--id", "--ref", "--consistency", "--" + dsOption, cfgFilePath, "--store", storePath,
"--dump", dump.getAbsolutePath());
checkCommand.execute(argsList.toArray(new String[0]));
@@ -170,9 +192,10 @@ public class DataStoreCheckTest {
@Test
public void testMissingOpParams() throws Exception {
+ setupDataStore.close();
File dump = temporaryFolder.newFolder();
List<String> argsList = Lists
- .newArrayList("--fds", cfgFilePath, "--store", storePath,
+ .newArrayList("--" + dsOption, cfgFilePath, "--store", storePath,
"--dump", dump.getAbsolutePath());
log.info("Running testMissinOpParams: {}", argsList);
testIncorrectParams(argsList, Lists.newArrayList("Missing required option(s)", "'id'", "'ref'", "'consistency'"));
@@ -180,6 +203,7 @@ public class DataStoreCheckTest {
@Test
public void testTarNoDS() throws Exception {
+ setupDataStore.close();
File dump = temporaryFolder.newFolder();
List<String> argsList = Lists
.newArrayList("--id", "--ref", "--consistency", "--store", storePath,
@@ -190,14 +214,15 @@ public class DataStoreCheckTest {
@Test
public void testOpNoStore() throws Exception {
+ setupDataStore.close();
File dump = temporaryFolder.newFolder();
List<String> argsList = Lists
- .newArrayList("--consistency", "--fds", cfgFilePath,
+ .newArrayList("--consistency", "--" + dsOption, cfgFilePath,
"--dump", dump.getAbsolutePath());
testIncorrectParams(argsList, Lists.newArrayList("Missing required option(s) ['store']"));
argsList = Lists
- .newArrayList("--ref", "--fds", cfgFilePath,
+ .newArrayList("--ref", "--" + dsOption, cfgFilePath,
"--dump", dump.getAbsolutePath());
testIncorrectParams(argsList, Lists.newArrayList("Missing required option(s) ['store']"));
}
@@ -236,4 +261,10 @@ public class DataStoreCheckTest {
r.nextBytes(data);
return new ByteArrayInputStream(data);
}
+
+ private static String createTempConfig(File cfgFile, Properties props) throws IOException {
+ FileOutputStream fos = FileUtils.openOutputStream(cfgFile);
+ ConfigurationHandler.write(fos, props);
+ return cfgFile.getAbsolutePath();
+ }
}
Re: svn commit: r1795497 - in /jackrabbit/oak/branches/1.6: ./
oak-run/pom.xml
oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java
Posted by Davide Giannella <da...@apache.org>.
On 22/05/2017 14:41, Amit Jain wrote:
> Do note that what you are talking about here is a commit to the 1.6 branch.
> The trunk version should already be pulling these dependencies from March
> when the aws dependency was updated.
As I said I didn't do a proper investigation but simply went through
commits looking for pom and dependencies update.
Nevertheless I increased it to 41 which is fine.
D.
Re: svn commit: r1795497 - in /jackrabbit/oak/branches/1.6: ./
oak-run/pom.xml
oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java
Posted by Davide Giannella <da...@apache.org>.
On 22/05/2017 14:41, Amit Jain wrote:
> Do note that what you are talking about here is a commit to the 1.6 branch.
> The trunk version should already be pulling these dependencies from March
> when the aws dependency was updated.
As I said I didn't do a proper investigation but simply went through
commits looking for pom and dependencies update.
Nevertheless I increased it to 41 which is fine.
D.
Re: svn commit: r1795497 - in /jackrabbit/oak/branches/1.6: ./
oak-run/pom.xml oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java
Posted by Amit Jain <am...@ieee.org>.
Do note that what you are talking about here is a commit to the 1.6 branch.
The trunk version should already be pulling these dependencies from March
when the aws dependency was updated.
Thanks
Amit
On Mon, May 22, 2017 at 5:06 PM, Davide Giannella <da...@apache.org> wrote:
> On 18/05/2017 11:00, amitj@apache.org wrote:
> > Author: amitj
> > Date: Thu May 18 10:00:08 2017
> > New Revision: 1795497
> >
> > URL: http://svn.apache.org/viewvc?rev=1795497&view=rev
> > Log:
> > OAK-6229: NPE when running datastorecheck command with S3
> >
> > - Merge r1795475 from trunk
> > - Added explicit dependency to httpcore-4.4.4 and httpclient-4.5.2 as
> stale versions not supported by the latest aws-java-sdk were being pulled
> in by solr
> > - Added explicit dependency to jackson core, annotations and databind as
> stale versions not supported by the latest aws-java-sdk, were being pulled
> in by oak-http
> >
>
> Looks like this is the best candidate for the following error. Didn't
> try reverting and there were few more commits after this one not on
> dependencies though.
>
> /oak-run-1.8-SNAPSHOT.jar size (40321219) too large. Max. is 38000000
>
> I'm going to increase the size to 41MB but I ask next time to run
> locally a build before committing
>
> Davide
>
>
>
Re: svn commit: r1795497 - in /jackrabbit/oak/branches/1.6: ./
oak-run/pom.xml oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java
Posted by Amit Jain <am...@ieee.org>.
Do note that what you are talking about here is a commit to the 1.6 branch.
The trunk version should already be pulling these dependencies from March
when the aws dependency was updated.
Thanks
Amit
On Mon, May 22, 2017 at 5:06 PM, Davide Giannella <da...@apache.org> wrote:
> On 18/05/2017 11:00, amitj@apache.org wrote:
> > Author: amitj
> > Date: Thu May 18 10:00:08 2017
> > New Revision: 1795497
> >
> > URL: http://svn.apache.org/viewvc?rev=1795497&view=rev
> > Log:
> > OAK-6229: NPE when running datastorecheck command with S3
> >
> > - Merge r1795475 from trunk
> > - Added explicit dependency to httpcore-4.4.4 and httpclient-4.5.2 as
> stale versions not supported by the latest aws-java-sdk were being pulled
> in by solr
> > - Added explicit dependency to jackson core, annotations and databind as
> stale versions not supported by the latest aws-java-sdk, were being pulled
> in by oak-http
> >
>
> Looks like this is the best candidate for the following error. Didn't
> try reverting and there were few more commits after this one not on
> dependencies though.
>
> /oak-run-1.8-SNAPSHOT.jar size (40321219) too large. Max. is 38000000
>
> I'm going to increase the size to 41MB but I ask next time to run
> locally a build before committing
>
> Davide
>
>
>
Re: svn commit: r1795497 - in /jackrabbit/oak/branches/1.6: ./
oak-run/pom.xml
oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java
Posted by Davide Giannella <da...@apache.org>.
On 18/05/2017 11:00, amitj@apache.org wrote:
> Author: amitj
> Date: Thu May 18 10:00:08 2017
> New Revision: 1795497
>
> URL: http://svn.apache.org/viewvc?rev=1795497&view=rev
> Log:
> OAK-6229: NPE when running datastorecheck command with S3
>
> - Merge r1795475 from trunk
> - Added explicit dependency to httpcore-4.4.4 and httpclient-4.5.2 as stale versions not supported by the latest aws-java-sdk were being pulled in by solr
> - Added explicit dependency to jackson core, annotations and databind as stale versions not supported by the latest aws-java-sdk, were being pulled in by oak-http
>
Looks like this is the best candidate for the following error. Didn't
try reverting and there were few more commits after this one not on
dependencies though.
/oak-run-1.8-SNAPSHOT.jar size (40321219) too large. Max. is 38000000
I'm going to increase the size to 41MB but I ask next time to run
locally a build before committing
Davide
Re: svn commit: r1795497 - in /jackrabbit/oak/branches/1.6: ./
oak-run/pom.xml
oak-run/src/main/java/org/apache/jackrabbit/oak/run/Utils.java
oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java
Posted by Davide Giannella <da...@apache.org>.
On 18/05/2017 11:00, amitj@apache.org wrote:
> Author: amitj
> Date: Thu May 18 10:00:08 2017
> New Revision: 1795497
>
> URL: http://svn.apache.org/viewvc?rev=1795497&view=rev
> Log:
> OAK-6229: NPE when running datastorecheck command with S3
>
> - Merge r1795475 from trunk
> - Added explicit dependency to httpcore-4.4.4 and httpclient-4.5.2 as stale versions not supported by the latest aws-java-sdk were being pulled in by solr
> - Added explicit dependency to jackson core, annotations and databind as stale versions not supported by the latest aws-java-sdk, were being pulled in by oak-http
>
Looks like this is the best candidate for the following error. Didn't
try reverting and there were few more commits after this one not on
dependencies though.
/oak-run-1.8-SNAPSHOT.jar size (40321219) too large. Max. is 38000000
I'm going to increase the size to 41MB but I ask next time to run
locally a build before committing
Davide