You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2020/05/20 09:47:12 UTC
svn commit: r1877944 - in /jackrabbit/oak/trunk:
oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/
oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/
oak-blob-plugins/src/...
Author: amitj
Date: Wed May 20 09:47:11 2020
New Revision: 1877944
URL: http://svn.apache.org/viewvc?rev=1877944&view=rev
Log:
OAK-9063: Use a custom metadata property to manage lastModified in AzureDataStore
- Use a custom metadata property for lastModified, added when adding blob normal and metadata
- Fallback to azure managed lastModified in case the custom metadata property not present for existing blobs
Modified:
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDS.java
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDSWithSmallCache.java
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDsCacheOff.java
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreTest.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandMetadataTest.java
Modified: jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java?rev=1877944&r1=1877943&r2=1877944&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java (original)
+++ jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java Wed May 20 09:47:11 2020
@@ -99,6 +99,7 @@ public class AzureBlobStoreBackend exten
private static final String META_KEY_PREFIX = META_DIR_NAME + "/";
private static final String REF_KEY = "reference.key";
+ private static final String LAST_MODIFIED_KEY = "lastModified";
private static final long BUFFERED_STREAM_THRESHHOLD = 1024 * 1024;
static final long MIN_MULTIPART_UPLOAD_PART_SIZE = 1024 * 1024 * 10; // 10MB
@@ -266,6 +267,8 @@ public class AzureBlobStoreBackend exten
LOG.debug("Blob write started. identifier={} length={}", key, len);
CloudBlockBlob blob = getAzureContainer().getBlockBlobReference(key);
if (!blob.exists()) {
+ addLastModified(blob);
+
BlobRequestOptions options = new BlobRequestOptions();
options.setConcurrentRequestCount(concurrentRequestCount);
boolean useBufferedStream = len < BUFFERED_STREAM_THRESHHOLD;
@@ -289,16 +292,13 @@ public class AzureBlobStoreBackend exten
" new length=" + len +
" old length=" + blob.getProperties().getLength());
}
- LOG.trace("Blob already exists. identifier={} lastModified={}", key, blob.getProperties().getLastModified().getTime());
- blob.startCopy(blob);
- //TODO: better way of updating lastModified (use custom metadata?)
- if (!waitForCopy(blob)) {
- throw new DataStoreException(
- String.format("Cannot update lastModified for blob. identifier=%s status=%s",
- key, blob.getCopyState().getStatusDescription()));
- }
+
+ LOG.trace("Blob already exists. identifier={} lastModified={}", key, getLastModified(blob));
+ addLastModified(blob);
+ blob.uploadMetadata();
+
LOG.debug("Blob updated. identifier={} lastModified={} duration={}", key,
- blob.getProperties().getLastModified().getTime(), (System.currentTimeMillis() - start));
+ getLastModified(blob), (System.currentTimeMillis() - start));
}
catch (StorageException e) {
LOG.info("Error writing blob. identifier={}", key, e);
@@ -307,9 +307,6 @@ public class AzureBlobStoreBackend exten
catch (URISyntaxException | IOException e) {
LOG.debug("Error writing blob. identifier={}", key, e);
throw new DataStoreException(String.format("Cannot write blob. identifier=%s", key), e);
- } catch (InterruptedException e) {
- LOG.debug("Error writing blob. identifier={}", key, e);
- throw new DataStoreException(String.format("Cannot copy blob. identifier=%s", key), e);
} finally {
if (null != contextClassLoader) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
@@ -387,7 +384,7 @@ public class AzureBlobStoreBackend exten
connectionString,
containerName,
new DataIdentifier(getIdentifierName(blob.getName())),
- blob.getProperties().getLastModified().getTime(),
+ getLastModified(blob),
blob.getProperties().getLength());
LOG.debug("Data record read for blob. identifier={} duration={} record={}",
key, (System.currentTimeMillis() - start), record);
@@ -553,6 +550,7 @@ public class AzureBlobStoreBackend exten
try {
CloudBlobDirectory metaDir = getAzureContainer().getDirectoryReference(META_DIR_NAME);
CloudBlockBlob blob = metaDir.getBlockBlobReference(name);
+ addLastModified(blob);
blob.upload(input, recordLength);
}
catch (StorageException e) {
@@ -578,7 +576,7 @@ public class AzureBlobStoreBackend exten
return null;
}
blob.downloadAttributes();
- long lastModified = blob.getProperties().getLastModified().getTime();
+ long lastModified = getLastModified(blob);
long length = blob.getProperties().getLength();
AzureBlobStoreDataRecord record = new AzureBlobStoreDataRecord(this,
connectionString,
@@ -617,12 +615,13 @@ public class AzureBlobStoreBackend exten
for (ListBlobItem item : metaDir.listBlobs(prefix)) {
if (item instanceof CloudBlob) {
CloudBlob blob = (CloudBlob) item;
+ blob.downloadAttributes();
records.add(new AzureBlobStoreDataRecord(
this,
connectionString,
containerName,
new DataIdentifier(stripMetaKeyPrefix(blob.getName())),
- blob.getProperties().getLastModified().getTime(),
+ getLastModified(blob),
blob.getProperties().getLength(),
true));
}
@@ -762,6 +761,17 @@ public class AzureBlobStoreBackend exten
return name;
}
+ private static void addLastModified(CloudBlockBlob blob) {
+ blob.getMetadata().put(LAST_MODIFIED_KEY, String.valueOf(System.currentTimeMillis()));
+ }
+
+ private static long getLastModified(CloudBlob blob) {
+ if (blob.getMetadata().containsKey(LAST_MODIFIED_KEY)) {
+ return Long.parseLong(blob.getMetadata().get(LAST_MODIFIED_KEY));
+ }
+ return blob.getProperties().getLastModified().getTime();
+ }
+
void setHttpDownloadURIExpirySeconds(int seconds) {
httpDownloadURIExpirySeconds = seconds;
}
@@ -787,7 +797,7 @@ public class AzureBlobStoreBackend exten
// When running unit test from Maven, it doesn't always honor the @NotNull decorators
if (null == identifier) throw new NullPointerException("identifier");
if (null == downloadOptions) throw new NullPointerException("downloadOptions");
-
+
if (httpDownloadURIExpirySeconds > 0) {
String domain = getDirectDownloadBlobStorageDomain(downloadOptions.isDomainOverrideIgnored());
@@ -1012,6 +1022,7 @@ public class AzureBlobStoreBackend exten
AccessCondition.generateEmptyCondition(),
null,
null);
+ addLastModified(blob);
blob.commitBlockList(blocks);
long size = 0L;
for (BlockEntry block : blocks) {
@@ -1022,7 +1033,7 @@ public class AzureBlobStoreBackend exten
connectionString,
containerName,
blobId,
- blob.getProperties().getLastModified().getTime(),
+ getLastModified(blob),
size);
}
else {
@@ -1182,9 +1193,10 @@ public class AzureBlobStoreBackend exten
return length;
}
- public static AzureBlobInfo fromCloudBlob(CloudBlob cloudBlob) {
+ public static AzureBlobInfo fromCloudBlob(CloudBlob cloudBlob) throws StorageException {
+ cloudBlob.downloadAttributes();
return new AzureBlobInfo(cloudBlob.getName(),
- cloudBlob.getProperties().getLastModified().getTime(),
+ AzureBlobStoreBackend.getLastModified(cloudBlob),
cloudBlob.getProperties().getLength());
}
}
Modified: jackrabbit/oak/trunk/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDS.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDS.java?rev=1877944&r1=1877943&r2=1877944&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDS.java (original)
+++ jackrabbit/oak/trunk/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDS.java Wed May 20 09:47:11 2020
@@ -42,7 +42,7 @@ import javax.jcr.RepositoryException;
public class TestAzureDS extends AbstractDataStoreTest {
protected static final Logger LOG = LoggerFactory.getLogger(TestAzureDS.class);
- protected Properties props;
+ protected Properties props = new Properties();
protected String container;
@BeforeClass
@@ -53,7 +53,7 @@ public class TestAzureDS extends Abstrac
@Override
@Before
public void setUp() throws Exception {
- props = AzureDataStoreUtils.getAzureConfig();
+ props.putAll(AzureDataStoreUtils.getAzureConfig());
container = String.valueOf(randomGen.nextInt(9999)) + "-" + String.valueOf(randomGen.nextInt(9999))
+ "-test";
props.setProperty(AzureConstants.AZURE_BLOB_CONTAINER_NAME, container);
Modified: jackrabbit/oak/trunk/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDSWithSmallCache.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDSWithSmallCache.java?rev=1877944&r1=1877943&r2=1877944&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDSWithSmallCache.java (original)
+++ jackrabbit/oak/trunk/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDSWithSmallCache.java Wed May 20 09:47:11 2020
@@ -38,7 +38,7 @@ public class TestAzureDSWithSmallCache e
@Override
@Before
public void setUp() throws Exception {
- super.setUp();
props.setProperty("cacheSize", String.valueOf(dataLength * 10));
+ super.setUp();
}
}
Modified: jackrabbit/oak/trunk/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDsCacheOff.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDsCacheOff.java?rev=1877944&r1=1877943&r2=1877944&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDsCacheOff.java (original)
+++ jackrabbit/oak/trunk/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDsCacheOff.java Wed May 20 09:47:11 2020
@@ -36,7 +36,7 @@ public class TestAzureDsCacheOff extends
@Override
@Before
public void setUp() throws Exception {
- super.setUp();
props.setProperty("cacheSize", "0");
+ super.setUp();
}
}
Modified: jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreTest.java?rev=1877944&r1=1877943&r2=1877944&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreTest.java (original)
+++ jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreTest.java Wed May 20 09:47:11 2020
@@ -120,6 +120,25 @@ public abstract class AbstractDataStoreT
}
/**
+ * Testcase to validate {@link DataStore#addRecord(InputStream)} API.
+ */
+ @Test
+ public void testAddDuplicateRecord() {
+ try {
+ long start = System.currentTimeMillis();
+ LOG.info("Testcase: " + this.getClass().getName()
+ + "#testAddDuplicateRecord, testDir=" + dataStoreDir);
+ doAddDuplicateRecordTest();
+ LOG.info("Testcase: " + this.getClass().getName()
+ + "#testAddDuplicateRecord finished, time taken = ["
+ + (System.currentTimeMillis() - start) + "]ms");
+ } catch (Exception e) {
+ LOG.error("error:", e);
+ fail(e.getMessage());
+ }
+ }
+
+ /**
* Testcase to validate {@link DataStore#getRecord(DataIdentifier)} API.
*/
@Test
@@ -309,6 +328,24 @@ public abstract class AbstractDataStoreT
}
/**
+ * Test {@link DataStore#addRecord(InputStream)} and assert length & last modified of copied
+ * record.
+ */
+ protected void doAddDuplicateRecordTest() throws Exception {
+ byte[] data = new byte[dataLength];
+ randomGen.nextBytes(data);
+ DataRecord rec = ds.addRecord(new ByteArrayInputStream(data));
+ Assert.assertEquals(data.length, rec.getLength());
+ assertRecord(data, rec);
+
+ DataRecord rec2 = ds.addRecord(new ByteArrayInputStream(data));
+ Assert.assertEquals(data.length, rec2.getLength());
+ assertRecord(data, rec2);
+
+ assertTrue("Copied record last modified not greater", rec.getLastModified() <= rec2.getLastModified());
+ }
+
+ /**
* Test {@link DataStore#getRecord(DataIdentifier)} and assert length and
* inputstream.
*/
Modified: jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandMetadataTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandMetadataTest.java?rev=1877944&r1=1877943&r2=1877944&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandMetadataTest.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandMetadataTest.java Wed May 20 09:47:11 2020
@@ -165,12 +165,12 @@ public class DataStoreCommandMetadataTes
REFERENCES.getNameFromIdPrefix(rep2Id, sessionId));
List<String> expectations = Lists.newArrayList();
- expectations.add(Joiner.on("|").join(rep2Id, MILLISECONDS.toSeconds(expectAuxMetadataRecord.getLastModified()),
- MILLISECONDS.toSeconds(expectAuxMarkerMetadataRecord.getLastModified()), "-"));
- expectations.add(Joiner.on("|").join(repoId, MILLISECONDS.toSeconds(expectMainMetadataRecord.getLastModified()),
- MILLISECONDS.toSeconds(expectMainMarkerMetadataRecord.getLastModified()), "*"));
- expectations.add(Joiner.on("|").join(rep3Id, MILLISECONDS.toSeconds(expectAux2MetadataRecord.getLastModified()),
- MILLISECONDS.toSeconds(expectAux2MarkerMetadataRecord.getLastModified()), "-"));
+ expectations.add(Joiner.on("|").join(rep2Id, MILLISECONDS.toSeconds(expectAuxMarkerMetadataRecord.getLastModified()),
+ MILLISECONDS.toSeconds(expectAuxMetadataRecord.getLastModified()), "-"));
+ expectations.add(Joiner.on("|").join(repoId, MILLISECONDS.toSeconds(expectMainMarkerMetadataRecord.getLastModified()),
+ MILLISECONDS.toSeconds(expectMainMetadataRecord.getLastModified()), "*"));
+ expectations.add(Joiner.on("|").join(rep3Id, MILLISECONDS.toSeconds(expectAux2MarkerMetadataRecord.getLastModified()),
+ MILLISECONDS.toSeconds(expectAux2MetadataRecord.getLastModified()), "-"));
storeFixture.close();
return expectations;