You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2016/10/24 04:53:55 UTC
svn commit: r1766350 - in /jackrabbit/oak/trunk:
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/
oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datas...
Author: amitj
Date: Mon Oct 24 04:53:55 2016
New Revision: 1766350
URL: http://svn.apache.org/viewvc?rev=1766350&view=rev
Log:
OAK-4971: Implement caching for FileDataStore
* Implementation of Caching FileDataStore using the AbstractSharedCachingDataStore introduced by OAK-4837
* Corresponding Tests
* Switched on by default when cacheSize param > 0. Can be switched to the older one with a system flag.
Added:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/CachingFileDataStore.java (with props)
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackend.java (with props)
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/CachingFileDataStoreTest.java (with props)
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/AbstractSharedCachingDataStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FileDataStoreService.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreServiceTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakCachingFDSTest.java
jackrabbit/oak/trunk/oak-it-osgi/test-bundles.xml
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/AbstractSharedCachingDataStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/AbstractSharedCachingDataStore.java?rev=1766350&r1=1766349&r2=1766350&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/AbstractSharedCachingDataStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/AbstractSharedCachingDataStore.java Mon Oct 24 04:53:55 2016
@@ -243,6 +243,13 @@ public abstract class AbstractSharedCach
}
}
+ /**
+ * In rare cases may include some duplicates in cases where async staged uploads complete
+ * during iteration.
+ *
+ * @return Iterator over all ids available
+ * @throws DataStoreException
+ */
@Override
public Iterator<DataIdentifier> getAllIdentifiers() throws DataStoreException {
return Iterators.concat(Iterators.transform(cache.getStagingCache().getAllIdentifiers(),
Added: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/CachingFileDataStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/CachingFileDataStore.java?rev=1766350&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/CachingFileDataStore.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/CachingFileDataStore.java Mon Oct 24 04:53:55 2016
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.blob.datastore;
+
+import java.util.Properties;
+
+import org.apache.jackrabbit.oak.plugins.blob.AbstractSharedCachingDataStore;
+import org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend;
+
+/**
+ * File system implementation of {@link AbstractSharedCachingDataStore}.
+ */
+public class CachingFileDataStore extends AbstractSharedCachingDataStore {
+ private Properties properties;
+
+ /**
+ * The minimum size of an object that should be stored in this data store.
+ */
+ private int minRecordLength = 16 * 1024;
+
+ protected AbstractSharedBackend createBackend() {
+ FSBackend
+ backend = new FSBackend();
+ if(this.properties != null) {
+ backend.setProperties(this.properties);
+ }
+
+ return backend;
+ }
+
+ /*------------------------------------------- Getters & Setters-----------------------------**/
+
+ /**
+ * Properties required to configure the Backend
+ */
+ public void setProperties(Properties properties) {
+ this.properties = properties;
+ }
+
+ protected AbstractSharedBackend getBackend() {
+ return backend;
+ }
+
+ @Override
+ public int getMinRecordLength() {
+ return minRecordLength;
+ }
+
+ public void setMinRecordLength(int minRecordLength) {
+ this.minRecordLength = minRecordLength;
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/CachingFileDataStore.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackend.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackend.java?rev=1766350&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackend.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackend.java Mon Oct 24 04:53:55 2016
@@ -0,0 +1,445 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.blob.datastore;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.RandomAccessFile;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Properties;
+
+import javax.annotation.Nonnull;
+
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+import com.google.common.io.Closeables;
+import com.google.common.io.Files;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.filefilter.FileFilterUtils;
+import org.apache.jackrabbit.core.data.DataIdentifier;
+import org.apache.jackrabbit.core.data.DataRecord;
+import org.apache.jackrabbit.core.data.DataStoreException;
+import org.apache.jackrabbit.core.data.LazyFileInputStream;
+import org.apache.jackrabbit.oak.spi.blob.AbstractDataRecord;
+import org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.commons.io.FilenameUtils.normalizeNoEndSeparator;
+
+/**
+ */
+public class FSBackend extends AbstractSharedBackend {
+ private static final Logger LOG = LoggerFactory.getLogger(FSBackend.class);
+
+ public static final String FS_BACKEND_PATH = "fsBackendPath";
+ /**
+ * The maximum last modified time resolution of the file system.
+ */
+ private static final int ACCESS_TIME_RESOLUTION = 2000;
+
+ private Properties properties;
+
+ private String fsPath;
+
+ private File fsPathDir;
+
+ @Override
+ public void init() throws DataStoreException {
+ fsPath = properties.getProperty(FS_BACKEND_PATH);
+ if (this.fsPath == null || "".equals(this.fsPath)) {
+ throw new DataStoreException(
+ "Could not initialize FSBackend from " + properties + ". [" + FS_BACKEND_PATH
+ + "] property not found.");
+ }
+ this.fsPath = normalizeNoEndSeparator(fsPath);
+ fsPathDir = new File(this.fsPath);
+ if (fsPathDir.exists() && fsPathDir.isFile()) {
+ throw new DataStoreException(
+ "Can not create a directory " + "because a file exists with the same name: "
+ + this.fsPath);
+ }
+ if (!fsPathDir.exists()) {
+ boolean created = fsPathDir.mkdirs();
+ if (!created) {
+ throw new DataStoreException(
+ "Could not create directory: " + fsPathDir.getAbsolutePath());
+ }
+ }
+ }
+
+ @Override
+ public InputStream read(DataIdentifier identifier) throws DataStoreException {
+ File file = getFile(identifier, fsPathDir);
+ try {
+ return new LazyFileInputStream(file);
+ } catch (IOException e) {
+ throw new DataStoreException("Error opening input stream of " + file.getAbsolutePath(),
+ e);
+ }
+ }
+
+ @Override
+ public void write(DataIdentifier identifier, File file) throws DataStoreException {
+ File dest = getFile(identifier, fsPathDir);
+ synchronized (this) {
+ if (dest.exists()) {
+ long now = System.currentTimeMillis();
+ if (getLastModified(dest) < now + ACCESS_TIME_RESOLUTION) {
+ setLastModified(dest, now + ACCESS_TIME_RESOLUTION);
+ }
+ } else {
+ try {
+ FileUtils.copyFile(file, dest);
+ } catch (IOException ie) {
+ LOG.error("failed to copy [{}] to [{}]", file.getAbsolutePath(),
+ dest.getAbsolutePath());
+ throw new DataStoreException("Not able to write file [" + identifier + "]", ie);
+ }
+ }
+ }
+ }
+
+ @Override
+ public DataRecord getRecord(DataIdentifier identifier) throws DataStoreException {
+ long start = System.currentTimeMillis();
+
+ File file = getFile(identifier, fsPathDir);
+ if (!file.exists() || !file.isFile()) {
+ LOG.info("getRecord:Identifier [{}] not found. Took [{}] ms.", identifier,
+ (System.currentTimeMillis() - start));
+ throw new DataStoreException("Identifier [" + identifier + "] not found.");
+ }
+ return new FSBackendDataRecord(this, identifier, file);
+ }
+
+ @Override
+ public Iterator<DataIdentifier> getAllIdentifiers() throws DataStoreException {
+ return Files.fileTreeTraverser().postOrderTraversal(fsPathDir)
+ .filter(new Predicate<File>() {
+ @Override public boolean apply(File input) {
+ return input.isFile() && !normalizeNoEndSeparator(input.getParent())
+ .equals(fsPath);
+ }
+ }).transform(new Function<File, DataIdentifier>() {
+ @Override public DataIdentifier apply(File input) {
+ return new DataIdentifier(input.getName());
+ }
+ }).iterator();
+ }
+
+ @Override
+ public boolean exists(DataIdentifier identifier) throws DataStoreException {
+ File file = getFile(identifier, fsPathDir);
+ return file.exists() && file.isFile();
+ }
+
+ @Override
+ public void deleteRecord(DataIdentifier identifier) throws DataStoreException {
+ File file = getFile(identifier, fsPathDir);
+ synchronized (this) {
+ if (file.exists()) {
+ if (file.delete()) {
+ deleteEmptyParentDirs(file);
+ } else {
+ LOG.warn("Failed to delete file " + file.getAbsolutePath());
+ }
+ }
+ }
+ }
+
+ @Override
+ public void addMetadataRecord(InputStream input, String name)
+ throws DataStoreException {
+ try {
+ File file = new File(fsPathDir, name);
+ FileOutputStream os = new FileOutputStream(file);
+ try {
+ IOUtils.copyLarge(input, os);
+ } finally {
+ Closeables.close(os, true);
+ Closeables.close(input, true);
+ }
+ } catch (IOException e) {
+ LOG.error("Exception while adding metadata record with name {}, {}",
+ new Object[] {name, e});
+ throw new DataStoreException("Could not add root record", e);
+ }
+ }
+
+ @Override
+ public void addMetadataRecord(File input, String name) throws DataStoreException {
+ try {
+ File file = new File(fsPathDir, name);
+ FileUtils.copyFile(input, file);
+ } catch (IOException e) {
+ LOG.error("Exception while adding metadata record file {} with name {}, {}",
+ input, name, e);
+ throw new DataStoreException("Could not add root record", e);
+ }
+ }
+
+ @Override
+ public DataRecord getMetadataRecord(String name) {
+ for (File file : FileFilterUtils
+ .filter(FileFilterUtils.nameFileFilter(name), fsPathDir.listFiles())) {
+ if (!file.isDirectory()) {
+ return new FSBackendDataRecord(this, new DataIdentifier(file.getName()), file);
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public List<DataRecord> getAllMetadataRecords(String prefix) {
+ List<DataRecord> rootRecords = new ArrayList<DataRecord>();
+ for (File file : FileFilterUtils
+ .filterList(FileFilterUtils.prefixFileFilter(prefix), fsPathDir.listFiles())) {
+ if (!file.isDirectory()) { // skip directories which are actual data store files
+ rootRecords
+ .add(new FSBackendDataRecord(this, new DataIdentifier(file.getName()), file));
+ }
+ }
+ return rootRecords;
+ }
+
+ @Override
+ public boolean deleteMetadataRecord(String name) {
+ for (File file : FileFilterUtils
+ .filterList(FileFilterUtils.nameFileFilter(name), fsPathDir.listFiles())) {
+ if (!file.isDirectory()) { // skip directories which are actual data store files
+ if (!file.delete()) {
+ LOG.warn("Failed to delete root record {} ",
+ new Object[] {file.getAbsolutePath()});
+ } else {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public void deleteAllMetadataRecords(String prefix) {
+ for (File file : FileFilterUtils
+ .filterList(FileFilterUtils.prefixFileFilter(prefix), fsPathDir.listFiles())) {
+ if (!file.isDirectory()) { // skip directories which are actual data store files
+ if (!file.delete()) {
+ LOG.warn("Failed to delete root record {} ",
+ new Object[] {file.getAbsolutePath()});
+ }
+ }
+ }
+ }
+
+ @Override
+ public Iterator<DataRecord> getAllRecords() {
+ final AbstractSharedBackend backend = this;
+ return Files.fileTreeTraverser().postOrderTraversal(fsPathDir)
+ .filter(new Predicate<File>() {
+ @Override public boolean apply(File input) {
+ return input.isFile() && !normalizeNoEndSeparator(input.getParent())
+ .equals(fsPath);
+ }
+ }).transform(new Function<File, DataRecord>() {
+ @Override public DataRecord apply(File input) {
+ return new FSBackendDataRecord(backend, new DataIdentifier(input.getName()),
+ input);
+ }
+ }).iterator();
+ }
+
+
+ @Override
+ public void close() throws DataStoreException {
+ }
+
+ @Override
+ public byte[] getOrCreateReferenceKey() throws DataStoreException {
+ File file = new File(fsPathDir, "reference.key");
+ try {
+ if (file.exists()) {
+ return FileUtils.readFileToByteArray(file);
+ } else {
+ byte[] key = super.getOrCreateReferenceKey();
+ FileUtils.writeByteArrayToFile(file, key);
+ return key;
+ }
+ } catch (IOException e) {
+ throw new DataStoreException("Unable to access reference key file " + file.getPath(),
+ e);
+ }
+ }
+
+ /*----------------------------------- Helper Methods-- -------------------------------------**/
+
+ /**
+ * Returns the identified file. This method implements the pattern used to
+ * avoid problems with too many files in a single directory.
+ * <p>
+ * No sanity checks are performed on the given identifier.
+ *
+ * @param identifier data identifier
+ * @return identified file
+ */
+ private static File getFile(DataIdentifier identifier, File root) {
+ String string = identifier.toString();
+ File file = root;
+ file = new File(file, string.substring(0, 2));
+ file = new File(file, string.substring(2, 4));
+ file = new File(file, string.substring(4, 6));
+ return new File(file, string);
+ }
+
+ /**
+ * Get the last modified date of a file.
+ *
+ * @param file the file
+ * @return the last modified date
+ * @throws DataStoreException if reading fails
+ */
+ private static long getLastModified(File file) throws DataStoreException {
+ long lastModified = file.lastModified();
+ if (lastModified == 0) {
+ throw new DataStoreException(
+ "Failed to read record modified date: " + file.getAbsolutePath());
+ }
+ return lastModified;
+ }
+
+ /**
+ * Set the last modified date of a file, if the file is writable.
+ *
+ * @param file the file
+ * @param time the new last modified date
+ * @throws DataStoreException if the file is writable but modifying the date
+ * fails
+ */
+ private static void setLastModified(File file, long time) throws DataStoreException {
+ if (!file.setLastModified(time)) {
+ if (!file.canWrite()) {
+ // if we can't write to the file, so garbage collection will
+ // also not delete it
+ // (read only files or file systems)
+ return;
+ }
+ try {
+ // workaround for Windows: if the file is already open for
+ // reading
+ // (in this or another process), then setting the last modified
+ // date
+ // doesn't work - see also JCR-2872
+ RandomAccessFile r = new RandomAccessFile(file, "rw");
+ try {
+ r.setLength(r.length());
+ } finally {
+ r.close();
+ }
+ } catch (IOException e) {
+ throw new DataStoreException(
+ "An IO Exception occurred while trying to set the last modified date: " + file
+ .getAbsolutePath(), e);
+ }
+ }
+ }
+
+ private void deleteEmptyParentDirs(File file) {
+ File parent = file.getParentFile();
+ try {
+ // Only iterate & delete if parent directory of the blob file is
+ // child
+ // of the base directory and if it is empty
+ while (FileUtils.directoryContains(fsPathDir, parent)) {
+ String[] entries = parent.list();
+ if (entries == null) {
+ LOG.warn("Failed to list directory {}", parent.getAbsolutePath());
+ break;
+ }
+ if (entries.length > 0) {
+ break;
+ }
+ boolean deleted = parent.delete();
+ LOG.debug("Deleted parent [{}] of file [{}]: {}",
+ parent, file.getAbsolutePath(), deleted);
+ parent = parent.getParentFile();
+ }
+ } catch (IOException e) {
+ LOG.warn("Error in parents deletion for " + file.getAbsoluteFile(), e);
+ }
+ }
+
+ /*--------------------------------- Gettters & Setters -------------------------------------**/
+
+ /**
+ * Properties used to configure the backend. These are mandatorily to be provided explicitly
+ * before calling {{@link #init()} is invoked.
+ *
+ * @param properties to configure Backend
+ */
+ public void setProperties(Properties properties) {
+ this.properties = properties;
+ }
+
+ /*-------------------------------- Inner classes -------------------------------------------**/
+
+ /**
+ * FSBackendDataRecord which lazily retrieves the input stream of the record.
+ */
+ class FSBackendDataRecord extends AbstractDataRecord {
+ private long length;
+ private long lastModified;
+ private File file;
+
+ public FSBackendDataRecord(AbstractSharedBackend backend,
+ @Nonnull DataIdentifier identifier, @Nonnull File file) {
+ super(backend, identifier);
+ this.file = file;
+ this.length = file.length();
+ this.lastModified = file.lastModified();
+ }
+
+ @Override public long getLength() throws DataStoreException {
+ return length;
+ }
+
+ @Override public InputStream getStream() throws DataStoreException {
+ try {
+ return new LazyFileInputStream(file);
+ } catch (FileNotFoundException e) {
+ LOG.error("Error in returning stream", e);
+ throw new DataStoreException(e);
+ }
+ }
+
+ @Override public long getLastModified() {
+ return lastModified;
+ }
+
+ @Override public String toString() {
+ return "S3DataRecord{" + "identifier=" + getIdentifier() + ", length=" + length
+ + ", lastModified=" + lastModified + '}';
+ }
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackend.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FileDataStoreService.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FileDataStoreService.java?rev=1766350&r1=1766349&r2=1766350&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FileDataStoreService.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FileDataStoreService.java Mon Oct 24 04:53:55 2016
@@ -22,14 +22,17 @@ package org.apache.jackrabbit.oak.plugin
import com.google.common.base.Preconditions;
import org.apache.felix.scr.annotations.Component;
import org.apache.felix.scr.annotations.ConfigurationPolicy;
-import org.apache.jackrabbit.core.data.CachingFDS;
import org.apache.jackrabbit.core.data.DataStore;
import org.apache.jackrabbit.oak.commons.PropertiesUtil;
+import org.apache.jackrabbit.oak.plugins.blob.AbstractSharedCachingDataStore;
+import org.osgi.framework.Constants;
+import org.osgi.framework.ServiceRegistration;
import org.osgi.service.component.ComponentContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.jcr.RepositoryException;
+import java.util.Dictionary;
+import java.util.Hashtable;
import java.util.Map;
import java.util.Properties;
@@ -37,11 +40,15 @@ import java.util.Properties;
public class FileDataStoreService extends AbstractDataStoreService {
public static final String NAME = "org.apache.jackrabbit.oak.plugins.blob.datastore.FileDataStore";
+ private static final String DESCRIPTION = "oak.datastore.description";
+
public static final String CACHE_PATH = "cachePath";
public static final String CACHE_SIZE = "cacheSize";
public static final String FS_BACKEND_PATH = "fsBackendPath";
public static final String PATH = "path";
+ private ServiceRegistration delegateReg;
+
private Logger log = LoggerFactory.getLogger(getClass());
@Override
@@ -53,11 +60,7 @@ public class FileDataStoreService extend
String fsBackendPath = PropertiesUtil.toString(config.get(PATH), null);
Preconditions.checkNotNull(fsBackendPath, "Cannot create " +
"FileDataStoreService with caching. [{path}] property not configured.");
- OakCachingFDS dataStore = new OakCachingFDS();
- dataStore.setFsBackendPath(fsBackendPath);
- // Disabling asyncUpload by default
- dataStore.setAsyncUploadLimit(PropertiesUtil.toInteger(config.get("asyncUploadLimit"), 0));
config.remove(PATH);
config.remove(CACHE_SIZE);
config.put(FS_BACKEND_PATH, fsBackendPath);
@@ -69,15 +72,41 @@ public class FileDataStoreService extend
}
Properties properties = new Properties();
properties.putAll(config);
- dataStore.setProperties(properties);
- log.info("CachingFDS initialized with properties " + properties);
- return dataStore;
+ log.info("Initializing with properties " + properties);
+
+ if (JR2_CACHING) {
+ OakCachingFDS dataStore = new OakCachingFDS();
+ dataStore.setFsBackendPath(fsBackendPath);
+
+ // Disabling asyncUpload by default
+ dataStore.setAsyncUploadLimit(
+ PropertiesUtil.toInteger(config.get("asyncUploadLimit"), 0));
+ dataStore.setProperties(properties);
+ return dataStore;
+ }
+ return getCachingDataStore(properties, context);
} else {
log.info("OakFileDataStore initialized");
return new OakFileDataStore();
}
}
+ private DataStore getCachingDataStore(Properties props, ComponentContext context) {
+ CachingFileDataStore dataStore = new CachingFileDataStore();
+ dataStore.setStagingSplitPercentage(
+ PropertiesUtil.toInteger(props.get("stagingSplitPercentage"), 0));
+ dataStore.setProperties(props);
+ Dictionary<String, Object> config = new Hashtable<String, Object>();
+ config.put(Constants.SERVICE_PID, dataStore.getClass().getName());
+ config.put(DESCRIPTION, getDescription());
+
+ delegateReg = context.getBundleContext().registerService(new String[] {
+ AbstractSharedCachingDataStore.class.getName(),
+ AbstractSharedCachingDataStore.class.getName()
+ }, dataStore , config);
+ return dataStore;
+ }
+
@Override
protected String[] getDescription() {
return new String[]{"type=filesystem"};
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreTest.java?rev=1766350&r1=1766349&r2=1766350&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreTest.java Mon Oct 24 04:53:55 2016
@@ -30,6 +30,7 @@ import java.util.Random;
import javax.jcr.RepositoryException;
+import com.google.common.collect.Sets;
import org.apache.jackrabbit.core.data.DataIdentifier;
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStore;
@@ -85,13 +86,13 @@ public abstract class AbstractDataStoreT
* Delete temporary directory.
*/
@Before
- protected void setUp() throws Exception {
+ public void setUp() throws Exception {
dataStoreDir = folder.newFolder().getAbsolutePath();
ds = createDataStore();
}
@After
- protected void tearDown() {
+ public void tearDown() {
try {
ds.close();
} catch (DataStoreException e) {
@@ -318,7 +319,10 @@ public abstract class AbstractDataStoreT
DataRecord rec3 = ds.addRecord(new ByteArrayInputStream(data3));
((MultiDataStoreAware)ds).deleteRecord(rec2.getIdentifier());
-
+ // Try again if async uploads
+ if (ds.getRecordIfStored(rec2.getIdentifier()) != null) {
+ ((MultiDataStoreAware)ds).deleteRecord(rec2.getIdentifier());
+ }
assertNull("rec2 should be null",
ds.getRecordIfStored(rec2.getIdentifier()));
assertEquals(new ByteArrayInputStream(data1),
@@ -349,7 +353,7 @@ public abstract class AbstractDataStoreT
rec = ds.addRecord(new ByteArrayInputStream(data));
list.add(rec.getIdentifier());
- Iterator<DataIdentifier> itr = ds.getAllIdentifiers();
+ Iterator<DataIdentifier> itr = Sets.newHashSet(ds.getAllIdentifiers()).iterator();
while (itr.hasNext()) {
assertTrue("record found on list", list.remove(itr.next()));
}
@@ -533,7 +537,7 @@ public abstract class AbstractDataStoreT
RandomInputStream in = new RandomInputStream(size + offset, size);
DataRecord rec = ds.addRecord(in);
list.add(rec);
- map.put(rec, new Integer(size));
+ map.put(rec, size);
}
Random random = new Random(1);
for (int i = 0; i < list.size(); i++) {
Added: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/CachingFileDataStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/CachingFileDataStoreTest.java?rev=1766350&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/CachingFileDataStoreTest.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/CachingFileDataStoreTest.java Mon Oct 24 04:53:55 2016
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.blob.datastore;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.util.Map;
+import java.util.Properties;
+
+import javax.jcr.RepositoryException;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.jackrabbit.core.data.DataRecord;
+import org.apache.jackrabbit.core.data.DataStore;
+import org.apache.jackrabbit.oak.commons.PropertiesUtil;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests for {@link CachingFileDataStore}.
+ */
+public class CachingFileDataStoreTest extends AbstractDataStoreTest {
+ protected static final Logger LOG = LoggerFactory.getLogger(CachingFileDataStoreTest.class);
+
+ private Properties props;
+ private String fsBackendPath;
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ fsBackendPath = folder.newFolder().getAbsolutePath();
+ props = new Properties();
+ props.setProperty("fsBackendPath", fsBackendPath);
+ super.setUp();
+ }
+
+ protected DataStore createDataStore() throws RepositoryException {
+ CachingFileDataStore ds = null;
+ try {
+ ds = new CachingFileDataStore();
+ Map<String, ?> config = DataStoreUtils.getConfig();
+ props.putAll(config);
+ PropertiesUtil.populate(ds, config, false);
+ ds.setProperties(props);
+ ds.init(dataStoreDir);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return ds;
+ }
+
+ @Test
+ public void assertReferenceKey() throws Exception {
+ byte[] data = new byte[dataLength];
+ randomGen.nextBytes(data);
+ DataRecord rec = ds.addRecord(new ByteArrayInputStream(data));
+ Assert.assertEquals(data.length, rec.getLength());
+ assertRecord(data, rec);
+ DataRecord refRec = ds.getRecordFromReference(rec.getReference());
+ assertRecord(data, refRec);
+
+ // Check bytes retrieved from reference.key file
+ File refFile = new File(fsBackendPath, "reference.key");
+ assertTrue(refFile.exists());
+ byte[] keyRet = FileUtils.readFileToByteArray(refFile);
+ assertTrue(keyRet.length != 0);
+ }
+
+ @Override
+ @After
+ public void tearDown() {
+ super.tearDown();
+ }
+
+ /**
+ * ---------- skip -----------
+ **/
+ @Override
+ public void testUpdateLastModifiedOnAccess() {
+ }
+
+ @Override
+ public void testDeleteAllOlderThan() {
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/CachingFileDataStoreTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreServiceTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreServiceTest.java?rev=1766350&r1=1766349&r2=1766350&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreServiceTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreServiceTest.java Mon Oct 24 04:53:55 2016
@@ -19,6 +19,9 @@
package org.apache.jackrabbit.oak.plugins.blob.datastore;
+import static org.apache.jackrabbit.oak.plugins.blob.datastore.AbstractDataStoreService
+ .JR2_CACHING_PROP;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -29,7 +32,10 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
+import javax.jcr.RepositoryException;
+
import com.google.common.collect.ImmutableMap;
+import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.core.data.Backend;
import org.apache.jackrabbit.core.data.CachingFDS;
import org.apache.jackrabbit.core.data.DataStore;
@@ -37,6 +43,7 @@ import org.apache.jackrabbit.core.data.F
import org.apache.jackrabbit.core.data.FileDataStore;
import org.apache.jackrabbit.oak.api.jmx.CacheStatsMBean;
import org.apache.jackrabbit.oak.commons.PropertiesUtil;
+import org.apache.jackrabbit.oak.spi.blob.SharedBackend;
import org.apache.jackrabbit.oak.spi.blob.stats.BlobStoreStatsMBean;
import org.apache.jackrabbit.oak.stats.StatisticsProvider;
import org.apache.sling.testing.mock.osgi.MockOsgi;
@@ -72,12 +79,58 @@ public class DataStoreServiceTest {
*/
@Test
public void configCachingFDS() throws Exception {
+ System.setProperty(JR2_CACHING_PROP, "true");
+ try {
+ String nasPath = folder.getRoot().getAbsolutePath() + "/NASPath";
+ String cachePath = folder.getRoot().getAbsolutePath() + "/cachePath";
+ long cacheSize = 100L;
+ Map<String, Object> config = new HashMap<String, Object>();
+ config.put("repository.home", folder.getRoot().getAbsolutePath());
+ config.put(FileDataStoreService.CACHE_SIZE, cacheSize);
+ config.put(FileDataStoreService.PATH, nasPath);
+ config.put(FileDataStoreService.CACHE_PATH, cachePath);
+ FileDataStoreService fdsSvc = new FileDataStoreService();
+
+ DataStore ds = fdsSvc.createDataStore(context.componentContext(), config);
+ PropertiesUtil.populate(ds, config, false);
+ ds.init(folder.getRoot().getAbsolutePath());
+ assertTrue("not instance of CachingFDS", ds instanceof CachingFDS);
+ CachingFDS cds = (CachingFDS) ds;
+ assertEquals("cachesize not equal", cacheSize, cds.getCacheSize());
+ assertEquals("cachepath not equal", cachePath, cds.getPath());
+ Backend backend = cds.getBackend();
+ Properties props = (Properties) getField(backend);
+ assertEquals("path not equal", nasPath, props.getProperty(FSBackend.FS_BACKEND_PATH));
+ } finally {
+ System.clearProperty(JR2_CACHING_PROP);
+ }
+ }
+
+ /**
+ *
+ * Test {@link CachingFileDataStore} is returned when cacheSize > 0 by default.
+ */
+ @Test
+ public void configCachingFileDataStore() throws Exception {
+ String nasPath = folder.getRoot().getAbsolutePath() + "/NASPath";
+ String cachePath = folder.getRoot().getAbsolutePath() + "/cachePath";
+ DataStore ds = getAssertCachingFileDataStore(nasPath, cachePath);
+ CachingFileDataStore cds = (CachingFileDataStore) ds;
+ SharedBackend backend = cds.getBackend();
+ Properties props = (Properties) getField(backend);
+ assertEquals("path not equal", nasPath, props.getProperty(FSBackend.FS_BACKEND_PATH));
+ }
+
+ /**
+ *
+ * Test to verify @FileDataStore is returned if cacheSize is not configured.
+ */
+ @Test
+ public void configFileDataStore() throws Exception {
String nasPath = folder.getRoot().getAbsolutePath() + "/NASPath";
String cachePath = folder.getRoot().getAbsolutePath() + "/cachePath";
- long cacheSize = 100l;
Map<String, Object> config = new HashMap<String, Object>();
config.put("repository.home", folder.getRoot().getAbsolutePath());
- config.put(FileDataStoreService.CACHE_SIZE, cacheSize);
config.put(FileDataStoreService.PATH, nasPath);
config.put(FileDataStoreService.CACHE_PATH, cachePath);
FileDataStoreService fdsSvc = new FileDataStoreService();
@@ -85,27 +138,41 @@ public class DataStoreServiceTest {
DataStore ds = fdsSvc.createDataStore(context.componentContext(), config);
PropertiesUtil.populate(ds, config, false);
ds.init(folder.getRoot().getAbsolutePath());
- assertTrue("not instance of CachingFDS", ds instanceof CachingFDS);
- CachingFDS cds = (CachingFDS) ds;
- assertEquals("cachesize not equal", cacheSize, cds.getCacheSize());
- assertEquals("cachepath not equal", cachePath, cds.getPath());
- Backend backend = cds.getBackend();
- Properties props = (Properties) getField(backend, "properties");
- assertEquals("path not equal", nasPath, props.getProperty(FSBackend.FS_BACKEND_PATH));
-
-
+ assertTrue("not instance of FileDataStore", ds instanceof FileDataStore);
+ FileDataStore fds = (FileDataStore) ds;
+ assertEquals("path not equal", nasPath, fds.getPath());
}
/**
- *
- * Test to verify @FileDataStore is returned if cacheSize is not configured.
+ * Tests the regitration of CachingFileDataStore and checks existence of
+ * reference.key file on first access of getOrCreateReference.
+ * @throws Exception
*/
@Test
- public void configFileDataStore() throws Exception {
+ public void registerAndCheckReferenceKey() throws Exception {
+ context.registerService(StatisticsProvider.class, StatisticsProvider.NOOP);
+
String nasPath = folder.getRoot().getAbsolutePath() + "/NASPath";
String cachePath = folder.getRoot().getAbsolutePath() + "/cachePath";
+ DataStore ds = getAssertCachingFileDataStore(nasPath, cachePath);
+ final CachingFileDataStore dataStore = (CachingFileDataStore) ds;
+
+ byte[] key = dataStore.getBackend().getOrCreateReferenceKey();
+
+ // Check bytes retrieved from reference.key file
+ File refFile = new File(nasPath, "reference.key");
+ byte[] keyRet = FileUtils.readFileToByteArray(refFile);
+ assertArrayEquals(key, keyRet);
+
+ assertArrayEquals(key, dataStore.getBackend().getOrCreateReferenceKey());
+ }
+
+ private DataStore getAssertCachingFileDataStore(String nasPath, String cachePath)
+ throws RepositoryException {
+ long cacheSize = 100L;
Map<String, Object> config = new HashMap<String, Object>();
config.put("repository.home", folder.getRoot().getAbsolutePath());
+ config.put(FileDataStoreService.CACHE_SIZE, cacheSize);
config.put(FileDataStoreService.PATH, nasPath);
config.put(FileDataStoreService.CACHE_PATH, cachePath);
FileDataStoreService fdsSvc = new FileDataStoreService();
@@ -113,13 +180,12 @@ public class DataStoreServiceTest {
DataStore ds = fdsSvc.createDataStore(context.componentContext(), config);
PropertiesUtil.populate(ds, config, false);
ds.init(folder.getRoot().getAbsolutePath());
- assertTrue("not instance of FileDataStore", ds instanceof FileDataStore);
- FileDataStore fds = (FileDataStore) ds;
- assertEquals("path not equal", nasPath, fds.getPath());
+ assertTrue("not instance of CachingFDS", ds instanceof CachingFileDataStore);
+ return ds;
}
- private static Object getField(Object obj, String fieldName) throws Exception {
- Field f = obj.getClass().getDeclaredField(fieldName); //NoSuchFieldException
+ private static Object getField(Object obj) throws Exception {
+ Field f = obj.getClass().getDeclaredField("properties"); //NoSuchFieldException
f.setAccessible(true);
return f.get(obj);
}
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakCachingFDSTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakCachingFDSTest.java?rev=1766350&r1=1766349&r2=1766350&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakCachingFDSTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakCachingFDSTest.java Mon Oct 24 04:53:55 2016
@@ -28,12 +28,15 @@ import org.apache.jackrabbit.core.data.D
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.stats.StatisticsProvider;
import org.apache.sling.testing.mock.osgi.junit.OsgiContext;
+import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import static com.google.common.collect.Maps.newHashMap;
+import static org.apache.jackrabbit.oak.plugins.blob.datastore.AbstractDataStoreService
+ .JR2_CACHING_PROP;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertNotNull;
@@ -59,10 +62,15 @@ public class OakCachingFDSTest {
@Before
public void setup() throws Exception {
+ System.setProperty(JR2_CACHING_PROP, "true");
fsBackendPath = folder.newFolder().getAbsolutePath();
path = folder.newFolder().getAbsolutePath();
}
+ @After
+ public void tear() throws Exception {
+ System.clearProperty(JR2_CACHING_PROP);
+ }
@Test
public void createAndCheckReferenceKey() throws Exception {
createCachingFDS();
@@ -76,7 +84,7 @@ public class OakCachingFDSTest {
assertReferenceKey();
}
- public void assertReferenceKey() throws Exception {
+ private void assertReferenceKey() throws Exception {
byte[] key = dataStore.getOrCreateReferenceKey();
// Check bytes retrieved from reference.key file
Modified: jackrabbit/oak/trunk/oak-it-osgi/test-bundles.xml
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-it-osgi/test-bundles.xml?rev=1766350&r1=1766349&r2=1766350&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-it-osgi/test-bundles.xml (original)
+++ jackrabbit/oak/trunk/oak-it-osgi/test-bundles.xml Mon Oct 24 04:53:55 2016
@@ -34,6 +34,7 @@
<include>commons-io:commons-io</include>
<include>org.apache.jackrabbit:jackrabbit-api</include>
<include>org.apache.jackrabbit:jackrabbit-jcr-commons</include>
+ <include>org.apache.jackrabbit:jackrabbit-data</include>
<include>org.apache.jackrabbit:oak-commons</include>
<include>org.apache.jackrabbit:oak-core</include>
<include>org.apache.jackrabbit:oak-segment</include>