You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by re...@apache.org on 2026/05/14 10:56:18 UTC
(jackrabbit-oak) branch trunk updated: OAK-12209: oak-upgrade: remove unused code (LengthCachingDataStore) (#2897)
This is an automated email from the ASF dual-hosted git repository.
reschke pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 7396ed90b9 OAK-12209: oak-upgrade: remove unused code (LengthCachingDataStore) (#2897)
7396ed90b9 is described below
commit 7396ed90b9ea93fc904e6c8e2ad5768f4a06e979
Author: Julian Reschke <re...@apache.org>
AuthorDate: Thu May 14 12:56:12 2026 +0200
OAK-12209: oak-upgrade: remove unused code (LengthCachingDataStore) (#2897)
---
.../oak/upgrade/blob/LengthCachingDataStore.java | 352 ---------------------
.../upgrade/blob/LengthCachingDataStoreTest.java | 185 -----------
2 files changed, 537 deletions(-)
diff --git a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/blob/LengthCachingDataStore.java b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/blob/LengthCachingDataStore.java
deleted file mode 100644
index f4a1dd3f38..0000000000
--- a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/blob/LengthCachingDataStore.java
+++ /dev/null
@@ -1,352 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.jackrabbit.oak.upgrade.blob;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStreamWriter;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.ConcurrentHashMap;
-
-import javax.jcr.RepositoryException;
-
-import org.apache.commons.io.FilenameUtils;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.io.LineIterator;
-import org.apache.jackrabbit.core.data.AbstractDataRecord;
-import org.apache.jackrabbit.core.data.AbstractDataStore;
-import org.apache.jackrabbit.core.data.DataIdentifier;
-import org.apache.jackrabbit.core.data.DataRecord;
-import org.apache.jackrabbit.core.data.DataStore;
-import org.apache.jackrabbit.core.data.DataStoreException;
-import org.apache.jackrabbit.oak.commons.PropertiesUtil;
-import org.apache.jackrabbit.oak.commons.conditions.Validate;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.jackrabbit.oak.commons.conditions.Validate.checkArgument;
-import static java.util.Objects.requireNonNull;
-
-/**
- * A DelegatingDataStore can avoid performing expensive file system access by making
- * use of pre computed data related to files in DataStore.
- * <p>
- * During repository migration actual blob content is not accessed and instead
- * only the blob length and blob references are accessed. DelegatingDataStore can be
- * configured with a mapping file which would be used to determine the length of given
- * blob reference.
- * <p>
- * Mapping file format
- * <pre>{@code
- * #< length >| < identifier >
- * 4432|dd10bca036f3134352c63e534d4568a3d2ac2fdc
- * 32167|dd10bca036f3134567c63e534d4568a3d2ac2fdc
- * }</pre>
- * <p>
- * The Configuration:
- * <pre>{@code
- * <DataStore class="org.apache.jackrabbit.oak.upgrade.blob.LengthCachingDataStore">
- * <param name="mappingFilePath" value="/path/to/mapping/file" />
- * <param name="delegateClass" value="org.apache.jackrabbit.core.data.FileDataStore" />
- * </DataStore>
- * }</pre>
- */
-public class LengthCachingDataStore extends AbstractDataStore {
- private static final Logger log = LoggerFactory.getLogger(LengthCachingDataStore.class);
- /**
- * Separator used while writing length and identifier to the mapping file
- */
- public static final char SEPARATOR = '|';
-
- //TODO For now using an in memory map. For very large repositories
- //this might consume lots of memory. For such case we would need to switch to
- //some off heap map
- private Map<String, Long> existingMappings = Collections.emptyMap();
- private Map<String, Long> newMappings = new ConcurrentHashMap<>();
-
- private String mappingFilePath = "datastore-list.txt";
- private String delegateClass;
- private String delegateConfigFilePath;
- private DataStore delegate;
- private boolean readOnly = true;
- private File mappingFile;
-
- @Override
- public void init(String homeDir) throws RepositoryException {
- initializeDelegate(homeDir);
- initializeMappingData(homeDir);
- }
-
- @Override
- public DataRecord getRecordIfStored(DataIdentifier dataIdentifier) throws DataStoreException {
- if (existingMappings.containsKey(dataIdentifier.toString())) {
- return new DelegateDataRecord(this, dataIdentifier, existingMappings);
- } else if (newMappings.containsKey(dataIdentifier.toString())) {
- return new DelegateDataRecord(this, dataIdentifier, newMappings);
- }
- DataRecord result = getDelegate().getRecordIfStored(dataIdentifier);
- addNewMapping(result);
- return result;
- }
-
- @Override
- public DataRecord getRecordFromReference(String reference) throws DataStoreException {
- //Override the getRecordFromReference so that reference handling does not
- //trigger a call to FS
- if (reference != null) {
- int colon = reference.indexOf(':');
- if (colon != -1) {
- return getRecordIfStored(new DataIdentifier(reference.substring(0, colon)));
- }
- }
- return null;
- }
-
- @Override
- public DataRecord addRecord(InputStream inputStream) throws DataStoreException {
- checkIfReadOnly();
- DataRecord result = getDelegate().addRecord(inputStream);
- addNewMapping(result);
- return result;
- }
-
- @Override
- public void updateModifiedDateOnAccess(long before) {
- checkIfReadOnly();
- getDelegate().updateModifiedDateOnAccess(before);
-
- }
-
- @Override
- public int deleteAllOlderThan(long min) throws DataStoreException {
- checkIfReadOnly();
- return getDelegate().deleteAllOlderThan(min);
- }
-
- @Override
- public Iterator<DataIdentifier> getAllIdentifiers() throws DataStoreException {
- return getDelegate().getAllIdentifiers();
- }
-
- @Override
- public int getMinRecordLength() {
- return getDelegate().getMinRecordLength();
- }
-
- @Override
- public void close() throws DataStoreException {
- existingMappings.clear();
- saveNewMappingsToFile();
-
- if (delegate != null) {
- delegate.close();
- }
- }
-
- @Override
- public void clearInUse() {
- getDelegate().clearInUse();
- }
-
- File getMappingFile() {
- return mappingFile;
- }
-
- //~---------------------------------< Setters >
-
- public void setMappingFilePath(String mappingFilePath) {
- this.mappingFilePath = mappingFilePath;
- }
-
- public void setReadOnly(boolean readOnly) {
- this.readOnly = readOnly;
- }
-
- public void setDelegateClass(String delegateClass) {
- this.delegateClass = delegateClass;
- }
-
- public void setDelegateConfigFilePath(String delegateConfigFilePath) {
- this.delegateConfigFilePath = delegateConfigFilePath;
- }
-
- //~---------------------------------< DelegateDataRecord >
-
- private class DelegateDataRecord extends AbstractDataRecord {
- private final Map<String, Long> mapping;
- private DataRecord delegateRecord;
-
- public DelegateDataRecord(AbstractDataStore store, DataIdentifier identifier,
- Map<String, Long> recordSizeMapping) {
- super(store, identifier);
- this.mapping = recordSizeMapping;
- }
-
- @Override
- public long getLength() throws DataStoreException {
- Long size = mapping.get(getIdentifier().toString());
- if (size == null) {
- log.info("No size mapping found for {}. Checking with delegate", getIdentifier());
- return getDelegateRecord().getLength();
- }
- return size;
- }
-
- @Override
- public InputStream getStream() throws DataStoreException {
- return getDelegateRecord().getStream();
- }
-
- @Override
- public long getLastModified() {
- try {
- return getDelegateRecord().getLastModified();
- } catch (DataStoreException e) {
- throw new RuntimeException(e);
- }
- }
-
- private DataRecord getDelegateRecord() throws DataStoreException {
- //Lazily load the delegateRecord to avoid FS access
- if (delegateRecord == null) {
- delegateRecord = getDelegate().getRecord(getIdentifier());
- }
- return delegateRecord;
- }
- }
-
- //~---------------------------------< internal >
-
- private void checkIfReadOnly() {
- Validate.checkState(!readOnly, "Read only DataStore in use");
- }
-
- private DataStore getDelegate() {
- return requireNonNull(delegate, "Delegate DataStore not configured");
- }
-
- private void addNewMapping(DataRecord dr) throws DataStoreException {
- if (dr != null) {
- newMappings.put(dr.getIdentifier().toString(), dr.getLength());
- }
- }
-
- private void initializeMappingData(String homeDir) {
- mappingFile = new File(FilenameUtils.concat(homeDir, mappingFilePath));
- if (mappingFile.exists()) {
- try {
- existingMappings = loadMappingData(mappingFile);
- } catch (IOException e) {
- throw new RuntimeException("Failed to read mapping data from " + mappingFile, e);
- }
- } else {
- log.info("Mapping file {} not found. Would create a new one.", mappingFile);
- }
- }
-
- private void initializeDelegate(String homeDir) throws RepositoryException {
- requireNonNull(delegateClass, "No delegate DataStore class defined via 'delegateClass' property");
- try {
- delegate = (DataStore) getClass().getClassLoader().loadClass(delegateClass).newInstance();
- } catch (InstantiationException e) {
- throw new RepositoryException("Cannot load delegate class " + delegateClass, e);
- } catch (IllegalAccessException e) {
- throw new RepositoryException("Cannot load delegate class " + delegateClass, e);
- } catch (ClassNotFoundException e) {
- throw new RepositoryException("Cannot load delegate class " + delegateClass, e);
- }
-
- log.info("Using {} as the delegating DataStore", delegateClass);
- if (delegateConfigFilePath != null) {
- File configFile = new File(delegateConfigFilePath);
- checkArgument(configFile.exists(), "Delegate DataStore config file %s does not exist", configFile.getAbsolutePath());
-
- try (InputStream is = new FileInputStream(configFile)) {
- Properties props = new Properties();
- props.load(is);
- PropertiesUtil.populate(delegate, propsToMap(props), false);
- log.info("Configured the delegating DataStore via {}", configFile.getAbsolutePath());
- } catch (IOException e) {
- throw new RepositoryException("Error reading from config file " + configFile.getAbsolutePath(), e);
- }
- }
-
- delegate.init(homeDir);
- }
-
- private void saveNewMappingsToFile() {
- if (!newMappings.isEmpty()) {
- BufferedWriter w = null;
- try {
- w = new BufferedWriter(
- new OutputStreamWriter(new FileOutputStream(mappingFile, true), StandardCharsets.UTF_8));
- for (Map.Entry<String, Long> e : newMappings.entrySet()) {
- w.write(String.valueOf(e.getValue()));
- w.write(SEPARATOR);
- w.write(e.getKey());
- w.newLine();
- }
- log.info("Added {} new entries to the mapping file {}", newMappings.size(), mappingFile);
- newMappings.clear();
- } catch (IOException e) {
- log.warn("Error occurred while writing mapping data to {}", mappingFile, e);
- } finally {
- IOUtils.closeQuietly(w);
- }
- }
- }
-
- private static Map<String, Long> loadMappingData(File mappingFile) throws IOException {
- Map<String, Long> mapping = new HashMap<String, Long>();
- log.info("Reading mapping data from {}", mappingFile.getAbsolutePath());
- LineIterator itr = new LineIterator(Files.newBufferedReader(mappingFile.toPath()));
- try {
- while (itr.hasNext()) {
- String line = itr.nextLine();
- int indexOfBar = line.indexOf(SEPARATOR);
- Validate.checkState(indexOfBar > 0, "Malformed entry found [%s]", line);
- String length = line.substring(0, indexOfBar);
- String id = line.substring(indexOfBar + 1);
- mapping.put(id.trim(), Long.valueOf(length));
- }
- log.info("Total {} mapping entries found", mapping.size());
- } finally {
- LineIterator.closeQuietly(itr);
- }
- return mapping;
- }
-
- private static Map<String, Object> propsToMap(Properties p) {
- Map<String, Object> result = new HashMap<>();
- for (String keyName : p.stringPropertyNames()) {
- result.put(keyName, p.getProperty(keyName));
- }
- return result;
- }
-}
diff --git a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/blob/LengthCachingDataStoreTest.java b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/blob/LengthCachingDataStoreTest.java
deleted file mode 100644
index dd05787e0f..0000000000
--- a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/blob/LengthCachingDataStoreTest.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.jackrabbit.oak.upgrade.blob;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Files;
-import java.util.Properties;
-import java.util.Random;
-
-import org.apache.jackrabbit.core.data.DataIdentifier;
-import org.apache.jackrabbit.core.data.DataRecord;
-import org.apache.jackrabbit.core.data.DataStoreException;
-import org.apache.jackrabbit.core.data.FileDataStore;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-public class LengthCachingDataStoreTest {
-
- @Rule
- public final TemporaryFolder tempFolder = new TemporaryFolder(new File("target"));
-
- @Test
- public void mappingFileData() throws Exception {
- File root = tempFolder.getRoot();
- File mappingFile = new File(root, "mapping.txt");
- String text = "1000|foo\n2000|bar";
- Files.writeString(mappingFile.toPath(), text);
-
- LengthCachingDataStore fds = new LengthCachingDataStore();
- fds.setDelegateClass(FileDataStore.class.getName());
- fds.setMappingFilePath(mappingFile.getAbsolutePath());
- fds.init(tempFolder.getRoot().getAbsolutePath());
-
- DataRecord dr = fds.getRecord(new DataIdentifier("foo"));
- assertNotNull(dr);
- assertEquals(1000, dr.getLength());
-
- assertEquals(2000, fds.getRecord(new DataIdentifier("bar")).getLength());
- }
-
- @Test
- public void configDelegate() throws Exception{
- //1. Store the config in a file
- Properties p = new Properties();
- p.setProperty("minRecordLength", "4972");
- File configFile = tempFolder.newFile();
- FileOutputStream fos = new FileOutputStream(configFile);
- p.store(fos, null);
- fos.close();
-
- //2. Configure the delegate and config file
- LengthCachingDataStore fds = new LengthCachingDataStore();
- fds.setDelegateClass(FileDataStore.class.getName());
- fds.setDelegateConfigFilePath(configFile.getAbsolutePath());
- fds.init(tempFolder.getRoot().getAbsolutePath());
-
- assertEquals(4972, fds.getMinRecordLength());
- }
-
- @Test
- public void delegateRecordTest() throws Exception{
- FileDataStore ds = new FileDataStore();
- byte[] data = bytes(ds.getMinRecordLength() + 10);
- ds.init(tempFolder.getRoot().getAbsolutePath());
- DataRecord dr = ds.addRecord(new ByteArrayInputStream(data));
-
- File mappingFile = new File(tempFolder.getRoot(), "mapping.txt");
- String text = String.format("%s|%s", data.length, dr.getIdentifier().toString());
- Files.writeString(mappingFile.toPath(), text);
-
- LengthCachingDataStore fds = new LengthCachingDataStore();
- fds.setDelegateClass(FileDataStore.class.getName());
- fds.setMappingFilePath(mappingFile.getAbsolutePath());
- fds.init(tempFolder.getRoot().getAbsolutePath());
-
- DataRecord dr2 = fds.getRecordIfStored(dr.getIdentifier());
- assertEquals(dr, dr2);
-
- assertEquals(dr.getLength(), dr2.getLength());
- assertArrayEquals(streamAsByteArray(dr), streamAsByteArray(dr2));
- }
-
- @Test
- public void writeBackNewEntries() throws Exception{
- //1. Add some entries to FDS
- FileDataStore fds1 = new FileDataStore();
- File fds1Dir = tempFolder.newFolder();
- int minSize = fds1.getMinRecordLength();
- fds1.init(fds1Dir.getAbsolutePath());
- DataRecord dr1 = fds1.addRecord(byteStream(minSize + 10));
- DataRecord dr2 = fds1.addRecord(byteStream(minSize + 100));
-
- //2. Try reading them so as to populate the new mappings
- LengthCachingDataStore fds2 = new LengthCachingDataStore();
- fds2.setDelegateClass(FileDataStore.class.getName());
- fds2.init(fds1Dir.getAbsolutePath());
-
- fds2.getRecord(new DataIdentifier(dr1.getIdentifier().toString()));
- fds2.getRecord(new DataIdentifier(dr2.getIdentifier().toString()));
-
- File mappingFile = fds2.getMappingFile();
-
- //3. Get the mappings pushed to file
- fds2.close();
-
- //4. Open a new FDS pointing to new directory. Read should still work fine
- //as they would be served by the mapping data
- LengthCachingDataStore fds3 = new LengthCachingDataStore();
- fds3.setDelegateClass(FileDataStore.class.getName());
- fds3.setMappingFilePath(mappingFile.getAbsolutePath());
- fds3.init(tempFolder.newFolder().getAbsolutePath());
- fds3.setReadOnly(false);
-
- assertEquals(dr1.getLength(), fds3.getRecord(dr1.getIdentifier()).getLength());
- assertEquals(dr2.getLength(), fds3.getRecord(dr2.getIdentifier()).getLength());
-
- DataRecord dr3 = fds3.addRecord(byteStream(minSize + 200));
- //5. Close again so see if update of existing file works
- fds3.close();
-
- LengthCachingDataStore fds4 = new LengthCachingDataStore();
- fds4.setDelegateClass(FileDataStore.class.getName());
- fds4.setMappingFilePath(mappingFile.getAbsolutePath());
- fds4.init(tempFolder.newFolder().getAbsolutePath());
-
- assertEquals(dr3.getLength(), fds4.getRecord(dr3.getIdentifier()).getLength());
- assertEquals(dr2.getLength(), fds4.getRecord(dr2.getIdentifier()).getLength());
-
-
- }
-
- @Test
- public void referenceHandling() throws Exception{
- int minSize = new FileDataStore().getMinRecordLength();
- LengthCachingDataStore fds = new LengthCachingDataStore();
- fds.setDelegateClass(FileDataStore.class.getName());
- fds.init(tempFolder.newFolder().getAbsolutePath());
- fds.setReadOnly(false);
-
- DataRecord dr1 = fds.addRecord(byteStream(minSize + 10));
- assertNotNull(fds.getRecordFromReference(dr1.getReference()));
- assertEquals(dr1.getIdentifier(), fds.getRecordFromReference(dr1.getReference()).getIdentifier());
- }
-
- private InputStream byteStream(int size) {
- return new ByteArrayInputStream(bytes(size));
- }
-
- private byte[] bytes(int size) {
- byte[] data = new byte[size];
- new Random().nextBytes(data);
- return data;
- }
-
- private static byte[] streamAsByteArray(final DataRecord dr) throws IOException, DataStoreException {
- try (InputStream is = dr.getStream()) {
- return is.readAllBytes();
- }
- }
-}