You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2017/08/11 10:22:56 UTC
svn commit: r1804763 - in /jackrabbit/oak/trunk/oak-run/src:
main/java/org/apache/jackrabbit/oak/exporter/
test/java/org/apache/jackrabbit/oak/exporter/
Author: chetanm
Date: Fri Aug 11 10:22:56 2017
New Revision: 1804763
URL: http://svn.apache.org/viewvc?rev=1804763&view=rev
Log:
OAK-6545 - Tooling to serialize NodeState as json along with blobs
NodeStateSerializer supports serializing the NodeState values to json
file with support for serializing blobs to FileDataStore
Added:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/FSBlobSerializer.java (with props)
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/JsopStreamWriter.java (with props)
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/NodeStateSerializer.java (with props)
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/exporter/
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/exporter/FSBlobSerializerTest.java (with props)
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/exporter/NodeStateSerializerTest.java (with props)
Added: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/FSBlobSerializer.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/FSBlobSerializer.java?rev=1804763&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/FSBlobSerializer.java (added)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/FSBlobSerializer.java Fri Aug 11 10:22:56 2017
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.exporter;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.jackrabbit.core.data.DataStoreException;
+import org.apache.jackrabbit.core.data.FileDataStore;
+import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.json.BlobDeserializer;
+import org.apache.jackrabbit.oak.json.BlobSerializer;
+import org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.OakFileDataStore;
+
+import static org.apache.commons.io.FileUtils.ONE_KB;
+
+/**
+ * Serializer which stores blobs in a FileDataStore format
+ */
+public class FSBlobSerializer extends BlobSerializer implements BlobDeserializer, Closeable {
+ private final File dir;
+ private final int maxInlineSize;
+ private final DataStoreBlobStore dataStore;
+
+ public FSBlobSerializer(File dir) {
+ this(dir, (int) ONE_KB * 4);
+ }
+
+ public FSBlobSerializer(File dir, int maxInlineSize) {
+ this.dir = dir;
+ this.maxInlineSize = maxInlineSize;
+ this.dataStore = createDataStore();
+ }
+
+ @Override
+ public String serialize(Blob blob) {
+ try {
+ try (InputStream is = blob.getNewStream()) {
+ return dataStore.writeBlob(is);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException("Error occurred while serializing Blob " +
+ "with id " + blob.getContentIdentity(), e);
+ }
+ }
+
+ private DataStoreBlobStore createDataStore() {
+ FileDataStore fds = new OakFileDataStore();
+ fds.setPath(dir.getAbsolutePath());
+ fds.setMinRecordLength(maxInlineSize);
+ fds.init(null);
+ return new DataStoreBlobStore(fds);
+ }
+
+ @Override
+ public Blob deserialize(String value) {
+ return new BlobStoreBlob(dataStore, value);
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (dataStore != null) {
+ try {
+ dataStore.close();
+ } catch (DataStoreException e) {
+ throw new IOException(e);
+ }
+ }
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/FSBlobSerializer.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/JsopStreamWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/JsopStreamWriter.java?rev=1804763&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/JsopStreamWriter.java (added)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/JsopStreamWriter.java Fri Aug 11 10:22:56 2017
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.exporter;
+
+import java.io.IOException;
+
+import com.google.gson.stream.JsonWriter;
+import org.apache.jackrabbit.oak.commons.json.JsopWriter;
+
+/**
+ * A streaming JsopWriter which uses gson JsonWriter
+ */
+class JsopStreamWriter implements JsopWriter {
+ private final JsonWriter w;
+
+ public JsopStreamWriter(JsonWriter w) {
+ this.w = w;
+ }
+
+ @Override
+ public JsopWriter array() {
+ try {
+ w.beginArray();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return this;
+ }
+
+ @Override
+ public JsopWriter object() {
+ try {
+ w.beginObject();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return this;
+ }
+
+ @Override
+ public JsopWriter key(String key) {
+ try {
+ w.name(key);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return this;
+ }
+
+ @Override
+ public JsopWriter value(String value) {
+ try {
+ w.value(value);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return this;
+ }
+
+ @Override
+ public JsopWriter encodedValue(String raw) {
+ try {
+ //EncodedValue call is used in JsonSerializer for
+ //double values
+ try {
+ double d = Double.parseDouble(raw);
+ w.value(d);
+ } catch (NumberFormatException e) {
+ w.value(raw);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return this;
+ }
+
+ @Override
+ public JsopWriter endObject() {
+ try {
+ w.endObject();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return this;
+ }
+
+ @Override
+ public JsopWriter endArray() {
+ try {
+ w.endArray();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return this;
+ }
+
+ @Override
+ public JsopWriter value(long x) {
+ try {
+ w.value(x);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return this;
+ }
+
+ @Override
+ public JsopWriter value(boolean b) {
+ try {
+ w.value(b);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return this;
+ }
+
+ //Unsupported operation. These are also not used by JsonSerializer
+
+ @Override
+ public JsopWriter tag(char tag) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public JsopWriter append(JsopWriter diff) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public JsopWriter newline() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void resetWriter() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void setLineLength(int length) {
+ throw new UnsupportedOperationException();
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/JsopStreamWriter.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/NodeStateSerializer.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/NodeStateSerializer.java?rev=1804763&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/NodeStateSerializer.java (added)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/NodeStateSerializer.java Fri Aug 11 10:22:56 2017
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.exporter;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.io.Writer;
+
+import com.google.common.io.Files;
+import com.google.gson.stream.JsonWriter;
+import org.apache.jackrabbit.oak.commons.json.JsopWriter;
+import org.apache.jackrabbit.oak.json.Base64BlobSerializer;
+import org.apache.jackrabbit.oak.json.BlobSerializer;
+import org.apache.jackrabbit.oak.json.JsonSerializer;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStateUtils;
+
+import static com.google.common.base.Charsets.UTF_8;
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+
+public class NodeStateSerializer {
+ public enum Format {JSON, TXT}
+
+ private final NodeState nodeState;
+ private String blobDirName = "blobs";
+ private String jsonFileName = "nodestates.json";
+ private String txtFileName = "nodestates.txt";
+ private int depth = Integer.MAX_VALUE;
+ private int maxChildNodes = Integer.MAX_VALUE;
+ private String filter = "{}";
+ private File filterFile;
+ private String path = "/";
+ private Format format = Format.JSON;
+ private boolean serializeBlobContent;
+ private boolean prettyPrint = true;
+ private FSBlobSerializer blobSerializer;
+
+ public NodeStateSerializer(NodeState nodeState) {
+ this.nodeState = nodeState;
+ }
+
+ public String serialize() throws IOException {
+ StringWriter sw = new StringWriter();
+ serialize(sw, createBlobSerializer());
+ closeSerializer();
+ return sw.toString();
+ }
+
+ public void serialize(File dir) throws IOException {
+ if (dir.exists()) {
+ checkArgument(dir.isDirectory(), "Input file must be directory [%s]", dir.getAbsolutePath());
+ } else {
+ checkState(dir.mkdirs(), "Cannot create directory [%s]", dir.getAbsolutePath());
+ }
+ File file = new File(dir, getFileName());
+ try (Writer writer = Files.newWriter(file, UTF_8)){
+ serialize(writer, createBlobSerializer(dir));
+ }
+ closeSerializer();
+ }
+
+ private void serialize(Writer writer, BlobSerializer blobSerializer) throws IOException {
+ JsopWriter jsopWriter = null;
+ if (format == Format.JSON) {
+ JsonWriter jw = new JsonWriter(writer);
+ if (prettyPrint) {
+ jw.setIndent(" ");
+ }
+ jsopWriter = new JsopStreamWriter(jw);
+ }
+
+ serialize(jsopWriter, blobSerializer);
+ }
+
+ private void serialize(JsopWriter writer, BlobSerializer blobSerializer) throws IOException {
+ JsonSerializer serializer = new JsonSerializer(writer, depth, 0, maxChildNodes, getFilter(), blobSerializer);
+ NodeState state = NodeStateUtils.getNode(nodeState, path);
+ serializer.serialize(state);
+ }
+
+ private BlobSerializer createBlobSerializer(File dir) {
+ if (!serializeBlobContent) {
+ return new BlobSerializer();
+ }
+ File blobs = new File(dir, blobDirName);
+ blobSerializer = new FSBlobSerializer(blobs);
+ return blobSerializer;
+ }
+
+ private void closeSerializer() throws IOException {
+ if (blobSerializer != null) {
+ blobSerializer.close();
+ }
+ }
+
+ private BlobSerializer createBlobSerializer() {
+ return serializeBlobContent ? new Base64BlobSerializer() : new BlobSerializer();
+ }
+
+ private String getFilter() throws IOException {
+ return filterFile != null ? Files.toString(filterFile, UTF_8) : filter;
+ }
+
+ public String getFileName() {
+ return format == Format.JSON ? jsonFileName : txtFileName;
+ }
+
+ public String getBlobDirName() {
+ return blobDirName;
+ }
+
+ public void setBlobDirName(String blobDirName) {
+ this.blobDirName = blobDirName;
+ }
+
+ public void setJsonFileName(String jsonFileName) {
+ this.jsonFileName = jsonFileName;
+ }
+
+ public void setTxtFileName(String txtFileName) {
+ this.txtFileName = txtFileName;
+ }
+
+ public void setDepth(int depth) {
+ this.depth = depth;
+ }
+
+ public void setMaxChildNodes(int maxChildNodes) {
+ this.maxChildNodes = maxChildNodes;
+ }
+
+ public void setFilter(String filter) {
+ this.filter = filter;
+ }
+
+ public void setFilterFile(File filterFile) {
+ this.filterFile = filterFile;
+ }
+
+ public void setPath(String path) {
+ this.path = path;
+ }
+
+ public void setFormat(Format format) {
+ this.format = format;
+ }
+
+ public void setSerializeBlobContent(boolean serializeBlobContent) {
+ this.serializeBlobContent = serializeBlobContent;
+ }
+
+ public void setPrettyPrint(boolean prettyPrint) {
+ this.prettyPrint = prettyPrint;
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/exporter/NodeStateSerializer.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/exporter/FSBlobSerializerTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/exporter/FSBlobSerializerTest.java?rev=1804763&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/exporter/FSBlobSerializerTest.java (added)
+++ jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/exporter/FSBlobSerializerTest.java Fri Aug 11 10:22:56 2017
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.exporter;
+
+import java.io.File;
+
+import joptsimple.internal.Strings;
+import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.plugins.memory.AbstractBlob;
+import org.apache.jackrabbit.oak.plugins.memory.ArrayBasedBlob;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import static com.google.common.base.Charsets.UTF_8;
+import static org.junit.Assert.assertTrue;
+
+public class FSBlobSerializerTest {
+
+ @Rule
+ public final TemporaryFolder folder = new TemporaryFolder(new File("target"));
+
+ @Test
+ public void blobs() throws Exception{
+ int maxInlineSize = 100;
+ FSBlobSerializer serializer = new FSBlobSerializer(folder.getRoot(), maxInlineSize);
+ String data = Strings.repeat('x', maxInlineSize * 10);
+
+ Blob b = new ArrayBasedBlob(data.getBytes(UTF_8));
+
+ String id = serializer.serialize(b);
+ Blob b2 = serializer.deserialize(id);
+
+ assertTrue(AbstractBlob.equal(b, b2));
+ }
+
+}
\ No newline at end of file
Propchange: jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/exporter/FSBlobSerializerTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/exporter/NodeStateSerializerTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/exporter/NodeStateSerializerTest.java?rev=1804763&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/exporter/NodeStateSerializerTest.java (added)
+++ jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/exporter/NodeStateSerializerTest.java Fri Aug 11 10:22:56 2017
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.exporter;
+
+import java.io.File;
+
+import com.google.common.io.Files;
+import org.apache.jackrabbit.oak.json.BlobDeserializer;
+import org.apache.jackrabbit.oak.json.JsonDeserializer;
+import org.apache.jackrabbit.oak.spi.state.EqualsDiff;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import static com.google.common.base.Charsets.UTF_8;
+import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.mock;
+
+public class NodeStateSerializerTest {
+ private NodeBuilder builder = EMPTY_NODE.builder();
+ private BlobDeserializer blobHandler = mock(BlobDeserializer.class);
+
+ @Rule
+ public final TemporaryFolder folder = new TemporaryFolder(new File("target"));
+
+ @Test
+ public void basics() throws Exception{
+ builder.child("a").setProperty("foo", "bar");
+
+ NodeStateSerializer serializer = new NodeStateSerializer(builder.getNodeState());
+ String json = serializer.serialize();
+ NodeState nodeState2 = deserialize(json);
+ assertTrue(EqualsDiff.equals(builder.getNodeState(), nodeState2));
+ }
+
+ @Test
+ public void serializeToFile() throws Exception{
+ builder.child("a").setProperty("foo", "bar");
+
+ NodeStateSerializer serializer = new NodeStateSerializer(builder.getNodeState());
+ serializer.serialize(folder.getRoot());
+
+ File json = new File(folder.getRoot(), serializer.getFileName());
+ assertTrue(json.exists());
+
+ String text = Files.toString(json, UTF_8);
+ NodeState nodeState2 = deserialize(text);
+ assertTrue(EqualsDiff.equals(builder.getNodeState(), nodeState2));
+
+ }
+
+ private NodeState deserialize(String json) {
+ JsonDeserializer deserializer = new JsonDeserializer(blobHandler);
+ return deserializer.deserialize(json);
+ }
+
+}
\ No newline at end of file
Propchange: jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/exporter/NodeStateSerializerTest.java
------------------------------------------------------------------------------
svn:eol-style = native