You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ch...@apache.org on 2018/11/30 11:50:58 UTC
[1/2] ignite git commit: IGNITE-10287: [ML] Model storage
Repository: ignite
Updated Branches:
refs/heads/master 19772109d -> 957b59c35
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/test/java/org/apache/ignite/ml/inference/storage/model/DefaultModelStorageTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/inference/storage/model/DefaultModelStorageTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/inference/storage/model/DefaultModelStorageTest.java
new file mode 100644
index 0000000..2c2a678
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/inference/storage/model/DefaultModelStorageTest.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.inference.storage.model;
+
+import java.util.concurrent.locks.Lock;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+
+/**
+ * Tests for {@link DefaultModelStorage}.
+ */
+public class DefaultModelStorageTest extends AbstractModelStorageTest {
+ /** {@inheritDoc} */
+ @Override ModelStorage getModelStorage() {
+ ModelStorageProvider provider = new LocalModelStorageProvider();
+ return new DefaultModelStorage(provider);
+ }
+
+ /** */
+ @Test
+ public void testSynchronize() {
+ Lock[] locks = new Lock[10];
+ for (int i = 0; i < locks.length; i++)
+ locks[i] = mock(Lock.class);
+
+ DefaultModelStorage.synchronize(() -> {}, locks);
+
+ for (Lock lock : locks) {
+ verify(lock, times(1)).lock();
+ verify(lock, times(1)).unlock();
+ verifyNoMoreInteractions(lock);
+ }
+ }
+
+ /** */
+ @Test
+ public void testSynchronizeWithExceptionInTask() {
+ Lock[] locks = new Lock[10];
+ for (int i = 0; i < locks.length; i++)
+ locks[i] = mock(Lock.class);
+
+ RuntimeException ex = new RuntimeException();
+
+ try {
+ DefaultModelStorage.synchronize(() -> { throw ex; }, locks);
+ fail();
+ }
+ catch (RuntimeException e) {
+ assertEquals(ex, e);
+ }
+
+ for (Lock lock : locks) {
+ verify(lock, times(1)).lock();
+ verify(lock, times(1)).unlock();
+ verifyNoMoreInteractions(lock);
+ }
+ }
+
+ /** */
+ @Test
+ public void testSynchronizeWithExceptionInLock() {
+ Lock[] locks = new Lock[10];
+ for (int i = 0; i < locks.length; i++)
+ locks[i] = mock(Lock.class);
+
+ RuntimeException ex = new RuntimeException();
+
+ doThrow(ex).when(locks[5]).lock();
+
+ try {
+ DefaultModelStorage.synchronize(() -> {}, locks);
+ fail();
+ }
+ catch (RuntimeException e) {
+ assertEquals(ex, e);
+ }
+
+ for (int i = 0; i < locks.length; i++) {
+ if (i <= 4) {
+ verify(locks[i], times(1)).lock();
+ verify(locks[i], times(1)).unlock();
+ }
+ else if (i > 5) {
+ verify(locks[i], times(0)).lock();
+ verify(locks[i], times(0)).unlock();
+ }
+ else {
+ verify(locks[i], times(1)).lock();
+ verify(locks[i], times(0)).unlock();
+ }
+
+ verifyNoMoreInteractions(locks[i]);
+ }
+ }
+
+ /** */
+ @Test
+ public void testSynchronizeWithExceptionInUnlock() {
+ Lock[] locks = new Lock[10];
+ for (int i = 0; i < locks.length; i++)
+ locks[i] = mock(Lock.class);
+
+ RuntimeException ex = new RuntimeException();
+
+ doThrow(ex).when(locks[5]).unlock();
+
+ try {
+ DefaultModelStorage.synchronize(() -> {}, locks);
+ fail();
+ }
+ catch (RuntimeException e) {
+ assertEquals(ex, e);
+ }
+
+ for (Lock lock : locks) {
+ verify(lock, times(1)).lock();
+ verify(lock, times(1)).unlock();
+ verifyNoMoreInteractions(lock);
+ }
+ }
+}
[2/2] ignite git commit: IGNITE-10287: [ML] Model storage
Posted by ch...@apache.org.
IGNITE-10287: [ML] Model storage
This closes #5507
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/957b59c3
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/957b59c3
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/957b59c3
Branch: refs/heads/master
Commit: 957b59c35e8bc832f6638d565db5341d422d30eb
Parents: 1977210
Author: Anton Dmitriev <dm...@gmail.com>
Authored: Fri Nov 30 14:50:50 2018 +0300
Committer: Yury Babak <yb...@gridgain.com>
Committed: Fri Nov 30 14:50:50 2018 +0300
----------------------------------------------------------------------
examples/config/example-ignite-ml.xml | 38 +++
.../ml/inference/ModelStorageExample.java | 122 ++++++++
.../ignite/ml/inference/ModelDescriptor.java | 8 +
.../reader/FileSystemInfModelReader.java | 2 +-
.../reader/ModelStorageInfModelReader.java | 64 ++++
.../storage/IgniteModelDescriptorStorage.java | 57 ----
.../storage/LocalModelDescriptorStorage.java | 45 ---
.../storage/ModelDescriptorStorage.java | 48 ---
.../IgniteModelDescriptorStorage.java | 66 ++++
.../descriptor/LocalModelDescriptorStorage.java | 52 +++
.../descriptor/ModelDescriptorStorage.java | 58 ++++
.../ModelDescriptorStorageFactory.java | 43 +++
.../storage/descriptor/package-info.java | 22 ++
.../storage/model/DefaultModelStorage.java | 313 +++++++++++++++++++
.../ml/inference/storage/model/Directory.java | 42 +++
.../ignite/ml/inference/storage/model/File.java | 48 +++
.../storage/model/FileOrDirectory.java | 41 +++
.../model/IgniteModelStorageProvider.java | 58 ++++
.../model/LocalModelStorageProvider.java | 56 ++++
.../inference/storage/model/ModelStorage.java | 115 +++++++
.../storage/model/ModelStorageFactory.java | 42 +++
.../storage/model/ModelStorageProvider.java | 56 ++++
.../inference/storage/model/package-info.java | 22 ++
.../ml/inference/storage/package-info.java | 2 +-
.../apache/ignite/ml/util/plugin/MLPlugin.java | 30 ++
.../ml/util/plugin/MLPluginConfiguration.java | 77 +++++
.../ignite/ml/util/plugin/MLPluginProvider.java | 195 ++++++++++++
.../org.apache.ignite.plugin.PluginProvider | 1 +
.../ignite/ml/inference/InferenceTestSuite.java | 4 +-
.../storage/model/AbstractModelStorageTest.java | 142 +++++++++
.../storage/model/DefaultModelStorageTest.java | 143 +++++++++
31 files changed, 1859 insertions(+), 153 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/examples/config/example-ignite-ml.xml
----------------------------------------------------------------------
diff --git a/examples/config/example-ignite-ml.xml b/examples/config/example-ignite-ml.xml
new file mode 100644
index 0000000..a8ac705
--- /dev/null
+++ b/examples/config/example-ignite-ml.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ Ignite configuration with all defaults and enabled p2p deployment and enabled events.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans.xsd">
+ <!-- Imports default Ignite configuration -->
+ <import resource="example-default.xml"/>
+
+ <bean parent="ignite.cfg">
+ <property name="pluginConfigurations">
+ <bean class="org.apache.ignite.ml.util.plugin.MLPluginConfiguration">
+ <property name="withMdlStorage" value="#{true}" />
+ <property name="withMdlDescStorage" value="#{true}" />
+ </bean>
+ </property>
+ </bean>
+</beans>
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/examples/src/main/java/org/apache/ignite/examples/ml/inference/ModelStorageExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/inference/ModelStorageExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/inference/ModelStorageExample.java
new file mode 100644
index 0000000..5b704f3
--- /dev/null
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/inference/ModelStorageExample.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.examples.ml.inference;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.ml.inference.InfModel;
+import org.apache.ignite.ml.inference.ModelDescriptor;
+import org.apache.ignite.ml.inference.ModelSignature;
+import org.apache.ignite.ml.inference.builder.SingleInfModelBuilder;
+import org.apache.ignite.ml.inference.parser.IgniteFunctionInfModelParser;
+import org.apache.ignite.ml.inference.reader.ModelStorageInfModelReader;
+import org.apache.ignite.ml.inference.storage.descriptor.ModelDescriptorStorage;
+import org.apache.ignite.ml.inference.storage.descriptor.ModelDescriptorStorageFactory;
+import org.apache.ignite.ml.inference.storage.model.ModelStorage;
+import org.apache.ignite.ml.inference.storage.model.ModelStorageFactory;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
+
+/**
+ * This example demonstrates how to work with {@link ModelStorage}.
+ */
+public class ModelStorageExample {
+ /** Run example. */
+ public static void main(String... args) throws IOException, ClassNotFoundException {
+ try (Ignite ignite = Ignition.start("examples/config/example-ignite-ml.xml")) {
+ System.out.println(">>> Ignite grid started.");
+
+ ModelStorage storage = new ModelStorageFactory().getModelStorage(ignite);
+ ModelDescriptorStorage descStorage = new ModelDescriptorStorageFactory().getModelDescriptorStorage(ignite);
+
+ System.out.println("Saving model into model storage...");
+ byte[] mdl = serialize((IgniteFunction<byte[], byte[]>)i -> i);
+ storage.mkdirs("/");
+ storage.putFile("/my_model", mdl);
+
+ System.out.println("Saving model descriptor into model descriptor storage...");
+ ModelDescriptor desc = new ModelDescriptor(
+ "MyModel",
+ "My Cool Model",
+ new ModelSignature("", "", ""),
+ new ModelStorageInfModelReader("/my_model"),
+ new IgniteFunctionInfModelParser<>()
+ );
+ descStorage.put("my_model", desc);
+
+ System.out.println("List saved models...");
+ for (IgniteBiTuple<String, ModelDescriptor> model : descStorage)
+ System.out.println("-> {'" + model.getKey() + "' : " + model.getValue() + "}");
+
+ System.out.println("Load saved model descriptor...");
+ desc = descStorage.get("my_model");
+
+ System.out.println("Build inference model...");
+ SingleInfModelBuilder mdlBuilder = new SingleInfModelBuilder();
+ try (InfModel<byte[], byte[]> infMdl = mdlBuilder.build(desc.getReader(), desc.getParser())) {
+
+ System.out.println("Make inference...");
+ for (int i = 0; i < 10; i++) {
+ Integer res = deserialize(infMdl.predict(serialize(i)));
+ System.out.println(i + " -> " + res);
+ }
+ }
+ }
+ }
+
+ /**
+ * Serialized the specified object.
+ *
+ * @param o Object to be serialized.
+ * @return Serialized object as byte array.
+ * @throws IOException In case of exception.
+ */
+ private static <T extends Serializable> byte[] serialize(T o) throws IOException {
+ try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ ObjectOutputStream oos = new ObjectOutputStream(baos)) {
+ oos.writeObject(o);
+ oos.flush();
+
+ return baos.toByteArray();
+ }
+ }
+
+ /**
+ * Deserialized object represented as a byte array.
+ *
+ * @param o Serialized object.
+ * @param <T> Type of serialized object.
+ * @return Deserialized object.
+ * @throws IOException In case of exception.
+ * @throws ClassNotFoundException In case of exception.
+ */
+ @SuppressWarnings("unchecked")
+ private static <T extends Serializable> T deserialize(byte[] o) throws IOException, ClassNotFoundException {
+ try (ByteArrayInputStream bais = new ByteArrayInputStream(o);
+ ObjectInputStream ois = new ObjectInputStream(bais)) {
+
+ return (T)ois.readObject();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/main/java/org/apache/ignite/ml/inference/ModelDescriptor.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/inference/ModelDescriptor.java b/modules/ml/src/main/java/org/apache/ignite/ml/inference/ModelDescriptor.java
index e156063..49a2593 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/inference/ModelDescriptor.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/inference/ModelDescriptor.java
@@ -83,4 +83,12 @@ public class ModelDescriptor implements Serializable {
public InfModelParser<byte[], byte[]> getParser() {
return parser;
}
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return "ModelDescriptor{" +
+ "name='" + name + '\'' +
+ ", desc='" + desc + '\'' +
+ '}';
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/main/java/org/apache/ignite/ml/inference/reader/FileSystemInfModelReader.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/inference/reader/FileSystemInfModelReader.java b/modules/ml/src/main/java/org/apache/ignite/ml/inference/reader/FileSystemInfModelReader.java
index 1ad2161..0f04270 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/inference/reader/FileSystemInfModelReader.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/inference/reader/FileSystemInfModelReader.java
@@ -24,7 +24,7 @@ import java.nio.file.Paths;
import org.apache.ignite.ml.inference.util.DirectorySerializer;
/**
- * Model reader that reads directory and serializes it using {@link DirectorySerializer}.
+ * Model reader that reads directory or file and serializes it using {@link DirectorySerializer}.
*/
public class FileSystemInfModelReader implements InfModelReader {
/** */
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/main/java/org/apache/ignite/ml/inference/reader/ModelStorageInfModelReader.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/inference/reader/ModelStorageInfModelReader.java b/modules/ml/src/main/java/org/apache/ignite/ml/inference/reader/ModelStorageInfModelReader.java
new file mode 100644
index 0000000..2a4d4b4
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/inference/reader/ModelStorageInfModelReader.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.inference.reader;
+
+import org.apache.ignite.Ignition;
+import org.apache.ignite.ml.inference.storage.model.ModelStorage;
+import org.apache.ignite.ml.inference.storage.model.ModelStorageFactory;
+import org.apache.ignite.ml.inference.util.DirectorySerializer;
+import org.apache.ignite.ml.math.functions.IgniteSupplier;
+
+/**
+ * Model reader that reads directory or file from model storage and serializes it using {@link DirectorySerializer}.
+ */
+public class ModelStorageInfModelReader implements InfModelReader {
+ /** */
+ private static final long serialVersionUID = -5878564742783562872L;
+
+ /** Path to the directory or file. */
+ private final String path;
+
+ /** Model storage supplier. */
+ private final IgniteSupplier<ModelStorage> mdlStorageSupplier;
+
+ /**
+ * Constructs a new instance of model storage inference model builder.
+ *
+ * @param path Path to the directory or file.
+ */
+ public ModelStorageInfModelReader(String path, IgniteSupplier<ModelStorage> mdlStorageSupplier) {
+ this.path = path;
+ this.mdlStorageSupplier = mdlStorageSupplier;
+ }
+
+ /**
+ * Constructs a new instance of model storage inference model builder.
+ *
+ * @param path Path to the directory or file.
+ */
+ public ModelStorageInfModelReader(String path) {
+ this(path, () -> new ModelStorageFactory().getModelStorage(Ignition.ignite()));
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte[] read() {
+ ModelStorage storage = mdlStorageSupplier.get();
+
+ return storage.getFile(path);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/IgniteModelDescriptorStorage.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/IgniteModelDescriptorStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/IgniteModelDescriptorStorage.java
deleted file mode 100644
index a198190..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/IgniteModelDescriptorStorage.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.inference.storage;
-
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.ml.inference.ModelDescriptor;
-
-/**
- * Model descriptor storage based on Apache Ignite cache.
- */
-public class IgniteModelDescriptorStorage implements ModelDescriptorStorage {
- /** Apache Ignite cache name to keep model descriptors. */
- private static final String MODEL_DESCRIPTOR_CACHE_NAME = "MODEL_DESCRIPTOR_CACHE";
-
- /** Apache Ignite cache to keep model descriptors. */
- private final IgniteCache<String, ModelDescriptor> models;
-
- /**
- * Constructs a new instance of Ignite model descriptor storage.
- *
- * @param ignite Ignite instance.
- */
- public IgniteModelDescriptorStorage(Ignite ignite) {
- models = ignite.getOrCreateCache(MODEL_DESCRIPTOR_CACHE_NAME);
- }
-
- /** {@inheritDoc} */
- @Override public void put(String mdlId, ModelDescriptor mdl) {
- models.put(mdlId, mdl);
- }
-
- /** {@inheritDoc} */
- @Override public ModelDescriptor get(String mdlId) {
- return models.get(mdlId);
- }
-
- /** {@inheritDoc} */
- @Override public void remove(String mdlId) {
- models.remove(mdlId);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/LocalModelDescriptorStorage.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/LocalModelDescriptorStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/LocalModelDescriptorStorage.java
deleted file mode 100644
index 99e3dac..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/LocalModelDescriptorStorage.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.inference.storage;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import org.apache.ignite.ml.inference.ModelDescriptor;
-
-/**
- * Model descriptor storage based on local hash map.
- */
-public class LocalModelDescriptorStorage implements ModelDescriptorStorage {
- /** Hash map model storage. */
- private final Map<String, ModelDescriptor> models = new ConcurrentHashMap<>();
-
- /** {@inheritDoc} */
- @Override public void put(String name, ModelDescriptor mdl) {
- models.put(name, mdl);
- }
-
- /** {@inheritDoc} */
- @Override public ModelDescriptor get(String name) {
- return models.get(name);
- }
-
- /** {@inheritDoc} */
- @Override public void remove(String name) {
- models.remove(name);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/ModelDescriptorStorage.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/ModelDescriptorStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/ModelDescriptorStorage.java
deleted file mode 100644
index 46b6b65..0000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/ModelDescriptorStorage.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.inference.storage;
-
-import org.apache.ignite.ml.inference.ModelDescriptor;
-
-/**
- * Storage that allows to load, keep and get access to model descriptors (see {@link ModelDescriptor}).
- */
-public interface ModelDescriptorStorage {
- /**
- * Saves the specified model descriptor with the specified model identifier.
- *
- * @param mdlId Model identifier.
- * @param mdl Model descriptor.
- */
- public void put(String mdlId, ModelDescriptor mdl);
-
- /**
- * Returns model descriptor saved for the specified model identifier.
- *
- * @param mdlId Model identifier.
- * @return Model descriptor.
- */
- public ModelDescriptor get(String mdlId);
-
- /**
- * Removes model descriptor for the specified model descriptor.
- *
- * @param mdlId Model identifier.
- */
- public void remove(String mdlId);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/descriptor/IgniteModelDescriptorStorage.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/descriptor/IgniteModelDescriptorStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/descriptor/IgniteModelDescriptorStorage.java
new file mode 100644
index 0000000..07503c5
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/descriptor/IgniteModelDescriptorStorage.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.inference.storage.descriptor;
+
+import java.util.Iterator;
+import java.util.Spliterator;
+import java.util.Spliterators;
+import java.util.stream.StreamSupport;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.ml.inference.ModelDescriptor;
+
+/**
+ * Model descriptor storage based on Apache Ignite cache.
+ */
+public class IgniteModelDescriptorStorage implements ModelDescriptorStorage {
+ /** Apache Ignite cache to keep model descriptors. */
+ private final IgniteCache<String, ModelDescriptor> models;
+
+ /**
+ * Constructs a new instance of Ignite model descriptor storage.
+ *
+ * @param models Ignite cache with model descriptors.
+ */
+ public IgniteModelDescriptorStorage(IgniteCache<String, ModelDescriptor> models) {
+ this.models = models;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void put(String mdlId, ModelDescriptor mdl) {
+ models.put(mdlId, mdl);
+ }
+
+ /** {@inheritDoc} */
+ @Override public ModelDescriptor get(String mdlId) {
+ return models.get(mdlId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void remove(String mdlId) {
+ models.remove(mdlId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Iterator<IgniteBiTuple<String, ModelDescriptor>> iterator() {
+ return StreamSupport.stream(Spliterators.spliteratorUnknownSize(models.iterator(), Spliterator.ORDERED), false)
+ .map(e -> new IgniteBiTuple<>(e.getKey(), e.getValue()))
+ .iterator();
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/descriptor/LocalModelDescriptorStorage.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/descriptor/LocalModelDescriptorStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/descriptor/LocalModelDescriptorStorage.java
new file mode 100644
index 0000000..df54ab7
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/descriptor/LocalModelDescriptorStorage.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.inference.storage.descriptor;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.ml.inference.ModelDescriptor;
+
+/**
+ * Model descriptor storage based on local hash map.
+ */
+public class LocalModelDescriptorStorage implements ModelDescriptorStorage {
+ /** Hash map model storage. */
+ private final Map<String, ModelDescriptor> models = new ConcurrentHashMap<>();
+
+ /** {@inheritDoc} */
+ @Override public void put(String name, ModelDescriptor mdl) {
+ models.put(name, mdl);
+ }
+
+ /** {@inheritDoc} */
+ @Override public ModelDescriptor get(String name) {
+ return models.get(name);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void remove(String name) {
+ models.remove(name);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Iterator<IgniteBiTuple<String, ModelDescriptor>> iterator() {
+ return models.entrySet().stream().map(e -> new IgniteBiTuple<>(e.getKey(), e.getValue())).iterator();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/descriptor/ModelDescriptorStorage.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/descriptor/ModelDescriptorStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/descriptor/ModelDescriptorStorage.java
new file mode 100644
index 0000000..4bb8cfd
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/descriptor/ModelDescriptorStorage.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.inference.storage.descriptor;
+
+import java.util.Iterator;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.ml.inference.ModelDescriptor;
+
+/**
+ * Storage that allows to load, keep and get access to model descriptors (see {@link ModelDescriptor}).
+ */
+public interface ModelDescriptorStorage extends Iterable<IgniteBiTuple<String, ModelDescriptor>> {
+ /**
+ * Saves the specified model descriptor with the specified model identifier.
+ *
+ * @param mdlId Model identifier.
+ * @param mdl Model descriptor.
+ */
+ public void put(String mdlId, ModelDescriptor mdl);
+
+ /**
+ * Returns model descriptor saved for the specified model identifier.
+ *
+ * @param mdlId Model identifier.
+ * @return Model descriptor.
+ */
+ public ModelDescriptor get(String mdlId);
+
+ /**
+ * Removes model descriptor for the specified model descriptor.
+ *
+ * @param mdlId Model identifier.
+ */
+ public void remove(String mdlId);
+
+ /**
+ * Returns iterator of model descriptors stored in this model descriptor storage. The objects produces by the
+ * iterator are pairs of model identifier and model descriptor.
+ *
+ * @return Iterator of pairs of model identifier and model descriptor.
+ */
+ public Iterator<IgniteBiTuple<String, ModelDescriptor>> iterator();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/descriptor/ModelDescriptorStorageFactory.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/descriptor/ModelDescriptorStorageFactory.java b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/descriptor/ModelDescriptorStorageFactory.java
new file mode 100644
index 0000000..7f5daf4
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/descriptor/ModelDescriptorStorageFactory.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.inference.storage.descriptor;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.ml.inference.ModelDescriptor;
+
+/**
+ * Model descriptor storage factory. Provides {@link ModelDescriptorStorage}.
+ */
+public class ModelDescriptorStorageFactory {
+ /** Model descriptor storage cache name. */
+ public static final String MODEL_DESCRIPTOR_STORAGE_CACHE_NAME = "MODEL_DESCRIPTOR_STORAGE";
+
+ /**
+ * Returns model descriptor storage based on Apache Ignite cache.
+ *
+ * @param ignite Ignite instance.
+ * @return Model descriptor storage.
+ */
+ public ModelDescriptorStorage getModelDescriptorStorage(Ignite ignite) {
+ IgniteCache<String, ModelDescriptor> cache = ignite.cache(MODEL_DESCRIPTOR_STORAGE_CACHE_NAME);
+
+ return new IgniteModelDescriptorStorage(cache);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/descriptor/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/descriptor/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/descriptor/package-info.java
new file mode 100644
index 0000000..b94273b
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/descriptor/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Root package for inference model descriptor storages.
+ */
+package org.apache.ignite.ml.inference.storage.descriptor;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/DefaultModelStorage.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/DefaultModelStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/DefaultModelStorage.java
new file mode 100644
index 0000000..b765198
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/DefaultModelStorage.java
@@ -0,0 +1,313 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.inference.storage.model;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Deque;
+import java.util.LinkedList;
+import java.util.Set;
+import java.util.concurrent.locks.Lock;
+import java.util.function.Supplier;
+import org.apache.ignite.lang.IgniteBiTuple;
+
+/**
+ * Default implementation of {@link ModelStorage} that can use any {@link ModelStorageProvider} as a backend storage
+ * system.
+ */
+public class DefaultModelStorage implements ModelStorage {
+ /** Ignite Cache that is used to store model storage files. */
+ private final ModelStorageProvider storageProvider;
+
+ /**
+ * Constructs a new instance of Ignite model storage.
+ *
+ * @param storageProvider Model storage provider.
+ */
+ public DefaultModelStorage(ModelStorageProvider storageProvider) {
+ this.storageProvider = storageProvider;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void putFile(String path, byte[] data, boolean onlyIfNotExist) {
+ String parentPath = getParent(path);
+
+ // Paths are locked in child-first order.
+ Lock pathLock = storageProvider.lock(path);
+ Lock parentPathLock = storageProvider.lock(parentPath);
+
+ synchronize(() -> {
+ if (exists(path) && onlyIfNotExist)
+ throw new IllegalArgumentException("File already exists [path=" + path + "]");
+
+ FileOrDirectory parent = storageProvider.get(parentPath);
+
+ // If parent doesn't exist throw an exception.
+ if (parent == null)
+ throw new IllegalArgumentException("Cannot create file because directory doesn't exist [path="
+ + path + "]");
+
+ // If parent is not a directory throw an exception.
+ if (!parent.isDirectory())
+ throw new IllegalArgumentException("Cannot create file because parent is not a directory [path="
+ + path + "]");
+
+ Directory dir = (Directory) parent;
+ // Update parent if it's a new file.
+ if (!dir.getFiles().contains(path)) {
+ dir.getFiles().add(path);
+ storageProvider.put(parentPath, parent);
+ }
+
+ // Save file into cache.
+ storageProvider.put(path, new File(data));
+ }, pathLock, parentPathLock);
+ }
+
+ /** {@inheritDoc}*/
+ @Override public byte[] getFile(String path) {
+ FileOrDirectory fileOrDir = storageProvider.get(path);
+
+ Lock pathLock = storageProvider.lock(path);
+
+ return synchronize(() -> {
+ // If file doesn't exist throw an exception.
+ if (fileOrDir == null)
+ throw new IllegalArgumentException("File doesn't exist [path=" + path + "]");
+
+ // If file is not a regular file throw an exception.
+ if (!fileOrDir.isFile())
+ throw new IllegalArgumentException("File is not a regular file [path=" + path + "]");
+
+ return ((File) fileOrDir).getData();
+ }, pathLock);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void mkdir(String path, boolean onlyIfNotExist) {
+ String parentPath = getParent(path);
+
+ // Paths are locked in child-first order.
+ Lock pathLock = storageProvider.lock(path);
+ Lock parentPathLock = storageProvider.lock(parentPath);
+
+ synchronize(() -> {
+ // If a directory associated with specified path exists return.
+ if (isDirectory(path)) {
+ if (onlyIfNotExist)
+ throw new IllegalArgumentException("Directory already exists [path=" + path + "]");
+
+ return;
+ }
+
+ // If a regular file associated with specified path exists throw an exception.
+ if (isFile(path))
+ throw new IllegalArgumentException("File with specified path already exists [path=" + path + "]");
+
+ FileOrDirectory parent = storageProvider.get(parentPath);
+
+ // If parent doesn't exist throw an exception.
+ if (parent == null)
+ throw new IllegalArgumentException("Cannot create directory because parent directory does not exist"
+ + " [path=" + path + "]");
+
+ // If parent is not a directory throw an exception.
+ if (!parent.isDirectory())
+ throw new IllegalArgumentException("Cannot create directory because parent is not a directory [path="
+ + path + "]");
+
+ Directory dir = (Directory) parent;
+ dir.getFiles().add(path);
+
+ // Update parent and save directory into cache.
+ storageProvider.put(parentPath, parent);
+ storageProvider.put(path, new Directory());
+ }, pathLock, parentPathLock);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void mkdirs(String path) {
+ Deque<IgniteBiTuple<String, Lock>> pathsToBeCreated = new LinkedList<>();
+
+ IgniteBiTuple<String, Lock> parentWithLock = null;
+
+ try {
+ while (path != null) {
+ // Paths are locked in child-first order.
+ Lock lock = storageProvider.lock(path);
+ lock.lock();
+
+ pathsToBeCreated.push(new IgniteBiTuple<>(path, lock));
+
+ if (exists(path)) {
+ if (isDirectory(path)) {
+ parentWithLock = pathsToBeCreated.pop();
+ break;
+ }
+
+ throw new IllegalArgumentException("Cannot create directory because parent is not a directory "
+ + "[path=" + path + "]");
+ }
+
+ path = getParent(path);
+ }
+
+ while (!pathsToBeCreated.isEmpty()) {
+ IgniteBiTuple<String, Lock> pathWithLock = pathsToBeCreated.pop();
+
+ storageProvider.put(pathWithLock.get1(), new Directory());
+
+ if (parentWithLock != null) {
+ Directory parentDir = (Directory)storageProvider.get(parentWithLock.get1());
+ parentDir.getFiles().add(pathWithLock.get1());
+ storageProvider.put(parentWithLock.get1(), parentDir);
+ parentWithLock.get2().unlock();
+ }
+
+ parentWithLock = pathWithLock;
+ }
+
+ if (parentWithLock != null)
+ parentWithLock.get2().unlock();
+ }
+ finally {
+ for (IgniteBiTuple<String, Lock> pathWithLock : pathsToBeCreated)
+ pathWithLock.get2().unlock();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public Set<String> listFiles(String path) {
+ Lock pathLock = storageProvider.lock(path);
+
+ pathLock.lock();
+
+ return synchronize(() -> {
+ FileOrDirectory dir = storageProvider.get(path);
+
+ // If directory doesn't exist throw an exception.
+ if (dir == null)
+ throw new IllegalArgumentException("Directory doesn't exist [path=" + path + "]");
+
+ // If directory isn't a directory throw an exception.
+ if (!dir.isDirectory())
+ throw new IllegalArgumentException("Specified path is not associated with directory [path=" + path
+ + "]");
+
+ return ((Directory) dir).getFiles();
+ }, pathLock);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void remove(String path) {
+ Lock pathLock = storageProvider.lock(path);
+
+ synchronize(() -> {
+ FileOrDirectory file = storageProvider.get(path);
+ storageProvider.remove(path);
+
+ if (file.isDirectory()) {
+ for (String s : ((Directory) file).getFiles())
+ remove(s);
+ }
+ }, pathLock);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean exists(String path) {
+ return storageProvider.get(path) != null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isDirectory(String path) {
+ FileOrDirectory file = storageProvider.get(path);
+
+ return file != null && file.isDirectory();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isFile(String path) {
+ FileOrDirectory file = storageProvider.get(path);
+
+ return file != null && file.isFile();
+ }
+
+ /**
+ * Returns parent directory for the specified path.
+ *
+ * @param path Path.
+ * @return Parent directory path.
+ */
+ private String getParent(String path) {
+ Path parentPath = Paths.get(path).getParent();
+ return parentPath == null ? null : parentPath.toString();
+ }
+
+ /**
+ * Wraps task execution into locks.
+ *
+ * @param task Runnable task.
+ * @param locks List of locks.
+ */
+ static void synchronize(Runnable task, Lock... locks) {
+ synchronize(() -> {
+ task.run();
+ return null;
+ }, locks);
+ }
+
+ /**
+ * Wraps task execution into locks. Util method.
+ * @param task Task to executed.
+ * @param locks List of locks.
+ */
+ static <T> T synchronize(Supplier<T> task, Lock... locks) {
+ Throwable ex = null;
+ T res;
+
+ int i = 0;
+ try {
+ for (; i < locks.length; i++)
+ locks[i].lock();
+
+ res = task.get();
+ }
+ finally {
+ for (i = i - 1; i >= 0; i--) {
+ try {
+ locks[i].unlock();
+ }
+ catch (RuntimeException | Error e) {
+ ex = e;
+ }
+ }
+ }
+
+ if (ex != null) {
+ if (ex instanceof RuntimeException)
+ throw (RuntimeException) ex;
+
+ if (ex instanceof Error)
+ throw (Error) ex;
+
+ throw new IllegalStateException("Unexpected type of throwable");
+ }
+
+ return res;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/Directory.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/Directory.java b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/Directory.java
new file mode 100644
index 0000000..303760f
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/Directory.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.inference.storage.model;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Implementation of directory {@link ModelStorageProvider} works with.
+ */
+class Directory implements FileOrDirectory {
+ /** */
+ private static final long serialVersionUID = -6441963559954107245L;
+
+ /** List of files in the directory. */
+ private final Set<String> files = new HashSet<>();
+
+ /** {@inheritDoc} */
+ @Override public boolean isFile() {
+ return false;
+ }
+
+ /** */
+ Set<String> getFiles() {
+ return files;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/File.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/File.java b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/File.java
new file mode 100644
index 0000000..774bdaf
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/File.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.inference.storage.model;
+
+/**
+ * Implementation of file {@link ModelStorageProvider} works with.
+ */
+class File implements FileOrDirectory {
+ /** */
+ private static final long serialVersionUID = -7739751667495712802L;
+
+ /** File content. */
+ private final byte[] data;
+
+ /**
+ * Constructs a new instance of file.
+ *
+ * @param data File content.
+ */
+ protected File(byte[] data) {
+ this.data = data;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isFile() {
+ return true;
+ }
+
+ /** */
+ protected byte[] getData() {
+ return data;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/FileOrDirectory.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/FileOrDirectory.java b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/FileOrDirectory.java
new file mode 100644
index 0000000..de16751
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/FileOrDirectory.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.inference.storage.model;
+
+import java.io.Serializable;
+
+/**
+ * Base interface for file or directory {@link ModelStorageProvider} works with.
+ */
+public interface FileOrDirectory extends Serializable {
+ /**
+ * Returns {@code true} if this object is a regular file, otherwise {@code false}.
+ *
+ * @return {@code true} if this object is a regular file, otherwise {@code false}.
+ */
+ public boolean isFile();
+
+ /**
+ * Return {@code true} if this object is a directory, otherwise {@code false}.
+ *
+ * @return {@code true} if this object is a directory, otherwise {@code false}.
+ */
+ public default boolean isDirectory() {
+ return !isFile();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/IgniteModelStorageProvider.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/IgniteModelStorageProvider.java b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/IgniteModelStorageProvider.java
new file mode 100644
index 0000000..7e40aed
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/IgniteModelStorageProvider.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.inference.storage.model;
+
+import java.util.concurrent.locks.Lock;
+import org.apache.ignite.IgniteCache;
+
+/**
+ * Implementation of {@link ModelStorageProvider} based on Apache Ignite cache.
+ */
+public class IgniteModelStorageProvider implements ModelStorageProvider {
+ /** Storage of the files and directories. */
+ private final IgniteCache<String, FileOrDirectory> cache;
+
+ /**
+ * Constructs a new instance of Ignite model storage provider.
+ *
+ * @param cache Storage of the files and directories.
+ */
+ public IgniteModelStorageProvider(IgniteCache<String, FileOrDirectory> cache) {
+ this.cache = cache;
+ }
+
+ /** {@inheritDoc} */
+ @Override public FileOrDirectory get(String path) {
+ return cache.get(path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void put(String path, FileOrDirectory file) {
+ cache.put(path, file);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void remove(String path) {
+ cache.remove(path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Lock lock(String path) {
+ return cache.lock(path);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/LocalModelStorageProvider.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/LocalModelStorageProvider.java b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/LocalModelStorageProvider.java
new file mode 100644
index 0000000..e86533d
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/LocalModelStorageProvider.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.inference.storage.model;
+
+import java.lang.ref.WeakReference;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * Implementation of {@link ModelStorageProvider} based on local {@link ConcurrentHashMap}.
+ */
+public class LocalModelStorageProvider implements ModelStorageProvider {
+ /** Storage of the files and directories. */
+ private final ConcurrentMap<String, FileOrDirectory> storage = new ConcurrentHashMap<>();
+
+ /** Storage of the locks. */
+ private final ConcurrentMap<String, WeakReference<Lock>> locks = new ConcurrentHashMap<>();
+
+ /** {@inheritDoc} */
+ @Override public FileOrDirectory get(String key) {
+ return storage.get(key);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void put(String key, FileOrDirectory file) {
+ storage.put(key, file);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void remove(String key) {
+ storage.remove(key);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Lock lock(String key) {
+ Lock lock = new ReentrantLock();
+ return locks.computeIfAbsent(key, k -> new WeakReference<>(lock)).get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/ModelStorage.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/ModelStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/ModelStorage.java
new file mode 100644
index 0000000..f4e6f85
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/ModelStorage.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.inference.storage.model;
+
+import java.util.Set;
+
+/**
+ * Storage that allows to load, keep and get access to model in byte representation.
+ */
+public interface ModelStorage {
+ /**
+ * Creates a new or replaces existing file.
+ *
+ * @param path Path to file.
+ * @param data File content.
+ * @param onlyIfNotExist If file already exists throw an exception.
+ */
+ public void putFile(String path, byte[] data, boolean onlyIfNotExist);
+
+ /**
+ * Creates a new or replaces existing file.
+ *
+ * @param path Path to file.
+ * @param data File content.
+ */
+ public default void putFile(String path, byte[] data) {
+ putFile(path, data, false);
+ }
+
+ /**
+ * Returns file content.
+ *
+ * @param path Path to file.
+ * @return File content.
+ */
+ public byte[] getFile(String path);
+
+ /**
+ * Creates directory.
+ *
+ * @param path Path to directory.
+ * @param onlyIfNotExist If directory already exists throw an exception.
+ */
+ public void mkdir(String path, boolean onlyIfNotExist);
+
+ /**
+ * Creates directory.
+ *
+ * @param path Path to directory.
+ */
+ public default void mkdir(String path) {
+ mkdir(path, false);
+ }
+
+ /**
+ * Creates directory and all required parent directories in the path.
+ *
+ * @param path Path to directory.
+ */
+ public void mkdirs(String path);
+
+ /**
+ * Returns list of files in the specified directory.
+ *
+ * @param path Path to directory.
+ * @return List of files in the specified directory.
+ */
+ public Set<String> listFiles(String path);
+
+ /**
+ * Removes specified directory or file.
+ *
+ * @param path Path to directory or file.
+ */
+ public void remove(String path);
+
+ /**
+ * Returns {@code true} if a regular file or directory exist, otherwise {@code false}.
+ *
+ * @param path Path to directory or file.
+ * @return {@code true} if a regular file or directory exist, otherwise {@code false}.
+ */
+ public boolean exists(String path);
+
+ /**
+ * Returns {@code true} if the specified path associated with a directory.
+ *
+ * @param path Path to directory or file.
+ * @return {@code true} if the specified path associated with a directory.
+ */
+ public boolean isDirectory(String path);
+
+ /**
+ * Returns {@code true} if the specified path associated with a regular file.
+ *
+ * @param path Path to directory or file.
+ * @return {@code true} if the specified path associated with a regular file.
+ */
+ public boolean isFile(String path);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/ModelStorageFactory.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/ModelStorageFactory.java b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/ModelStorageFactory.java
new file mode 100644
index 0000000..96246ff
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/ModelStorageFactory.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.inference.storage.model;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+
+/**
+ * Model storage factory. Provides {@link ModelStorage}.
+ */
+public class ModelStorageFactory {
+ /** Model storage cache name. */
+ public static final String MODEL_STORAGE_CACHE_NAME = "MODEL_STORAGE";
+
+ /**
+ * Returns model storage based on Apache Ignite cache.
+ *
+ * @param ignite Ignite instance.
+ * @return Model storage.
+ */
+ public ModelStorage getModelStorage(Ignite ignite) {
+ IgniteCache<String, FileOrDirectory> cache = ignite.cache(MODEL_STORAGE_CACHE_NAME);
+ ModelStorageProvider storageProvider = new IgniteModelStorageProvider(cache);
+
+ return new DefaultModelStorage(storageProvider);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/ModelStorageProvider.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/ModelStorageProvider.java b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/ModelStorageProvider.java
new file mode 100644
index 0000000..af11755
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/ModelStorageProvider.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.inference.storage.model;
+
+import java.util.concurrent.locks.Lock;
+
+/**
+ * Model storage provider that keeps files and directories presented as {@link FileOrDirectory} files and correspondent
+ * locks.
+ */
+public interface ModelStorageProvider {
+ /**
+ * Returns file or directory associated with the specified path.
+ *
+ * @param path Path of file or directory.
+ * @return File or directory associated with the specified path.
+ */
+ public FileOrDirectory get(String path);
+
+ /**
+ * Saves file or directory associated with the specified path.
+ *
+ * @param path Path to the file or directory.
+ * @param file File or directory to be saved.
+ */
+ public void put(String path, FileOrDirectory file);
+
+ /**
+ * Removes file or directory associated with the specified path.
+ *
+ * @param path Path to the file or directory.
+ */
+ public void remove(String path);
+
+ /**
+ * Locks the specified path.
+ *
+ * @param path Path to be locked.
+ */
+ public Lock lock(String path);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/package-info.java
new file mode 100644
index 0000000..c203307
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/model/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Root package for inference model storages.
+ */
+package org.apache.ignite.ml.inference.storage.model;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/package-info.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/package-info.java
index 168f4e4..c310e62 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/package-info.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/inference/storage/package-info.java
@@ -17,6 +17,6 @@
/**
* <!-- Package description. -->
- * Root package for model inference descriptor storages.
+ * Root package for inference model storages.
*/
package org.apache.ignite.ml.inference.storage;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/main/java/org/apache/ignite/ml/util/plugin/MLPlugin.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/util/plugin/MLPlugin.java b/modules/ml/src/main/java/org/apache/ignite/ml/util/plugin/MLPlugin.java
new file mode 100644
index 0000000..81c42e2
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/util/plugin/MLPlugin.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.util.plugin;
+
+import org.apache.ignite.plugin.IgnitePlugin;
+
+/**
+ * ML plugin.
+ */
+class MLPlugin implements IgnitePlugin {
+ /**
+ * Creates a new instance of ML inference plugin.
+ */
+ MLPlugin() {}
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/main/java/org/apache/ignite/ml/util/plugin/MLPluginConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/util/plugin/MLPluginConfiguration.java b/modules/ml/src/main/java/org/apache/ignite/ml/util/plugin/MLPluginConfiguration.java
new file mode 100644
index 0000000..58fe601
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/util/plugin/MLPluginConfiguration.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.util.plugin;
+
+import org.apache.ignite.plugin.PluginConfiguration;
+
+/**
+ * Configuration of ML plugin that defines which ML inference services should be start up on Ignite startup.
+ */
+public class MLPluginConfiguration implements PluginConfiguration {
+ /** Model storage should be created on startup. */
+ private boolean withMdlStorage;
+
+ /** Model descriptor storage should be created on startup. */
+ private boolean withMdlDescStorage;
+
+ /** Number of backups in model storage cache. */
+ private Integer mdlStorageBackups;
+
+ /** Number of backups in model descriptor storage cache. */
+ private Integer mdlDescStorageBackups;
+
+ /** */
+ public boolean isWithMdlStorage() {
+ return withMdlStorage;
+ }
+
+ /** */
+ public void setWithMdlStorage(boolean withMdlStorage) {
+ this.withMdlStorage = withMdlStorage;
+ }
+
+ /** */
+ public boolean isWithMdlDescStorage() {
+ return withMdlDescStorage;
+ }
+
+ /** */
+ public void setWithMdlDescStorage(boolean withMdlDescStorage) {
+ this.withMdlDescStorage = withMdlDescStorage;
+ }
+
+ /** */
+ public Integer getMdlStorageBackups() {
+ return mdlStorageBackups;
+ }
+
+ /** */
+ public void setMdlStorageBackups(Integer mdlStorageBackups) {
+ this.mdlStorageBackups = mdlStorageBackups;
+ }
+
+ /** */
+ public Integer getMdlDescStorageBackups() {
+ return mdlDescStorageBackups;
+ }
+
+ /** */
+ public void setMdlDescStorageBackups(Integer mdlDescStorageBackups) {
+ this.mdlDescStorageBackups = mdlDescStorageBackups;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/main/java/org/apache/ignite/ml/util/plugin/MLPluginProvider.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/util/plugin/MLPluginProvider.java b/modules/ml/src/main/java/org/apache/ignite/ml/util/plugin/MLPluginProvider.java
new file mode 100644
index 0000000..de039fb
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/util/plugin/MLPluginProvider.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.util.plugin;
+
+import java.io.Serializable;
+import java.util.UUID;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.ml.inference.storage.descriptor.ModelDescriptorStorageFactory;
+import org.apache.ignite.ml.inference.storage.model.ModelStorageFactory;
+import org.apache.ignite.plugin.CachePluginContext;
+import org.apache.ignite.plugin.CachePluginProvider;
+import org.apache.ignite.plugin.ExtensionRegistry;
+import org.apache.ignite.plugin.IgnitePlugin;
+import org.apache.ignite.plugin.PluginConfiguration;
+import org.apache.ignite.plugin.PluginContext;
+import org.apache.ignite.plugin.PluginProvider;
+import org.apache.ignite.plugin.PluginValidationException;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Machine learning inference plugin provider.
+ */
+public class MLPluginProvider implements PluginProvider<MLPluginConfiguration> {
+ /** Plugin name. */
+ private static final String ML_INFERENCE_PLUGIN_NAME = "ml-inference-plugin";
+
+ /** Plugin version/ */
+ private static final String ML_INFERENCE_PLUGIN_VERSION = "1.0.0";
+
+ /** Default number of model storage backups. */
+ private static final int MODEL_STORAGE_DEFAULT_BACKUPS = 1;
+
+ /** Default number of model descriptor storage backups. */
+ private static final int MODEL_DESCRIPTOR_STORAGE_DEFAULT_BACKUPS = 1;
+
+ /** Plugin configuration. */
+ private MLPluginConfiguration cfg;
+
+ /** Ignite instance. */
+ private Ignite ignite;
+
+ /** Ignite logger. */
+ private IgniteLogger log;
+
+ /** {@inheritDoc} */
+ @Override public String name() {
+ return ML_INFERENCE_PLUGIN_NAME;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String version() {
+ return ML_INFERENCE_PLUGIN_VERSION;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String copyright() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
+ @Override public <T extends IgnitePlugin> T plugin() {
+ return (T)new MLPlugin();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void initExtensions(PluginContext ctx, ExtensionRegistry registry) {
+ IgniteConfiguration igniteCfg = ctx.igniteConfiguration();
+
+ this.ignite = ctx.grid();
+ this.log = ctx.log(this.getClass());
+
+ if (igniteCfg.getPluginConfigurations() != null) {
+ for (PluginConfiguration pluginCfg : igniteCfg.getPluginConfigurations()) {
+ if (pluginCfg instanceof MLPluginConfiguration) {
+ cfg = (MLPluginConfiguration)pluginCfg;
+ break;
+ }
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public <T> @Nullable T createComponent(PluginContext ctx, Class<T> cls) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public CachePluginProvider createCacheProvider(CachePluginContext ctx) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void start(PluginContext ctx) {
+ // Do nothing.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void stop(boolean cancel) {
+ // Do nothing.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onIgniteStart() {
+ if (ignite == null || log == null)
+ throw new RuntimeException("Plugin provider has not been initialized");
+
+ if (cfg != null) {
+ if (cfg.isWithMdlStorage())
+ startModelStorage(cfg);
+
+ if (cfg.isWithMdlDescStorage())
+ startModelDescriptorStorage(cfg);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onIgniteStop(boolean cancel) {
+
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public Serializable provideDiscoveryData(UUID nodeId) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void receiveDiscoveryData(UUID nodeId, Serializable data) {
+ // Do nothing.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void validateNewNode(ClusterNode node) throws PluginValidationException {
+ // Do nothing.
+ }
+
+ /**
+ * Starts model storage.
+ */
+ private void startModelStorage(MLPluginConfiguration cfg) {
+ CacheConfiguration<String, byte[]> storageCfg = new CacheConfiguration<>();
+
+ storageCfg.setName(ModelStorageFactory.MODEL_STORAGE_CACHE_NAME);
+ storageCfg.setCacheMode(CacheMode.PARTITIONED);
+ storageCfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+
+ if (cfg.getMdlStorageBackups() == null)
+ storageCfg.setBackups(MODEL_STORAGE_DEFAULT_BACKUPS);
+ else
+ storageCfg.setBackups(cfg.getMdlStorageBackups());
+
+ ignite.getOrCreateCache(storageCfg);
+
+ log.info("ML model storage is ready");
+ }
+
+ /**
+ * Starts model descriptor storage.
+ */
+ private void startModelDescriptorStorage(MLPluginConfiguration cfg) {
+ CacheConfiguration<String, byte[]> storageCfg = new CacheConfiguration<>();
+
+ storageCfg.setName(ModelDescriptorStorageFactory.MODEL_DESCRIPTOR_STORAGE_CACHE_NAME);
+ storageCfg.setCacheMode(CacheMode.PARTITIONED);
+ storageCfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+
+ if (cfg.getMdlDescStorageBackups() == null)
+ storageCfg.setBackups(MODEL_DESCRIPTOR_STORAGE_DEFAULT_BACKUPS);
+
+ ignite.getOrCreateCache(storageCfg);
+
+ log.info("ML model descriptor storage is ready");
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/main/resources/META-INF/services/org.apache.ignite.plugin.PluginProvider
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/resources/META-INF/services/org.apache.ignite.plugin.PluginProvider b/modules/ml/src/main/resources/META-INF/services/org.apache.ignite.plugin.PluginProvider
new file mode 100644
index 0000000..03cd54b
--- /dev/null
+++ b/modules/ml/src/main/resources/META-INF/services/org.apache.ignite.plugin.PluginProvider
@@ -0,0 +1 @@
+org.apache.ignite.ml.util.plugin.MLPluginProvider
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/test/java/org/apache/ignite/ml/inference/InferenceTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/inference/InferenceTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/inference/InferenceTestSuite.java
index f2ead5a..7546b16 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/inference/InferenceTestSuite.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/inference/InferenceTestSuite.java
@@ -20,6 +20,7 @@ package org.apache.ignite.ml.inference;
import org.apache.ignite.ml.inference.builder.IgniteDistributedInfModelBuilderTest;
import org.apache.ignite.ml.inference.builder.SingleInfModelBuilderTest;
import org.apache.ignite.ml.inference.builder.ThreadedInfModelBuilderTest;
+import org.apache.ignite.ml.inference.storage.model.DefaultModelStorageTest;
import org.apache.ignite.ml.inference.util.DirectorySerializerTest;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
@@ -32,7 +33,8 @@ import org.junit.runners.Suite;
IgniteDistributedInfModelBuilderTest.class,
SingleInfModelBuilderTest.class,
ThreadedInfModelBuilderTest.class,
- DirectorySerializerTest.class
+ DirectorySerializerTest.class,
+ DefaultModelStorageTest.class
})
public class InferenceTestSuite {
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/957b59c3/modules/ml/src/test/java/org/apache/ignite/ml/inference/storage/model/AbstractModelStorageTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/inference/storage/model/AbstractModelStorageTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/inference/storage/model/AbstractModelStorageTest.java
new file mode 100644
index 0000000..84e2a85
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/inference/storage/model/AbstractModelStorageTest.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.inference.storage.model;
+
+import java.util.Set;
+import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Base tests for all implementation of {@link ModelStorage}.
+ */
+public abstract class AbstractModelStorageTest {
+ /**
+ * Returns model storage to be tested.
+ *
+ * @return Model storage to be tested.
+ */
+ abstract ModelStorage getModelStorage();
+
+ /** */
+ @Test
+ public void testPutGetRemoveFile() {
+ ModelStorage mdlStorage = getModelStorage();
+
+ byte[] data = new byte[]{1, 2, 3, 4, 5};
+
+ mdlStorage.mkdirs("/");
+ mdlStorage.putFile("/test", data);
+
+ assertTrue(mdlStorage.exists("/test"));
+ assertArrayEquals(data, mdlStorage.getFile("/test"));
+
+ mdlStorage.remove("/test");
+
+ assertFalse(mdlStorage.exists("/test"));
+ }
+
+ /** */
+ @Test
+ public void testListDirectory() {
+ ModelStorage mdlStorage = getModelStorage();
+
+ mdlStorage.mkdirs("/a/b");
+ mdlStorage.mkdirs("/a/c");
+ mdlStorage.putFile("/a/test", new byte[0]);
+
+ Set<String> aFiles = mdlStorage.listFiles("/a");
+ Set<String> bFiles = mdlStorage.listFiles("/a/b");
+ Set<String> cFiles = mdlStorage.listFiles("/a/c");
+
+ assertEquals(3, aFiles.size());
+ assertTrue(bFiles.isEmpty());
+ assertTrue(cFiles.isEmpty());
+
+ assertTrue(aFiles.contains("/a/b"));
+ assertTrue(aFiles.contains("/a/c"));
+ assertTrue(aFiles.contains("/a/test"));
+ }
+
+ /** */
+ @Test
+ public void testIsDirectory() {
+ ModelStorage mdlStorage = getModelStorage();
+
+ mdlStorage.mkdirs("/a");
+
+ assertTrue(mdlStorage.exists("/a"));
+ assertTrue(mdlStorage.isDirectory("/a"));
+ assertFalse(mdlStorage.isFile("/a"));
+ }
+
+ /** */
+ @Test
+ public void testIsFile() {
+ ModelStorage mdlStorage = getModelStorage();
+
+ mdlStorage.mkdirs("/");
+ mdlStorage.putFile("/test", new byte[0]);
+
+ assertTrue(mdlStorage.exists("/test"));
+ assertTrue(mdlStorage.isFile("/test"));
+ assertFalse(mdlStorage.isDirectory("/test"));
+ }
+
+ /** */
+ @Test
+ public void testRemoveDirectory() {
+ ModelStorage mdlStorage = getModelStorage();
+
+ mdlStorage.mkdirs("/a/b/c");
+ mdlStorage.mkdirs("/a/b/d");
+ mdlStorage.mkdirs("/a/c");
+ mdlStorage.putFile("/a/b/c/test", new byte[0]);
+ mdlStorage.putFile("/a/b/test", new byte[0]);
+
+ mdlStorage.remove("/a/b");
+
+ assertFalse(mdlStorage.exists("/a/b"));
+ assertFalse(mdlStorage.exists("/a/b/c"));
+ assertFalse(mdlStorage.exists("/a/b/d"));
+ assertFalse(mdlStorage.exists("/a/b/test"));
+ assertFalse(mdlStorage.exists("/a/b/c/test"));
+
+ assertTrue(mdlStorage.exists("/a"));
+ assertTrue(mdlStorage.exists("/a/c"));
+ }
+
+ /** */
+ @Test(expected = IllegalArgumentException.class)
+ public void testPutFileIntoNonExistingDirectory() {
+ ModelStorage mdlStorage = getModelStorage();
+
+ mdlStorage.putFile("/test", new byte[0]);
+ }
+
+ /** */
+ @Test(expected = IllegalArgumentException.class)
+ public void testMakeDirInNonExistingDirectory() {
+ ModelStorage mdlStorage = getModelStorage();
+
+ mdlStorage.mkdir("/test");
+ }
+}