You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by hg...@apache.org on 2016/02/19 06:29:03 UTC
[1/4] drill git commit: DRILL-4275: create TransientStore for
short-lived objects;
refactor PersistentStore to introduce pagination mechanism
Repository: drill
Updated Branches:
refs/heads/master 9a3a5c4ff -> 8126927fd
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/zk/ZkAbstractStore.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/zk/ZkAbstractStore.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/zk/ZkAbstractStore.java
deleted file mode 100644
index 0d2fb38..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/zk/ZkAbstractStore.java
+++ /dev/null
@@ -1,291 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.sys.zk;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map.Entry;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.api.ACLBackgroundPathAndBytesable;
-import org.apache.curator.framework.recipes.cache.ChildData;
-import org.apache.curator.framework.recipes.cache.PathChildrenCache;
-import org.apache.curator.framework.recipes.cache.PathChildrenCache.StartMode;
-import org.apache.drill.exec.store.sys.PStoreConfig;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException;
-
-/**
- * This is the abstract class that is shared by ZkPStore (Persistent store) and ZkEStore (Ephemeral Store)
- * @param <V>
- */
-public abstract class ZkAbstractStore<V> implements AutoCloseable {
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ZkAbstractStore.class);
-
- protected final CuratorFramework framework;
- protected final PStoreConfig<V> config;
- private final PathChildrenCache childrenCache;
- private final String prefix;
- private final String parent;
-
- public ZkAbstractStore(CuratorFramework framework, PStoreConfig<V> config)
- throws IOException {
- this.parent = "/" + config.getName();
- this.prefix = parent + "/";
- this.framework = framework;
- this.config = config;
- this.childrenCache = new PathChildrenCache(framework, parent, true);
-
- // make sure the parent node exists.
- createOrUpdate(parent, null, CreateMode.PERSISTENT);
- try {
- childrenCache.start(StartMode.BUILD_INITIAL_CACHE);
- } catch (Exception e) {
- throw new RuntimeException("Failure while initializing Zookeeper for PStore", e);
- }
- }
-
- public Iterator<Entry<String, V>> iterator() {
- try {
- return new Iter(childrenCache.getCurrentData());
- } catch (Exception e) {
- throw new RuntimeException("Failure while accessing Zookeeper.", e);
- }
- }
-
- protected String withPrefix(String key) {
- Preconditions.checkArgument(!key.contains("/"),
- "You cannot use keys that have slashes in them when using the Zookeeper SystemTable storage interface.");
- return prefix + key;
- }
-
- public V get(String key) {
- try {
- ChildData d = childrenCache.getCurrentData(withPrefix(key));
- if(d == null || d.getData() == null){
- return null;
- }
- byte[] bytes = d.getData();
- return config.getSerializer().deserialize(bytes);
-
- } catch (Exception e) {
- throw new RuntimeException("Failure while accessing Zookeeper. " + e.getMessage(), e);
- }
- }
-
- public void put(String key, V value) {
- try {
- if (childrenCache.getCurrentData(withPrefix(key)) != null) {
- framework.setData().forPath(withPrefix(key), config.getSerializer().serialize(value));
- } else {
- createWithPrefix(key, value);
- }
- childrenCache.rebuildNode(withPrefix(key));
-
- } catch (Exception e) {
- throw new RuntimeException("Failure while accessing Zookeeper. " + e.getMessage(), e);
- }
- }
-
- public void delete(String key) {
- try {
- if (framework.checkExists().forPath(withPrefix(key)) != null) {
- framework.delete().forPath(withPrefix(key));
- childrenCache.rebuildNode(withPrefix(key));
- }
- } catch (Exception e) {
- throw new RuntimeException("Failure while accessing Zookeeper. " + e.getMessage(), e);
- }
- }
-
- public boolean putIfAbsent(String key, V value) {
- try {
- if (childrenCache.getCurrentData(withPrefix(key)) != null) {
- return false;
- } else {
- createWithPrefix(key, value);
- childrenCache.rebuildNode(withPrefix(key));
- return true;
- }
-
- } catch (Exception e) {
- throw new RuntimeException("Failure while accessing Zookeeper", e);
- }
- }
-
- /**
- * Default {@link CreateMode create mode} that will be used in create operations referred in the see also section.
- *
- * @see #createOrUpdate(String, Object)
- * @see #createWithPrefix(String, Object)
- */
- protected abstract CreateMode getCreateMode();
-
-
- /**
- * Creates a node in zookeeper with the {@link #getCreateMode() default create mode} and sets its value if supplied.
- *
- * @param path target path
- * @param value value to set, null if none available
- *
- * @see #getCreateMode()
- * @see #createOrUpdate(String, Object)
- * @see #withPrefix(String)
- */
- protected void createWithPrefix(String path, V value) {
- createOrUpdate(withPrefix(path), value);
- }
-
- /**
- * Creates a node in zookeeper with the {@link #getCreateMode() default create mode} and sets its value if supplied
- * or updates its value if the node already exists.
- *
- * Note that if node exists, its mode will not be changed.
- *
- * @param path target path
- * @param value value to set, null if none available
- *
- * @see #getCreateMode()
- * @see #createOrUpdate(String, Object, CreateMode)
- */
- protected void createOrUpdate(String path, V value) {
- createOrUpdate(path, value, getCreateMode());
- }
-
- /**
- * Creates a node in zookeeper with the given mode and sets its value if supplied or updates its value if the node
- * already exists.
- *
- * Note that if the node exists, its mode will not be changed.
- *
- * Internally, the method suppresses {@link org.apache.zookeeper.KeeperException.NodeExistsException}. It is
- * safe to do so since the implementation is idempotent.
- *
- * @param path target path
- * @param value value to set, null if none available
- * @param mode creation mode
- * @throws RuntimeException throws a {@link RuntimeException} wrapping the root cause.
- */
- protected void createOrUpdate(String path, V value, CreateMode mode) {
- try {
- final boolean isUpdate = value != null;
- final byte[] valueInBytes = isUpdate ? config.getSerializer().serialize(value) : null;
- final boolean nodeExists = framework.checkExists().forPath(path) != null;
- if (!nodeExists) {
- final ACLBackgroundPathAndBytesable<String> creator = framework.create().withMode(mode);
- if (isUpdate) {
- creator.forPath(path, valueInBytes);
- } else {
- creator.forPath(path);
- }
- } else if (isUpdate) {
- framework.setData().forPath(path, valueInBytes);
- }
- } catch (KeeperException.NodeExistsException ex) {
- logger.warn("Node already exists in Zookeeper. Skipping... -- [path: {}, mode: {}]", path, mode);
- } catch (Exception e) {
- final String msg = String.format("Failed to create/update Zookeeper node. [path: %s, mode: %s]", path, mode);
- throw new RuntimeException(msg, e);
- }
- }
-
- private class Iter implements Iterator<Entry<String, V>>{
-
- private Iterator<ChildData> keys;
- private ChildData current;
-
- public Iter(List<ChildData> children) {
- super();
- List<ChildData> sortedChildren = Lists.newArrayList(children);
- Collections.sort(sortedChildren, new Comparator<ChildData>(){
- @Override
- public int compare(ChildData o1, ChildData o2) {
- return o1.getPath().compareTo(o2.getPath());
- }});
- this.keys = sortedChildren.iterator();
- }
-
- @Override
- public boolean hasNext() {
- return keys.hasNext();
- }
-
- @Override
- public Entry<String, V> next() {
- current = keys.next();
- return new DeferredEntry(current);
- }
-
- @Override
- public void remove() {
- delete(keyFromPath(current));
- }
-
- private String keyFromPath(ChildData data){
- String path = data.getPath();
- return path.substring(prefix.length(), path.length());
- }
-
- private class DeferredEntry implements Entry<String, V>{
-
- private ChildData data;
-
- public DeferredEntry(ChildData data) {
- super();
- this.data = data;
- }
-
- @Override
- public String getKey() {
- return keyFromPath(data);
- }
-
- @Override
- public V getValue() {
- try {
- return config.getSerializer().deserialize(data.getData());
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public V setValue(V value) {
- throw new UnsupportedOperationException();
- }
-
- }
-
- }
-
- @Override
- public void close() {
- try{
- childrenCache.close();
- }catch(IOException e){
- logger.warn("Failure while closing out abstract store.", e);
- }
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/zk/ZkEStore.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/zk/ZkEStore.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/zk/ZkEStore.java
deleted file mode 100644
index 4706287..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/zk/ZkEStore.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.sys.zk;
-
-import java.io.IOException;
-
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.drill.exec.store.sys.EStore;
-import org.apache.drill.exec.store.sys.PStoreConfig;
-import org.apache.zookeeper.CreateMode;
-
-/**
- * Implementation of EStore using Zookeeper's EPHEMERAL node.
- * @param <V>
- */
-public class ZkEStore<V> extends ZkAbstractStore<V> implements EStore<V> {
-
- public ZkEStore(CuratorFramework framework, PStoreConfig<V> config) throws IOException {
- super(framework,config);
- }
-
- @Override
- protected CreateMode getCreateMode() {
- return CreateMode.EPHEMERAL;
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/zk/ZkEStoreProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/zk/ZkEStoreProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/zk/ZkEStoreProvider.java
deleted file mode 100644
index 60277aa..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/zk/ZkEStoreProvider.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.drill.exec.store.sys.zk;
-
-import java.io.IOException;
-
-import com.google.common.base.Preconditions;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.drill.exec.store.sys.EStore;
-import org.apache.drill.exec.store.sys.EStoreProvider;
-import org.apache.drill.exec.store.sys.PStoreConfig;
-import org.apache.drill.exec.store.sys.PStoreConfig.Mode;
-
-public class ZkEStoreProvider implements EStoreProvider{
- private final CuratorFramework curator;
-
- public ZkEStoreProvider(CuratorFramework curator) {
- this.curator = curator;
- }
-
- @Override
- public <V> EStore<V> getStore(PStoreConfig<V> store) throws IOException {
- Preconditions.checkArgument(store.getMode() == Mode.EPHEMERAL);
- return new ZkEStore<V>(curator,store);
- }
-
- @Override
- public void start() throws IOException {
- }
-
- @Override
- public void close() throws Exception {
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/zk/ZkPStore.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/zk/ZkPStore.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/zk/ZkPStore.java
deleted file mode 100644
index da22996..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/zk/ZkPStore.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.sys.zk;
-
-import java.io.IOException;
-
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.drill.exec.store.sys.PStore;
-import org.apache.drill.exec.store.sys.PStoreConfig;
-import org.apache.zookeeper.CreateMode;
-
-/**
- * Implementation of PStore using Zookeeper's PERSISTENT node.
- * @param <V>
- */
-public class ZkPStore<V> extends ZkAbstractStore<V> implements PStore<V> {
-
- public ZkPStore(CuratorFramework framework, PStoreConfig<V> config) throws IOException {
- super(framework, config);
- }
-
- @Override
- protected CreateMode getCreateMode() {
- return CreateMode.PERSISTENT;
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/zk/ZkPStoreProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/zk/ZkPStoreProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/zk/ZkPStoreProvider.java
index eb5df43..8cc2fde 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/zk/ZkPStoreProvider.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/zk/ZkPStoreProvider.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ * <p/>
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -17,83 +17,19 @@
*/
package org.apache.drill.exec.store.sys.zk;
-import java.io.IOException;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.exec.coord.ClusterCoordinator;
import org.apache.drill.exec.coord.zk.ZKClusterCoordinator;
-import org.apache.drill.exec.exception.DrillbitStartupException;
-import org.apache.drill.exec.store.dfs.DrillFileSystem;
-import org.apache.drill.exec.store.sys.EStoreProvider;
-import org.apache.drill.exec.store.sys.PStore;
-import org.apache.drill.exec.store.sys.PStoreConfig;
-import org.apache.drill.exec.store.sys.PStoreProvider;
-import org.apache.drill.exec.store.sys.PStoreRegistry;
-import org.apache.drill.exec.store.sys.local.FilePStore;
-import org.apache.hadoop.fs.Path;
-
-public class ZkPStoreProvider implements PStoreProvider {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ZkPStoreProvider.class);
-
- private static final String DRILL_EXEC_SYS_STORE_PROVIDER_ZK_BLOBROOT = "drill.exec.sys.store.provider.zk.blobroot";
-
- private final CuratorFramework curator;
+import org.apache.drill.exec.exception.StoreException;
+import org.apache.drill.exec.store.sys.PersistentStoreRegistry;
+import org.apache.drill.exec.store.sys.store.provider.ZookeeperPersistentStoreProvider;
- private final DrillFileSystem fs;
- private final Path blobRoot;
- private final EStoreProvider zkEStoreProvider;
-
- public ZkPStoreProvider(PStoreRegistry registry) throws DrillbitStartupException {
- ClusterCoordinator coord = registry.getClusterCoordinator();
- if (!(coord instanceof ZKClusterCoordinator)) {
- throw new DrillbitStartupException("A ZkPStoreProvider was created without a ZKClusterCoordinator.");
- }
- this.curator = ((ZKClusterCoordinator)registry.getClusterCoordinator()).getCurator();
-
- if (registry.getConfig().hasPath(DRILL_EXEC_SYS_STORE_PROVIDER_ZK_BLOBROOT)) {
- blobRoot = new Path(registry.getConfig().getString(DRILL_EXEC_SYS_STORE_PROVIDER_ZK_BLOBROOT));
- }else{
- blobRoot = FilePStore.getLogDir();
- }
-
- try{
- this.fs = FilePStore.getFileSystem(registry.getConfig(), blobRoot);
- }catch(IOException e){
- throw new DrillbitStartupException("Failure while attempting to set up blob store.", e);
- }
-
- this.zkEStoreProvider = new ZkEStoreProvider(curator);
- }
-
- @VisibleForTesting
- public ZkPStoreProvider(DrillConfig config, CuratorFramework curator) throws IOException {
- this.curator = curator;
- this.blobRoot = FilePStore.getLogDir();
- this.fs = FilePStore.getFileSystem(config, blobRoot);
- this.zkEStoreProvider = new ZkEStoreProvider(curator);
- }
-
- @Override
- public <V> PStore<V> getStore(PStoreConfig<V> config) throws IOException {
- switch(config.getMode()){
- case BLOB_PERSISTENT:
- return new FilePStore<V>(fs, blobRoot, config);
- case EPHEMERAL:
- return zkEStoreProvider.getStore(config);
- case PERSISTENT:
- return new ZkPStore<V>(curator, config);
- default:
- throw new IllegalStateException();
- }
- }
-
- @Override
- public void start() {
- }
-
- @Override
- public void close() {
+/**
+ * Kept for possible references to old class name in configuration.
+ *
+ * @deprecated will be removed in 1.7
+ * use {@link ZookeeperPersistentStoreProvider} instead.
+ */
+public class ZkPStoreProvider extends ZookeeperPersistentStoreProvider {
+ public ZkPStoreProvider(PersistentStoreRegistry<ZKClusterCoordinator> registry) throws StoreException {
+ super(registry);
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/testing/store/NoWriteLocalStore.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/testing/store/NoWriteLocalStore.java b/exec/java-exec/src/main/java/org/apache/drill/exec/testing/store/NoWriteLocalStore.java
new file mode 100644
index 0000000..ff14f6d
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/testing/store/NoWriteLocalStore.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.testing.store;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
+import org.apache.drill.exec.store.sys.BasePersistentStore;
+import org.apache.drill.exec.store.sys.PersistentStoreMode;
+
+public class NoWriteLocalStore<V> extends BasePersistentStore<V> {
+ private final ConcurrentMap<String, V> store = Maps.newConcurrentMap();
+
+ public void delete(final String key) {
+ store.remove(key);
+ }
+
+ @Override
+ public PersistentStoreMode getMode() {
+ return PersistentStoreMode.PERSISTENT;
+ }
+
+ @Override
+ public V get(final String key) {
+ return store.get(key);
+ }
+
+ @Override
+ public void put(final String key, final V value) {
+ store.put(key, value);
+ }
+
+ @Override
+ public boolean putIfAbsent(final String key, final V value) {
+ final V old = store.putIfAbsent(key, value);
+ return value != old;
+ }
+
+ @Override
+ public Iterator<Map.Entry<String, V>> getRange(final int skip, final int take) {
+ return Iterables.limit(Iterables.skip(store.entrySet(), skip), take).iterator();
+ }
+
+ @Override
+ public void close() throws Exception {
+ store.clear();
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java
index 5fd6f1c..f2305c4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java
@@ -39,7 +39,7 @@ import org.apache.drill.exec.rpc.control.WorkEventBus;
import org.apache.drill.exec.rpc.data.DataConnectionCreator;
import org.apache.drill.exec.server.BootStrapContext;
import org.apache.drill.exec.server.DrillbitContext;
-import org.apache.drill.exec.store.sys.PStoreProvider;
+import org.apache.drill.exec.store.sys.PersistentStoreProvider;
import org.apache.drill.exec.work.batch.ControlMessageHandler;
import org.apache.drill.exec.work.foreman.Foreman;
import org.apache.drill.exec.work.foreman.QueryManager;
@@ -101,7 +101,7 @@ public class WorkManager implements AutoCloseable {
final Controller controller,
final DataConnectionCreator data,
final ClusterCoordinator coord,
- final PStoreProvider provider) {
+ final PersistentStoreProvider provider) {
dContext = new DrillbitContext(endpoint, bContext, coord, controller, data, workBus, provider);
statusThread.start();
@@ -153,6 +153,8 @@ public class WorkManager implements AutoCloseable {
}
}
}
+
+ getContext().close();
}
public DrillbitContext getContext() {
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
index bfc9dff..8527850 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
@@ -163,8 +163,8 @@ public class Foreman implements Runnable {
closeFuture.addListener(closeListener);
queryContext = new QueryContext(connection.getSession(), drillbitContext, queryId);
- queryManager = new QueryManager(queryId, queryRequest, drillbitContext.getPersistentStoreProvider(),
- stateListener, this); // TODO reference escapes before ctor is complete via stateListener, this
+ queryManager = new QueryManager(queryId, queryRequest, drillbitContext.getStoreProvider(),
+ drillbitContext.getClusterCoordinator(), stateListener, this); // TODO reference escapes before ctor is complete via stateListener, this
final OptionManager optionManager = queryContext.getOptions();
queuingEnabled = optionManager.getOption(ExecConstants.ENABLE_QUEUE);
@@ -768,6 +768,12 @@ public class Foreman implements Runnable {
bee.retireForeman(Foreman.this);
try {
+ queryManager.close();
+ } catch (final Exception e) {
+ logger.warn("unable to close query manager", e);
+ }
+
+ try {
releaseLease();
} finally {
isClosed = true;
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/QueryManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/QueryManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/QueryManager.java
index 060cffc..39fa5cb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/QueryManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/QueryManager.java
@@ -19,15 +19,18 @@ package org.apache.drill.exec.work.foreman;
import io.netty.buffer.ByteBuf;
-import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.drill.common.AutoCloseables;
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.exceptions.UserRemoteException;
+import org.apache.drill.exec.coord.ClusterCoordinator;
+import org.apache.drill.exec.coord.store.TransientStore;
+import org.apache.drill.exec.coord.store.TransientStoreConfig;
import org.apache.drill.exec.proto.BitControl.FragmentStatus;
import org.apache.drill.exec.proto.BitControl.PlanFragment;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
@@ -45,9 +48,9 @@ import org.apache.drill.exec.proto.helper.QueryIdHelper;
import org.apache.drill.exec.rpc.RpcException;
import org.apache.drill.exec.rpc.control.Controller;
import org.apache.drill.exec.server.DrillbitContext;
-import org.apache.drill.exec.store.sys.PStore;
-import org.apache.drill.exec.store.sys.PStoreConfig;
-import org.apache.drill.exec.store.sys.PStoreProvider;
+import org.apache.drill.exec.store.sys.PersistentStore;
+import org.apache.drill.exec.store.sys.PersistentStoreConfig;
+import org.apache.drill.exec.store.sys.PersistentStoreProvider;
import org.apache.drill.exec.work.EndpointListener;
import org.apache.drill.exec.work.foreman.Foreman.StateListener;
@@ -59,20 +62,18 @@ import com.google.common.collect.Maps;
/**
* Each Foreman holds its own QueryManager. This manages the events associated with execution of a particular query across all fragments.
*/
-public class QueryManager {
+public class QueryManager implements AutoCloseable {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(QueryManager.class);
- public static final PStoreConfig<QueryProfile> QUERY_PROFILE = PStoreConfig.
+ public static final PersistentStoreConfig<QueryProfile> QUERY_PROFILE = PersistentStoreConfig.
newProtoBuilder(SchemaUserBitShared.QueryProfile.WRITE, SchemaUserBitShared.QueryProfile.MERGE)
.name("profiles")
.blob()
- .max(100)
.build();
- public static final PStoreConfig<QueryInfo> RUNNING_QUERY_INFO = PStoreConfig.
- newProtoBuilder(SchemaUserBitShared.QueryInfo.WRITE, SchemaUserBitShared.QueryInfo.MERGE)
+ public static final TransientStoreConfig<QueryInfo> RUNNING_QUERY_INFO = TransientStoreConfig
+ .newProtoBuilder(SchemaUserBitShared.QueryInfo.WRITE, SchemaUserBitShared.QueryInfo.MERGE)
.name("running")
- .ephemeral()
.build();
private final Map<DrillbitEndpoint, NodeTracker> nodeMap = Maps.newHashMap();
@@ -90,8 +91,8 @@ public class QueryManager {
new IntObjectHashMap<>();
private final List<FragmentData> fragmentDataSet = Lists.newArrayList();
- private final PStore<QueryProfile> profilePStore;
- private final PStore<QueryInfo> profileEStore;
+ private final PersistentStore<QueryProfile> profileStore;
+ private final TransientStore<QueryInfo> transientProfiles;
// the following mutable variables are used to capture ongoing query status
private String planText;
@@ -104,8 +105,8 @@ public class QueryManager {
// How many fragments have finished their execution.
private final AtomicInteger finishedFragments = new AtomicInteger(0);
- public QueryManager(final QueryId queryId, final RunQuery runQuery, final PStoreProvider pStoreProvider,
- final StateListener stateListener, final Foreman foreman) {
+ public QueryManager(final QueryId queryId, final RunQuery runQuery, final PersistentStoreProvider storeProvider,
+ final ClusterCoordinator coordinator, final StateListener stateListener, final Foreman foreman) {
this.queryId = queryId;
this.runQuery = runQuery;
this.stateListener = stateListener;
@@ -113,11 +114,11 @@ public class QueryManager {
stringQueryId = QueryIdHelper.getQueryId(queryId);
try {
- profilePStore = pStoreProvider.getStore(QUERY_PROFILE);
- profileEStore = pStoreProvider.getStore(RUNNING_QUERY_INFO);
- } catch (final IOException e) {
+ profileStore = storeProvider.getOrCreateStore(QUERY_PROFILE);
+ } catch (final Exception e) {
throw new DrillRuntimeException(e);
}
+ transientProfiles = coordinator.getOrCreateTransientStore(RUNNING_QUERY_INFO);
}
private static boolean isTerminal(final FragmentState state) {
@@ -237,11 +238,14 @@ public class QueryManager {
}
}
+ @Override
+ public void close() throws Exception { }
+
/*
- * This assumes that the FragmentStatusListener implementation takes action when it hears
- * that the target fragment has acknowledged the signal. As a result, this listener doesn't do anything
- * but log messages.
- */
+ * This assumes that the FragmentStatusListener implementation takes action when it hears
+ * that the target fragment has acknowledged the signal. As a result, this listener doesn't do anything
+ * but log messages.
+ */
private static class SignalListener extends EndpointListener<Ack, FragmentHandle> {
/**
* An enum of possible signals that {@link SignalListener} listens to.
@@ -281,14 +285,14 @@ public class QueryManager {
case STARTING:
case RUNNING:
case CANCELLATION_REQUESTED:
- profileEStore.put(stringQueryId, getQueryInfo()); // store as ephemeral query profile.
+ transientProfiles.put(stringQueryId, getQueryInfo()); // store as ephemeral query profile.
break;
case COMPLETED:
case CANCELED:
case FAILED:
try {
- profileEStore.delete(stringQueryId);
+ transientProfiles.remove(stringQueryId);
} catch(final Exception e) {
logger.warn("Failure while trying to delete the estore profile for this query.", e);
}
@@ -305,7 +309,7 @@ public class QueryManager {
void writeFinalProfile(UserException ex) {
try {
// TODO(DRILL-2362) when do these ever get deleted?
- profilePStore.put(stringQueryId, getQueryProfile(ex));
+ profileStore.put(stringQueryId, getQueryProfile(ex));
} catch (Exception e) {
logger.error("Failure while storing Query Profile", e);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/resources/drill-module.conf
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf
index d6ba99a..b1b9b46 100644
--- a/exec/java-exec/src/main/resources/drill-module.conf
+++ b/exec/java-exec/src/main/resources/drill-module.conf
@@ -120,7 +120,7 @@ drill.exec: {
affinity.factor: 1.2
},
sys.store.provider: {
- class: "org.apache.drill.exec.store.sys.zk.ZkPStoreProvider",
+ class: "org.apache.drill.exec.store.sys.store.provider.ZookeeperPersistentStoreProvider",
local: {
path: "/tmp/drill",
write: true
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
index 26b7464..7ab73dc 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
@@ -248,7 +248,7 @@ public class BaseTestQuery extends ExecTest {
}
@AfterClass
- public static void closeClient() throws IOException {
+ public static void closeClient() throws Exception {
if (client != null) {
client.close();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java b/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java
index 24c8c63..9f39d15 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java
@@ -48,7 +48,7 @@ import org.apache.drill.exec.server.options.SystemOptionManager;
import org.apache.drill.exec.store.SchemaConfig;
import org.apache.drill.exec.store.StoragePluginRegistry;
import org.apache.drill.exec.store.StoragePluginRegistryImpl;
-import org.apache.drill.exec.store.sys.local.LocalPStoreProvider;
+import org.apache.drill.exec.store.sys.store.provider.LocalPersistentStoreProvider;
import org.apache.drill.exec.testing.ExecutionControls;
import org.junit.Rule;
import org.junit.rules.TestRule;
@@ -76,7 +76,7 @@ public class PlanningBase extends ExecTest{
protected void testSqlPlan(String sqlCommands) throws Exception {
final String[] sqlStrings = sqlCommands.split(";");
- final LocalPStoreProvider provider = new LocalPStoreProvider(config);
+ final LocalPersistentStoreProvider provider = new LocalPersistentStoreProvider(config);
provider.start();
final ScanResult scanResult = ClassPathScanner.fromPrescan(config);
final LogicalPlanPersistence logicalPlanPersistence = new LogicalPlanPersistence(config, scanResult);
@@ -97,7 +97,7 @@ public class PlanningBase extends ExecTest{
result = config;
dbContext.getOptionManager();
result = systemOptions;
- dbContext.getPersistentStoreProvider();
+ dbContext.getStoreProvider();
result = provider;
dbContext.getClasspathScan();
result = scanResult;
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/test/java/org/apache/drill/exec/compile/CodeCompilerTestFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/CodeCompilerTestFactory.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/CodeCompilerTestFactory.java
index 9032946..f4903d8 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/CodeCompilerTestFactory.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/CodeCompilerTestFactory.java
@@ -19,19 +19,17 @@ package org.apache.drill.exec.compile;
import static com.google.common.base.Preconditions.checkNotNull;
-import java.io.IOException;
-
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.config.LogicalPlanPersistence;
import org.apache.drill.common.scanner.ClassPathScanner;
import org.apache.drill.exec.server.options.SystemOptionManager;
-import org.apache.drill.exec.store.sys.local.LocalPStoreProvider;
+import org.apache.drill.exec.store.sys.store.provider.LocalPersistentStoreProvider;
public class CodeCompilerTestFactory {
- public static CodeCompiler getTestCompiler(DrillConfig c) throws IOException {
+ public static CodeCompiler getTestCompiler(DrillConfig c) throws Exception {
DrillConfig config = checkNotNull(c);
LogicalPlanPersistence persistence = new LogicalPlanPersistence(config, ClassPathScanner.fromPrescan(config));
- LocalPStoreProvider provider = new LocalPStoreProvider(config);
+ LocalPersistentStoreProvider provider = new LocalPersistentStoreProvider(config);
SystemOptionManager systemOptionManager = new SystemOptionManager(persistence, provider);
return new CodeCompiler(config, systemOptionManager.init());
}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java
index 8486801..9b507fe 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java
@@ -25,10 +25,9 @@ import java.net.URL;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.compile.DrillCheckClassAdapter;
import org.apache.drill.exec.compile.QueryClassLoader;
-import org.apache.drill.exec.planner.PhysicalPlanReader;
import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory;
import org.apache.drill.exec.server.options.SystemOptionManager;
-import org.apache.drill.exec.store.sys.local.LocalPStoreProvider;
+import org.apache.drill.exec.store.sys.store.provider.LocalPersistentStoreProvider;
import org.objectweb.asm.ClassReader;
import org.objectweb.asm.ClassVisitor;
import org.objectweb.asm.ClassWriter;
@@ -57,7 +56,7 @@ public class ReplaceMethodInvoke {
check(output);
final DrillConfig c = DrillConfig.forClient();
- final SystemOptionManager m = new SystemOptionManager(PhysicalPlanReaderTestFactory.defaultLogicalPlanPersistence(c), new LocalPStoreProvider(c));
+ final SystemOptionManager m = new SystemOptionManager(PhysicalPlanReaderTestFactory.defaultLogicalPlanPersistence(c), new LocalPersistentStoreProvider(c));
m.init();
try (QueryClassLoader ql = new QueryClassLoader(DrillConfig.create(), m)) {
ql.injectByteCode("org.apache.drill.Pickle$OutgoingBatch", output);
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestEphemeralStore.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestEphemeralStore.java b/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestEphemeralStore.java
new file mode 100644
index 0000000..021a0b7
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestEphemeralStore.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.coord.zk;
+
+import java.io.IOException;
+
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.framework.listen.ListenerContainer;
+import org.apache.curator.framework.recipes.cache.PathChildrenCache;
+import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
+import org.apache.curator.retry.RetryNTimes;
+import org.apache.curator.test.TestingServer;
+import org.apache.drill.exec.coord.store.TransientStoreConfig;
+import org.apache.drill.exec.serialization.InstanceSerializer;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+public class TestEphemeralStore {
+ private final static String root = "/test";
+ private final static String path = "test-key";
+ private final static String value = "testing";
+
+ private TestingServer server;
+ private CuratorFramework curator;
+ private TransientStoreConfig<String> config;
+ private ZkEphemeralStore<String> store;
+
+ static class StoreWithMockClient<V> extends ZkEphemeralStore<V> {
+ private final ZookeeperClient client = Mockito.mock(ZookeeperClient.class);
+
+ public StoreWithMockClient(final TransientStoreConfig<V> config, final CuratorFramework curator) {
+ super(config, curator);
+ }
+
+ @Override
+ protected ZookeeperClient getClient() {
+ return client;
+ }
+ }
+
+
+ @Before
+ public void setUp() throws Exception {
+ server = new TestingServer();
+ final RetryPolicy policy = new RetryNTimes(2, 1000);
+ curator = CuratorFrameworkFactory.newClient(server.getConnectString(), policy);
+
+ config = Mockito.mock(TransientStoreConfig.class);
+ Mockito
+ .when(config.getName())
+ .thenReturn(root);
+
+ Mockito
+ .when(config.getSerializer())
+ .thenReturn(new InstanceSerializer<String>() {
+ @Override
+ public byte[] serialize(final String instance) throws IOException {
+ if (instance == null) {
+ return null;
+ }
+ return instance.getBytes();
+ }
+
+ @Override
+ public String deserialize(final byte[] raw) throws IOException {
+ if (raw == null) {
+ return null;
+ }
+ return new String(raw);
+ }
+ });
+
+ store = new ZkEphemeralStore<>(config, curator);
+
+ server.start();
+ curator.start();
+ store.start();
+ }
+
+ /**
+ * This test ensures store subscribes to receive events from underlying client. Dispatcher tests ensures listeners
+ * are fired on incoming events. These two sets of tests ensure observer pattern in {@code TransientStore} works fine.
+ */
+ @Test
+ public void testStoreRegistersDispatcherAndStartsItsClient() throws Exception {
+ final StoreWithMockClient<String> store = new StoreWithMockClient<>(config, curator);
+
+ final PathChildrenCache cache = Mockito.mock(PathChildrenCache.class);
+ final ZookeeperClient client = store.getClient();
+ Mockito
+ .when(client.getCache())
+ .thenReturn(cache);
+
+ final ListenerContainer<PathChildrenCacheListener> container = Mockito.mock(ListenerContainer.class);
+ Mockito
+ .when(cache.getListenable())
+ .thenReturn(container);
+
+ store.start();
+
+ Mockito
+ .verify(container)
+ .addListener(store.dispatcher);
+
+ Mockito
+ .verify(client)
+ .start();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ store.close();
+ curator.close();
+ server.close();
+ }
+
+ @Test
+ public void testPutAndGetWorksAntagonistacally() {
+ store.put(path, value);
+ final String actual = store.get(path);
+ Assert.assertEquals("value mismatch", value, actual);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestEventDispatcher.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestEventDispatcher.java b/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestEventDispatcher.java
new file mode 100644
index 0000000..f83ec00
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestEventDispatcher.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.coord.zk;
+
+import java.io.IOException;
+
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
+import org.apache.drill.exec.coord.store.TransientStoreConfig;
+import org.apache.drill.exec.coord.store.TransientStoreEvent;
+import org.apache.drill.exec.serialization.InstanceSerializer;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+public class TestEventDispatcher {
+
+ private final static String key = "some-key";
+ private final static String value = "some-data";
+ private final static byte[] data = "some-data".getBytes();
+
+ private ZkEphemeralStore<String> store;
+ private EventDispatcher<String> dispatcher;
+ private ChildData child;
+
+ @Before
+ public void setUp() {
+ store = Mockito.mock(ZkEphemeralStore.class);
+ final TransientStoreConfig<String> config = Mockito.mock(TransientStoreConfig.class);
+ Mockito
+ .when(store.getConfig())
+ .thenReturn(config);
+
+ Mockito
+ .when(config.getSerializer())
+ .thenReturn(new InstanceSerializer<String>() {
+ @Override
+ public byte[] serialize(String instance) throws IOException {
+ return instance.getBytes();
+ }
+
+ @Override
+ public String deserialize(byte[] raw) throws IOException {
+ return new String(raw);
+ }
+ });
+
+ dispatcher = new EventDispatcher<>(store);
+ child = Mockito.mock(ChildData.class);
+ Mockito
+ .when(child.getPath())
+ .thenReturn(key);
+
+ Mockito
+ .when(child.getData())
+ .thenReturn(data);
+ }
+
+ @Test
+ public void testDispatcherPropagatesEvents() throws Exception {
+ final PathChildrenCacheEvent.Type[] types = new PathChildrenCacheEvent.Type[] {
+ PathChildrenCacheEvent.Type.CHILD_ADDED,
+ PathChildrenCacheEvent.Type.CHILD_REMOVED,
+ PathChildrenCacheEvent.Type.CHILD_UPDATED
+ };
+
+ for (final PathChildrenCacheEvent.Type type:types) {
+ dispatcher.childEvent(null, new PathChildrenCacheEvent(type, child));
+
+ final TransientStoreEvent event = TransientStoreEvent.of(EventDispatcher.MAPPINGS.get(type), key, value);
+ Mockito
+ .verify(store)
+ .fireListeners(event);
+ }
+
+ Assert.assertEquals("Number of event types that dispatcher can handle is different", types.length, EventDispatcher.MAPPINGS.size());
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestPathUtils.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestPathUtils.java b/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestPathUtils.java
new file mode 100644
index 0000000..d966744
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestPathUtils.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.coord.zk;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestPathUtils {
+
+ @Test(expected = NullPointerException.class)
+ public void testNullSegmentThrowsNPE() {
+ PathUtils.join("", null, "");
+ }
+
+ @Test
+ public void testJoinPreservesAbsoluteOrRelativePaths() {
+ final String actual = PathUtils.join("/a", "/b", "/c");
+ final String expected = "/a/b/c";
+ Assert.assertEquals("invalid path", expected, actual);
+
+ final String actual2 = PathUtils.join("/a", "b", "c");
+ final String expected2 = "/a/b/c";
+ Assert.assertEquals("invalid path", expected2, actual2);
+
+ final String actual3 = PathUtils.join("a", "b", "c");
+ final String expected3 = "a/b/c";
+ Assert.assertEquals("invalid path", expected3, actual3);
+
+ final String actual4 = PathUtils.join("a", "", "c");
+ final String expected4 = "a/c";
+ Assert.assertEquals("invalid path", expected4, actual4);
+
+ final String actual5 = PathUtils.join("", "", "c");
+ final String expected5 = "c";
+ Assert.assertEquals("invalid path", expected5, actual5);
+
+ final String actual6 = PathUtils.join("", "", "");
+ final String expected6 = "";
+ Assert.assertEquals("invalid path", expected6, actual6);
+
+ final String actual7 = PathUtils.join("", "", "/");
+ final String expected7 = "/";
+ Assert.assertEquals("invalid path", expected7, actual7);
+
+ final String actual8 = PathUtils.join("", "", "c/");
+ final String expected8 = "c/";
+ Assert.assertEquals("invalid path", expected8, actual8);
+ }
+
+
+ @Test
+ public void testNormalizeRemovesRedundantForwardSlashes() {
+ final String actual = PathUtils.normalize("/a/b/c");
+ final String expected = "/a/b/c";
+ Assert.assertEquals("invalid path", expected, actual);
+
+ final String actual2 = PathUtils.normalize("//a//b//c");
+ final String expected2 = "/a/b/c";
+ Assert.assertEquals("invalid path", expected2, actual2);
+
+ final String actual3 = PathUtils.normalize("///");
+ final String expected3 = "/";
+ Assert.assertEquals("invalid path", expected3, actual3);
+
+ final String actual4 = PathUtils.normalize("/a");
+ final String expected4 = "/a";
+ Assert.assertEquals("invalid path", expected4, actual4);
+
+ final String actual5 = PathUtils.normalize("//////");
+ final String expected5 = "/";
+ Assert.assertEquals("invalid path", expected5, actual5);
+
+ final String actual6 = PathUtils.normalize("");
+ final String expected6 = "";
+ Assert.assertEquals("invalid path", expected6, actual6);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestZookeeperClient.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestZookeeperClient.java b/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestZookeeperClient.java
new file mode 100644
index 0000000..3007566
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestZookeeperClient.java
@@ -0,0 +1,201 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.coord.zk;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.collect.Lists;
+import org.apache.curator.CuratorZookeeperClient;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.framework.api.ACLBackgroundPathAndBytesable;
+import org.apache.curator.framework.api.CreateBuilder;
+import org.apache.curator.framework.api.DeleteBuilder;
+import org.apache.curator.framework.api.SetDataBuilder;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.PathChildrenCache;
+import org.apache.curator.retry.RetryNTimes;
+import org.apache.curator.test.TestingServer;
+import org.apache.curator.utils.EnsurePath;
+import org.apache.drill.common.collections.ImmutableEntry;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.zookeeper.CreateMode;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+public class TestZookeeperClient {
+ private final static String root = "/test";
+ private final static String path = "test-key";
+ private final static String abspath = PathUtils.join(root, path);
+ private final static byte[] data = "testing".getBytes();
+ private final static CreateMode mode = CreateMode.PERSISTENT;
+
+ private TestingServer server;
+ private CuratorFramework curator;
+ private ZookeeperClient client;
+
+ static class ClientWithMockCache extends ZookeeperClient {
+ private final PathChildrenCache cacheMock = Mockito.mock(PathChildrenCache.class);
+
+ public ClientWithMockCache(final CuratorFramework curator, final String root, final CreateMode mode) {
+ super(curator, root, mode);
+ }
+
+ @Override
+ public PathChildrenCache getCache() {
+ return cacheMock;
+ }
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ server = new TestingServer();
+ final RetryPolicy policy = new RetryNTimes(1, 1000);
+ curator = CuratorFrameworkFactory.newClient(server.getConnectString(), policy);
+ client = new ClientWithMockCache(curator, root, mode);
+
+ server.start();
+ curator.start();
+ client.start();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ client.close();
+ curator.close();
+ server.close();
+ }
+
+ @Test
+ public void testStartingClientEnablesCacheAndEnsuresRootNodeExists() throws Exception {
+ Assert.assertTrue("start must create the root node", client.hasPath("", true));
+
+ Mockito
+ .verify(client.getCache())
+ .start();
+ }
+
+ @Test
+ public void testHasPathWithEventualConsistencyHitsCache() {
+ final String path = "test-key";
+ final String absPath = PathUtils.join(root, path);
+
+ Mockito
+ .when(client.getCache().getCurrentData(absPath))
+ .thenReturn(null);
+
+ Assert.assertFalse(client.hasPath(path)); // test convenience method
+
+ Mockito
+ .when(client.getCache().getCurrentData(absPath))
+ .thenReturn(new ChildData(absPath, null, null));
+
+ Assert.assertTrue(client.hasPath(path, false)); // test actual method
+ }
+
+ @Test(expected = DrillRuntimeException.class)
+ public void testHasPathThrowsDrillRuntimeException() {
+ final String path = "test-key";
+ final String absPath = PathUtils.join(root, path);
+
+ Mockito
+ .when(client.getCache().getCurrentData(absPath))
+ .thenThrow(Exception.class);
+
+ client.hasPath(path);
+ }
+
+ @Test
+ public void testPutAndGetWorks() {
+ client.put(path, data);
+ final byte[] actual = client.get(path, true);
+ Assert.assertArrayEquals("data mismatch", data, actual);
+ }
+
+ @Test
+ public void testGetWithEventualConsistencyHitsCache() {
+ Mockito
+ .when(client.getCache().getCurrentData(abspath))
+ .thenReturn(null);
+
+ Assert.assertEquals("get should return null", null, client.get(path));
+
+ Mockito
+ .when(client.getCache().getCurrentData(abspath))
+ .thenReturn(new ChildData(abspath, null, data));
+
+ Assert.assertEquals("get should return data", data, client.get(path, false));
+ }
+
+ @Test
+ public void testCreate() throws Exception {
+ client.create(path);
+ Assert.assertTrue("path must exist", client.hasPath(path, true));
+
+ // ensure invoking create also rebuilds cache
+ Mockito
+ .verify(client.getCache(), Mockito.times(1))
+ .rebuildNode(abspath);
+ }
+
+ @Test
+ public void testDelete() throws Exception {
+ client.create(path);
+ Assert.assertTrue("path must exist", client.hasPath(path, true));
+ client.delete(path);
+ Assert.assertFalse("path must not exist", client.hasPath(path, true));
+
+ // ensure cache is rebuilt
+ Mockito
+ .verify(client.getCache(), Mockito.times(2))
+ .rebuildNode(abspath);
+ }
+
+
+ @Test
+ public void testEntriesReturnsRelativePaths() throws Exception {
+ final ChildData child = Mockito.mock(ChildData.class);
+ Mockito
+ .when(child.getPath())
+ .thenReturn(abspath);
+
+ Mockito
+ .when(child.getData())
+ .thenReturn(data);
+
+ final List<ChildData> children = Lists.newArrayList(child);
+ Mockito
+ .when(client.getCache().getCurrentData())
+ .thenReturn(children);
+
+ final Iterator<Map.Entry<String, byte[]>> entries = client.entries();
+
+ // returned entry must contain the given relative path
+ final Map.Entry<String, byte[]> expected = new ImmutableEntry<>(path, data);
+
+ Assert.assertEquals("entries do not match", expected, entries.next());
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
index 29aaf3a..bf56eb6 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
@@ -55,7 +55,7 @@ import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.server.RemoteServiceSet;
import org.apache.drill.exec.store.StoragePluginRegistry;
import org.apache.drill.exec.store.StoragePluginRegistryImpl;
-import org.apache.drill.exec.store.sys.local.LocalPStoreProvider;
+import org.apache.drill.exec.store.sys.store.provider.LocalPersistentStoreProvider;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.VarBinaryVector;
import org.junit.Ignore;
@@ -112,7 +112,7 @@ public class TestOptiqPlans extends ExecTest {
controller,
com,
workBus,
- new LocalPStoreProvider(config));
+ new LocalPersistentStoreProvider(config));
final QueryContext qc = new QueryContext(UserSession.Builder.newBuilder().setSupportComplexTypes(true).build(),
bitContext, QueryId.getDefaultInstance());
final PhysicalPlanReader reader = bitContext.getPlanReader();
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoin.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoin.java
index bd3145f..bed71f9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoin.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoin.java
@@ -26,7 +26,6 @@ import java.util.List;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.config.LogicalPlanPersistence;
import org.apache.drill.common.scanner.ClassPathScanner;
-import org.apache.drill.common.scanner.persistence.ScanResult;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.common.util.TestTools;
import org.apache.drill.exec.client.DrillClient;
@@ -51,7 +50,7 @@ import org.apache.drill.exec.server.Drillbit;
import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.server.RemoteServiceSet;
import org.apache.drill.exec.server.options.SystemOptionManager;
-import org.apache.drill.exec.store.sys.local.LocalPStoreProvider;
+import org.apache.drill.exec.store.sys.store.provider.LocalPersistentStoreProvider;
import org.apache.drill.exec.vector.ValueVector;
import org.junit.Rule;
import org.junit.Test;
@@ -73,7 +72,7 @@ public class TestHashJoin extends PopUnitTestBase {
private final DrillConfig c = DrillConfig.create();
private void testHJMockScanCommon(final DrillbitContext bitContext, UserServer.UserClientConnection connection, String physicalPlan, int expectedRows) throws Throwable {
- final LocalPStoreProvider provider = new LocalPStoreProvider(c);
+ final LocalPersistentStoreProvider provider = new LocalPersistentStoreProvider(c);
provider.start();
final SystemOptionManager opt = new SystemOptionManager(PhysicalPlanReaderTestFactory.defaultLogicalPlanPersistence(c), provider);
opt.init();
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/PStoreTestUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/PStoreTestUtil.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/PStoreTestUtil.java
index 1f5aa22..bcd2f5e 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/PStoreTestUtil.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/PStoreTestUtil.java
@@ -31,8 +31,8 @@ import com.google.common.collect.Maps;
public class PStoreTestUtil {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PStoreTestUtil.class);
- public static void test(PStoreProvider provider) throws Exception{
- PStore<String> store = provider.getStore(PStoreConfig.newJacksonBuilder(new ObjectMapper(), String.class).name("sys.test").build());
+ public static void test(PersistentStoreProvider provider) throws Exception{
+ PersistentStore<String> store = provider.getOrCreateStore(PersistentStoreConfig.newJacksonBuilder(new ObjectMapper(), String.class).name("sys.test").build());
String[] keys = {"first", "second"};
String[] values = {"value1", "value2"};
Map<String, String> expectedMap = Maps.newHashMap();
@@ -43,7 +43,7 @@ public class PStoreTestUtil {
}
// allow one second for puts to propagate back to cache
{
- Iterator<Map.Entry<String, String>> iter = store.iterator();
+ Iterator<Map.Entry<String, String>> iter = store.getAll();
for(int i =0; i < keys.length; i++){
Entry<String, String> e = iter.next();
assertTrue(expectedMap.containsKey(e.getKey()));
@@ -54,15 +54,15 @@ public class PStoreTestUtil {
}
{
- Iterator<Map.Entry<String, String>> iter = store.iterator();
+ Iterator<Map.Entry<String, String>> iter = store.getAll();
while(iter.hasNext()){
- iter.next();
- iter.remove();
+ final String key = iter.next().getKey();
+ store.delete(key);
}
}
// allow one second for deletes to propagate back to cache
- assertFalse(store.iterator().hasNext());
+ assertFalse(store.getAll().hasNext());
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestPStoreProviders.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestPStoreProviders.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestPStoreProviders.java
index cbfbcd3..93e2497 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestPStoreProviders.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/sys/TestPStoreProviders.java
@@ -23,18 +23,18 @@ import org.apache.curator.retry.RetryNTimes;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.TestWithZookeeper;
-import org.apache.drill.exec.store.sys.local.LocalPStoreProvider;
-import org.apache.drill.exec.store.sys.zk.ZkPStoreProvider;
+import org.apache.drill.exec.store.sys.store.provider.LocalPersistentStoreProvider;
+import org.apache.drill.exec.store.sys.store.provider.ZookeeperPersistentStoreProvider;
import org.junit.Test;
public class TestPStoreProviders extends TestWithZookeeper {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestPStoreProviders.class);
- static LocalPStoreProvider provider;
+ static LocalPersistentStoreProvider provider;
@Test
public void verifyLocalStore() throws Exception {
- try(LocalPStoreProvider provider = new LocalPStoreProvider(DrillConfig.create())){
+ try(LocalPersistentStoreProvider provider = new LocalPersistentStoreProvider(DrillConfig.create())){
PStoreTestUtil.test(provider);
}
}
@@ -51,7 +51,7 @@ public class TestPStoreProviders extends TestWithZookeeper {
try(CuratorFramework curator = builder.build()){
curator.start();
- ZkPStoreProvider provider = new ZkPStoreProvider(config, curator);
+ ZookeeperPersistentStoreProvider provider = new ZookeeperPersistentStoreProvider(config, curator);
PStoreTestUtil.test(provider);
}
}
[2/4] drill git commit: DRILL-4275: create TransientStore for
short-lived objects;
refactor PersistentStore to introduce pagination mechanism
Posted by hg...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/EStoreProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/EStoreProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/EStoreProvider.java
deleted file mode 100644
index 4c79a28..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/EStoreProvider.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.drill.exec.store.sys;
-
-
-/**
- * Interface to define the provider which return EStore.
- */
-
-public interface EStoreProvider extends PStoreProvider {
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java
index 85a1c3c..5c8a641 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java
@@ -21,15 +21,14 @@ import java.util.Collections;
import java.util.Iterator;
import java.util.List;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.server.options.DrillConfigIterator;
import org.apache.drill.exec.server.options.OptionManager;
import org.apache.drill.exec.server.options.OptionValue;
import org.apache.drill.exec.server.options.OptionValue.Kind;
import org.apache.drill.exec.server.options.OptionValue.OptionType;
-
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Lists;
import org.apache.drill.exec.server.options.SystemOptionManager;
public class OptionIterator implements Iterator<Object> {
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PStore.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PStore.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PStore.java
deleted file mode 100644
index b629645..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PStore.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.sys;
-
-import java.util.Map;
-
-
-/**
- * Interface for reading and writing values to a persistent storage provider. Iterators are guaranteed to be returned in key order.
- * @param <V>
- */
-public interface PStore<V> extends Iterable<Map.Entry<String, V>> {
- public V get(String key);
- public void put(String key, V value);
- public boolean putIfAbsent(String key, V value);
- public void delete(String key);
- public void close();
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PStoreConfig.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PStoreConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PStoreConfig.java
deleted file mode 100644
index bd9d977..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PStoreConfig.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.sys;
-
-import org.apache.drill.exec.store.sys.serialize.JacksonSerializer;
-import org.apache.drill.exec.store.sys.serialize.PClassSerializer;
-import org.apache.drill.exec.store.sys.serialize.ProtoSerializer;
-
-import com.dyuproject.protostuff.Schema;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import com.google.protobuf.Message;
-import com.google.protobuf.Message.Builder;
-
-public class PStoreConfig<V> {
-
- private final String name;
- private final PClassSerializer<V> valueSerializer;
- private final Mode mode;
- private final int maxIteratorSize;
-
- public static enum Mode {PERSISTENT, EPHEMERAL, BLOB_PERSISTENT};
-
- private PStoreConfig(String name, PClassSerializer<V> valueSerializer, Mode mode, int maxIteratorSize) {
- super();
- this.name = name;
- this.valueSerializer = valueSerializer;
- this.mode = mode;
- this.maxIteratorSize = Math.abs(maxIteratorSize);
- }
-
- public Mode getMode() {
- return mode;
- }
-
- public int getMaxIteratorSize() {
- return maxIteratorSize;
- }
-
- public String getName() {
- return name;
- }
-
- public PClassSerializer<V> getSerializer() {
- return valueSerializer;
- }
-
- public static <V extends Message, X extends Builder> PStoreConfigBuilder<V> newProtoBuilder(Schema<V> writeSchema, Schema<X> readSchema) {
- return new PStoreConfigBuilder<V>(new ProtoSerializer<V, X>(writeSchema, readSchema));
- }
-
- public static <V> PStoreConfigBuilder<V> newJacksonBuilder(ObjectMapper mapper, Class<V> clazz) {
- return new PStoreConfigBuilder<V>(new JacksonSerializer<V>(mapper, clazz));
- }
-
- public static class PStoreConfigBuilder<V> {
- String name;
- PClassSerializer<V> serializer;
- Mode mode = Mode.PERSISTENT;
- int maxIteratorSize = Integer.MAX_VALUE;
-
- PStoreConfigBuilder(PClassSerializer<V> serializer) {
- super();
- this.serializer = serializer;
- }
-
- public PStoreConfigBuilder<V> name(String name) {
- this.name = name;
- return this;
- }
-
- public PStoreConfigBuilder<V> persist(){
- this.mode = Mode.PERSISTENT;
- return this;
- }
-
- public PStoreConfigBuilder<V> ephemeral(){
- this.mode = Mode.EPHEMERAL;
- return this;
- }
-
- public PStoreConfigBuilder<V> blob(){
- this.mode = Mode.BLOB_PERSISTENT;
- return this;
- }
-
- /**
- * Set the maximum size of the iterator. Positive numbers start from the start of the list. Negative numbers start from the end of the list.
- * @param size
- * @return
- */
- public PStoreConfigBuilder<V> max(int size){
- this.maxIteratorSize = size;
- return this;
- }
-
- public PStoreConfig<V> build(){
- Preconditions.checkNotNull(name);
- return new PStoreConfig<V>(name, serializer, mode, maxIteratorSize);
- }
-
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + maxIteratorSize;
- result = prime * result + ((mode == null) ? 0 : mode.hashCode());
- result = prime * result + ((name == null) ? 0 : name.hashCode());
- result = prime * result + ((valueSerializer == null) ? 0 : valueSerializer.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- PStoreConfig other = (PStoreConfig) obj;
- if (maxIteratorSize != other.maxIteratorSize) {
- return false;
- }
- if (mode != other.mode) {
- return false;
- }
- if (name == null) {
- if (other.name != null) {
- return false;
- }
- } else if (!name.equals(other.name)) {
- return false;
- }
- if (valueSerializer == null) {
- if (other.valueSerializer != null) {
- return false;
- }
- } else if (!valueSerializer.equals(other.valueSerializer)) {
- return false;
- }
- return true;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PStoreProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PStoreProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PStoreProvider.java
deleted file mode 100644
index efa223e..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PStoreProvider.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.sys;
-
-import java.io.IOException;
-
-public interface PStoreProvider extends AutoCloseable {
- public <V> PStore<V> getStore(PStoreConfig<V> config) throws IOException;
- public void start() throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PStoreRegistry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PStoreRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PStoreRegistry.java
deleted file mode 100644
index 532e6be..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PStoreRegistry.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.sys;
-
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.coord.ClusterCoordinator;
-
-import com.typesafe.config.ConfigException;
-
-public class PStoreRegistry {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PStoreRegistry.class);
-
- private DrillConfig config;
- private ClusterCoordinator coord;
-
- public PStoreRegistry(ClusterCoordinator coord, DrillConfig config) {
- this.coord = coord;
- this.config = config;
- }
-
- public ClusterCoordinator getClusterCoordinator() {
- return this.coord;
- }
-
- public DrillConfig getConfig() {
- return this.config;
- }
-
- @SuppressWarnings("unchecked")
- public PStoreProvider newPStoreProvider() throws ExecutionSetupException {
- try {
- String storeProviderClassName = config.getString(ExecConstants.SYS_STORE_PROVIDER_CLASS);
- logger.info("Using the configured PStoreProvider class: '{}'.", storeProviderClassName);
- Class<? extends PStoreProvider> storeProviderClass = (Class<? extends PStoreProvider>) Class.forName(storeProviderClassName);
- Constructor<? extends PStoreProvider> c = storeProviderClass.getConstructor(PStoreRegistry.class);
- return new CachingStoreProvider(c.newInstance(this));
- } catch (ConfigException.Missing | ClassNotFoundException | NoSuchMethodException | SecurityException
- | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
- logger.error(e.getMessage(), e);
- throw new ExecutionSetupException("A System Table provider was either not specified or could not be found or instantiated", e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PersistentStore.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PersistentStore.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PersistentStore.java
new file mode 100644
index 0000000..767b1d5
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PersistentStore.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys;
+
+import java.util.Iterator;
+import java.util.Map;
+
+/**
+ * An abstraction used to store and retrieve instances of given value type.
+ *
+ * @param <V> value type
+ */
+public interface PersistentStore<V> extends AutoCloseable {
+ /**
+ * Returns storage {@link PersistentStoreMode mode} of this store.
+ */
+ PersistentStoreMode getMode();
+
+ /**
+ * Returns the value for the given key if exists, null otherwise.
+ * @param key lookup key
+ */
+ V get(String key);
+
+ /**
+ * Stores the (key, value) tuple in the store. Lifetime of the tuple depends upon store {@link #getMode mode}.
+ *
+ * @param key lookup key
+ * @param value value to store
+ */
+ void put(String key, V value);
+
+
+ /**
+ * Removes the value corresponding to the given key if exists, nothing happens otherwise.
+ * @param key lookup key
+ */
+ void delete(String key);
+
+ /**
+ * Stores the (key, value) tuple in the store only if it does not exists.
+ *
+ * @param key lookup key
+ * @param value value to store
+ * @return true if put takes place, false otherwise.
+ */
+ boolean putIfAbsent(String key, V value);
+
+ /**
+ * Returns an iterator of desired number of entries offsetting by the skip value.
+ *
+ * @param skip number of records to skip from beginning
+ * @param take max number of records to return
+ */
+ Iterator<Map.Entry<String, V>> getRange(int skip, int take);
+
+ /**
+ * Returns an iterator of entries.
+ */
+ Iterator<Map.Entry<String, V>> getAll();
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PersistentStoreConfig.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PersistentStoreConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PersistentStoreConfig.java
new file mode 100644
index 0000000..ca319f2
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PersistentStoreConfig.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys;
+
+import com.dyuproject.protostuff.Schema;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import com.google.protobuf.Message;
+import com.google.protobuf.Message.Builder;
+import org.apache.drill.exec.serialization.InstanceSerializer;
+import org.apache.drill.exec.serialization.JacksonSerializer;
+import org.apache.drill.exec.serialization.ProtoSerializer;
+
+
+/**
+ * An abstraction for configurations that are used to create a {@link PersistentStore store}.
+ *
+ * @param <V> value type of which {@link PersistentStore} uses to store & retrieve instances
+ */
+public class PersistentStoreConfig<V> {
+
+ private final String name;
+ private final InstanceSerializer<V> valueSerializer;
+ private final PersistentStoreMode mode;
+
+ protected PersistentStoreConfig(String name, InstanceSerializer<V> valueSerializer, PersistentStoreMode mode) {
+ this.name = name;
+ this.valueSerializer = valueSerializer;
+ this.mode = mode;
+ }
+
+ public PersistentStoreMode getMode() {
+ return mode;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public InstanceSerializer<V> getSerializer() {
+ return valueSerializer;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(name, valueSerializer, mode);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof PersistentStoreConfig) {
+ final PersistentStoreConfig other = PersistentStoreConfig.class.cast(obj);
+ return Objects.equal(name, other.name)
+ && Objects.equal(valueSerializer, other.valueSerializer)
+ && Objects.equal(mode, other.mode);
+ }
+ return false;
+ }
+
+ public static <V extends Message, X extends Builder> StoreConfigBuilder<V> newProtoBuilder(Schema<V> writeSchema, Schema<X> readSchema) {
+ return new StoreConfigBuilder<>(new ProtoSerializer<>(readSchema, writeSchema));
+ }
+
+ public static <V> StoreConfigBuilder<V> newJacksonBuilder(ObjectMapper mapper, Class<V> clazz) {
+ return new StoreConfigBuilder<>(new JacksonSerializer<>(mapper, clazz));
+ }
+
+ public static class StoreConfigBuilder<V> {
+ private String name;
+ private InstanceSerializer<V> serializer;
+ private PersistentStoreMode mode = PersistentStoreMode.PERSISTENT;
+
+ protected StoreConfigBuilder(InstanceSerializer<V> serializer) {
+ super();
+ this.serializer = serializer;
+ }
+
+ public StoreConfigBuilder<V> name(String name) {
+ this.name = name;
+ return this;
+ }
+
+ public StoreConfigBuilder<V> persist(){
+ this.mode = PersistentStoreMode.PERSISTENT;
+ return this;
+ }
+
+ public StoreConfigBuilder<V> blob(){
+ this.mode = PersistentStoreMode.BLOB_PERSISTENT;
+ return this;
+ }
+
+ public PersistentStoreConfig<V> build(){
+ Preconditions.checkNotNull(name);
+ return new PersistentStoreConfig<>(name, serializer, mode);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PersistentStoreMode.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PersistentStoreMode.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PersistentStoreMode.java
new file mode 100644
index 0000000..68d0cb6
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PersistentStoreMode.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys;
+
+/**
+ * Defines operation mode of a {@link PersistentStore} instance.
+ */
+public enum PersistentStoreMode {
+ PERSISTENT,
+ BLOB_PERSISTENT
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PersistentStoreProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PersistentStoreProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PersistentStoreProvider.java
new file mode 100644
index 0000000..75b89b4
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PersistentStoreProvider.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys;
+
+import org.apache.drill.exec.exception.StoreException;
+
+/**
+ * A factory used to create {@link PersistentStore store} instances.
+ *
+ */
+public interface PersistentStoreProvider extends AutoCloseable {
+ /**
+ * Gets or creates a {@link PersistentStore persistent store} for the given configuration.
+ *
+ * Note that implementors have liberty to cache previous {@link PersistentStore store} instances.
+ *
+ * @param config store configuration
+ * @param <V> store value type
+ */
+ <V> PersistentStore<V> getOrCreateStore(PersistentStoreConfig<V> config) throws StoreException;
+
+
+ /**
+ * Sets up the provider.
+ */
+ void start() throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PersistentStoreRegistry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PersistentStoreRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PersistentStoreRegistry.java
new file mode 100644
index 0000000..b117513
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PersistentStoreRegistry.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+
+import com.google.common.base.Preconditions;
+import com.typesafe.config.ConfigException;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.coord.ClusterCoordinator;
+import org.apache.drill.exec.store.sys.store.provider.CachingPersistentStoreProvider;
+
+public class PersistentStoreRegistry<C extends ClusterCoordinator> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PersistentStoreRegistry.class);
+
+ private final DrillConfig config;
+ private final C coordinator;
+
+ public PersistentStoreRegistry(C coordinator, DrillConfig config) {
+ this.coordinator = Preconditions.checkNotNull(coordinator, "coordinator cannot be null");
+ this.config = Preconditions.checkNotNull(config, "config cannot be null");
+ }
+
+ public C getCoordinator() {
+ return this.coordinator;
+ }
+
+ public DrillConfig getConfig() {
+ return this.config;
+ }
+
+ @SuppressWarnings("unchecked")
+ public PersistentStoreProvider newPStoreProvider() throws ExecutionSetupException {
+ try {
+ String storeProviderClassName = config.getString(ExecConstants.SYS_STORE_PROVIDER_CLASS);
+ logger.info("Using the configured PStoreProvider class: '{}'.", storeProviderClassName);
+ Class<? extends PersistentStoreProvider> storeProviderClass = (Class<? extends PersistentStoreProvider>) Class.forName(storeProviderClassName);
+ Constructor<? extends PersistentStoreProvider> c = storeProviderClass.getConstructor(PersistentStoreRegistry.class);
+ return new CachingPersistentStoreProvider(c.newInstance(this));
+ } catch (ConfigException.Missing | ClassNotFoundException | NoSuchMethodException | SecurityException
+ | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
+ logger.error(e.getMessage(), e);
+ throw new ExecutionSetupException("A System Table provider was either not specified or could not be found or instantiated", e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/StaticDrillTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/StaticDrillTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/StaticDrillTable.java
index 8b7225e..0a9b9b3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/StaticDrillTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/StaticDrillTable.java
@@ -17,12 +17,11 @@
*/
package org.apache.drill.exec.store.sys;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.drill.exec.planner.logical.DrillTable;
import org.apache.drill.exec.store.RecordDataType;
import org.apache.drill.exec.store.StoragePlugin;
-import org.apache.drill.exec.store.pojo.PojoDataType;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
/**
* A {@link org.apache.drill.exec.planner.logical.DrillTable} with a defined schema
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTablePlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTablePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTablePlugin.java
index 33f030b..4fb0475 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTablePlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTablePlugin.java
@@ -34,8 +34,8 @@ import org.apache.drill.exec.planner.logical.DrillTable;
import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.store.AbstractSchema;
import org.apache.drill.exec.store.AbstractStoragePlugin;
-import org.apache.drill.exec.store.pojo.PojoDataType;
import org.apache.drill.exec.store.SchemaConfig;
+import org.apache.drill.exec.store.pojo.PojoDataType;
/**
* A "storage" plugin for system tables.
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTableScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTableScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTableScan.java
index 8ade25c..2c3bba4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTableScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTableScan.java
@@ -21,7 +21,11 @@ import java.io.IOException;
import java.util.Collections;
import java.util.List;
+import com.fasterxml.jackson.annotation.JacksonInject;
import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
import com.google.common.collect.Lists;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.SchemaPath;
@@ -36,11 +40,6 @@ import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType;
import org.apache.drill.exec.store.StoragePluginRegistry;
-import com.fasterxml.jackson.annotation.JacksonInject;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-
@JsonTypeName("sys")
public class SystemTableScan extends AbstractGroupScan implements SubScan {
// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SystemTableScan.class);
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ThreadsIterator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ThreadsIterator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ThreadsIterator.java
index e9bc7ff..681119d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ThreadsIterator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ThreadsIterator.java
@@ -17,13 +17,13 @@
*/
package org.apache.drill.exec.store.sys;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
-
import java.lang.management.ManagementFactory;
import java.lang.management.ThreadMXBean;
import java.util.Iterator;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
+
public class ThreadsIterator implements Iterator<Object> {
private boolean beforeFirst = true;
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/VersionIterator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/VersionIterator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/VersionIterator.java
index 5620ece..9bfb700 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/VersionIterator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/VersionIterator.java
@@ -19,10 +19,8 @@ package org.apache.drill.exec.store.sys;
import java.io.IOException;
import java.net.URL;
-import java.util.Enumeration;
import java.util.Iterator;
import java.util.Properties;
-import java.util.jar.Manifest;
import com.google.common.io.Resources;
import org.apache.drill.common.util.DrillVersionInfo;
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/FilePStore.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/FilePStore.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/FilePStore.java
deleted file mode 100644
index ebee7a8..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/FilePStore.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.sys.local;
-
-import static org.apache.drill.exec.ExecConstants.DRILL_SYS_FILE_SUFFIX;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map.Entry;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.exec.store.dfs.DrillFileSystem;
-import org.apache.drill.exec.store.sys.PStore;
-import org.apache.drill.exec.store.sys.PStoreConfig;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
-public class FilePStore<V> implements PStore<V> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FilePStore.class);
-
-
- private final Path basePath;
- private final PStoreConfig<V> config;
- private final DrillFileSystem fs;
-
- public FilePStore(DrillFileSystem fs, Path base, PStoreConfig<V> config) {
- super();
- this.basePath = new Path(base, config.getName());
- this.config = config;
- this.fs = fs;
-
- try {
- mkdirs(basePath);
- } catch (IOException e) {
- throw new RuntimeException("Failure setting pstore configuration path.");
- }
- }
-
- private void mkdirs(Path path) throws IOException{
- fs.mkdirs(path);
- }
-
- public static Path getLogDir(){
- String drillLogDir = System.getenv("DRILL_LOG_DIR");
- if (drillLogDir == null) {
- drillLogDir = "/var/log/drill";
- }
- return new Path(new File(drillLogDir).getAbsoluteFile().toURI());
- }
-
- public static DrillFileSystem getFileSystem(DrillConfig config, Path root) throws IOException{
- Path blobRoot = root == null ? getLogDir() : root;
- Configuration fsConf = new Configuration();
- if(blobRoot.toUri().getScheme() != null){
- fsConf.set(FileSystem.FS_DEFAULT_NAME_KEY, blobRoot.toUri().toString());
- }
-
-
- DrillFileSystem fs = new DrillFileSystem(fsConf);
- fs.mkdirs(blobRoot);
- return fs;
- }
-
- @Override
- public Iterator<Entry<String, V>> iterator() {
- try{
- List<FileStatus> f = fs.list(false, basePath);
- if (f == null || f.isEmpty()) {
- return Collections.emptyIterator();
- }
- List<String> files = Lists.newArrayList();
-
- for (FileStatus stat : f) {
- String s = stat.getPath().getName();
- if (s.endsWith(DRILL_SYS_FILE_SUFFIX)) {
- files.add(s.substring(0, s.length() - DRILL_SYS_FILE_SUFFIX.length()));
- }
- }
-
- Collections.sort(files);
- files = files.subList(0, Math.min(files.size(), config.getMaxIteratorSize()));
- return new Iter(files.iterator());
-
- }catch(IOException e){
- throw new RuntimeException(e);
- }
- }
-
- private Path makePath(String name) {
- Preconditions.checkArgument(
- !name.contains("/") &&
- !name.contains(":") &&
- !name.contains(".."));
-
- final Path path = new Path(basePath, name + DRILL_SYS_FILE_SUFFIX);
- // do this to check file name.
- return path;
- }
-
- public V get(String key) {
- try{
- Path path = makePath(key);
- if(!fs.exists(path)){
- return null;
- }
- }catch(IOException e){
- throw new RuntimeException(e);
- }
-
- final Path path = makePath(key);
- try (InputStream is = fs.open(path)) {
- return config.getSerializer().deserialize(IOUtils.toByteArray(is));
- } catch (IOException e) {
- throw new RuntimeException("Unable to deserialize \"" + path + "\"", e);
- }
- }
-
- public void put(String key, V value) {
- try (OutputStream os = fs.create(makePath(key))) {
- IOUtils.write(config.getSerializer().serialize(value), os);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public boolean putIfAbsent(String key, V value) {
- try {
- Path p = makePath(key);
- if (fs.exists(p)) {
- return false;
- } else {
- put(key, value);
- return true;
- }
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- public void delete(String key) {
- try {
- fs.delete(makePath(key), false);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- private class Iter implements Iterator<Entry<String, V>>{
-
- private Iterator<String> keys;
- private String current;
-
- public Iter(Iterator<String> keys) {
- super();
- this.keys = keys;
- }
-
- @Override
- public boolean hasNext() {
- return keys.hasNext();
- }
-
- @Override
- public Entry<String, V> next() {
- current = keys.next();
- return new DeferredEntry(current);
- }
-
- @Override
- public void remove() {
- delete(current);
- keys.remove();
- }
-
- private class DeferredEntry implements Entry<String, V> {
-
- private String name;
-
-
- public DeferredEntry(String name) {
- super();
- this.name = name;
- }
-
- @Override
- public String getKey() {
- return name;
- }
-
- @Override
- public V getValue() {
- return get(name);
- }
-
- @Override
- public V setValue(V value) {
- throw new UnsupportedOperationException();
- }
-
- }
- }
-
- @Override
- public void close() {
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/LocalEStoreProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/LocalEStoreProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/LocalEStoreProvider.java
deleted file mode 100644
index e7c2f94..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/LocalEStoreProvider.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.drill.exec.store.sys.local;
-
-import java.io.IOException;
-
-import org.apache.drill.exec.store.sys.EStore;
-import org.apache.drill.exec.store.sys.EStoreProvider;
-import org.apache.drill.exec.store.sys.PStoreConfig;
-import org.apache.drill.exec.store.sys.PStoreConfig.Mode;
-
-import com.google.common.base.Preconditions;
-
-public class LocalEStoreProvider implements EStoreProvider{
-
- @Override
- public <V> EStore<V> getStore(PStoreConfig<V> storeConfig) throws IOException {
- Preconditions.checkArgument(storeConfig.getMode() == Mode.EPHEMERAL, "Estore configurations must be set ephemeral.");
-
- return new MapEStore<V>();
- }
-
- @Override
- public void start() throws IOException {
- }
-
- @Override
- public void close() {
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/LocalPStoreProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/LocalPStoreProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/LocalPStoreProvider.java
index 3131290..cda1180 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/LocalPStoreProvider.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/LocalPStoreProvider.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ * <p/>
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -17,77 +17,18 @@
*/
package org.apache.drill.exec.store.sys.local;
-import java.io.IOException;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.store.dfs.DrillFileSystem;
-import org.apache.drill.exec.store.sys.PStore;
-import org.apache.drill.exec.store.sys.PStoreConfig;
-import org.apache.drill.exec.store.sys.PStoreProvider;
-import org.apache.drill.exec.store.sys.PStoreRegistry;
-import org.apache.hadoop.fs.Path;
+import org.apache.drill.exec.exception.StoreException;
+import org.apache.drill.exec.store.sys.PersistentStoreRegistry;
+import org.apache.drill.exec.store.sys.store.provider.LocalPersistentStoreProvider;
/**
- * A really simple provider that stores data in the local file system, one value per file.
+ * Kept for possible references to old class name in configuration.
+ *
+ * @deprecated will be removed in 1.7
+ * use {@link org.apache.drill.exec.store.sys.store.provider.LocalPersistentStoreProvider} instead.
*/
-public class LocalPStoreProvider implements PStoreProvider {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(LocalPStoreProvider.class);
-
- private final Path path;
- private final boolean enableWrite;
- private final ConcurrentMap<PStoreConfig<?>, PStore<?>> pstores;
- private final LocalEStoreProvider estoreProvider;
- private final DrillFileSystem fs;
-
- public LocalPStoreProvider(DrillConfig config) throws IOException {
- this.path = new Path(config.getString(ExecConstants.SYS_STORE_PROVIDER_LOCAL_PATH));
- this.enableWrite = config.getBoolean(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE);
- this.pstores = enableWrite ? null : new ConcurrentHashMap<PStoreConfig<?>, PStore<?>>();
- this.estoreProvider = new LocalEStoreProvider();
- this.fs = FilePStore.getFileSystem(config, path);
- }
-
- public LocalPStoreProvider(PStoreRegistry registry) throws IOException {
- this(registry.getConfig());
- }
-
- @Override
- public void close() {
+public class LocalPStoreProvider extends LocalPersistentStoreProvider {
+ public LocalPStoreProvider(PersistentStoreRegistry registry) throws StoreException {
+ super(registry);
}
-
- @Override
- public <V> PStore<V> getStore(PStoreConfig<V> storeConfig) throws IOException {
- switch(storeConfig.getMode()){
- case EPHEMERAL:
- return estoreProvider.getStore(storeConfig);
- case BLOB_PERSISTENT:
- case PERSISTENT:
- return getPStore(storeConfig);
- default:
- throw new IllegalStateException();
- }
-
- }
-
- private <V> PStore<V> getPStore(PStoreConfig<V> storeConfig) throws IOException {
- if (enableWrite) {
- return new FilePStore<V>(fs, path, storeConfig);
- } else {
- PStore<V> p = new NoWriteLocalPStore<V>();
- PStore<?> p2 = pstores.putIfAbsent(storeConfig, p);
- if(p2 != null) {
- return (PStore<V>) p2;
- }
- return p;
- }
- }
-
-
- @Override
- public void start() {
- }
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/MapEStore.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/MapEStore.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/MapEStore.java
deleted file mode 100644
index 96e51e6..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/MapEStore.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.drill.exec.store.sys.local;
-
-import org.apache.drill.exec.store.sys.EStore;
-
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * Implementation of EStore using ConcurrentHashMap.
- * @param <V>
- */
-public class MapEStore<V> implements EStore<V> {
- ConcurrentHashMap<String, V> store = new ConcurrentHashMap<>();
-
- @Override
- public V get(String key) {
- return store.get(key);
- }
-
- @Override
- public void put(String key, V value) {
- store.put(key, value);
- }
-
- @Override
- public void delete(String key) {
- store.remove(key);
- }
-
- @Override
- public Iterator<Map.Entry<String, V>> iterator() {
- return store.entrySet().iterator();
- }
-
- @Override
- public boolean putIfAbsent(String key, V value) {
- V out = store.putIfAbsent(key, value);
- return out == null;
- }
-
- @Override
- public void close() {
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/NoWriteLocalPStore.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/NoWriteLocalPStore.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/NoWriteLocalPStore.java
deleted file mode 100644
index c675618..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/local/NoWriteLocalPStore.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.sys.local;
-
-import java.util.Iterator;
-import java.util.Map.Entry;
-import java.util.concurrent.ConcurrentMap;
-
-import org.apache.drill.exec.store.sys.PStore;
-
-import com.google.common.collect.Maps;
-
-public class NoWriteLocalPStore<V> implements PStore<V>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(NoWriteLocalPStore.class);
-
- private ConcurrentMap<String, V> map = Maps.newConcurrentMap();
-
- private ConcurrentMap<String, V> blobMap = Maps.newConcurrentMap();
-
- public NoWriteLocalPStore() {
- super();
- }
-
- @Override
- public Iterator<Entry<String, V>> iterator() {
- return map.entrySet().iterator();
- }
-
- @Override
- public V get(String key) {
- return map.get(key);
- }
-
- @Override
- public void put(String key, V value) {
- map.put(key, value);
- }
-
- @Override
- public boolean putIfAbsent(String key, V value) {
- return null == map.putIfAbsent(key, value);
- }
-
- @Override
- public void delete(String key) {
- map.remove(key);
- blobMap.remove(key);
- }
-
- @Override
- public void close() {
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/serialize/JacksonSerializer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/serialize/JacksonSerializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/serialize/JacksonSerializer.java
deleted file mode 100644
index 53452f3..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/serialize/JacksonSerializer.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.sys.serialize;
-
-import java.io.IOException;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-import com.fasterxml.jackson.databind.ObjectWriter;
-
-public class JacksonSerializer<X> implements PClassSerializer<X> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JacksonSerializer.class);
-
- private ObjectWriter writer;
- private ObjectReader reader;
-
- public JacksonSerializer(ObjectMapper mapper, Class<X> clazz){
- this.reader = mapper.reader(clazz);
- this.writer = mapper.writer();
- }
-
- @Override
- public byte[] serialize(X val) throws IOException {
- return writer.writeValueAsBytes(val);
- }
-
- @Override
- public X deserialize(byte[] bytes) throws IOException {
- return reader.readValue(bytes);
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((reader == null) ? 0 : reader.hashCode());
- result = prime * result + ((writer == null) ? 0 : writer.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- JacksonSerializer other = (JacksonSerializer) obj;
- if (reader == null) {
- if (other.reader != null) {
- return false;
- }
- } else if (!reader.equals(other.reader)) {
- return false;
- }
- if (writer == null) {
- if (other.writer != null) {
- return false;
- }
- } else if (!writer.equals(other.writer)) {
- return false;
- }
- return true;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/serialize/PClassSerializer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/serialize/PClassSerializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/serialize/PClassSerializer.java
deleted file mode 100644
index a3b0c1b..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/serialize/PClassSerializer.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.sys.serialize;
-
-import java.io.IOException;
-
-public interface PClassSerializer<X> {
- public byte[] serialize(X val) throws IOException;
- public X deserialize(byte[] bytes) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/serialize/ProtoSerializer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/serialize/ProtoSerializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/serialize/ProtoSerializer.java
deleted file mode 100644
index 52df7a4..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/serialize/ProtoSerializer.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.sys.serialize;
-
-import java.io.IOException;
-
-import org.apache.commons.io.output.ByteArrayOutputStream;
-
-import com.dyuproject.protostuff.JsonIOUtil;
-import com.dyuproject.protostuff.Schema;
-import com.google.protobuf.Message;
-
-public class ProtoSerializer<X, B extends Message.Builder> implements PClassSerializer<X> {
-
- private final Schema<X> writeSchema;
- private final Schema<B> readSchema;
-
- public ProtoSerializer(Schema<X> writeSchema, Schema<B> readSchema) {
- super();
- this.writeSchema = writeSchema;
- this.readSchema = readSchema;
- }
-
- @Override
- public byte[] serialize(X val) throws IOException {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- JsonIOUtil.writeTo(baos, val, writeSchema, false);
- return baos.toByteArray();
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public X deserialize(byte[] bytes) throws IOException {
- B b = readSchema.newMessage();
- JsonIOUtil.mergeFrom(bytes, b, readSchema, false);
- return (X) b.build();
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((readSchema == null) ? 0 : readSchema.hashCode());
- result = prime * result + ((writeSchema == null) ? 0 : writeSchema.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- ProtoSerializer other = (ProtoSerializer) obj;
- if (readSchema == null) {
- if (other.readSchema != null) {
- return false;
- }
- } else if (!readSchema.equals(other.readSchema)) {
- return false;
- }
- if (writeSchema == null) {
- if (other.writeSchema != null) {
- return false;
- }
- } else if (!writeSchema.equals(other.writeSchema)) {
- return false;
- }
- return true;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/LocalPersistentStore.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/LocalPersistentStore.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/LocalPersistentStore.java
new file mode 100644
index 0000000..1ef8d12
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/LocalPersistentStore.java
@@ -0,0 +1,196 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys.store;
+
+import static org.apache.drill.exec.ExecConstants.DRILL_SYS_FILE_SUFFIX;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import javax.annotation.Nullable;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import org.apache.commons.io.IOUtils;
+import org.apache.drill.common.collections.ImmutableEntry;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.store.dfs.DrillFileSystem;
+import org.apache.drill.exec.store.sys.BasePersistentStore;
+import org.apache.drill.exec.store.sys.PersistentStore;
+import org.apache.drill.exec.store.sys.PersistentStoreConfig;
+import org.apache.drill.exec.store.sys.PersistentStoreMode;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class LocalPersistentStore<V> extends BasePersistentStore<V> {
+ private static final Logger logger = LoggerFactory.getLogger(LocalPersistentStore.class);
+
+ private final Path basePath;
+ private final PersistentStoreConfig<V> config;
+ private final DrillFileSystem fs;
+
+ public LocalPersistentStore(DrillFileSystem fs, Path base, PersistentStoreConfig<V> config) {
+ super();
+ this.basePath = new Path(base, config.getName());
+ this.config = config;
+ this.fs = fs;
+
+ try {
+ mkdirs(basePath);
+ } catch (IOException e) {
+ throw new RuntimeException("Failure setting pstore configuration path.");
+ }
+ }
+
+ @Override
+ public PersistentStoreMode getMode() {
+ return PersistentStoreMode.PERSISTENT;
+ }
+
+ private void mkdirs(Path path) throws IOException{
+ fs.mkdirs(path);
+ }
+
+ public static Path getLogDir(){
+ String drillLogDir = System.getenv("DRILL_LOG_DIR");
+ if (drillLogDir == null) {
+ drillLogDir = "/var/log/drill";
+ }
+ return new Path(new File(drillLogDir).getAbsoluteFile().toURI());
+ }
+
+ public static DrillFileSystem getFileSystem(DrillConfig config, Path root) throws IOException{
+ Path blobRoot = root == null ? getLogDir() : root;
+ Configuration fsConf = new Configuration();
+ if(blobRoot.toUri().getScheme() != null){
+ fsConf.set(FileSystem.FS_DEFAULT_NAME_KEY, blobRoot.toUri().toString());
+ }
+
+
+ DrillFileSystem fs = new DrillFileSystem(fsConf);
+ fs.mkdirs(blobRoot);
+ return fs;
+ }
+
+ @Override
+ public Iterator<Map.Entry<String, V>> getRange(int skip, int take) {
+ try{
+ List<FileStatus> f = fs.list(false, basePath);
+ if (f == null || f.isEmpty()) {
+ return Collections.emptyIterator();
+ }
+ List<String> files = Lists.newArrayList();
+
+ for (FileStatus stat : f) {
+ String s = stat.getPath().getName();
+ if (s.endsWith(DRILL_SYS_FILE_SUFFIX)) {
+ files.add(s.substring(0, s.length() - DRILL_SYS_FILE_SUFFIX.length()));
+ }
+ }
+
+ Collections.sort(files);
+ return Iterables.transform(Iterables.limit(Iterables.skip(files, skip), take), new Function<String, Entry<String, V>>() {
+ @Nullable
+ @Override
+ public Entry<String, V> apply(String key) {
+ return new ImmutableEntry<>(key, get(key));
+ }
+ }).iterator();
+ }catch(IOException e){
+ throw new RuntimeException(e);
+ }
+ }
+
+ private Path makePath(String name) {
+ Preconditions.checkArgument(
+ !name.contains("/") &&
+ !name.contains(":") &&
+ !name.contains(".."));
+
+ final Path path = new Path(basePath, name + DRILL_SYS_FILE_SUFFIX);
+ // do this to check file name.
+ return path;
+ }
+
+ public V get(String key) {
+ try{
+ Path path = makePath(key);
+ if(!fs.exists(path)){
+ return null;
+ }
+ }catch(IOException e){
+ throw new RuntimeException(e);
+ }
+
+ final Path path = makePath(key);
+ try (InputStream is = fs.open(path)) {
+ return config.getSerializer().deserialize(IOUtils.toByteArray(is));
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to deserialize \"" + path + "\"", e);
+ }
+ }
+
+ public void put(String key, V value) {
+ try (OutputStream os = fs.create(makePath(key))) {
+ IOUtils.write(config.getSerializer().serialize(value), os);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public boolean putIfAbsent(String key, V value) {
+ try {
+ Path p = makePath(key);
+ if (fs.exists(p)) {
+ return false;
+ } else {
+ put(key, value);
+ return true;
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public void delete(String key) {
+ try {
+ fs.delete(makePath(key), false);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void close() {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/ZookeeperPersistentStore.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/ZookeeperPersistentStore.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/ZookeeperPersistentStore.java
new file mode 100644
index 0000000..3dde4b8
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/ZookeeperPersistentStore.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys.store;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterators;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.drill.common.collections.ImmutableEntry;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.exec.coord.zk.PathUtils;
+import org.apache.drill.exec.coord.zk.ZookeeperClient;
+import org.apache.drill.exec.exception.StoreException;
+import org.apache.drill.exec.serialization.InstanceSerializer;
+import org.apache.drill.exec.store.sys.BasePersistentStore;
+import org.apache.drill.exec.store.sys.PersistentStoreConfig;
+import org.apache.drill.exec.store.sys.PersistentStoreMode;
+import org.apache.zookeeper.CreateMode;
+
+/**
+ * Zookeeper based implementation of {@link org.apache.drill.exec.store.sys.PersistentStore}.
+ */
+public class ZookeeperPersistentStore<V> extends BasePersistentStore<V> {
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ZookeeperPersistentStore.class);
+
+ private final PersistentStoreConfig<V> config;
+ private final ZookeeperClient client;
+
+ public ZookeeperPersistentStore(final CuratorFramework framework, final PersistentStoreConfig<V> config) throws StoreException {
+ this.config = Preconditions.checkNotNull(config);
+ this.client = new ZookeeperClient(framework, PathUtils.join("/", config.getName()), CreateMode.PERSISTENT);
+ }
+
+ public void start() throws Exception {
+ client.start();
+ }
+
+ @Override
+ public PersistentStoreMode getMode() {
+ return config.getMode();
+ }
+
+ @Override
+ public V get(final String key) {
+ final byte[] bytes = client.get(key);
+ if (bytes == null) {
+ return null;
+ }
+ try {
+ return config.getSerializer().deserialize(bytes);
+ } catch (final IOException e) {
+ throw new DrillRuntimeException(String.format("unable to deserialize value at %s", key), e);
+ }
+ }
+
+ @Override
+ public void put(final String key, final V value) {
+ final InstanceSerializer<V> serializer = config.getSerializer();
+ try {
+ final byte[] bytes = serializer.serialize(value);
+ client.put(key, bytes);
+ } catch (final IOException e) {
+ throw new DrillRuntimeException(String.format("unable to de/serialize value of type %s", value.getClass()), e);
+ }
+ }
+
+ @Override
+ public boolean putIfAbsent(final String key, final V value) {
+ final V old = get(key);
+ if (old == null) {
+ try {
+ final byte[] bytes = config.getSerializer().serialize(value);
+ client.put(key, bytes);
+ return true;
+ } catch (final IOException e) {
+ throw new DrillRuntimeException(String.format("unable to serialize value of type %s", value.getClass()), e);
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public void delete(final String key) {
+ client.delete(key);
+ }
+
+ @Override
+ public Iterator<Map.Entry<String, V>> getRange(final int skip, final int take) {
+ final Iterator<Map.Entry<String, byte[]>> entries = client.entries();
+ Iterators.advance(entries, skip);
+ return Iterators.transform(Iterators.limit(entries, take), new Function<Map.Entry<String, byte[]>, Map.Entry<String, V>>() {
+ @Nullable
+ @Override
+ public Map.Entry<String, V> apply(@Nullable Map.Entry<String, byte[]> input) {
+ try {
+ final V value = config.getSerializer().deserialize(input.getValue());
+ return new ImmutableEntry<>(input.getKey(), value);
+ } catch (final IOException e) {
+ throw new DrillRuntimeException(String.format("unable to deserialize value at key %s", input.getKey()), e);
+ }
+ }
+ });
+ }
+
+ @Override
+ public void close() {
+ try{
+ client.close();
+ } catch(final Exception e) {
+ logger.warn("Failure while closing out %s.", getClass().getSimpleName(), e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/BasePersistentStoreProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/BasePersistentStoreProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/BasePersistentStoreProvider.java
new file mode 100644
index 0000000..e497a4c
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/BasePersistentStoreProvider.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys.store.provider;
+
+import org.apache.drill.exec.store.sys.PersistentStoreProvider;
+
+public abstract class BasePersistentStoreProvider implements PersistentStoreProvider {
+ @Override
+ public void start() throws Exception { }
+
+ @Override
+ public void close() throws Exception { }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/CachingPersistentStoreProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/CachingPersistentStoreProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/CachingPersistentStoreProvider.java
new file mode 100644
index 0000000..99ccc8e
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/CachingPersistentStoreProvider.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys.store.provider;
+
+import java.util.List;
+import java.util.concurrent.ConcurrentMap;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.drill.common.AutoCloseables;
+import org.apache.drill.exec.exception.StoreException;
+import org.apache.drill.exec.store.sys.PersistentStore;
+import org.apache.drill.exec.store.sys.PersistentStoreConfig;
+import org.apache.drill.exec.store.sys.PersistentStoreProvider;
+
+public class CachingPersistentStoreProvider extends BasePersistentStoreProvider {
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CachingPersistentStoreProvider.class);
+
+ private final ConcurrentMap<PersistentStoreConfig<?>, PersistentStore<?>> storeCache = Maps.newConcurrentMap();
+ private final PersistentStoreProvider provider;
+
+ public CachingPersistentStoreProvider(PersistentStoreProvider provider) {
+ this.provider = provider;
+ }
+
+ @SuppressWarnings("unchecked")
+ public <V> PersistentStore<V> getOrCreateStore(final PersistentStoreConfig<V> config) throws StoreException {
+ final PersistentStore<?> store = storeCache.get(config);
+ if (store == null) {
+ final PersistentStore<?> newStore = provider.getOrCreateStore(config);
+ final PersistentStore<?> finalStore = storeCache.putIfAbsent(config, newStore);
+ if (finalStore == null) {
+ return (PersistentStore<V>)newStore;
+ }
+ try {
+ newStore.close();
+ } catch (Exception ex) {
+ throw new StoreException(ex);
+ }
+ }
+
+ return (PersistentStore<V>) store;
+ }
+
+ @Override
+ public void start() throws Exception {
+ provider.start();
+ }
+
+ @Override
+ public void close() throws Exception {
+ final List<AutoCloseable> closeables = Lists.newArrayList();
+ for (final AutoCloseable store : storeCache.values()) {
+ closeables.add(store);
+ }
+ closeables.add(provider);
+ storeCache.clear();
+ AutoCloseables.close(closeables);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/LocalPersistentStoreProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/LocalPersistentStoreProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/LocalPersistentStoreProvider.java
new file mode 100644
index 0000000..9bf18ab
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/LocalPersistentStoreProvider.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys.store.provider;
+
+import java.io.IOException;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.exception.StoreException;
+import org.apache.drill.exec.store.dfs.DrillFileSystem;
+import org.apache.drill.exec.store.sys.PersistentStoreRegistry;
+import org.apache.drill.exec.store.sys.PersistentStore;
+import org.apache.drill.exec.store.sys.PersistentStoreConfig;
+import org.apache.drill.exec.store.sys.store.LocalPersistentStore;
+import org.apache.drill.exec.testing.store.NoWriteLocalStore;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A really simple provider that stores data in the local file system, one value per file.
+ */
+public class LocalPersistentStoreProvider extends BasePersistentStoreProvider {
+ private static final Logger logger = LoggerFactory.getLogger(LocalPersistentStoreProvider.class);
+
+ private final Path path;
+ private final DrillFileSystem fs;
+ // This flag is used in testing. Ideally, tests should use a specific PersistentStoreProvider that knows
+ // how to handle this flag.
+ private final boolean enableWrite;
+
+ public LocalPersistentStoreProvider(final PersistentStoreRegistry registry) throws StoreException {
+ this(registry.getConfig());
+ }
+
+ public LocalPersistentStoreProvider(final DrillConfig config) throws StoreException {
+ this.path = new Path(config.getString(ExecConstants.SYS_STORE_PROVIDER_LOCAL_PATH));
+ this.enableWrite = config.getBoolean(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE);
+ try {
+ this.fs = LocalPersistentStore.getFileSystem(config, path);
+ } catch (IOException e) {
+ throw new StoreException("unable to get filesystem", e);
+ }
+ }
+
+ @Override
+ public <V> PersistentStore<V> getOrCreateStore(PersistentStoreConfig<V> storeConfig) {
+ switch(storeConfig.getMode()){
+ case BLOB_PERSISTENT:
+ case PERSISTENT:
+ if (enableWrite) {
+ return new LocalPersistentStore<>(fs, path, storeConfig);
+ }
+ return new NoWriteLocalStore<>();
+ default:
+ throw new IllegalStateException();
+ }
+ }
+
+
+ @Override
+ public void close() throws Exception {
+ fs.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/ZookeeperPersistentStoreProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/ZookeeperPersistentStoreProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/ZookeeperPersistentStoreProvider.java
new file mode 100644
index 0000000..58c46a7
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/store/provider/ZookeeperPersistentStoreProvider.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys.store.provider;
+
+import java.io.IOException;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.coord.zk.ZKClusterCoordinator;
+import org.apache.drill.exec.exception.StoreException;
+import org.apache.drill.exec.store.dfs.DrillFileSystem;
+import org.apache.drill.exec.store.sys.PersistentStore;
+import org.apache.drill.exec.store.sys.PersistentStoreRegistry;
+import org.apache.drill.exec.store.sys.PersistentStoreConfig;
+import org.apache.drill.exec.store.sys.store.LocalPersistentStore;
+import org.apache.drill.exec.store.sys.store.ZookeeperPersistentStore;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ZookeeperPersistentStoreProvider extends BasePersistentStoreProvider {
+ private static final Logger logger = LoggerFactory.getLogger(ZookeeperPersistentStoreProvider.class);
+
+ private static final String DRILL_EXEC_SYS_STORE_PROVIDER_ZK_BLOBROOT = "drill.exec.sys.store.provider.zk.blobroot";
+
+ private final CuratorFramework curator;
+ private final DrillFileSystem fs;
+ private final Path blobRoot;
+
+ public ZookeeperPersistentStoreProvider(final PersistentStoreRegistry<ZKClusterCoordinator> registry) throws StoreException {
+ this(registry.getConfig(), registry.getCoordinator().getCurator());
+ }
+
+ @VisibleForTesting
+ public ZookeeperPersistentStoreProvider(final DrillConfig config, final CuratorFramework curator) throws StoreException {
+ this.curator = curator;
+
+ if (config.hasPath(DRILL_EXEC_SYS_STORE_PROVIDER_ZK_BLOBROOT)) {
+ blobRoot = new Path(config.getString(DRILL_EXEC_SYS_STORE_PROVIDER_ZK_BLOBROOT));
+ }else{
+ blobRoot = LocalPersistentStore.getLogDir();
+ }
+
+ try {
+ this.fs = LocalPersistentStore.getFileSystem(config, blobRoot);
+ } catch (IOException ex) {
+ throw new StoreException("unable to get filesystem", ex);
+ }
+ }
+
+ @Override
+ public <V> PersistentStore<V> getOrCreateStore(final PersistentStoreConfig<V> config) throws StoreException {
+ switch(config.getMode()){
+ case BLOB_PERSISTENT:
+ return new LocalPersistentStore<>(fs, blobRoot, config);
+ case PERSISTENT:
+ final ZookeeperPersistentStore<V> store = new ZookeeperPersistentStore<>(curator, config);
+ try {
+ store.start();
+ } catch (Exception e) {
+ throw new StoreException("unable to start zookeeper store", e);
+ }
+ return store;
+ default:
+ throw new IllegalStateException();
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+ fs.close();
+ }
+}
[4/4] drill git commit: DRILL-4275: create TransientStore for
short-lived objects;
refactor PersistentStore to introduce pagination mechanism
Posted by hg...@apache.org.
DRILL-4275: create TransientStore for short-lived objects; refactor PersistentStore to introduce pagination mechanism
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/8126927f
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/8126927f
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/8126927f
Branch: refs/heads/master
Commit: 8126927fd6f538000a28fb77e704a37e20abbe6c
Parents: 9a3a5c4
Author: Hanifi Gunes <ha...@gmail.com>
Authored: Mon Feb 15 15:16:47 2016 -0800
Committer: Hanifi Gunes <ha...@gmail.com>
Committed: Thu Feb 18 16:47:02 2016 -0800
----------------------------------------------------------------------
.../common/collections/ImmutableEntry.java | 62 ++++
.../exec/store/hbase/config/HBasePStore.java | 236 ---------------
.../store/hbase/config/HBasePStoreProvider.java | 157 +---------
.../hbase/config/HBasePersistentStore.java | 238 +++++++++++++++
.../config/HBasePersistentStoreProvider.java | 135 +++++++++
.../drill/hbase/TestHBaseTableProvider.java | 23 +-
.../exec/store/mongo/config/MongoPStore.java | 184 ------------
.../store/mongo/config/MongoPStoreProvider.java | 88 ------
.../mongo/config/MongoPersistentStore.java | 190 ++++++++++++
.../config/MongoPersistentStoreProvider.java | 84 ++++++
.../src/resources/drill-override-example.conf | 2 +-
exec/java-exec/pom.xml | 6 +
.../apache/drill/exec/client/DrillClient.java | 2 +-
.../apache/drill/exec/compile/CodeCompiler.java | 3 -
.../drill/exec/coord/ClusterCoordinator.java | 18 +-
.../coord/local/LocalClusterCoordinator.java | 22 +-
.../drill/exec/coord/local/MapBackedStore.java | 85 ++++++
.../exec/coord/store/BaseTransientStore.java | 83 ++++++
.../store/CachingTransientStoreFactory.java | 63 ++++
.../drill/exec/coord/store/TransientStore.java | 106 +++++++
.../exec/coord/store/TransientStoreConfig.java | 74 +++++
.../store/TransientStoreConfigBuilder.java | 50 ++++
.../exec/coord/store/TransientStoreEvent.java | 71 +++++
.../coord/store/TransientStoreEventType.java | 27 ++
.../exec/coord/store/TransientStoreFactory.java | 36 +++
.../coord/store/TransientStoreListener.java | 32 ++
.../drill/exec/coord/zk/EventDispatcher.java | 60 ++++
.../apache/drill/exec/coord/zk/PathUtils.java | 73 +++++
.../exec/coord/zk/ZKClusterCoordinator.java | 57 ++--
.../drill/exec/coord/zk/ZkEphemeralStore.java | 145 +++++++++
.../exec/coord/zk/ZkTransientStoreFactory.java | 50 ++++
.../drill/exec/coord/zk/ZookeeperClient.java | 238 +++++++++++++++
.../drill/exec/exception/StoreException.java | 42 +++
.../exec/serialization/InstanceSerializer.java | 25 ++
.../exec/serialization/JacksonSerializer.java | 59 ++++
.../exec/serialization/ProtoSerializer.java | 65 +++++
.../org/apache/drill/exec/server/Drillbit.java | 14 +-
.../drill/exec/server/DrillbitContext.java | 14 +-
.../drill/exec/server/RemoteServiceSet.java | 7 +-
.../drill/exec/server/options/OptionValue.java | 3 +-
.../server/options/SystemOptionManager.java | 34 ++-
.../drill/exec/server/rest/DrillRestServer.java | 4 +-
.../exec/server/rest/StorageResources.java | 2 +-
.../server/rest/profile/ProfileResources.java | 97 ++++---
.../drill/exec/store/StoragePluginRegistry.java | 4 +-
.../exec/store/StoragePluginRegistryImpl.java | 22 +-
.../exec/store/sys/BasePersistentStore.java | 32 ++
.../exec/store/sys/CachingStoreProvider.java | 70 -----
.../org/apache/drill/exec/store/sys/EStore.java | 28 --
.../drill/exec/store/sys/EStoreProvider.java | 27 --
.../drill/exec/store/sys/OptionIterator.java | 5 +-
.../org/apache/drill/exec/store/sys/PStore.java | 33 ---
.../drill/exec/store/sys/PStoreConfig.java | 166 -----------
.../drill/exec/store/sys/PStoreProvider.java | 25 --
.../drill/exec/store/sys/PStoreRegistry.java | 64 ----
.../drill/exec/store/sys/PersistentStore.java | 77 +++++
.../exec/store/sys/PersistentStoreConfig.java | 115 ++++++++
.../exec/store/sys/PersistentStoreMode.java | 26 ++
.../exec/store/sys/PersistentStoreProvider.java | 42 +++
.../exec/store/sys/PersistentStoreRegistry.java | 65 +++++
.../drill/exec/store/sys/StaticDrillTable.java | 5 +-
.../drill/exec/store/sys/SystemTablePlugin.java | 2 +-
.../drill/exec/store/sys/SystemTableScan.java | 9 +-
.../drill/exec/store/sys/ThreadsIterator.java | 6 +-
.../drill/exec/store/sys/VersionIterator.java | 2 -
.../drill/exec/store/sys/local/FilePStore.java | 234 ---------------
.../store/sys/local/LocalEStoreProvider.java | 47 ---
.../store/sys/local/LocalPStoreProvider.java | 83 +-----
.../drill/exec/store/sys/local/MapEStore.java | 64 ----
.../store/sys/local/NoWriteLocalPStore.java | 69 -----
.../store/sys/serialize/JacksonSerializer.java | 86 ------
.../store/sys/serialize/PClassSerializer.java | 25 --
.../store/sys/serialize/ProtoSerializer.java | 93 ------
.../store/sys/store/LocalPersistentStore.java | 196 +++++++++++++
.../sys/store/ZookeeperPersistentStore.java | 135 +++++++++
.../provider/BasePersistentStoreProvider.java | 28 ++
.../CachingPersistentStoreProvider.java | 76 +++++
.../provider/LocalPersistentStoreProvider.java | 80 +++++
.../ZookeeperPersistentStoreProvider.java | 89 ++++++
.../exec/store/sys/zk/ZkAbstractStore.java | 291 -------------------
.../drill/exec/store/sys/zk/ZkEStore.java | 41 ---
.../exec/store/sys/zk/ZkEStoreProvider.java | 50 ----
.../drill/exec/store/sys/zk/ZkPStore.java | 41 ---
.../exec/store/sys/zk/ZkPStoreProvider.java | 92 +-----
.../exec/testing/store/NoWriteLocalStore.java | 66 +++++
.../org/apache/drill/exec/work/WorkManager.java | 6 +-
.../apache/drill/exec/work/foreman/Foreman.java | 10 +-
.../drill/exec/work/foreman/QueryManager.java | 52 ++--
.../src/main/resources/drill-module.conf | 2 +-
.../java/org/apache/drill/BaseTestQuery.java | 2 +-
.../java/org/apache/drill/PlanningBase.java | 6 +-
.../exec/compile/CodeCompilerTestFactory.java | 8 +-
.../compile/bytecode/ReplaceMethodInvoke.java | 5 +-
.../drill/exec/coord/zk/TestEphemeralStore.java | 144 +++++++++
.../exec/coord/zk/TestEventDispatcher.java | 94 ++++++
.../drill/exec/coord/zk/TestPathUtils.java | 92 ++++++
.../exec/coord/zk/TestZookeeperClient.java | 201 +++++++++++++
.../exec/physical/impl/TestOptiqPlans.java | 4 +-
.../exec/physical/impl/join/TestHashJoin.java | 5 +-
.../drill/exec/store/sys/PStoreTestUtil.java | 14 +-
.../exec/store/sys/TestPStoreProviders.java | 10 +-
101 files changed, 3967 insertions(+), 2465 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/common/src/main/java/org/apache/drill/common/collections/ImmutableEntry.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/collections/ImmutableEntry.java b/common/src/main/java/org/apache/drill/common/collections/ImmutableEntry.java
new file mode 100644
index 0000000..99ea544
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/collections/ImmutableEntry.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.collections;
+
+import java.util.Map;
+
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+
+public class ImmutableEntry<K, V> implements Map.Entry<K, V> {
+ private final K key;
+ private final V value;
+
+ public ImmutableEntry(final K key, final V value) {
+ this.key = Preconditions.checkNotNull(key, "key is required");
+ this.value = Preconditions.checkNotNull(value, "value is required");
+ }
+
+ @Override
+ public K getKey() {
+ return key;
+ }
+
+ @Override
+ public V getValue() {
+ return value;
+ }
+
+ @Override
+ public V setValue(final V value) {
+ throw new UnsupportedOperationException("entry is immutable");
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (other instanceof ImmutableEntry && other.getClass() == getClass()) {
+ final ImmutableEntry<K, V> entry = (ImmutableEntry<K, V>)other;
+ return Objects.equal(key, entry.key) && Objects.equal(value, entry.value);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(key, value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePStore.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePStore.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePStore.java
deleted file mode 100644
index 17ddcb1..0000000
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePStore.java
+++ /dev/null
@@ -1,236 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.hbase.config;
-
-import static org.apache.drill.exec.store.hbase.config.HBasePStoreProvider.FAMILY;
-import static org.apache.drill.exec.store.hbase.config.HBasePStoreProvider.QUALIFIER;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.Map.Entry;
-import java.util.NoSuchElementException;
-
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-import org.apache.drill.exec.store.sys.PStore;
-import org.apache.drill.exec.store.sys.PStoreConfig;
-import org.apache.drill.exec.store.sys.PStoreConfig.Mode;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.util.Bytes;
-
-public class HBasePStore<V> implements PStore<V> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HBasePStore.class);
-
- private final PStoreConfig<V> config;
-
- private final HTableInterface table;
-
- private final String tableName;
- private final byte[] tableNameStartKey;
- private final byte[] tableNameStopKey;
-
- public HBasePStore(PStoreConfig<V> config, HTableInterface table) throws IOException {
- this.tableName = config.getName() + '\0';
- this.tableNameStartKey = Bytes.toBytes(tableName); // "tableName\x00"
- this.tableNameStopKey = this.tableNameStartKey.clone();
- this.tableNameStopKey[tableNameStartKey.length-1] = 1;
- this.config = config;
- this.table = table;
- }
-
- @Override
- public V get(String key) {
- return get(key, FAMILY);
- }
-
- protected synchronized V get(String key, byte[] family) {
- try {
- Get get = new Get(row(key));
- get.addColumn(family, QUALIFIER);
- Result r = table.get(get);
- if(r.isEmpty()){
- return null;
- }
- return value(r);
- } catch (IOException e) {
- throw new DrillRuntimeException("Caught error while getting row '" + key + "' from for table:" + Bytes.toString(table.getTableName()), e);
- }
- }
-
- @Override
- public void put(String key, V value) {
- put(key, FAMILY, value);
- }
-
- protected synchronized void put(String key, byte[] family, V value) {
- try {
- Put put = new Put(row(key));
- put.add(family, QUALIFIER, bytes(value));
- table.put(put);
- } catch (IOException e) {
- throw new DrillRuntimeException("Caught error while putting row '" + key + "' from for table:" + Bytes.toString(table.getTableName()), e);
- }
- }
-
- @Override
- public synchronized boolean putIfAbsent(String key, V value) {
- try {
- Put put = new Put(row(key));
- put.add(FAMILY, QUALIFIER, bytes(value));
- return table.checkAndPut(put.getRow(), FAMILY, QUALIFIER, null /*absent*/, put);
- } catch (IOException e) {
- throw new DrillRuntimeException("Caught error while putting row '" + key + "' from for table:" + Bytes.toString(table.getTableName()), e);
- }
- }
-
- @Override
- public synchronized void delete(String key) {
- delete(row(key));
- }
-
- @Override
- public Iterator<Entry<String, V>> iterator() {
- return new Iter();
- }
-
- private byte[] row(String key) {
- return Bytes.toBytes(this.tableName + key);
- }
-
- private byte[] bytes(V value) {
- try {
- return config.getSerializer().serialize(value);
- } catch (IOException e) {
- throw new DrillRuntimeException(e);
- }
- }
-
- private V value(Result result) {
- try {
- return config.getSerializer().deserialize(result.value());
- } catch (IOException e) {
- throw new DrillRuntimeException(e);
- }
- }
-
- private void delete(byte[] row) {
- try {
- Delete del = new Delete(row);
- table.delete(del);
- } catch (IOException e) {
- throw new DrillRuntimeException("Caught error while deleting row '" + Bytes.toStringBinary(row)
- + "' from for table:" + Bytes.toString(table.getTableName()), e);
- }
- }
-
- private class Iter implements Iterator<Entry<String, V>> {
- private ResultScanner scanner;
- private Result current = null;
- private Result last = null;
- private boolean done = false;
- private int rowsRead = 0;
-
- Iter() {
- try {
- Scan scan = new Scan(tableNameStartKey, tableNameStopKey);
- scan.addColumn(FAMILY, QUALIFIER);
- scan.setCaching(config.getMaxIteratorSize() > 100 ? 100 : config.getMaxIteratorSize());
- scanner = table.getScanner(scan);
- } catch (IOException e) {
- throw new DrillRuntimeException("Caught error while creating HBase scanner for table:" + Bytes.toString(table.getTableName()), e);
- }
- }
-
- @Override
- public boolean hasNext() {
- if (config.getMode() == Mode.BLOB_PERSISTENT
- && rowsRead >= config.getMaxIteratorSize()) {
- done = true;
- } else if (!done && current == null) {
- try {
- if ((current = scanner.next()) == null) {
- done = true;
- }
- } catch (IOException e) {
- throw new DrillRuntimeException("Caught error while fetching rows from for table:" + Bytes.toString(table.getTableName()), e);
- }
- }
-
- if (done && scanner != null) {
- scanner.close();
- }
- return (current != null);
- }
-
- @Override
- public Entry<String, V> next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
- last = current;
- current = null;
- rowsRead++;
- return new DeferredEntry(last);
- }
-
- @Override
- public void remove() {
- if (last == null) {
- throw new IllegalStateException("remove() called on HBase persistent store iterator, but there is no last row.");
- }
- delete(last.getRow());
- }
-
- }
-
- private class DeferredEntry implements Entry<String, V>{
-
- private Result result;
-
- public DeferredEntry(Result result) {
- super();
- this.result = result;
- }
-
- @Override
- public String getKey() {
- return Bytes.toString(result.getRow(), tableNameStartKey.length, result.getRow().length-tableNameStartKey.length);
- }
-
- @Override
- public V getValue() {
- return value(result);
- }
-
- @Override
- public V setValue(V value) {
- throw new UnsupportedOperationException();
- }
-
- }
-
- @Override
- public void close() {
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePStoreProvider.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePStoreProvider.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePStoreProvider.java
index e57f2b3..df32b87 100644
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePStoreProvider.java
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePStoreProvider.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ * <p/>
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -17,149 +17,16 @@
*/
package org.apache.drill.exec.store.hbase.config;
-import java.io.IOException;
-import java.util.Map;
-
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-import org.apache.drill.exec.coord.ClusterCoordinator;
-import org.apache.drill.exec.coord.zk.ZKClusterCoordinator;
-import org.apache.drill.exec.store.hbase.DrillHBaseConstants;
-import org.apache.drill.exec.store.sys.PStore;
-import org.apache.drill.exec.store.sys.PStoreConfig;
-import org.apache.drill.exec.store.sys.PStoreProvider;
-import org.apache.drill.exec.store.sys.PStoreRegistry;
-import org.apache.drill.exec.store.sys.local.LocalEStoreProvider;
-import org.apache.drill.exec.store.sys.zk.ZkEStoreProvider;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.util.Bytes;
-
-import com.google.common.annotations.VisibleForTesting;
-
-public class HBasePStoreProvider implements PStoreProvider {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HBasePStoreProvider.class);
-
- static final byte[] FAMILY = Bytes.toBytes("s");
-
- static final byte[] QUALIFIER = Bytes.toBytes("d");
-
- private final String storeTableName;
-
- private Configuration hbaseConf;
+import org.apache.drill.exec.store.sys.PersistentStoreRegistry;
- private HConnection connection;
-
- private HTableInterface table;
-
- private final boolean zkAvailable;
- private final LocalEStoreProvider localEStoreProvider;
- private final ZkEStoreProvider zkEStoreProvider;
-
- public HBasePStoreProvider(PStoreRegistry registry) {
- @SuppressWarnings("unchecked")
- Map<String, Object> config = (Map<String, Object>) registry.getConfig().getAnyRef(DrillHBaseConstants.SYS_STORE_PROVIDER_HBASE_CONFIG);
- this.hbaseConf = HBaseConfiguration.create();
- this.hbaseConf.set(HConstants.HBASE_CLIENT_INSTANCE_ID, "drill-hbase-persistent-store-client");
- if (config != null) {
- for (Map.Entry<String, Object> entry : config.entrySet()) {
- this.hbaseConf.set(entry.getKey(), String.valueOf(entry.getValue()));
- }
- }
- this.storeTableName = registry.getConfig().getString(DrillHBaseConstants.SYS_STORE_PROVIDER_HBASE_TABLE);
-
- ClusterCoordinator coord = registry.getClusterCoordinator();
- if ((coord instanceof ZKClusterCoordinator)) {
- this.localEStoreProvider = null;
- this.zkEStoreProvider = new ZkEStoreProvider(((ZKClusterCoordinator)registry.getClusterCoordinator()).getCurator());
- this.zkAvailable = true;
- } else {
- this.localEStoreProvider = new LocalEStoreProvider();
- this.zkEStoreProvider = null;
- this.zkAvailable = false;
- }
-
- }
-
- @VisibleForTesting
- public HBasePStoreProvider(Configuration conf, String storeTableName) {
- this.hbaseConf = conf;
- this.storeTableName = storeTableName;
- this.localEStoreProvider = new LocalEStoreProvider();
- this.zkEStoreProvider = null;
- this.zkAvailable = false;
- }
-
-
-
- @Override
- public <V> PStore<V> getStore(PStoreConfig<V> config) throws IOException {
- switch(config.getMode()){
- case EPHEMERAL:
- if (this.zkAvailable) {
- return zkEStoreProvider.getStore(config);
- } else {
- return localEStoreProvider.getStore(config);
- }
-
- case BLOB_PERSISTENT:
- case PERSISTENT:
- return new HBasePStore<V>(config, this.table);
-
- default:
- throw new IllegalStateException();
- }
- }
-
-
- @Override
- @SuppressWarnings("deprecation")
- public void start() throws IOException {
- this.connection = HConnectionManager.createConnection(hbaseConf);
-
- try(HBaseAdmin admin = new HBaseAdmin(connection)) {
- if (!admin.tableExists(storeTableName)) {
- HTableDescriptor desc = new HTableDescriptor(storeTableName);
- desc.addFamily(new HColumnDescriptor(FAMILY).setMaxVersions(1));
- admin.createTable(desc);
- } else {
- HTableDescriptor desc = admin.getTableDescriptor(Bytes.toBytes(storeTableName));
- if (!desc.hasFamily(FAMILY)) {
- throw new DrillRuntimeException("The HBase table " + storeTableName
- + " specified as persistent store exists but does not contain column family: "
- + (Bytes.toString(FAMILY)));
- }
- }
- }
-
- this.table = connection.getTable(storeTableName);
- this.table.setAutoFlush(true);
- }
-
- @Override
- public synchronized void close() {
- if (this.table != null) {
- try {
- this.table.close();
- this.table = null;
- } catch (IOException e) {
- logger.warn("Caught exception while closing HBase table.", e);
- }
- }
- if (this.connection != null && !this.connection.isClosed()) {
- try {
- this.connection.close();
- } catch (IOException e) {
- logger.warn("Caught exception while closing HBase connection.", e);
- }
- this.connection = null;
- }
+/**
+ * Kept for possible references to old class name in configuration.
+ *
+ * @deprecated will be removed in 1.7
+ * use {@link HBasePersistentStoreProvider} instead.
+ */
+public class HBasePStoreProvider extends HBasePersistentStoreProvider {
+ public HBasePStoreProvider(PersistentStoreRegistry registry) {
+ super(registry);
}
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePersistentStore.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePersistentStore.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePersistentStore.java
new file mode 100644
index 0000000..ac78eb0
--- /dev/null
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePersistentStore.java
@@ -0,0 +1,238 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hbase.config;
+
+import static org.apache.drill.exec.store.hbase.config.HBasePersistentStoreProvider.FAMILY;
+import static org.apache.drill.exec.store.hbase.config.HBasePersistentStoreProvider.QUALIFIER;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.NoSuchElementException;
+
+import com.google.common.collect.Iterators;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.exec.store.sys.BasePersistentStore;
+import org.apache.drill.exec.store.sys.PersistentStoreConfig;
+import org.apache.drill.exec.store.sys.PersistentStoreMode;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+
+public class HBasePersistentStore<V> extends BasePersistentStore<V> {
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HBasePersistentStore.class);
+
+ private final PersistentStoreConfig<V> config;
+ private final HTableInterface table;
+ private final String tableName;
+ private final byte[] tableNameStartKey;
+ private final byte[] tableNameStopKey;
+
+ public HBasePersistentStore(PersistentStoreConfig<V> config, HTableInterface table) {
+ this.tableName = config.getName() + '\0';
+ this.tableNameStartKey = Bytes.toBytes(tableName); // "tableName\x00"
+ this.tableNameStopKey = this.tableNameStartKey.clone();
+ this.tableNameStopKey[tableNameStartKey.length-1] = 1;
+ this.config = config;
+ this.table = table;
+ }
+
+ @Override
+ public PersistentStoreMode getMode() {
+ return PersistentStoreMode.PERSISTENT;
+ }
+
+ @Override
+ public V get(String key) {
+ return get(key, FAMILY);
+ }
+
+ protected synchronized V get(String key, byte[] family) {
+ try {
+ Get get = new Get(row(key));
+ get.addColumn(family, QUALIFIER);
+ Result r = table.get(get);
+ if(r.isEmpty()){
+ return null;
+ }
+ return value(r);
+ } catch (IOException e) {
+ throw new DrillRuntimeException("Caught error while getting row '" + key + "' from for table:" + Bytes.toString(table.getTableName()), e);
+ }
+ }
+
+ @Override
+ public void put(String key, V value) {
+ put(key, FAMILY, value);
+ }
+
+ protected synchronized void put(String key, byte[] family, V value) {
+ try {
+ Put put = new Put(row(key));
+ put.add(family, QUALIFIER, bytes(value));
+ table.put(put);
+ } catch (IOException e) {
+ throw new DrillRuntimeException("Caught error while putting row '" + key + "' from for table:" + Bytes.toString(table.getTableName()), e);
+ }
+ }
+
+ @Override
+ public synchronized boolean putIfAbsent(String key, V value) {
+ try {
+ Put put = new Put(row(key));
+ put.add(FAMILY, QUALIFIER, bytes(value));
+ return table.checkAndPut(put.getRow(), FAMILY, QUALIFIER, null /*absent*/, put);
+ } catch (IOException e) {
+ throw new DrillRuntimeException("Caught error while putting row '" + key + "' from for table:" + Bytes.toString(table.getTableName()), e);
+ }
+ }
+
+ @Override
+ public synchronized void delete(String key) {
+ delete(row(key));
+ }
+
+ @Override
+ public Iterator<Entry<String, V>> getRange(int skip, int take) {
+ final Iterator<Entry<String, V>> iter = new Iter(take);
+ Iterators.advance(iter, skip);
+ return Iterators.limit(iter, take);
+ }
+
+ private byte[] row(String key) {
+ return Bytes.toBytes(this.tableName + key);
+ }
+
+ private byte[] bytes(V value) {
+ try {
+ return config.getSerializer().serialize(value);
+ } catch (IOException e) {
+ throw new DrillRuntimeException(e);
+ }
+ }
+
+ private V value(Result result) {
+ try {
+ return config.getSerializer().deserialize(result.value());
+ } catch (IOException e) {
+ throw new DrillRuntimeException(e);
+ }
+ }
+
+ private void delete(byte[] row) {
+ try {
+ Delete del = new Delete(row);
+ table.delete(del);
+ } catch (IOException e) {
+ throw new DrillRuntimeException("Caught error while deleting row '" + Bytes.toStringBinary(row)
+ + "' from for table:" + Bytes.toString(table.getTableName()), e);
+ }
+ }
+
+ private class Iter implements Iterator<Entry<String, V>> {
+ private ResultScanner scanner;
+ private Result current = null;
+ private Result last = null;
+ private boolean done = false;
+
+ Iter(int take) {
+ try {
+ Scan scan = new Scan(tableNameStartKey, tableNameStopKey);
+ scan.addColumn(FAMILY, QUALIFIER);
+ scan.setCaching(Math.min(take, 100));
+ scan.setBatch(take); // set batch size
+ scanner = table.getScanner(scan);
+ } catch (IOException e) {
+ throw new DrillRuntimeException("Caught error while creating HBase scanner for table:" + Bytes.toString(table.getTableName()), e);
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (!done && current == null) {
+ try {
+ if ((current = scanner.next()) == null) {
+ done = true;
+ }
+ } catch (IOException e) {
+ throw new DrillRuntimeException("Caught error while fetching rows from for table:" + Bytes.toString(table.getTableName()), e);
+ }
+ }
+
+ if (done && scanner != null) {
+ scanner.close();
+ }
+ return (current != null);
+ }
+
+ @Override
+ public Entry<String, V> next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ last = current;
+ current = null;
+ return new DeferredEntry(last);
+ }
+
+ @Override
+ public void remove() {
+ if (last == null) {
+ throw new IllegalStateException("remove() called on HBase persistent store iterator, but there is no last row.");
+ }
+ delete(last.getRow());
+ }
+
+ }
+
+ private class DeferredEntry implements Entry<String, V>{
+
+ private Result result;
+
+ public DeferredEntry(Result result) {
+ super();
+ this.result = result;
+ }
+
+ @Override
+ public String getKey() {
+ return Bytes.toString(result.getRow(), tableNameStartKey.length, result.getRow().length-tableNameStartKey.length);
+ }
+
+ @Override
+ public V getValue() {
+ return value(result);
+ }
+
+ @Override
+ public V setValue(V value) {
+ throw new UnsupportedOperationException();
+ }
+
+ }
+
+ @Override
+ public void close() {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePersistentStoreProvider.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePersistentStoreProvider.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePersistentStoreProvider.java
new file mode 100644
index 0000000..6e379c6
--- /dev/null
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePersistentStoreProvider.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hbase.config;
+
+import java.io.IOException;
+import java.util.Map;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.exec.exception.StoreException;
+import org.apache.drill.exec.store.hbase.DrillHBaseConstants;
+import org.apache.drill.exec.store.sys.PersistentStore;
+import org.apache.drill.exec.store.sys.PersistentStoreConfig;
+import org.apache.drill.exec.store.sys.PersistentStoreRegistry;
+import org.apache.drill.exec.store.sys.store.provider.BasePersistentStoreProvider;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.util.Bytes;
+
+public class HBasePersistentStoreProvider extends BasePersistentStoreProvider {
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HBasePersistentStoreProvider.class);
+
+ static final byte[] FAMILY = Bytes.toBytes("s");
+
+ static final byte[] QUALIFIER = Bytes.toBytes("d");
+
+ private final String storeTableName;
+
+ private Configuration hbaseConf;
+
+ private HConnection connection;
+
+ private HTableInterface table;
+
+ public HBasePersistentStoreProvider(PersistentStoreRegistry registry) {
+ @SuppressWarnings("unchecked")
+ final Map<String, Object> config = (Map<String, Object>) registry.getConfig().getAnyRef(DrillHBaseConstants.SYS_STORE_PROVIDER_HBASE_CONFIG);
+ this.hbaseConf = HBaseConfiguration.create();
+ this.hbaseConf.set(HConstants.HBASE_CLIENT_INSTANCE_ID, "drill-hbase-persistent-store-client");
+ if (config != null) {
+ for (Map.Entry<String, Object> entry : config.entrySet()) {
+ this.hbaseConf.set(entry.getKey(), String.valueOf(entry.getValue()));
+ }
+ }
+ this.storeTableName = registry.getConfig().getString(DrillHBaseConstants.SYS_STORE_PROVIDER_HBASE_TABLE);
+ }
+
+ @VisibleForTesting
+ public HBasePersistentStoreProvider(Configuration conf, String storeTableName) {
+ this.hbaseConf = conf;
+ this.storeTableName = storeTableName;
+ }
+
+
+
+ @Override
+ public <V> PersistentStore<V> getOrCreateStore(PersistentStoreConfig<V> config) throws StoreException {
+ switch(config.getMode()){
+ case BLOB_PERSISTENT:
+ case PERSISTENT:
+ return new HBasePersistentStore<>(config, this.table);
+
+ default:
+ throw new IllegalStateException();
+ }
+ }
+
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public void start() throws IOException {
+ this.connection = HConnectionManager.createConnection(hbaseConf);
+
+ try(HBaseAdmin admin = new HBaseAdmin(connection)) {
+ if (!admin.tableExists(storeTableName)) {
+ HTableDescriptor desc = new HTableDescriptor(storeTableName);
+ desc.addFamily(new HColumnDescriptor(FAMILY).setMaxVersions(1));
+ admin.createTable(desc);
+ } else {
+ HTableDescriptor desc = admin.getTableDescriptor(Bytes.toBytes(storeTableName));
+ if (!desc.hasFamily(FAMILY)) {
+ throw new DrillRuntimeException("The HBase table " + storeTableName
+ + " specified as persistent store exists but does not contain column family: "
+ + (Bytes.toString(FAMILY)));
+ }
+ }
+ }
+
+ this.table = connection.getTable(storeTableName);
+ this.table.setAutoFlush(true);
+ }
+
+ @Override
+ public synchronized void close() {
+ if (this.table != null) {
+ try {
+ this.table.close();
+ this.table = null;
+ } catch (IOException e) {
+ logger.warn("Caught exception while closing HBase table.", e);
+ }
+ }
+ if (this.connection != null && !this.connection.isClosed()) {
+ try {
+ this.connection.close();
+ } catch (IOException e) {
+ logger.warn("Caught exception while closing HBase connection.", e);
+ }
+ this.connection = null;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseTableProvider.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseTableProvider.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseTableProvider.java
index 9ff1b08..6b73283 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseTableProvider.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseTableProvider.java
@@ -19,32 +19,33 @@ package org.apache.drill.hbase;
import static org.junit.Assert.assertEquals;
-import java.io.IOException;
import java.util.Map.Entry;
+import com.google.common.collect.Lists;
import org.apache.drill.common.config.LogicalPlanPersistence;
+import org.apache.drill.exec.exception.StoreException;
import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory;
-import org.apache.drill.exec.store.hbase.config.HBasePStoreProvider;
-import org.apache.drill.exec.store.sys.PStore;
-import org.apache.drill.exec.store.sys.PStoreConfig;
+import org.apache.drill.exec.store.hbase.config.HBasePersistentStoreProvider;
+import org.apache.drill.exec.store.sys.PersistentStore;
+import org.apache.drill.exec.store.sys.PersistentStoreConfig;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
public class TestHBaseTableProvider extends BaseHBaseTest {
- private static HBasePStoreProvider provider;
+ private static HBasePersistentStoreProvider provider;
@BeforeClass // mask HBase cluster start function
public static void setUpBeforeTestHBaseTableProvider() throws Exception {
- provider = new HBasePStoreProvider(storagePluginConfig.getHBaseConf(), "drill_store");
+ provider = new HBasePersistentStoreProvider(storagePluginConfig.getHBaseConf(), "drill_store");
provider.start();
}
@Test
- public void testTableProvider() throws IOException {
+ public void testTableProvider() throws StoreException {
LogicalPlanPersistence lp = PhysicalPlanReaderTestFactory.defaultLogicalPlanPersistence(config);
- PStore<String> hbaseStore = provider.getStore(PStoreConfig.newJacksonBuilder(lp.getMapper(), String.class).name("hbase").build());
+ PersistentStore<String> hbaseStore = provider.getOrCreateStore(PersistentStoreConfig.newJacksonBuilder(lp.getMapper(), String.class).name("hbase").build());
hbaseStore.put("", "v0");
hbaseStore.put("k1", "v1");
hbaseStore.put("k2", "v2");
@@ -57,13 +58,13 @@ public class TestHBaseTableProvider extends BaseHBaseTest {
assertEquals("testValue", hbaseStore.get(".test"));
int rowCount = 0;
- for (Entry<String, String> entry : hbaseStore) {
+ for (Entry<String, String> entry : Lists.newArrayList(hbaseStore.getAll())) {
rowCount++;
System.out.println(entry.getKey() + "=" + entry.getValue());
}
assertEquals(7, rowCount);
- PStore<String> hbaseTestStore = provider.getStore(PStoreConfig.newJacksonBuilder(lp.getMapper(), String.class).name("hbase.test").build());
+ PersistentStore<String> hbaseTestStore = provider.getOrCreateStore(PersistentStoreConfig.newJacksonBuilder(lp.getMapper(), String.class).name("hbase.test").build());
hbaseTestStore.put("", "v0");
hbaseTestStore.put("k1", "v1");
hbaseTestStore.put("k2", "v2");
@@ -75,7 +76,7 @@ public class TestHBaseTableProvider extends BaseHBaseTest {
assertEquals("testValue", hbaseStore.get(".test"));
rowCount = 0;
- for (Entry<String, String> entry : hbaseTestStore) {
+ for (Entry<String, String> entry : Lists.newArrayList(hbaseTestStore.getAll())) {
rowCount++;
System.out.println(entry.getKey() + "=" + entry.getValue());
}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/config/MongoPStore.java
----------------------------------------------------------------------
diff --git a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/config/MongoPStore.java b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/config/MongoPStore.java
deleted file mode 100644
index 7883fc9..0000000
--- a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/config/MongoPStore.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.mongo.config;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.Map.Entry;
-import java.util.NoSuchElementException;
-
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-import org.apache.drill.exec.store.mongo.DrillMongoConstants;
-import org.apache.drill.exec.store.sys.PStore;
-import org.apache.drill.exec.store.sys.PStoreConfig;
-import org.bson.Document;
-import org.bson.conversions.Bson;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.mongodb.client.MongoCollection;
-import com.mongodb.client.MongoCursor;
-import com.mongodb.client.model.Filters;
-import com.mongodb.client.model.UpdateOptions;
-import com.mongodb.client.model.Updates;
-import com.mongodb.client.result.UpdateResult;
-
-import static org.apache.drill.exec.store.mongo.config.MongoPStoreProvider.pKey;
-
-public class MongoPStore<V> implements PStore<V>, DrillMongoConstants {
-
- static final Logger logger = LoggerFactory.getLogger(MongoPStore.class);
-
- private final PStoreConfig<V> config;
-
- private final MongoCollection<Document> collection;
-
- public MongoPStore(PStoreConfig<V> config, MongoCollection<Document> collection)
- throws IOException {
-// this.config = config;
-// this.collection = collection;
- throw new UnsupportedOperationException("Mongo DB PStore not currently supported");
- }
-
- @Override
- public V get(String key) {
- try {
- Bson query = Filters.eq(ID, key);
- Document document = collection.find(query).first();
- if (document != null) {
- return value((byte[]) document.get(pKey));
- } else {
- return null;
- }
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- throw new DrillRuntimeException(e.getMessage(), e);
- }
- }
-
- @Override
- public void put(String key, V value) {
- try {
- Document putObj = new Document(ID, key).append(pKey, bytes(value));
- collection.insertOne(putObj);
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- throw new DrillRuntimeException(e.getMessage(), e);
- }
- }
-
- @Override
- public boolean putIfAbsent(String key, V value) {
- try {
- Bson query = Filters.eq(ID, key);
- Bson update = Updates.set(pKey, bytes(value));
- UpdateResult updateResult = collection.updateOne(query, update, new UpdateOptions().upsert(true));
- return updateResult.getModifiedCount() == 1;
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- throw new DrillRuntimeException(e.getMessage(), e);
- }
- }
-
- @Override
- public void delete(String key) {
- try {
- Bson query = Filters.eq(ID, key);
- collection.deleteOne(query);
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- throw new DrillRuntimeException(e.getMessage(), e);
- }
- }
-
- private byte[] bytes(V value) {
- try {
- return config.getSerializer().serialize(value);
- } catch (IOException e) {
- throw new DrillRuntimeException(e.getMessage(), e);
- }
- }
-
- private V value(byte[] serialize) {
- try {
- return config.getSerializer().deserialize(serialize);
- } catch (IOException e) {
- throw new DrillRuntimeException(e.getMessage(), e);
- }
- }
-
- @Override
- public Iterator<Entry<String, V>> iterator() {
- return new MongoIterator();
- }
-
- private class MongoIterator implements Iterator<Entry<String, V>> {
-
- private MongoCursor<Document> cursor;
-
- public MongoIterator() {
- cursor = collection.find().iterator();
- }
-
- @Override
- public boolean hasNext() {
- return cursor.hasNext();
- }
-
- @Override
- public Entry<String, V> next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
- return new DeferredEntry(cursor.next());
- }
-
- @Override
- public void remove() {
- cursor.remove();
- }
- }
-
- private class DeferredEntry implements Entry<String, V> {
-
- private Document result;
-
- public DeferredEntry(Document result) {
- this.result = result;
- }
-
- @Override
- public String getKey() {
- return result.get(ID).toString();
- }
-
- @Override
- public V getValue() {
- return get(result.get(ID).toString());
- }
-
- @Override
- public V setValue(V value) {
- throw new UnsupportedOperationException();
- }
- }
-
- @Override
- public void close() {
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/config/MongoPStoreProvider.java
----------------------------------------------------------------------
diff --git a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/config/MongoPStoreProvider.java b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/config/MongoPStoreProvider.java
deleted file mode 100644
index ae9353a..0000000
--- a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/config/MongoPStoreProvider.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.mongo.config;
-
-import java.io.IOException;
-
-import org.apache.drill.exec.store.mongo.DrillMongoConstants;
-import org.apache.drill.exec.store.sys.PStore;
-import org.apache.drill.exec.store.sys.PStoreConfig;
-import org.apache.drill.exec.store.sys.PStoreProvider;
-import org.apache.drill.exec.store.sys.PStoreRegistry;
-import org.apache.drill.exec.store.sys.local.LocalEStoreProvider;
-import org.bson.Document;
-import org.bson.conversions.Bson;
-
-import com.mongodb.MongoClient;
-import com.mongodb.MongoClientURI;
-import com.mongodb.WriteConcern;
-import com.mongodb.client.MongoCollection;
-import com.mongodb.client.MongoDatabase;
-import com.mongodb.client.model.Indexes;
-
-public class MongoPStoreProvider implements PStoreProvider, DrillMongoConstants {
-
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory
- .getLogger(MongoPStoreProvider.class);
-
- static final String pKey = "pKey";
-
- private MongoClient client;
-
- private MongoCollection<Document> collection;
-
- private final String mongoURL;
- private final LocalEStoreProvider localEStoreProvider;
-
- public MongoPStoreProvider(PStoreRegistry registry) {
- mongoURL = registry.getConfig().getString(SYS_STORE_PROVIDER_MONGO_URL);
- localEStoreProvider = new LocalEStoreProvider();
- }
-
- @Override
- public void start() throws IOException {
- MongoClientURI clientURI = new MongoClientURI(mongoURL);
- client = new MongoClient(clientURI);
- MongoDatabase db = client.getDatabase(clientURI.getDatabase());
- collection = db.getCollection(clientURI.getCollection()).withWriteConcern(WriteConcern.JOURNALED);
- Bson index = Indexes.ascending(pKey);
- collection.createIndex(index);
- }
-
- @Override
- public <V> PStore<V> getStore(PStoreConfig<V> config) throws IOException {
- switch(config.getMode()){
- case BLOB_PERSISTENT:
- case PERSISTENT:
- return new MongoPStore<>(config, collection);
- case EPHEMERAL:
- return localEStoreProvider.getStore(config);
- default:
- throw new IllegalStateException();
-
- }
- }
-
- @Override
- public void close() {
- if (client != null) {
- client.close();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/config/MongoPersistentStore.java
----------------------------------------------------------------------
diff --git a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/config/MongoPersistentStore.java b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/config/MongoPersistentStore.java
new file mode 100644
index 0000000..b5cc3ee
--- /dev/null
+++ b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/config/MongoPersistentStore.java
@@ -0,0 +1,190 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.mongo.config;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.NoSuchElementException;
+
+import com.google.common.base.Preconditions;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.exec.store.mongo.DrillMongoConstants;
+import org.apache.drill.exec.store.sys.BasePersistentStore;
+import org.apache.drill.exec.store.sys.PersistentStoreConfig;
+import org.apache.drill.exec.store.sys.PersistentStoreMode;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoCursor;
+import com.mongodb.client.model.Filters;
+import com.mongodb.client.model.UpdateOptions;
+import com.mongodb.client.model.Updates;
+import com.mongodb.client.result.UpdateResult;
+
+import static org.apache.drill.exec.store.mongo.config.MongoPersistentStoreProvider.pKey;
+
+public class MongoPersistentStore<V> extends BasePersistentStore<V> {
+
+ private static final Logger logger = LoggerFactory.getLogger(MongoPersistentStore.class);
+
+ private final PersistentStoreConfig<V> config;
+ private final MongoCollection<Document> collection;
+
+ public MongoPersistentStore(PersistentStoreConfig<V> config, MongoCollection<Document> collection) {
+// this.config = config;
+// this.collection = collection;
+ throw new UnsupportedOperationException("Mongo DB PStore not currently supported");
+ }
+
+ @Override
+ public PersistentStoreMode getMode() {
+ return PersistentStoreMode.PERSISTENT;
+ }
+
+ @Override
+ public V get(String key) {
+ try {
+ Bson query = Filters.eq(DrillMongoConstants.ID, key);
+ Document document = collection.find(query).first();
+ if (document != null) {
+ return value((byte[]) document.get(pKey));
+ } else {
+ return null;
+ }
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ throw new DrillRuntimeException(e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public void put(String key, V value) {
+ try {
+ Document putObj = new Document(DrillMongoConstants.ID, key).append(pKey, bytes(value));
+ collection.insertOne(putObj);
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ throw new DrillRuntimeException(e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public boolean putIfAbsent(String key, V value) {
+ try {
+ Bson query = Filters.eq(DrillMongoConstants.ID, key);
+ Bson update = Updates.set(pKey, bytes(value));
+ UpdateResult updateResult = collection.updateOne(query, update, new UpdateOptions().upsert(true));
+ return updateResult.getModifiedCount() == 1;
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ throw new DrillRuntimeException(e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public void delete(String key) {
+ try {
+ Bson query = Filters.eq(DrillMongoConstants.ID, key);
+ collection.deleteOne(query);
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ throw new DrillRuntimeException(e.getMessage(), e);
+ }
+ }
+
+ private byte[] bytes(V value) {
+ try {
+ return config.getSerializer().serialize(value);
+ } catch (IOException e) {
+ throw new DrillRuntimeException(e.getMessage(), e);
+ }
+ }
+
+ private V value(byte[] serialize) {
+ try {
+ return config.getSerializer().deserialize(serialize);
+ } catch (IOException e) {
+ throw new DrillRuntimeException(e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public Iterator<Entry<String, V>> getRange(int skip, int take) {
+ final MongoCursor<Document> cursor = collection.find().skip(skip).limit(take).iterator();
+ return new MongoIterator(cursor);
+ }
+
+ private class MongoIterator implements Iterator<Entry<String, V>> {
+
+ private MongoCursor<Document> cursor;
+
+ public MongoIterator(final MongoCursor<Document> cursor) {
+ this.cursor = Preconditions.checkNotNull(cursor);
+ }
+
+ @Override
+ public boolean hasNext() {
+ return cursor.hasNext();
+ }
+
+ @Override
+ public Entry<String, V> next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ return new DeferredEntry(cursor.next());
+ }
+
+ @Override
+ public void remove() {
+ cursor.remove();
+ }
+ }
+
+ private class DeferredEntry implements Entry<String, V> {
+
+ private Document result;
+
+ public DeferredEntry(Document result) {
+ this.result = result;
+ }
+
+ @Override
+ public String getKey() {
+ return result.get(DrillMongoConstants.ID).toString();
+ }
+
+ @Override
+ public V getValue() {
+ return get(result.get(DrillMongoConstants.ID).toString());
+ }
+
+ @Override
+ public V setValue(V value) {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ @Override
+ public void close() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/config/MongoPersistentStoreProvider.java
----------------------------------------------------------------------
diff --git a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/config/MongoPersistentStoreProvider.java b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/config/MongoPersistentStoreProvider.java
new file mode 100644
index 0000000..33127bd
--- /dev/null
+++ b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/config/MongoPersistentStoreProvider.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.mongo.config;
+
+import java.io.IOException;
+
+import org.apache.drill.exec.exception.StoreException;
+import org.apache.drill.exec.store.mongo.DrillMongoConstants;
+import org.apache.drill.exec.store.sys.PersistentStore;
+import org.apache.drill.exec.store.sys.PersistentStoreConfig;
+import org.apache.drill.exec.store.sys.PersistentStoreRegistry;
+import org.apache.drill.exec.store.sys.store.provider.BasePersistentStoreProvider;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+
+import com.mongodb.MongoClient;
+import com.mongodb.MongoClientURI;
+import com.mongodb.WriteConcern;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.model.Indexes;
+
+public class MongoPersistentStoreProvider extends BasePersistentStoreProvider {
+
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory
+ .getLogger(MongoPersistentStoreProvider.class);
+
+ static final String pKey = "pKey";
+
+ private MongoClient client;
+
+ private MongoCollection<Document> collection;
+
+ private final String mongoURL;
+
+ public MongoPersistentStoreProvider(PersistentStoreRegistry registry) throws StoreException {
+ mongoURL = registry.getConfig().getString(DrillMongoConstants.SYS_STORE_PROVIDER_MONGO_URL);
+ }
+
+ @Override
+ public void start() throws IOException {
+ MongoClientURI clientURI = new MongoClientURI(mongoURL);
+ client = new MongoClient(clientURI);
+ MongoDatabase db = client.getDatabase(clientURI.getDatabase());
+ collection = db.getCollection(clientURI.getCollection()).withWriteConcern(WriteConcern.JOURNALED);
+ Bson index = Indexes.ascending(pKey);
+ collection.createIndex(index);
+ }
+
+ @Override
+ public <V> PersistentStore<V> getOrCreateStore(PersistentStoreConfig<V> config) {
+ switch(config.getMode()){
+ case BLOB_PERSISTENT:
+ case PERSISTENT:
+ return new MongoPersistentStore<>(config, collection);
+ default:
+ throw new IllegalStateException();
+
+ }
+ }
+
+ @Override
+ public void close() {
+ if (client != null) {
+ client.close();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/distribution/src/resources/drill-override-example.conf
----------------------------------------------------------------------
diff --git a/distribution/src/resources/drill-override-example.conf b/distribution/src/resources/drill-override-example.conf
index bc77245..9e29a5f 100644
--- a/distribution/src/resources/drill-override-example.conf
+++ b/distribution/src/resources/drill-override-example.conf
@@ -97,7 +97,7 @@ drill.exec: {
executor.threads: 4
},
sys.store.provider: {
- class: "org.apache.drill.exec.store.sys.zk.ZkPStoreProvider",
+ class: "org.apache.drill.exec.store.sys.store.provider.ZookeeperPersistentStoreProvider",
# The following section is used by ZkPStoreProvider
zk: {
blobroot: "file:///var/log/drill"
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index 751c810..604bc8e 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -336,6 +336,12 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-test</artifactId>
+ <version>2.7.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
<version>1.0.5-M3</version>
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java b/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
index 902b35b..f83285e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
@@ -288,7 +288,7 @@ public class DrillClient implements Closeable, ConnectionThrottle {
try {
clusterCoordinator.close();
clusterCoordinator = null;
- } catch (IOException e) {
+ } catch (Exception e) {
logger.warn("Error while closing Cluster Coordinator.", e);
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/compile/CodeCompiler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/CodeCompiler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/CodeCompiler.java
index 3f9da07..af328b1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/CodeCompiler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/CodeCompiler.java
@@ -25,10 +25,7 @@ import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.exception.ClassTransformationException;
import org.apache.drill.exec.expr.CodeGenerator;
-import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.server.options.OptionManager;
-import org.apache.drill.exec.server.options.SystemOptionManager;
-import org.apache.drill.exec.store.sys.local.LocalPStoreProvider;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.cache.CacheBuilder;
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ClusterCoordinator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ClusterCoordinator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ClusterCoordinator.java
index be2f3b1..ea9593e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ClusterCoordinator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ClusterCoordinator.java
@@ -22,6 +22,8 @@ import java.util.Collection;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.drill.exec.coord.store.TransientStore;
+import org.apache.drill.exec.coord.store.TransientStoreConfig;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.work.foreman.DrillbitStatusListener;
@@ -29,7 +31,7 @@ import org.apache.drill.exec.work.foreman.DrillbitStatusListener;
* Pluggable interface built to manage cluster coordination. Allows Drillbit or DrillClient to register its capabilities
* as well as understand other node's existence and capabilities.
**/
-public abstract class ClusterCoordinator implements Closeable {
+public abstract class ClusterCoordinator implements AutoCloseable {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ClusterCoordinator.class);
protected ConcurrentHashMap<DrillbitStatusListener, DrillbitStatusListener> listeners = new ConcurrentHashMap<>(
@@ -60,16 +62,26 @@ public abstract class ClusterCoordinator implements Closeable {
public abstract DistributedSemaphore getSemaphore(String name, int maximumLeases);
/**
+ * Returns a {@link TransientStore store} instance with the given {@link TransientStoreConfig configuration}.
+ *
+ * Note that implementor might cache the instance so new instance creation is not guaranteed.
+ *
+ * @param config store configuration
+ * @param <V> value type for this store
+ */
+ public abstract <V> TransientStore<V> getOrCreateTransientStore(TransientStoreConfig<V> config);
+
+ /**
* Actions to take when there are a set of new de-active drillbits.
* @param unregisteredBits
*/
- public void drillbitUnregistered(Set<DrillbitEndpoint> unregisteredBits) {
+ protected void drillbitUnregistered(Set<DrillbitEndpoint> unregisteredBits) {
for (DrillbitStatusListener listener : listeners.keySet()) {
listener.drillbitUnregistered(unregisteredBits);
}
}
- public void drillbitRegistered(Set<DrillbitEndpoint> registeredBits) {
+ protected void drillbitRegistered(Set<DrillbitEndpoint> registeredBits) {
for (DrillbitStatusListener listener : listeners.keySet()) {
listener.drillbitRegistered(registeredBits);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/coord/local/LocalClusterCoordinator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/local/LocalClusterCoordinator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/local/LocalClusterCoordinator.java
index 27868f0..8c13c42 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/local/LocalClusterCoordinator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/local/LocalClusterCoordinator.java
@@ -28,6 +28,10 @@ import java.util.concurrent.TimeUnit;
import org.apache.drill.exec.coord.ClusterCoordinator;
import org.apache.drill.exec.coord.DistributedSemaphore;
+import org.apache.drill.exec.coord.store.CachingTransientStoreFactory;
+import org.apache.drill.exec.coord.store.TransientStore;
+import org.apache.drill.exec.coord.store.TransientStoreConfig;
+import org.apache.drill.exec.coord.store.TransientStoreFactory;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import com.google.common.collect.Maps;
@@ -43,8 +47,19 @@ public class LocalClusterCoordinator extends ClusterCoordinator {
private final Map<RegistrationHandle, DrillbitEndpoint> endpoints = new ConcurrentHashMap<>();
private final ConcurrentMap<String, DistributedSemaphore> semaphores = Maps.newConcurrentMap();
+ private final TransientStoreFactory factory = CachingTransientStoreFactory.of(new TransientStoreFactory() {
+ @Override
+ public <V> TransientStore<V> getOrCreateStore(TransientStoreConfig<V> config) {
+ return new MapBackedStore<>(config);
+ }
+
+ @Override
+ public void close() throws Exception { }
+ });
+
@Override
- public void close() throws IOException {
+ public void close() throws Exception {
+ factory.close();
endpoints.clear();
}
@@ -125,6 +140,11 @@ public class LocalClusterCoordinator extends ClusterCoordinator {
return semaphores.get(name);
}
+ @Override
+ public <V> TransientStore<V> getOrCreateTransientStore(final TransientStoreConfig<V> config) {
+ return factory.getOrCreateStore(config);
+ }
+
public class LocalSemaphore implements DistributedSemaphore {
private final Semaphore semaphore;
private final LocalLease localLease = new LocalLease();
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/coord/local/MapBackedStore.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/local/MapBackedStore.java b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/local/MapBackedStore.java
new file mode 100644
index 0000000..c3c8f5a
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/local/MapBackedStore.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.coord.local;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
+
+import com.google.common.collect.Maps;
+import org.apache.drill.exec.coord.store.BaseTransientStore;
+import org.apache.drill.exec.coord.store.TransientStoreConfig;
+import org.apache.drill.exec.coord.store.TransientStoreEvent;
+import org.apache.drill.exec.coord.store.TransientStoreEventType;
+
+public class MapBackedStore<V> extends BaseTransientStore<V> {
+ private final ConcurrentMap<String, V> delegate = Maps.newConcurrentMap();
+
+ public MapBackedStore(final TransientStoreConfig<V> config) {
+ super(config);
+ }
+
+ @Override
+ public V get(final String key) {
+ return delegate.get(key);
+ }
+
+ @Override
+ public V put(final String key, final V value) {
+ final boolean hasKey = delegate.containsKey(key);
+ final V old = delegate.put(key, value);
+ if (old != value) {
+ final TransientStoreEventType type = hasKey ? TransientStoreEventType.UPDATE : TransientStoreEventType.CREATE;
+ fireListeners(TransientStoreEvent.of(type, key, value));
+ }
+ return old;
+ }
+
+ @Override
+ public V putIfAbsent(final String key, final V value) {
+ final V existing = delegate.putIfAbsent(key, value);
+ if (existing == null) {
+ fireListeners(TransientStoreEvent.of(TransientStoreEventType.CREATE, key, value));
+ }
+ return existing;
+ }
+
+ @Override
+ public V remove(final String key) {
+ final V existing = delegate.remove(key);
+ if (existing != null) {
+ fireListeners(TransientStoreEvent.of(TransientStoreEventType.DELETE, key, existing));
+ }
+ return existing;
+ }
+
+ @Override
+ public Iterator<Map.Entry<String, V>> entries() {
+ return delegate.entrySet().iterator();
+ }
+
+ @Override
+ public int size() {
+ return delegate.size();
+ }
+
+ @Override
+ public void close() throws Exception {
+ delegate.clear();
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/BaseTransientStore.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/BaseTransientStore.java b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/BaseTransientStore.java
new file mode 100644
index 0000000..463e1fb
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/BaseTransientStore.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.coord.store;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Maps;
+
+public abstract class BaseTransientStore<V> implements TransientStore<V> {
+ private final Set<TransientStoreListener> listeners = Collections.newSetFromMap(
+ Maps.<TransientStoreListener, Boolean>newConcurrentMap());
+
+ protected final TransientStoreConfig<V> config;
+
+ protected BaseTransientStore(final TransientStoreConfig<V> config) {
+ this.config = Preconditions.checkNotNull(config, "config cannot be null");
+ }
+
+ public TransientStoreConfig<V> getConfig() {
+ return config;
+ }
+
+ @Override
+ public Iterator<String> keys() {
+ return Iterators.transform(entries(), new Function<Map.Entry<String, V>, String>() {
+ @Nullable
+ @Override
+ public String apply(@Nullable Map.Entry<String, V> input) {
+ return input.getKey();
+ }
+ });
+ }
+
+ @Override
+ public Iterator<V> values() {
+ return Iterators.transform(entries(), new Function<Map.Entry<String, V>, V>() {
+ @Nullable
+ @Override
+ public V apply(final Map.Entry<String, V> entry) {
+ return entry.getValue();
+ }
+ });
+ }
+
+ protected void fireListeners(final TransientStoreEvent event) {
+ for (final TransientStoreListener listener:listeners) {
+ listener.onChange(event);
+ }
+ }
+
+ @Override
+ public void addListener(final TransientStoreListener listener) {
+ listeners.add(Preconditions.checkNotNull(listener, "listener cannot be null"));
+ }
+
+ @Override
+ public void removeListener(final TransientStoreListener listener) {
+ listeners.remove(Preconditions.checkNotNull(listener, "listener cannot be null"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/CachingTransientStoreFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/CachingTransientStoreFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/CachingTransientStoreFactory.java
new file mode 100644
index 0000000..b500c5d
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/CachingTransientStoreFactory.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.coord.store;
+
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.drill.common.AutoCloseables;
+import org.apache.drill.exec.store.sys.PersistentStore;
+
+public class CachingTransientStoreFactory implements TransientStoreFactory {
+ private final TransientStoreFactory delegate;
+ private final Map<TransientStoreConfig, TransientStore> cache = Maps.newHashMap();
+
+ public CachingTransientStoreFactory(final TransientStoreFactory delegate) {
+ this.delegate = Preconditions.checkNotNull(delegate, "delegate factory is required");
+ }
+
+ @Override
+ public <V> TransientStore<V> getOrCreateStore(final TransientStoreConfig<V> config) {
+ final TransientStore<V> store = cache.get(Preconditions.checkNotNull(config, "config is required"));
+ if (store != null) {
+ return store;
+ }
+
+ final TransientStore<V> newStore = delegate.getOrCreateStore(config);
+ cache.put(config, newStore);
+ return newStore;
+ }
+
+ @Override
+ public void close() throws Exception {
+ final List<AutoCloseable> closeables = Lists.newArrayList();
+ for(final AutoCloseable store : cache.values()){
+ closeables.add(store);
+ }
+ closeables.add(delegate);
+ cache.clear();
+ AutoCloseables.close(closeables);
+ }
+
+ public static TransientStoreFactory of(final TransientStoreFactory delegate) {
+ return new CachingTransientStoreFactory(delegate);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/TransientStore.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/TransientStore.java b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/TransientStore.java
new file mode 100644
index 0000000..ca9b028
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/TransientStore.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.coord.store;
+
+import java.util.Iterator;
+import java.util.Map;
+
+/**
+ * An abstraction for storing, retrieving and observing transient (key, value) pairs in a distributed environment.
+ *
+ * This abstraction diverges from {@link org.apache.drill.exec.store.sys.PersistentStore} in that the lifetime of
+ * a (key, value) tuple is bound to the lifetime of the node originally creating it. In other words, entries are evicted
+ * as soon as node/s originally stored them leaves the cluster. That should explain the reason for relocating this
+ * abstraction under cluster coordination package.
+ *
+ * Consumers of this interface can observe changes made to the store via attaching a {@link TransientStoreListener listener}.
+ *
+ */
+public interface TransientStore<V> extends AutoCloseable {
+ /**
+ * Returns a value corresponding to the given look-up key if exists, null otherwise.
+ * @param key look-up key
+ */
+ V get(String key);
+
+ /**
+ * Stores the given (key, value) in this store overriding the existing value.
+ *
+ * @param key look-up key
+ * @param value value to store
+ * @return the old value if the key exists, null otherwise
+ */
+ V put(String key, V value);
+
+ /**
+ * Stores the given (key, value) tuple in this store only if it does not exists.
+ *
+ * @param key look-up key
+ * @param value value to store
+ * @return the old value if the key exists, null otherwise
+ */
+ V putIfAbsent(String key, V value);
+
+
+ /**
+ * Removes the (key, value) tuple from this store if the key exists.
+ *
+ * @param key look-up key
+ * @return the removed value if key exists, null otherwise
+ */
+ V remove(String key);
+
+ /**
+ * Returns an iterator of (key, value) tuples.
+ */
+ Iterator<Map.Entry<String, V>> entries();
+
+ /**
+ * Returns an iterator of keys.
+ */
+ Iterator<String> keys();
+
+
+ /**
+ * Returns an iterator of values.
+ */
+ Iterator<V> values();
+
+ /**
+ * Returns number of entries.
+ */
+ int size();
+
+ /**
+ * Adds a listener that observes store {@link TransientStoreEvent events}.
+ *
+ * Note that
+ * i) Calling this method with the same listener instance more than once has no effect.
+ * ii) Listeners are not necessarily invoked from the calling thread. Consumer should consider thread safety issues.
+ * iii) Subclasses might hold a strong reference to the listener. It is important that consumer
+ * {@link #removeListener(TransientStoreListener) removes} its listener once it is done observing events.
+ *
+ * @see #removeListener(TransientStoreListener)
+ */
+ void addListener(TransientStoreListener listener);
+
+ /**
+ * Removes the given listener from this store if exists, has no effect otherwise.
+ */
+ void removeListener(TransientStoreListener listener);
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/TransientStoreConfig.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/TransientStoreConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/TransientStoreConfig.java
new file mode 100644
index 0000000..35c4a06
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/TransientStoreConfig.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.coord.store;
+
+import com.dyuproject.protostuff.Schema;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.protobuf.Message;
+import org.apache.drill.exec.serialization.JacksonSerializer;
+import org.apache.drill.exec.serialization.ProtoSerializer;
+import org.apache.drill.exec.serialization.InstanceSerializer;
+
+public class TransientStoreConfig<V> {
+ private final String name;
+ private final InstanceSerializer<V> serializer;
+
+ protected TransientStoreConfig(final String name, final InstanceSerializer<V> serializer) {
+ Preconditions.checkArgument(!Strings.isNullOrEmpty(name), "name is required");
+ this.name = name;
+ this.serializer = Preconditions.checkNotNull(serializer, "serializer cannot be null");
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public InstanceSerializer<V> getSerializer() {
+ return serializer;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(name, serializer);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof TransientStoreConfig && obj.getClass().equals(getClass())) {
+ final TransientStoreConfig<V> other = (TransientStoreConfig<V>)obj;
+ return Objects.equal(name, other.name) && Objects.equal(serializer, other.serializer);
+ }
+ return false;
+ }
+
+ public static <V> TransientStoreConfigBuilder<V> newBuilder() {
+ return new TransientStoreConfigBuilder<>();
+ }
+
+ public static <V extends Message, B extends Message.Builder> TransientStoreConfigBuilder<V> newProtoBuilder(final Schema<V> writeSchema, final Schema<B> readSchema) {
+ return TransientStoreConfig.<V>newBuilder().serializer(new ProtoSerializer<>(readSchema, writeSchema));
+ }
+
+ public static <V> TransientStoreConfigBuilder<V> newJacksonBuilder(final ObjectMapper mapper, final Class<V> klazz) {
+ return TransientStoreConfig.<V>newBuilder().serializer(new JacksonSerializer<>(mapper, klazz));
+ }
+
+}
[3/4] drill git commit: DRILL-4275: create TransientStore for
short-lived objects;
refactor PersistentStore to introduce pagination mechanism
Posted by hg...@apache.org.
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/TransientStoreConfigBuilder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/TransientStoreConfigBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/TransientStoreConfigBuilder.java
new file mode 100644
index 0000000..a019a77
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/TransientStoreConfigBuilder.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.coord.store;
+
+import com.google.common.base.Preconditions;
+import org.apache.drill.exec.serialization.InstanceSerializer;
+
+public class TransientStoreConfigBuilder<T> {
+ private String name;
+ private InstanceSerializer<T> serializer;
+
+ protected TransientStoreConfigBuilder() { }
+
+ public String name() {
+ return name;
+ }
+
+ public TransientStoreConfigBuilder<T> name(final String name) {
+ this.name = Preconditions.checkNotNull(name);
+ return this;
+ }
+
+ public InstanceSerializer<T> serializer() {
+ return serializer;
+ }
+
+ public TransientStoreConfigBuilder<T> serializer(final InstanceSerializer<T> serializer) {
+ this.serializer = Preconditions.checkNotNull(serializer);
+ return this;
+ }
+
+ public TransientStoreConfig<T> build() {
+ return new TransientStoreConfig<>(name, serializer);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/TransientStoreEvent.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/TransientStoreEvent.java b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/TransientStoreEvent.java
new file mode 100644
index 0000000..a0b5725
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/TransientStoreEvent.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.coord.store;
+
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+
+/**
+ * Represents an event created as a result of an operation over a particular (key, value) entry in a
+ * {@link TransientStore store} instance.
+ *
+ * Types of operations are enumerated in {@link TransientStoreEventType}
+ *
+ * @param <V> value type
+ */
+public class TransientStoreEvent<V> {
+ private final TransientStoreEventType type;
+ private final String key;
+ private final V value;
+
+ public TransientStoreEvent(final TransientStoreEventType type, final String key, final V value) {
+ this.type = Preconditions.checkNotNull(type);
+ this.key = Preconditions.checkNotNull(key);
+ this.value = Preconditions.checkNotNull(value);
+ }
+
+ public String getKey() {
+ return key;
+ }
+
+ public TransientStoreEventType getType() {
+ return type;
+ }
+
+ public V getValue() {
+ return value;
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (obj instanceof TransientStoreEvent && obj.getClass().equals(getClass())) {
+ final TransientStoreEvent<V> other = (TransientStoreEvent<V>)obj;
+ return Objects.equal(type, other.type) && Objects.equal(key, other.key) && Objects.equal(value, other.value);
+ }
+ return super.equals(obj);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(type, key, value);
+ }
+
+ public static <T> TransientStoreEvent<T>of(final TransientStoreEventType type, final String key, final T value) {
+ return new TransientStoreEvent<>(type, key, value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/TransientStoreEventType.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/TransientStoreEventType.java b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/TransientStoreEventType.java
new file mode 100644
index 0000000..51ae2c7
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/TransientStoreEventType.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.coord.store;
+
+/**
+ * Types of store events.
+ */
+public enum TransientStoreEventType {
+ CREATE,
+ UPDATE,
+ DELETE
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/TransientStoreFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/TransientStoreFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/TransientStoreFactory.java
new file mode 100644
index 0000000..c3d351d
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/TransientStoreFactory.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.coord.store;
+
+import org.apache.drill.exec.store.sys.PersistentStore;
+
+/**
+ * Factory that is used to obtain a {@link TransientStore store} instance.
+ */
+public interface TransientStoreFactory extends AutoCloseable {
+
+ /**
+ * Returns a {@link TransientStore transient store} instance for the given configuration.
+ *
+ * Note that implementors have liberty to cache previous {@link PersistentStore store} instances.
+ *
+ * @param config store configuration
+ * @param <V> store value type
+ */
+ <V> TransientStore<V> getOrCreateStore(TransientStoreConfig<V> config);
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/TransientStoreListener.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/TransientStoreListener.java b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/TransientStoreListener.java
new file mode 100644
index 0000000..ca8fa9d
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/store/TransientStoreListener.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.coord.store;
+
+/**
+ * A listener used for observing {@link TransientStore transient store} {@link TransientStoreEvent events}.
+ */
+public interface TransientStoreListener {
+
+ /**
+ * {@link TransientStore transient store} fires this method with event details upon an observed change.
+ *
+ * @param event event details
+ */
+ void onChange(TransientStoreEvent event);
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/EventDispatcher.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/EventDispatcher.java b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/EventDispatcher.java
new file mode 100644
index 0000000..580cfcd
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/EventDispatcher.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.coord.zk;
+
+import java.util.Map;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
+import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
+import org.apache.drill.exec.coord.store.TransientStoreEvent;
+import org.apache.drill.exec.coord.store.TransientStoreEventType;
+
+/**
+ * An abstraction used for dispatching store {@link TransientStoreEvent events}.
+ *
+ * @param <V> value type
+ */
+public class EventDispatcher<V> implements PathChildrenCacheListener {
+ public final static Map<PathChildrenCacheEvent.Type, TransientStoreEventType> MAPPINGS = ImmutableMap
+ .<PathChildrenCacheEvent.Type, TransientStoreEventType>builder()
+ .put(PathChildrenCacheEvent.Type.CHILD_ADDED, TransientStoreEventType.CREATE)
+ .put(PathChildrenCacheEvent.Type.CHILD_REMOVED, TransientStoreEventType.DELETE)
+ .put(PathChildrenCacheEvent.Type.CHILD_UPDATED, TransientStoreEventType.UPDATE)
+ .build();
+
+ private final ZkEphemeralStore<V> store;
+
+ protected EventDispatcher(final ZkEphemeralStore<V> store) {
+ this.store = Preconditions.checkNotNull(store, "store is required");
+ }
+
+ @Override
+ public void childEvent(final CuratorFramework client, final PathChildrenCacheEvent event) throws Exception {
+ final PathChildrenCacheEvent.Type original = event.getType();
+ final TransientStoreEventType mapped = MAPPINGS.get(original);
+ if (mapped != null) { // dispatch the event to listeners only if it can be mapped
+ final String path = event.getData().getPath();
+ final byte[] bytes = event.getData().getData();
+ final V value = store.getConfig().getSerializer().deserialize(bytes);
+ store.fireListeners(TransientStoreEvent.of(mapped, path, value));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/PathUtils.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/PathUtils.java b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/PathUtils.java
new file mode 100644
index 0000000..f01b989
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/PathUtils.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.coord.zk;
+
+import com.google.common.base.Preconditions;
+import org.apache.parquet.Strings;
+
+/**
+ * A convenience class used to expedite zookeeper paths manipulations.
+ */
+public final class PathUtils {
+
+ /**
+ * Returns a normalized, combined path out of the given path segments.
+ *
+ * @param parts path segments to combine
+ * @see #normalize(String)
+ */
+ public static final String join(final String... parts) {
+ final StringBuilder sb = new StringBuilder();
+ for (final String part:parts) {
+ Preconditions.checkNotNull(part, "parts cannot contain null");
+ if (!Strings.isNullOrEmpty(part)) {
+ sb.append(part).append("/");
+ }
+ }
+ if (sb.length() > 0) {
+ sb.deleteCharAt(sb.length() - 1);
+ }
+ final String path = sb.toString();
+ return normalize(path);
+ }
+
+ /**
+ * Normalizes the given path eliminating repeated forward slashes.
+ *
+ * @return normalized path
+ */
+ public static final String normalize(final String path) {
+ if (Strings.isNullOrEmpty(Preconditions.checkNotNull(path))) {
+ return path;
+ }
+
+ final StringBuilder builder = new StringBuilder();
+ char last = path.charAt(0);
+ builder.append(last);
+ for (int i=1; i<path.length(); i++) {
+ char cur = path.charAt(i);
+ if (last == '/' && cur == last) {
+ continue;
+ }
+ builder.append(cur);
+ last = cur;
+ }
+ return builder.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZKClusterCoordinator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZKClusterCoordinator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZKClusterCoordinator.java
index b831852..4926f9c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZKClusterCoordinator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZKClusterCoordinator.java
@@ -24,13 +24,14 @@ import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
+import java.util.List;
import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import com.google.common.collect.Lists;
import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
@@ -42,15 +43,19 @@ import org.apache.curator.x.discovery.ServiceDiscovery;
import org.apache.curator.x.discovery.ServiceDiscoveryBuilder;
import org.apache.curator.x.discovery.ServiceInstance;
import org.apache.curator.x.discovery.details.ServiceCacheListener;
+import org.apache.drill.common.AutoCloseables;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.coord.ClusterCoordinator;
import org.apache.drill.exec.coord.DistributedSemaphore;
import org.apache.drill.exec.coord.DrillServiceInstanceHelper;
+import org.apache.drill.exec.coord.store.CachingTransientStoreFactory;
+import org.apache.drill.exec.coord.store.TransientStore;
+import org.apache.drill.exec.coord.store.TransientStoreConfig;
+import org.apache.drill.exec.coord.store.TransientStoreFactory;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import com.google.common.base.Function;
-import org.apache.drill.exec.work.foreman.DrillbitStatusListener;
/**
* Manages cluster coordination utilizing zookeeper. *
@@ -60,15 +65,14 @@ public class ZKClusterCoordinator extends ClusterCoordinator {
private CuratorFramework curator;
private ServiceDiscovery<DrillbitEndpoint> discovery;
- private ServiceCache<DrillbitEndpoint> serviceCache;
private volatile Collection<DrillbitEndpoint> endpoints = Collections.emptyList();
private final String serviceName;
private final CountDownLatch initialConnection = new CountDownLatch(1);
+ private final TransientStoreFactory factory;
+ private ServiceCache<DrillbitEndpoint> serviceCache;
private static final Pattern ZK_COMPLEX_STRING = Pattern.compile("(^.*?)/(.*)/([^/]*)$");
-
-
public ZKClusterCoordinator(DrillConfig config) throws IOException{
this(config, null);
}
@@ -100,11 +104,8 @@ public class ZKClusterCoordinator extends ClusterCoordinator {
.build();
curator.getConnectionStateListenable().addListener(new InitialConnectionListener());
curator.start();
- discovery = getDiscovery();
- serviceCache = discovery.
- serviceCacheBuilder()
- .name(serviceName)
- .build();
+ discovery = newDiscovery();
+ factory = CachingTransientStoreFactory.of(new ZkTransientStoreFactory(curator));
}
public CuratorFramework getCurator() {
@@ -115,8 +116,6 @@ public class ZKClusterCoordinator extends ClusterCoordinator {
public void start(long millisToWait) throws Exception {
logger.debug("Starting ZKClusterCoordination.");
discovery.start();
- serviceCache.start();
- serviceCache.addListener(new ZKListener());
if(millisToWait != 0) {
boolean success = this.initialConnection.await(millisToWait, TimeUnit.MILLISECONDS);
@@ -127,6 +126,12 @@ public class ZKClusterCoordinator extends ClusterCoordinator {
this.initialConnection.await();
}
+ serviceCache = discovery
+ .serviceCacheBuilder()
+ .name(serviceName)
+ .build();
+ serviceCache.addListener(new EndpointListener());
+ serviceCache.start();
updateEndpoints();
}
@@ -142,29 +147,28 @@ public class ZKClusterCoordinator extends ClusterCoordinator {
}
- private class ZKListener implements ServiceCacheListener {
-
+ private class EndpointListener implements ServiceCacheListener {
@Override
- public void stateChanged(CuratorFramework client, ConnectionState newState) {
- }
+ public void stateChanged(CuratorFramework client, ConnectionState newState) { }
@Override
public void cacheChanged() {
- logger.debug("Cache changed, updating.");
+ logger.debug("Got cache changed --> updating endpoints");
updateEndpoints();
}
}
- public void close() throws IOException {
- serviceCache.close();
- discovery.close();
- curator.close();
+ public void close() throws Exception {
+ // discovery attempts to close its caches(ie serviceCache) already. however, being good citizens we make sure to
+ // explicitly close serviceCache. Not only that we make sure to close serviceCache before discovery to prevent
+ // double releasing and disallowing jvm to spit bothering warnings. simply put, we are great!
+ AutoCloseables.close(serviceCache, discovery, curator, factory);
}
@Override
public RegistrationHandle register(DrillbitEndpoint data) {
try {
- ServiceInstance<DrillbitEndpoint> serviceInstance = getServiceInstance(data);
+ ServiceInstance<DrillbitEndpoint> serviceInstance = newServiceInstance(data);
discovery.registerService(serviceInstance);
return new ZKRegistrationHandle(serviceInstance.getId());
} catch (Exception e) {
@@ -206,6 +210,11 @@ public class ZKClusterCoordinator extends ClusterCoordinator {
return new ZkDistributedSemaphore(curator, "/semaphore/" + name, maximumLeases);
}
+ @Override
+ public <V> TransientStore<V> getOrCreateTransientStore(final TransientStoreConfig<V> config) {
+ final ZkEphemeralStore<V> store = (ZkEphemeralStore<V>)factory.getOrCreateStore(config);
+ return store;
+ }
private synchronized void updateEndpoints() {
try {
@@ -253,7 +262,7 @@ public class ZKClusterCoordinator extends ClusterCoordinator {
}
}
- private ServiceInstance<DrillbitEndpoint> getServiceInstance(DrillbitEndpoint endpoint) throws Exception {
+ protected ServiceInstance<DrillbitEndpoint> newServiceInstance(DrillbitEndpoint endpoint) throws Exception {
return ServiceInstance.<DrillbitEndpoint>builder()
.name(serviceName)
.payload(endpoint)
@@ -261,7 +270,7 @@ public class ZKClusterCoordinator extends ClusterCoordinator {
}
- public ServiceDiscovery<DrillbitEndpoint> getDiscovery() {
+ protected ServiceDiscovery<DrillbitEndpoint> newDiscovery() {
return ServiceDiscoveryBuilder
.builder(DrillbitEndpoint.class)
.basePath("/")
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZkEphemeralStore.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZkEphemeralStore.java b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZkEphemeralStore.java
new file mode 100644
index 0000000..94e03ad
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZkEphemeralStore.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.coord.zk;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
+import org.apache.drill.common.collections.ImmutableEntry;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.exec.coord.store.BaseTransientStore;
+import org.apache.drill.exec.coord.store.TransientStoreConfig;
+import org.apache.drill.exec.coord.store.TransientStoreEvent;
+import org.apache.drill.exec.serialization.InstanceSerializer;
+import org.apache.zookeeper.CreateMode;
+
+public class ZkEphemeralStore<V> extends BaseTransientStore<V> {
+
+ @VisibleForTesting
+ protected final PathChildrenCacheListener dispatcher = new EventDispatcher<>(this);
+ private final ZookeeperClient client;
+
+ public ZkEphemeralStore(final TransientStoreConfig<V> config, final CuratorFramework curator) {
+ super(config);
+ this.client = new ZookeeperClient(curator, PathUtils.join("/", config.getName()), CreateMode.EPHEMERAL);
+ }
+
+ public void start() throws Exception {
+ getClient().getCache().getListenable().addListener(dispatcher);
+ getClient().start();
+ }
+
+ protected ZookeeperClient getClient() {
+ return client;
+ }
+
+ @Override
+ public V get(final String key) {
+ final byte[] bytes = getClient().get(key);
+ if (bytes == null) {
+ return null;
+ }
+ try {
+ return config.getSerializer().deserialize(bytes);
+ } catch (final IOException e) {
+ throw new DrillRuntimeException(String.format("unable to deserialize value at %s", key), e);
+ }
+ }
+
+ @Override
+ public V put(final String key, final V value) {
+ final InstanceSerializer<V> serializer = config.getSerializer();
+ try {
+ final byte[] old = getClient().get(key);
+ final byte[] bytes = serializer.serialize(value);
+ getClient().put(key, bytes);
+ if (old == null) {
+ return null;
+ }
+ return serializer.deserialize(old);
+ } catch (final IOException e) {
+ throw new DrillRuntimeException(String.format("unable to de/serialize value of type %s", value.getClass()), e);
+ }
+ }
+
+ @Override
+ public V putIfAbsent(final String key, final V value) {
+ final V old = get(key);
+ if (old == null) {
+ try {
+ final byte[] bytes = config.getSerializer().serialize(value);
+ getClient().put(key, bytes);
+ } catch (final IOException e) {
+ throw new DrillRuntimeException(String.format("unable to serialize value of type %s", value.getClass()), e);
+ }
+ }
+ return old;
+ }
+
+ @Override
+ public V remove(final String key) {
+ final V existing = get(key);
+ if (existing != null) {
+ getClient().delete(key);
+ }
+ return existing;
+ }
+
+ @Override
+ public Iterator<Map.Entry<String, V>> entries() {
+ return Iterators.transform(getClient().entries(), new Function<Map.Entry<String, byte[]>, Map.Entry<String, V>>() {
+ @Nullable
+ @Override
+ public Map.Entry<String, V> apply(@Nullable Map.Entry<String, byte[]> input) {
+ try {
+ final V value = config.getSerializer().deserialize(input.getValue());
+ return new ImmutableEntry<>(input.getKey(), value);
+ } catch (final IOException e) {
+ throw new DrillRuntimeException(String.format("unable to deserialize value at key %s", input.getKey()), e);
+ }
+ }
+ });
+ }
+
+ @Override
+ public int size() {
+ return getClient().getCache().getCurrentData().size();
+ }
+
+ @Override
+ public void close() throws Exception {
+ getClient().close();
+ }
+
+ /**
+ * This method override ensures package level method visibility.
+ */
+ @Override
+ protected void fireListeners(TransientStoreEvent event) {
+ super.fireListeners(event);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZkTransientStoreFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZkTransientStoreFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZkTransientStoreFactory.java
new file mode 100644
index 0000000..a58c376
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZkTransientStoreFactory.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.coord.zk;
+
+import com.google.common.base.Preconditions;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.exec.coord.store.TransientStore;
+import org.apache.drill.exec.coord.store.TransientStoreConfig;
+import org.apache.drill.exec.coord.store.TransientStoreFactory;
+
+public class ZkTransientStoreFactory implements TransientStoreFactory {
+
+ private final CuratorFramework curator;
+
+ public ZkTransientStoreFactory(final CuratorFramework curator) {
+ this.curator = Preconditions.checkNotNull(curator, "curator is required");
+ }
+
+ @Override
+ public <V> ZkEphemeralStore<V> getOrCreateStore(TransientStoreConfig<V> config) {
+ final ZkEphemeralStore<V> store = new ZkEphemeralStore<>(config, curator);
+ try {
+ store.start();
+ } catch (final Exception e) {
+ throw new DrillRuntimeException("unable to start zookeeper transient store", e);
+ }
+ return store;
+ }
+
+ @Override
+ public void close() throws Exception {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZookeeperClient.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZookeeperClient.java b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZookeeperClient.java
new file mode 100644
index 0000000..1c33f71
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZookeeperClient.java
@@ -0,0 +1,238 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.coord.zk;
+
+import java.util.Iterator;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.common.collect.Iterables;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.PathChildrenCache;
+import org.apache.drill.common.collections.ImmutableEntry;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.zookeeper.CreateMode;
+
+/**
+ * A namespace aware Zookeeper client.
+ *
+ * The implementation only operates under the given namespace and is safe to use.
+ *
+ * Note that instance of this class holds onto resources that must be released via {@code #close()}.
+ */
+public class ZookeeperClient implements AutoCloseable {
+ private final CuratorFramework curator;
+ private final String root;
+ private final PathChildrenCache cache;
+ private final CreateMode mode;
+
+ public ZookeeperClient(final CuratorFramework curator, final String root, final CreateMode mode) {
+ this.curator = Preconditions.checkNotNull(curator, "curator is required");
+ Preconditions.checkArgument(!Strings.isNullOrEmpty(root), "root path is required");
+ Preconditions.checkArgument(root.charAt(0) == '/', "root path must be absolute");
+ this.root = root;
+ this.mode = Preconditions.checkNotNull(mode, "mode is required");
+ this.cache = new PathChildrenCache(curator, root, true);
+ }
+
+ /**
+ * Starts the client. This call ensures the creation of the root path.
+ *
+ * @throws Exception if cache fails to start or root path creation fails.
+ * @see #close()
+ */
+ public void start() throws Exception {
+ curator.newNamespaceAwareEnsurePath(root).ensure(curator.getZookeeperClient()); // ensure root is created
+ getCache().start();
+ }
+
+ public PathChildrenCache getCache() {
+ return cache;
+ }
+
+ public String getRoot() {
+ return root;
+ }
+
+ public CreateMode getMode() {
+ return mode;
+ }
+
+ /**
+ * Returns true if path exists in the cache, false otherwise.
+ *
+ * Note that calls to this method are eventually consistent.
+ *
+ * @param path path to check
+ */
+ public boolean hasPath(final String path) {
+ return hasPath(path, false);
+ }
+
+ /**
+ * Checks if the given path exists.
+ *
+ * If the flag consistent is set, the check is consistent as it is made against Zookeeper directly. Otherwise,
+ * the check is eventually consistent.
+ *
+ * @param path path to check
+ * @param consistent whether the check should be consistent
+ * @return
+ */
+ public boolean hasPath(final String path, final boolean consistent) {
+ Preconditions.checkNotNull(path, "path is required");
+
+ final String target = PathUtils.join(root, path);
+ try {
+ if (consistent) {
+ return curator.checkExists().forPath(target) != null;
+ } else {
+ return getCache().getCurrentData(target) != null;
+ }
+ } catch (final Exception e) {
+ throw new DrillRuntimeException("error while checking path on zookeeper", e);
+ }
+ }
+
+ /**
+ * Returns a value corresponding to the given path if path exists in the cache, null otherwise.
+ *
+ * Note that calls to this method are eventually consistent.
+ *
+ * @param path target path
+ */
+ public byte[] get(final String path) {
+ return get(path, false);
+ }
+
+ /**
+ * Returns the value corresponding to the given key, null otherwise.
+ *
+ * If the flag consistent is set, the check is consistent as it is made against Zookeeper directly. Otherwise,
+ * the check is eventually consistent.
+ *
+ * @param path target path
+ */
+ public byte[] get(final String path, final boolean consistent) {
+ Preconditions.checkNotNull(path, "path is required");
+
+ final String target = PathUtils.join(root, path);
+ if (consistent) {
+ try {
+ return curator.getData().forPath(target);
+ } catch (final Exception ex) {
+ throw new DrillRuntimeException(String.format("error retrieving value for [%s]", path), ex);
+ }
+ } else {
+ final ChildData data = getCache().getCurrentData(target);
+ if (data != null) {
+ return data.getData();
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Creates the given path without placing any data in.
+ *
+ * @param path target path
+ */
+ public void create(final String path) {
+ Preconditions.checkNotNull(path, "path is required");
+
+ final String target = PathUtils.join(root, path);
+ try {
+ curator.create().withMode(mode).forPath(target);
+ getCache().rebuildNode(target);
+ } catch (final Exception e) {
+ throw new DrillRuntimeException("unable to put ", e);
+ }
+ }
+
+ /**
+ * Puts the given byte sequence into the given path.
+ *
+ * If path does not exists, this call creates it.
+ *
+ * @param path target path
+ * @param data data to store
+ */
+ public void put(final String path, final byte[] data) {
+ Preconditions.checkNotNull(path, "path is required");
+ Preconditions.checkNotNull(data, "data is required");
+
+ final String target = PathUtils.join(root, path);
+ try {
+ // we make a consistent read to ensure this call won't fail upon consecutive calls on the same path
+ // before cache is updated
+ if (hasPath(path, true)) {
+ curator.setData().forPath(target, data);
+ } else {
+ curator.create().withMode(mode).forPath(target, data);
+ }
+ getCache().rebuildNode(target);
+
+ } catch (final Exception e) {
+ throw new DrillRuntimeException("unable to put ", e);
+ }
+ }
+
+ /**
+ * Deletes the given node residing at the given path
+ *
+ * @param path target path to delete
+ */
+ public void delete(final String path) {
+ Preconditions.checkNotNull(path, "path is required");
+
+ final String target = PathUtils.join(root, path);
+ try {
+ curator.delete().forPath(target);
+ getCache().rebuildNode(target);
+ } catch (final Exception e) {
+ throw new DrillRuntimeException(String.format("unable to delete node at %s", target), e);
+ }
+ }
+
+ /**
+ * Returns an iterator of (key, value) pairs residing under {@link #getRoot() root} path.
+ */
+ public Iterator<Map.Entry<String, byte[]>> entries() {
+ final String prefix = PathUtils.join(root, "/");
+ return Iterables.transform(getCache().getCurrentData(), new Function<ChildData, Map.Entry<String, byte[]>>() {
+ @Nullable
+ @Override
+ public Map.Entry<String, byte[]> apply(final ChildData data) {
+ // normalize key name removing the root prefix. resultant key must be a relative path, not beginning with a '/'.
+ final String key = data.getPath().replace(prefix, "");
+ return new ImmutableEntry<>(key, data.getData());
+ }
+ }).iterator();
+ }
+
+ @Override
+ public void close() throws Exception {
+ getCache().close();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/exception/StoreException.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/exception/StoreException.java b/exec/java-exec/src/main/java/org/apache/drill/exec/exception/StoreException.java
new file mode 100644
index 0000000..506d485
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/exception/StoreException.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.exception;
+
+import org.apache.drill.common.exceptions.DrillException;
+
+public class StoreException extends DrillException {
+ public StoreException() {
+ super();
+ }
+
+ public StoreException(Throwable cause) {
+ super(cause);
+ }
+
+ public StoreException(String message) {
+ super(message);
+ }
+
+ public StoreException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public StoreException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/serialization/InstanceSerializer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/serialization/InstanceSerializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/serialization/InstanceSerializer.java
new file mode 100644
index 0000000..f44d835
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/serialization/InstanceSerializer.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.serialization;
+
+import java.io.IOException;
+
+public interface InstanceSerializer<T> {
+ byte[] serialize(T instance) throws IOException;
+ T deserialize(byte[] raw) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/serialization/JacksonSerializer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/serialization/JacksonSerializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/serialization/JacksonSerializer.java
new file mode 100644
index 0000000..676929d
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/serialization/JacksonSerializer.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.serialization;
+
+import java.io.IOException;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+import com.fasterxml.jackson.databind.ObjectWriter;
+import com.google.common.base.Objects;
+
+public class JacksonSerializer<T> implements InstanceSerializer<T> {
+ private final ObjectReader reader;
+ private final ObjectWriter writer;
+
+ public JacksonSerializer(final ObjectMapper mapper, final Class<T> klazz) {
+ this.reader = mapper.readerFor(klazz);
+ this.writer = mapper.writer();
+ }
+
+ @Override
+ public T deserialize(final byte[] raw) throws IOException {
+ return reader.readValue(raw);
+ }
+
+ @Override
+ public byte[] serialize(final T instance) throws IOException {
+ return writer.writeValueAsBytes(instance);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof JacksonSerializer && obj.getClass().equals(getClass())) {
+ final JacksonSerializer<T> other = (JacksonSerializer<T>)obj;
+ return Objects.equal(reader, other.reader) && Objects.equal(writer, other.writer);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(reader, writer);
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/serialization/ProtoSerializer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/serialization/ProtoSerializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/serialization/ProtoSerializer.java
new file mode 100644
index 0000000..e3ee5f6
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/serialization/ProtoSerializer.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.serialization;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+import com.dyuproject.protostuff.JsonIOUtil;
+import com.dyuproject.protostuff.Schema;
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import com.google.protobuf.Message;
+
+public class ProtoSerializer<T, B extends Message.Builder> implements InstanceSerializer<T> {
+ private final Schema<B> readSchema;
+ private final Schema<T> writeSchema;
+
+ public ProtoSerializer(final Schema<B> readSchema, final Schema<T> writeSchema) {
+ this.readSchema = Preconditions.checkNotNull(readSchema);
+ this.writeSchema = Preconditions.checkNotNull(writeSchema);
+ }
+
+ @Override
+ public T deserialize(final byte[] raw) throws IOException {
+ final B builder = readSchema.newMessage();
+ JsonIOUtil.mergeFrom(raw, builder, readSchema, false);
+ return (T)builder.build();
+ }
+
+ @Override
+ public byte[] serialize(final T instance) throws IOException {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ JsonIOUtil.writeTo(out, instance, writeSchema, false);
+ return out.toByteArray();
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(readSchema, writeSchema);
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (obj instanceof ProtoSerializer && obj.getClass().equals(getClass())) {
+ final ProtoSerializer<T, B> other = (ProtoSerializer<T, B>)obj;
+ return Objects.equal(readSchema, other.readSchema) && Objects.equal(writeSchema, other.writeSchema);
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java
index c781493..441fa91 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java
@@ -37,10 +37,10 @@ import org.apache.drill.exec.server.options.OptionValue.OptionType;
import org.apache.drill.exec.server.rest.WebServer;
import org.apache.drill.exec.service.ServiceEngine;
import org.apache.drill.exec.store.StoragePluginRegistry;
-import org.apache.drill.exec.store.sys.CachingStoreProvider;
-import org.apache.drill.exec.store.sys.PStoreProvider;
-import org.apache.drill.exec.store.sys.PStoreRegistry;
-import org.apache.drill.exec.store.sys.local.LocalPStoreProvider;
+import org.apache.drill.exec.store.sys.store.provider.CachingPersistentStoreProvider;
+import org.apache.drill.exec.store.sys.PersistentStoreProvider;
+import org.apache.drill.exec.store.sys.PersistentStoreRegistry;
+import org.apache.drill.exec.store.sys.store.provider.LocalPersistentStoreProvider;
import org.apache.drill.exec.work.WorkManager;
import org.apache.zookeeper.Environment;
@@ -63,7 +63,7 @@ public class Drillbit implements AutoCloseable {
private final ClusterCoordinator coord;
private final ServiceEngine engine;
- private final PStoreProvider storeProvider;
+ private final PersistentStoreProvider storeProvider;
private final WorkManager manager;
private final BootStrapContext context;
private final WebServer webServer;
@@ -93,10 +93,10 @@ public class Drillbit implements AutoCloseable {
if (serviceSet != null) {
coord = serviceSet.getCoordinator();
- storeProvider = new CachingStoreProvider(new LocalPStoreProvider(config));
+ storeProvider = new CachingPersistentStoreProvider(new LocalPersistentStoreProvider(config));
} else {
coord = new ZKClusterCoordinator(config);
- storeProvider = new PStoreRegistry(this.coord, config).newPStoreProvider();
+ storeProvider = new PersistentStoreRegistry(this.coord, config).newPStoreProvider();
}
logger.info("Construction completed ({} ms).", w.elapsed(TimeUnit.MILLISECONDS));
}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
index aa6a0da..1af6d11 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
@@ -39,11 +39,11 @@ import org.apache.drill.exec.rpc.data.DataConnectionCreator;
import org.apache.drill.exec.server.options.SystemOptionManager;
import org.apache.drill.exec.store.SchemaFactory;
import org.apache.drill.exec.store.StoragePluginRegistry;
-import org.apache.drill.exec.store.sys.PStoreProvider;
+import org.apache.drill.exec.store.sys.PersistentStoreProvider;
import com.codahale.metrics.MetricRegistry;
-public class DrillbitContext {
+public class DrillbitContext implements AutoCloseable {
// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillbitContext.class);
private final BootStrapContext context;
@@ -57,7 +57,7 @@ public class DrillbitContext {
private final WorkEventBus workBus;
private final FunctionImplementationRegistry functionRegistry;
private final SystemOptionManager systemOptions;
- private final PStoreProvider provider;
+ private final PersistentStoreProvider provider;
private final CodeCompiler compiler;
private final ScanResult classpathScan;
private final LogicalPlanPersistence lpPersistence;
@@ -70,7 +70,7 @@ public class DrillbitContext {
Controller controller,
DataConnectionCreator connectionsPool,
WorkEventBus workBus,
- PStoreProvider provider) {
+ PersistentStoreProvider provider) {
this.classpathScan = context.getClasspathScan();
this.workBus = workBus;
this.controller = checkNotNull(controller);
@@ -152,7 +152,7 @@ public class DrillbitContext {
return reader;
}
- public PStoreProvider getPersistentStoreProvider() {
+ public PersistentStoreProvider getStoreProvider() {
return provider;
}
@@ -180,4 +180,8 @@ public class DrillbitContext {
return classpathScan;
}
+ @Override
+ public void close() throws Exception {
+ getOptionManager().close();
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/server/RemoteServiceSet.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/RemoteServiceSet.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/RemoteServiceSet.java
index 72eb306..06bb686 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/RemoteServiceSet.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/RemoteServiceSet.java
@@ -17,15 +17,12 @@
*/
package org.apache.drill.exec.server;
-import java.io.Closeable;
-import java.io.IOException;
-
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.coord.ClusterCoordinator;
import org.apache.drill.exec.coord.local.LocalClusterCoordinator;
import org.apache.drill.exec.memory.BufferAllocator;
-public class RemoteServiceSet implements Closeable {
+public class RemoteServiceSet implements AutoCloseable {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RemoteServiceSet.class);
private final ClusterCoordinator coordinator;
@@ -41,7 +38,7 @@ public class RemoteServiceSet implements Closeable {
}
@Override
- public void close() throws IOException {
+ public void close() throws Exception {
coordinator.close();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java
index a2b2e93..8753a51 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java
@@ -23,11 +23,12 @@ import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
+import org.apache.drill.exec.store.sys.PersistentStore;
/**
* An {@link OptionValue option value} is used by an {@link OptionManager} to store a run-time setting. This setting,
* for example, could affect a query in execution stage. Instances of this class are JSON serializable and can be stored
- * in a {@link org.apache.drill.exec.store.sys.PStore persistent store} (see {@link SystemOptionManager#options}), or
+ * in a {@link PersistentStore persistent store} (see {@link SystemOptionManager#options}), or
* in memory (see {@link InMemoryOptionManager#options}).
*/
@JsonInclude(Include.NON_NULL)
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index e54b914..8b14076 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -24,6 +24,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.commons.collections.IteratorUtils;
import org.apache.drill.common.config.LogicalPlanPersistence;
@@ -34,9 +35,9 @@ import org.apache.drill.exec.compile.ClassTransformer;
import org.apache.drill.exec.compile.QueryClassLoader;
import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.server.options.OptionValue.OptionType;
-import org.apache.drill.exec.store.sys.PStore;
-import org.apache.drill.exec.store.sys.PStoreConfig;
-import org.apache.drill.exec.store.sys.PStoreProvider;
+import org.apache.drill.exec.store.sys.PersistentStore;
+import org.apache.drill.exec.store.sys.PersistentStoreConfig;
+import org.apache.drill.exec.store.sys.PersistentStoreProvider;
import org.apache.drill.exec.util.AssertionUtil;
import static com.google.common.base.Preconditions.checkArgument;
@@ -46,7 +47,7 @@ import static com.google.common.base.Preconditions.checkArgument;
* Only one instance of this class exists per drillbit. Options set at the system level affect the entire system and
* persist between restarts.
*/
-public class SystemOptionManager extends BaseOptionManager {
+public class SystemOptionManager extends BaseOptionManager implements AutoCloseable {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SystemOptionManager.class);
private static final CaseInsensitiveMap<OptionValidator> VALIDATORS;
@@ -143,19 +144,19 @@ public class SystemOptionManager extends BaseOptionManager {
VALIDATORS = CaseInsensitiveMap.newImmutableMap(tmp);
}
- private final PStoreConfig<OptionValue> config;
+ private final PersistentStoreConfig<OptionValue> config;
- private final PStoreProvider provider;
+ private final PersistentStoreProvider provider;
/**
* Persistent store for options that have been changed from default.
* NOTE: CRUD operations must use lowercase keys.
*/
- private PStore<OptionValue> options;
+ private PersistentStore<OptionValue> options;
- public SystemOptionManager(LogicalPlanPersistence lpPersistence, final PStoreProvider provider) {
+ public SystemOptionManager(LogicalPlanPersistence lpPersistence, final PersistentStoreProvider provider) {
this.provider = provider;
- this.config = PStoreConfig.newJacksonBuilder(lpPersistence.getMapper(), OptionValue.class)
+ this.config = PersistentStoreConfig.newJacksonBuilder(lpPersistence.getMapper(), OptionValue.class)
.name("sys.options")
.build();
}
@@ -166,10 +167,10 @@ public class SystemOptionManager extends BaseOptionManager {
* @return this option manager
* @throws IOException
*/
- public SystemOptionManager init() throws IOException {
- options = provider.getStore(config);
+ public SystemOptionManager init() throws Exception {
+ options = provider.getOrCreateStore(config);
// if necessary, deprecate and replace options from persistent store
- for (final Entry<String, OptionValue> option : options) {
+ for (final Entry<String, OptionValue> option : Lists.newArrayList(options.getAll())) {
final String name = option.getKey();
final OptionValidator validator = VALIDATORS.get(name);
if (validator == null) {
@@ -215,7 +216,7 @@ public class SystemOptionManager extends BaseOptionManager {
buildList.put(entry.getKey(), entry.getValue().getDefault());
}
// override if changed
- for (final Map.Entry<String, OptionValue> entry : options) {
+ for (final Map.Entry<String, OptionValue> entry : Lists.newArrayList(options.getAll())) {
buildList.put(entry.getKey(), entry.getValue());
}
return buildList.values().iterator();
@@ -260,7 +261,7 @@ public class SystemOptionManager extends BaseOptionManager {
public void deleteAllOptions(OptionType type) {
checkArgument(type == OptionType.SYSTEM, "OptionType must be SYSTEM.");
final Set<String> names = Sets.newHashSet();
- for (final Map.Entry<String, OptionValue> entry : options) {
+ for (final Map.Entry<String, OptionValue> entry : Lists.newArrayList(options.getAll())) {
names.add(entry.getKey());
}
for (final String name : names) {
@@ -272,4 +273,9 @@ public class SystemOptionManager extends BaseOptionManager {
public OptionList getOptionList() {
return (OptionList) IteratorUtils.toList(iterator());
}
+
+ @Override
+ public void close() throws Exception {
+ options.close();
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java
index 300c617..d8533b0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java
@@ -22,7 +22,7 @@ import org.apache.drill.exec.server.rest.auth.AuthDynamicFeature;
import org.apache.drill.exec.server.rest.auth.DrillUserPrincipal;
import org.apache.drill.exec.server.rest.profile.ProfileResources;
import org.apache.drill.exec.store.StoragePluginRegistry;
-import org.apache.drill.exec.store.sys.PStoreProvider;
+import org.apache.drill.exec.store.sys.PersistentStoreProvider;
import org.apache.drill.exec.work.WorkManager;
import org.glassfish.hk2.api.Factory;
import org.glassfish.hk2.utilities.binding.AbstractBinder;
@@ -80,7 +80,7 @@ public class DrillRestServer extends ResourceConfig {
protected void configure() {
bind(workManager).to(WorkManager.class);
bind(workManager.getContext().getLpPersistence().getMapper()).to(ObjectMapper.class);
- bind(workManager.getContext().getPersistentStoreProvider()).to(PStoreProvider.class);
+ bind(workManager.getContext().getStoreProvider()).to(PersistentStoreProvider.class);
bind(workManager.getContext().getStorage()).to(StoragePluginRegistry.class);
bindFactory(DrillUserPrincipalProvider.class).to(DrillUserPrincipal.class);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java
index 2af9cac..3266eda 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java
@@ -73,7 +73,7 @@ public class StorageResources {
public List<PluginConfigWrapper> getStoragePluginsJSON() {
List<PluginConfigWrapper> list = Lists.newArrayList();
- for (Map.Entry<String, StoragePluginConfig> entry : storage.getStore()) {
+ for (Map.Entry<String, StoragePluginConfig> entry : Lists.newArrayList(storage.getStore().getAll())) {
PluginConfigWrapper plugin = new PluginConfigWrapper(entry.getKey(), entry.getValue());
list.add(plugin);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileResources.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileResources.java
index 0c04c9e..ddc9da1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileResources.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/ProfileResources.java
@@ -17,7 +17,6 @@
*/
package org.apache.drill.exec.server.rest.profile;
-import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
@@ -36,7 +35,10 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.SecurityContext;
import javax.xml.bind.annotation.XmlRootElement;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.UserException;
+import org.apache.drill.exec.coord.ClusterCoordinator;
+import org.apache.drill.exec.coord.store.TransientStore;
import org.apache.drill.exec.proto.GeneralRPCProtos.Ack;
import org.apache.drill.exec.proto.UserBitShared.QueryId;
import org.apache.drill.exec.proto.UserBitShared.QueryInfo;
@@ -44,8 +46,8 @@ import org.apache.drill.exec.proto.UserBitShared.QueryProfile;
import org.apache.drill.exec.proto.helper.QueryIdHelper;
import org.apache.drill.exec.server.rest.ViewableWithPermissions;
import org.apache.drill.exec.server.rest.auth.DrillUserPrincipal;
-import org.apache.drill.exec.store.sys.PStore;
-import org.apache.drill.exec.store.sys.PStoreProvider;
+import org.apache.drill.exec.store.sys.PersistentStore;
+import org.apache.drill.exec.store.sys.PersistentStoreProvider;
import org.apache.drill.exec.work.WorkManager;
import org.apache.drill.exec.work.foreman.Foreman;
import org.apache.drill.exec.work.foreman.QueryManager;
@@ -58,6 +60,8 @@ import com.google.common.collect.Lists;
public class ProfileResources {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProfileResources.class);
+ public final static int MAX_PROFILES = 100;
+
@Inject WorkManager work;
@Inject DrillUserPrincipal principal;
@Inject SecurityContext sc;
@@ -119,8 +123,12 @@ public class ProfileResources {
}
- private PStoreProvider provider(){
- return work.getContext().getPersistentStoreProvider();
+ protected PersistentStoreProvider getProvider() {
+ return work.getContext().getStoreProvider();
+ }
+
+ protected ClusterCoordinator getCoordinator() {
+ return work.getContext().getClusterCoordinator();
}
@XmlRootElement
@@ -146,38 +154,37 @@ public class ProfileResources {
@Path("/profiles.json")
@Produces(MediaType.APPLICATION_JSON)
public QProfiles getProfilesJSON() {
- PStore<QueryProfile> completed = null;
- PStore<QueryInfo> running = null;
try {
- completed = provider().getStore(QueryManager.QUERY_PROFILE);
- running = provider().getStore(QueryManager.RUNNING_QUERY_INFO);
- } catch (IOException e) {
- logger.debug("Failed to get profiles from persistent or ephemeral store.");
- return new QProfiles(new ArrayList<ProfileInfo>(), new ArrayList<ProfileInfo>());
- }
+ final PersistentStore<QueryProfile> completed = getProvider().getOrCreateStore(QueryManager.QUERY_PROFILE);
+ final TransientStore<QueryInfo> running = getCoordinator().getOrCreateTransientStore(QueryManager.RUNNING_QUERY_INFO);
- List<ProfileInfo> runningQueries = Lists.newArrayList();
+ final List<ProfileInfo> runningQueries = Lists.newArrayList();
- for (Map.Entry<String, QueryInfo> entry : running) {
- QueryInfo profile = entry.getValue();
- if (principal.canManageProfileOf(profile.getUser())) {
- runningQueries.add(new ProfileInfo(entry.getKey(), profile.getStart(), profile.getForeman().getAddress(),
- profile.getQuery(), profile.getState().name(), profile.getUser()));
+ for (final Map.Entry<String, QueryInfo> entry: Lists.newArrayList(running.entries())) {
+ final QueryInfo profile = entry.getValue();
+ if (principal.canManageProfileOf(profile.getUser())) {
+ runningQueries.add(new ProfileInfo(entry.getKey(), profile.getStart(), profile.getForeman().getAddress(),
+ profile.getQuery(), profile.getState().name(), profile.getUser()));
+ }
}
- }
- Collections.sort(runningQueries, Collections.reverseOrder());
+ Collections.sort(runningQueries, Collections.reverseOrder());
- List<ProfileInfo> finishedQueries = Lists.newArrayList();
- for (Map.Entry<String, QueryProfile> entry : completed) {
- QueryProfile profile = entry.getValue();
- if (principal.canManageProfileOf(profile.getUser())) {
- finishedQueries.add(new ProfileInfo(entry.getKey(), profile.getStart(), profile.getForeman().getAddress(),
- profile.getQuery(), profile.getState().name(), profile.getUser()));
+ List<ProfileInfo> finishedQueries = Lists.newArrayList();
+ for (Map.Entry<String, QueryProfile> entry : Lists.newArrayList(completed.getRange(0, MAX_PROFILES))) {
+ QueryProfile profile = entry.getValue();
+ if (principal.canManageProfileOf(profile.getUser())) {
+ finishedQueries.add(new ProfileInfo(entry.getKey(), profile.getStart(), profile.getForeman().getAddress(),
+ profile.getQuery(), profile.getState().name(), profile.getUser()));
+ }
}
+
+ return new QProfiles(runningQueries, finishedQueries);
+ } catch (Exception e) {
+ logger.debug("Failed to get profiles from persistent or ephemeral store.");
+ return new QProfiles(new ArrayList<ProfileInfo>(), new ArrayList<ProfileInfo>());
}
- return new QProfiles(runningQueries, finishedQueries);
}
@GET
@@ -188,7 +195,7 @@ public class ProfileResources {
return ViewableWithPermissions.create("/rest/profile/list.ftl", sc, profiles);
}
- private QueryProfile getQueryProfile(String queryId) throws IOException {
+ private QueryProfile getQueryProfile(String queryId) {
QueryId id = QueryIdHelper.getQueryIdFromString(queryId);
// first check local running
@@ -200,9 +207,9 @@ public class ProfileResources {
}
// then check remote running
- try{
- PStore<QueryInfo> runningQueries = provider().getStore(QueryManager.RUNNING_QUERY_INFO);
- QueryInfo info = runningQueries.get(queryId);
+ try {
+ final TransientStore<QueryInfo> running = getCoordinator().getOrCreateTransientStore(QueryManager.RUNNING_QUERY_INFO);
+ final QueryInfo info = running.get(queryId);
if (info != null) {
QueryProfile queryProfile = work.getContext()
.getController()
@@ -217,11 +224,15 @@ public class ProfileResources {
}
// then check blob store
- PStore<QueryProfile> profiles = provider().getStore(QueryManager.QUERY_PROFILE);
- QueryProfile queryProfile = profiles.get(queryId);
- if (queryProfile != null) {
- checkOrThrowProfileViewAuthorization(queryProfile);
- return queryProfile;
+ try {
+ final PersistentStore<QueryProfile> profiles = getProvider().getOrCreateStore(QueryManager.QUERY_PROFILE);
+ final QueryProfile queryProfile = profiles.get(queryId);
+ if (queryProfile != null) {
+ checkOrThrowProfileViewAuthorization(queryProfile);
+ return queryProfile;
+ }
+ } catch (final Exception e) {
+ throw new DrillRuntimeException("error while retrieving profile", e);
}
throw UserException.validationError()
@@ -236,7 +247,7 @@ public class ProfileResources {
public String getProfileJSON(@PathParam("queryid") String queryId) {
try {
return new String(QueryManager.QUERY_PROFILE.getSerializer().serialize(getQueryProfile(queryId)));
- } catch (IOException e) {
+ } catch (Exception e) {
logger.debug("Failed to serialize profile for: " + queryId);
return ("{ 'message' : 'error (unable to serialize profile)' }");
}
@@ -245,7 +256,7 @@ public class ProfileResources {
@GET
@Path("/profiles/{queryid}")
@Produces(MediaType.TEXT_HTML)
- public Viewable getProfile(@PathParam("queryid") String queryId) throws IOException {
+ public Viewable getProfile(@PathParam("queryid") String queryId){
ProfileWrapper wrapper = new ProfileWrapper(getQueryProfile(queryId));
return ViewableWithPermissions.create("/rest/profile/profile.ftl", sc, wrapper);
}
@@ -254,7 +265,7 @@ public class ProfileResources {
@GET
@Path("/profiles/cancel/{queryid}")
@Produces(MediaType.TEXT_PLAIN)
- public String cancelQuery(@PathParam("queryid") String queryId) throws IOException {
+ public String cancelQuery(@PathParam("queryid") String queryId) {
QueryId id = QueryIdHelper.getQueryIdFromString(queryId);
@@ -267,9 +278,9 @@ public class ProfileResources {
}
// then check remote running
- try{
- PStore<QueryInfo> runningQueries = provider().getStore(QueryManager.RUNNING_QUERY_INFO);
- QueryInfo info = runningQueries.get(queryId);
+ try {
+ final TransientStore<QueryInfo> running = getCoordinator().getOrCreateTransientStore(QueryManager.RUNNING_QUERY_INFO);
+ final QueryInfo info = running.get(queryId);
checkOrThrowQueryCancelAuthorization(info.getUser(), queryId);
Ack a = work.getContext().getController().getTunnel(info.getForeman()).requestCancelQuery(id).checkedGet(2, TimeUnit.SECONDS);
if(a.getOk()){
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java
index c7d364b..b6eed2d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java
@@ -26,7 +26,7 @@ import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.exec.exception.DrillbitStartupException;
import org.apache.drill.exec.ops.OptimizerRulesContext;
import org.apache.drill.exec.store.dfs.FormatPlugin;
-import org.apache.drill.exec.store.sys.PStore;
+import org.apache.drill.exec.store.sys.PersistentStore;
public interface StoragePluginRegistry extends Iterable<Map.Entry<String, StoragePlugin>>, AutoCloseable {
final String SYS_PLUGIN = "sys";
@@ -104,7 +104,7 @@ public interface StoragePluginRegistry extends Iterable<Map.Entry<String, Storag
* Get the PStore for this StoragePluginRegistry. (Used in the management layer.)
* @return PStore for StoragePlugin configuration objects.
*/
- PStore<StoragePluginConfig> getStore();
+ PersistentStore<StoragePluginConfig> getStore();
/**
* Return StoragePlugin rule sets.
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistryImpl.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistryImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistryImpl.java
index fefa183..e680502 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistryImpl.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistryImpl.java
@@ -46,6 +46,7 @@ import org.apache.drill.common.scanner.ClassPathScanner;
import org.apache.drill.common.scanner.persistence.ScanResult;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.exception.DrillbitStartupException;
+import org.apache.drill.exec.exception.StoreException;
import org.apache.drill.exec.ops.OptimizerRulesContext;
import org.apache.drill.exec.planner.logical.DrillRuleSets;
import org.apache.drill.exec.planner.logical.StoragePlugins;
@@ -54,8 +55,8 @@ import org.apache.drill.exec.store.dfs.FileSystemPlugin;
import org.apache.drill.exec.store.dfs.FormatPlugin;
import org.apache.drill.exec.store.ischema.InfoSchemaConfig;
import org.apache.drill.exec.store.ischema.InfoSchemaStoragePlugin;
-import org.apache.drill.exec.store.sys.PStore;
-import org.apache.drill.exec.store.sys.PStoreConfig;
+import org.apache.drill.exec.store.sys.PersistentStore;
+import org.apache.drill.exec.store.sys.PersistentStoreConfig;
import org.apache.drill.exec.store.sys.SystemTablePlugin;
import org.apache.drill.exec.store.sys.SystemTablePluginConfig;
@@ -82,7 +83,7 @@ public class StoragePluginRegistryImpl implements StoragePluginRegistry {
private DrillbitContext context;
private final DrillSchemaFactory schemaFactory = new DrillSchemaFactory();
- private final PStore<StoragePluginConfig> pluginSystemTable;
+ private final PersistentStore<StoragePluginConfig> pluginSystemTable;
private final LogicalPlanPersistence lpPersistence;
private final ScanResult classpathScan;
private final LoadingCache<StoragePluginConfig, StoragePlugin> ephemeralPlugins;
@@ -93,12 +94,12 @@ public class StoragePluginRegistryImpl implements StoragePluginRegistry {
this.classpathScan = checkNotNull(context.getClasspathScan());
try {
this.pluginSystemTable = context //
- .getPersistentStoreProvider() //
- .getStore(PStoreConfig //
+ .getStoreProvider() //
+ .getOrCreateStore(PersistentStoreConfig //
.newJacksonBuilder(lpPersistence.getMapper(), StoragePluginConfig.class) //
.name(PSTORE_NAME) //
.build());
- } catch (IOException | RuntimeException e) {
+ } catch (StoreException | RuntimeException e) {
logger.error("Failure while loading storage plugin registry.", e);
throw new RuntimeException("Failure while reading and loading storage plugin configuration.", e);
}
@@ -120,7 +121,7 @@ public class StoragePluginRegistryImpl implements StoragePluginRegistry {
});
}
- public PStore<StoragePluginConfig> getStore() {
+ public PersistentStore<StoragePluginConfig> getStore() {
return pluginSystemTable;
}
@@ -137,7 +138,7 @@ public class StoragePluginRegistryImpl implements StoragePluginRegistry {
* Check if the storage plugins system table has any entries. If not, load the boostrap-storage-plugin file into
* the system table.
*/
- if (!pluginSystemTable.iterator().hasNext()) {
+ if (!pluginSystemTable.getAll().hasNext()) {
// bootstrap load the config since no plugins are stored.
logger.info("No storage plugin instances configured in persistent store, loading bootstrap configuration.");
Collection<URL> urls = ClassPathScanner.forResource(ExecConstants.BOOTSTRAP_STORAGE_PLUGINS_FILE, false);
@@ -162,7 +163,7 @@ public class StoragePluginRegistryImpl implements StoragePluginRegistry {
}
Map<String, StoragePlugin> activePlugins = new HashMap<String, StoragePlugin>();
- for (Map.Entry<String, StoragePluginConfig> entry : pluginSystemTable) {
+ for (Map.Entry<String, StoragePluginConfig> entry : Lists.newArrayList(pluginSystemTable.getAll())) {
String name = entry.getKey();
StoragePluginConfig config = entry.getValue();
if (config.isEnabled()) {
@@ -385,7 +386,7 @@ public class StoragePluginRegistryImpl implements StoragePluginRegistry {
Set<String> currentPluginNames = Sets.newHashSet(plugins.names());
// iterate through the plugin instances in the persistence store adding
// any new ones and refreshing those whose configuration has changed
- for (Map.Entry<String, StoragePluginConfig> config : pluginSystemTable) {
+ for (Map.Entry<String, StoragePluginConfig> config : Lists.newArrayList(pluginSystemTable.getAll())) {
if (config.getValue().isEnabled()) {
getPlugin(config.getKey());
currentPluginNames.remove(config.getKey());
@@ -460,6 +461,7 @@ public class StoragePluginRegistryImpl implements StoragePluginRegistry {
public synchronized void close() throws Exception {
ephemeralPlugins.invalidateAll();
plugins.close();
+ pluginSystemTable.close();
}
/**
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/BasePersistentStore.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/BasePersistentStore.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/BasePersistentStore.java
new file mode 100644
index 0000000..248c3cb
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/BasePersistentStore.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys;
+
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.drill.common.collections.ImmutableEntry;
+
+public abstract class BasePersistentStore<V> implements PersistentStore<V> {
+
+ @Override
+ public Iterator<Map.Entry<String, V>> getAll() {
+ return getRange(0, Integer.MAX_VALUE);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/CachingStoreProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/CachingStoreProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/CachingStoreProvider.java
deleted file mode 100644
index 68440cb..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/CachingStoreProvider.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.sys;
-
-import java.io.IOException;
-import java.util.concurrent.ConcurrentMap;
-
-import org.apache.drill.exec.store.sys.PStoreConfig.Mode;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
-
-public class CachingStoreProvider implements PStoreProvider, AutoCloseable {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CachingStoreProvider.class);
-
- private final ConcurrentMap<PStoreConfig<?>, PStore<?>> storeCache = Maps.newConcurrentMap();
- private final PStoreProvider provider;
-
- public CachingStoreProvider(PStoreProvider provider) {
- super();
- this.provider = provider;
- }
-
- @SuppressWarnings("unchecked")
- public <V> PStore<V> getStore(PStoreConfig<V> config) throws IOException {
- PStore<?> s = storeCache.get(config);
- if(s == null){
- PStore<?> newStore = provider.getStore(config);
- s = storeCache.putIfAbsent(config, newStore);
- if(s == null){
- s = newStore;
- }else{
- newStore.close();
- }
- }
-
- return (PStore<V>) s;
-
- }
-
- @Override
- public void start() throws IOException {
- provider.start();
- }
-
- @Override
- public void close() throws Exception {
- for(PStore<?> store : storeCache.values()){
- store.close();
- }
- storeCache.clear();
- provider.close();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/8126927f/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/EStore.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/EStore.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/EStore.java
deleted file mode 100644
index 2d04957..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/EStore.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.drill.exec.store.sys;
-
-
-/**
- * Interfaces to define EStore, which is keep track of status/information for running queries. The information
- * would be gone, if the query is completed, or the foreman drillbit is not responding.
- * @param <V>
- */
-public interface EStore <V> extends PStore<V> {
-}