You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@crail.apache.org by ps...@apache.org on 2018/06/20 13:48:23 UTC
[2/3] incubator-crail git commit: Introducing StorageUtils as a
common ground for storage utility functions
Introducing StorageUtils as a common ground for storage utility
functions
Signed-off-by: Patrick Stuedi <ps...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-crail/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-crail/commit/847f64ee
Tree: http://git-wip-us.apache.org/repos/asf/incubator-crail/tree/847f64ee
Diff: http://git-wip-us.apache.org/repos/asf/incubator-crail/diff/847f64ee
Branch: refs/heads/master
Commit: 847f64eef55368963fc1bf51c521681de274a9da
Parents: 84f608e
Author: Patrick Stuedi <ps...@apache.org>
Authored: Wed Jun 20 14:11:04 2018 +0200
Committer: Patrick Stuedi <ps...@apache.org>
Committed: Wed Jun 20 14:11:04 2018 +0200
----------------------------------------------------------------------
.../apache/crail/memory/MappedBufferCache.java | 16 ++++---
.../org/apache/crail/storage/StorageClient.java | 1 -
.../test/java/org/apache/crail/ClientTest.java | 2 -
.../crail/storage/tcp/TcpStorageServer.java | 33 +------------
.../nvmf/client/NvmfStagingBufferCacheTest.java | 2 -
.../crail/storage/rdma/RdmaStorageServer.java | 29 +-----------
.../org/apache/crail/storage/StorageUtils.java | 49 ++++++++++++++++++++
7 files changed, 63 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/847f64ee/client/src/main/java/org/apache/crail/memory/MappedBufferCache.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/crail/memory/MappedBufferCache.java b/client/src/main/java/org/apache/crail/memory/MappedBufferCache.java
index 429bbe0..fb4fef2 100644
--- a/client/src/main/java/org/apache/crail/memory/MappedBufferCache.java
+++ b/client/src/main/java/org/apache/crail/memory/MappedBufferCache.java
@@ -53,13 +53,17 @@ public class MappedBufferCache extends BufferCache {
id = "" + System.currentTimeMillis();
directory = CrailUtils.getCacheDirectory(id);
dir = new File(directory);
- if (!dir.exists()){
- if (!dir.mkdirs()) {
- throw new IOException("Cannot create cache directory [crail.cachepath] set to path " + directory + ", check if crail.cachepath exists and has write permissions");
+ try {
+ if (!dir.exists()){
+ if (!dir.mkdirs()) {
+ throw new IOException("Cannot create cache directory [crail.cachepath] set to path " + directory + ", check if crail.cachepath exists and has write permissions");
+ }
}
- }
- for (File child : dir.listFiles()) {
- child.delete();
+ for (File child : dir.listFiles()) {
+ child.delete();
+ }
+ } catch(SecurityException e) {
+ throw new IOException("Security exception when trying to access " + directory + ", please check the directory permissions", e);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/847f64ee/client/src/main/java/org/apache/crail/storage/StorageClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/crail/storage/StorageClient.java b/client/src/main/java/org/apache/crail/storage/StorageClient.java
index 3f218c3..88c0981 100644
--- a/client/src/main/java/org/apache/crail/storage/StorageClient.java
+++ b/client/src/main/java/org/apache/crail/storage/StorageClient.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import org.apache.crail.CrailBufferCache;
import org.apache.crail.CrailStatistics;
-import org.apache.crail.conf.Configurable;
import org.apache.crail.conf.CrailConfiguration;
import org.apache.crail.metadata.DataNodeInfo;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/847f64ee/client/src/test/java/org/apache/crail/ClientTest.java
----------------------------------------------------------------------
diff --git a/client/src/test/java/org/apache/crail/ClientTest.java b/client/src/test/java/org/apache/crail/ClientTest.java
index 9123cf5..2e02a5e 100644
--- a/client/src/test/java/org/apache/crail/ClientTest.java
+++ b/client/src/test/java/org/apache/crail/ClientTest.java
@@ -26,10 +26,8 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.Assert;
-import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
import java.util.concurrent.ThreadLocalRandom;
public class ClientTest {
http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/847f64ee/storage-narpc/src/main/java/org/apache/crail/storage/tcp/TcpStorageServer.java
----------------------------------------------------------------------
diff --git a/storage-narpc/src/main/java/org/apache/crail/storage/tcp/TcpStorageServer.java b/storage-narpc/src/main/java/org/apache/crail/storage/tcp/TcpStorageServer.java
index 4685607..1ce51c5 100644
--- a/storage-narpc/src/main/java/org/apache/crail/storage/tcp/TcpStorageServer.java
+++ b/storage-narpc/src/main/java/org/apache/crail/storage/tcp/TcpStorageServer.java
@@ -19,23 +19,19 @@
package org.apache.crail.storage.tcp;
import java.io.File;
-import java.io.IOException;
import java.io.RandomAccessFile;
-import java.net.InetAddress;
import java.net.InetSocketAddress;
-import java.net.InterfaceAddress;
-import java.net.NetworkInterface;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileChannel.MapMode;
import java.nio.file.Paths;
-import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.crail.conf.CrailConfiguration;
import org.apache.crail.conf.CrailConstants;
import org.apache.crail.storage.StorageResource;
import org.apache.crail.storage.StorageServer;
+import org.apache.crail.storage.StorageUtils;
import org.apache.crail.utils.CrailUtils;
import org.slf4j.Logger;
@@ -62,7 +58,7 @@ public class TcpStorageServer implements Runnable, StorageServer, NaRPCService<T
this.serverGroup = new NaRPCServerGroup<TcpStorageRequest, TcpStorageResponse>(this, TcpStorageConstants.STORAGE_TCP_QUEUE_DEPTH, (int) CrailConstants.BLOCK_SIZE*2, false, TcpStorageConstants.STORAGE_TCP_CORES);
this.serverEndpoint = serverGroup.createServerEndpoint();
- this.address = getDataNodeAddress();
+ this.address = StorageUtils.getDataNodeAddress(TcpStorageConstants.STORAGE_TCP_INTERFACE, TcpStorageConstants.STORAGE_TCP_PORT);
serverEndpoint.bind(address);
this.alive = false;
this.regions = TcpStorageConstants.STORAGE_TCP_STORAGE_LIMIT/TcpStorageConstants.STORAGE_TCP_ALLOCATION_SIZE;
@@ -91,7 +87,6 @@ public class TcpStorageServer implements Runnable, StorageServer, NaRPCService<T
dataChannel.close();
long address = CrailUtils.getAddress(buffer);
resource = StorageResource.createResource(address, buffer.capacity(), fileId);
-// LOG.info("allocating resource, key " + resource.getKey() + ", address " + resource.getAddress() + ", length " + resource.getLength());
}
return resource;
}
@@ -161,30 +156,6 @@ public class TcpStorageServer implements Runnable, StorageServer, NaRPCService<T
}
}
- public static InetSocketAddress getDataNodeAddress() throws IOException {
- String ifname = TcpStorageConstants.STORAGE_TCP_INTERFACE;
- int port = TcpStorageConstants.STORAGE_TCP_PORT;
-
- NetworkInterface netif = NetworkInterface.getByName(ifname);
- if (netif == null){
- throw new IOException("Cannot find network interface with name " + TcpStorageConstants.STORAGE_TCP_INTERFACE);
- }
- List<InterfaceAddress> addresses = netif.getInterfaceAddresses();
- InetAddress addr = null;
- for (InterfaceAddress address: addresses){
- if (address.getBroadcast() != null){
- InetAddress _addr = address.getAddress();
- addr = _addr;
- }
- }
-
- if (addr == null){
- throw new IOException("Cannot find valid network interface with name " + TcpStorageConstants.STORAGE_TCP_INTERFACE);
- }
- InetSocketAddress inetAddr = new InetSocketAddress(addr, port);
- return inetAddr;
- }
-
public static String getDatanodeDirectory(InetSocketAddress address) throws IllegalArgumentException {
if (address == null) {
throw new IllegalArgumentException("Address paramater cannot be null!");
http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/847f64ee/storage-nvmf/src/test/java/org/apache/crail/storage/nvmf/client/NvmfStagingBufferCacheTest.java
----------------------------------------------------------------------
diff --git a/storage-nvmf/src/test/java/org/apache/crail/storage/nvmf/client/NvmfStagingBufferCacheTest.java b/storage-nvmf/src/test/java/org/apache/crail/storage/nvmf/client/NvmfStagingBufferCacheTest.java
index d91cdeb..5d8a995 100644
--- a/storage-nvmf/src/test/java/org/apache/crail/storage/nvmf/client/NvmfStagingBufferCacheTest.java
+++ b/storage-nvmf/src/test/java/org/apache/crail/storage/nvmf/client/NvmfStagingBufferCacheTest.java
@@ -23,8 +23,6 @@ import org.apache.crail.CrailBufferCache;
import org.apache.crail.conf.CrailConfiguration;
import org.apache.crail.conf.CrailConstants;
import org.apache.crail.memory.MappedBufferCache;
-import org.junit.Assert;
-import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/847f64ee/storage-rdma/src/main/java/org/apache/crail/storage/rdma/RdmaStorageServer.java
----------------------------------------------------------------------
diff --git a/storage-rdma/src/main/java/org/apache/crail/storage/rdma/RdmaStorageServer.java b/storage-rdma/src/main/java/org/apache/crail/storage/rdma/RdmaStorageServer.java
index ee7d90b..1e4eba2 100644
--- a/storage-rdma/src/main/java/org/apache/crail/storage/rdma/RdmaStorageServer.java
+++ b/storage-rdma/src/main/java/org/apache/crail/storage/rdma/RdmaStorageServer.java
@@ -19,22 +19,18 @@
package org.apache.crail.storage.rdma;
import java.io.File;
-import java.io.IOException;
import java.io.RandomAccessFile;
-import java.net.InetAddress;
import java.net.InetSocketAddress;
-import java.net.InterfaceAddress;
-import java.net.NetworkInterface;
import java.net.URI;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileChannel.MapMode;
-import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.crail.conf.CrailConfiguration;
import org.apache.crail.storage.StorageResource;
import org.apache.crail.storage.StorageServer;
+import org.apache.crail.storage.StorageUtils;
import org.apache.crail.utils.CrailUtils;
import org.slf4j.Logger;
@@ -65,7 +61,7 @@ public class RdmaStorageServer implements Runnable, StorageServer {
public void init(CrailConfiguration conf, String[] args) throws Exception {
RdmaConstants.init(conf, args);
- this.serverAddr = getDataNodeAddress();
+ this.serverAddr = StorageUtils.getDataNodeAddress(RdmaConstants.STORAGE_RDMA_INTERFACE, RdmaConstants.STORAGE_RDMA_PORT);
if (serverAddr == null){
LOG.info("Configured network interface " + RdmaConstants.STORAGE_RDMA_INTERFACE + " cannot be found..exiting!!!");
return;
@@ -158,27 +154,6 @@ public class RdmaStorageServer implements Runnable, StorageServer {
}
}
- public static InetSocketAddress getDataNodeAddress() throws IOException {
- String ifname = RdmaConstants.STORAGE_RDMA_INTERFACE;
- int port = RdmaConstants.STORAGE_RDMA_PORT;
-
- NetworkInterface netif = NetworkInterface.getByName(ifname);
- if (netif == null){
- return null;
- }
- List<InterfaceAddress> addresses = netif.getInterfaceAddresses();
- InetAddress addr = null;
- for (InterfaceAddress address: addresses){
-// LOG.info("address* " + address.toString() + ", _addr " + _addr.toString() + ", isSiteLocal " + _addr.isSiteLocalAddress() + ", tmp " + tmp + ", size " + tmp.length + ", broadcast " + address.getBroadcast());
- if (address.getBroadcast() != null){
- InetAddress _addr = address.getAddress();
- addr = _addr;
- }
- }
- InetSocketAddress inetAddr = new InetSocketAddress(addr, port);
- return inetAddr;
- }
-
@Override
public boolean isAlive() {
return isAlive;
http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/847f64ee/storage/src/main/java/org/apache/crail/storage/StorageUtils.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/crail/storage/StorageUtils.java b/storage/src/main/java/org/apache/crail/storage/StorageUtils.java
new file mode 100644
index 0000000..4b0becf
--- /dev/null
+++ b/storage/src/main/java/org/apache/crail/storage/StorageUtils.java
@@ -0,0 +1,49 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.crail.storage;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.InterfaceAddress;
+import java.net.NetworkInterface;
+import java.util.List;
+
+public class StorageUtils {
+ public static InetSocketAddress getDataNodeAddress(String ifname, int port) throws IOException {
+ NetworkInterface netif = NetworkInterface.getByName(ifname);
+ if (netif == null){
+ throw new IOException("Cannot find network interface with name " + ifname);
+ }
+ List<InterfaceAddress> addresses = netif.getInterfaceAddresses();
+ InetAddress addr = null;
+ for (InterfaceAddress address: addresses){
+ if (address.getBroadcast() != null){
+ InetAddress _addr = address.getAddress();
+ addr = _addr;
+ }
+ }
+
+ if (addr == null){
+ throw new IOException("Network interface with name " + ifname + " has no valid IP address");
+ }
+ InetSocketAddress inetAddr = new InetSocketAddress(addr, port);
+ return inetAddr;
+ }
+}