You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mh...@apache.org on 2018/03/16 08:08:05 UTC
asterixdb git commit: [NO ISSUE][API] Add Storage Stats API
Repository: asterixdb
Updated Branches:
refs/heads/master 297a31f81 -> e45528725
[NO ISSUE][API] Add Storage Stats API
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Add a new API on NCs for storage stats.
The API returns the total size of each index
as well as the id and size of each disk component.
Change-Id: Ie0345b00cfd825cbb77664c7617e8301c4f84e6d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2486
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <ti...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/e4552872
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/e4552872
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/e4552872
Branch: refs/heads/master
Commit: e455287257d1f5d194f6b56dc08bd9c8b2383536
Parents: 297a31f
Author: Murtadha Hubail <mh...@apache.org>
Authored: Thu Mar 15 13:45:32 2018 +0300
Committer: Murtadha Hubail <mh...@apache.org>
Committed: Fri Mar 16 01:07:13 2018 -0700
----------------------------------------------------------------------
.../api/http/server/StorageApiServlet.java | 13 ++++
.../common/storage/ResourceStorageStats.java | 71 ++++++++++++++++++++
.../PersistentLocalResourceRepository.java | 41 +++++++++++
3 files changed, 125 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e4552872/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java
index 142b99b..8dd4290 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java
@@ -31,6 +31,8 @@ import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.replication.IPartitionReplica;
import org.apache.asterix.common.storage.IReplicaManager;
import org.apache.asterix.common.storage.ReplicaIdentifier;
+import org.apache.asterix.common.storage.ResourceStorageStats;
+import org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.http.api.IServletRequest;
import org.apache.hyracks.http.api.IServletResponse;
@@ -69,6 +71,8 @@ public class StorageApiServlet extends AbstractServlet {
json = getStatus(p -> true);
} else if (path.startsWith("/partition")) {
json = getPartitionStatus(path);
+ } else if (path.startsWith("/stats")) {
+ json = getStats();
} else {
throw new IllegalArgumentException();
}
@@ -188,4 +192,13 @@ public class StorageApiServlet extends AbstractServlet {
appCtx.getReplicaManager().release(Integer.valueOf(partition));
response.setStatus(HttpResponseStatus.OK);
}
+
+ private JsonNode getStats() throws HyracksDataException {
+ final PersistentLocalResourceRepository localResourceRepository =
+ (PersistentLocalResourceRepository) appCtx.getLocalResourceRepository();
+ final ArrayNode result = OBJECT_MAPPER.createArrayNode();
+ final List<ResourceStorageStats> storageStats = localResourceRepository.getStorageStats();
+ storageStats.stream().map(ResourceStorageStats::asJson).forEach(result::add);
+ return result;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e4552872/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ResourceStorageStats.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ResourceStorageStats.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ResourceStorageStats.java
new file mode 100644
index 0000000..6dda5d1
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ResourceStorageStats.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.storage;
+
+import java.util.Map;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+public class ResourceStorageStats {
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+ private final DatasetResourceReference ref;
+ private final Map<String, Long> components;
+ private final long totalSize;
+
+ public ResourceStorageStats(DatasetResourceReference ref, Map<String, Long> components, long totalSize) {
+ this.ref = ref;
+ this.components = components;
+ this.totalSize = totalSize;
+ }
+
+ public DatasetResourceReference getRef() {
+ return ref;
+ }
+
+ public Map<String, Long> getComponents() {
+ return components;
+ }
+
+ public long getTotalSize() {
+ return totalSize;
+ }
+
+ public JsonNode asJson() {
+ final ObjectNode json = OBJECT_MAPPER.createObjectNode();
+ json.put("index", ref.getIndex());
+ json.put("dataset", ref.getDataset());
+ json.put("datasetId", ref.getDatasetId());
+ json.put("partition", ref.getPartitionId());
+ json.put("path", ref.getRelativePath().toString());
+ json.put("totalSize", totalSize);
+ final ArrayNode componentsJson = OBJECT_MAPPER.createArrayNode();
+ components.forEach((id, size) -> {
+ ObjectNode component = OBJECT_MAPPER.createObjectNode();
+ component.put("id", id);
+ component.put("size", size);
+ componentsJson.add(component);
+ });
+ json.set("components", componentsJson);
+ return json;
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e4552872/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
index 7206382..0375c30 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
@@ -58,6 +58,7 @@ import org.apache.asterix.common.storage.DatasetResourceReference;
import org.apache.asterix.common.storage.IIndexCheckpointManager;
import org.apache.asterix.common.storage.IIndexCheckpointManagerProvider;
import org.apache.asterix.common.storage.ResourceReference;
+import org.apache.asterix.common.storage.ResourceStorageStats;
import org.apache.asterix.common.utils.StorageConstants;
import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.commons.io.FileUtils;
@@ -395,6 +396,19 @@ public class PersistentLocalResourceRepository implements ILocalResourceReposito
}
}
+ public List<ResourceStorageStats> getStorageStats() throws HyracksDataException {
+ final List<DatasetResourceReference> allResources = loadAndGetAllResources().values().stream()
+ .map(DatasetResourceReference::of).collect(Collectors.toList());
+ final List<ResourceStorageStats> resourcesStats = new ArrayList<>();
+ for (DatasetResourceReference res : allResources) {
+ final ResourceStorageStats resourceStats = getResourceStats(res);
+ if (resourceStats != null) {
+ resourcesStats.add(resourceStats);
+ }
+ }
+ return resourcesStats;
+ }
+
private void deleteIndexMaskedFiles(File index) throws IOException {
File[] masks = index.listFiles(MASK_FILES_FILTER);
if (masks != null) {
@@ -463,6 +477,29 @@ public class PersistentLocalResourceRepository implements ILocalResourceReposito
}
}
+ private ResourceStorageStats getResourceStats(DatasetResourceReference resource) {
+ try {
+ final FileReference resolvedPath = ioManager.resolve(resource.getRelativePath().toString());
+ long totalSize = 0;
+ final File[] indexFiles = resolvedPath.getFile().listFiles();
+ final Map<String, Long> componentsStats = new HashMap<>();
+ if (indexFiles != null) {
+ for (File file : indexFiles) {
+ long fileSize = file.length();
+ totalSize += fileSize;
+ if (isComponentFile(resolvedPath.getFile(), file.getName())) {
+ String componentId = getComponentId(file.getAbsolutePath());
+ componentsStats.put(componentId, componentsStats.getOrDefault(componentId, 0L) + fileSize);
+ }
+ }
+ }
+ return new ResourceStorageStats(resource, componentsStats, totalSize);
+ } catch (Exception e) {
+ LOGGER.warn("Couldn't get stats for resource {}", resource.getRelativePath(), e);
+ }
+ return null;
+ }
+
/**
* Gets a component id based on its unique timestamp.
* e.g. a component file 2018-01-08-01-08-50-439_2018-01-08-01-08-50-439_b
@@ -479,4 +516,8 @@ public class PersistentLocalResourceRepository implements ILocalResourceReposito
private static boolean isComponentMask(File mask) {
return mask.getName().startsWith(StorageConstants.COMPONENT_MASK_FILE_PREFIX);
}
+
+ private static boolean isComponentFile(File indexDir, String fileName) {
+ return COMPONENT_FILES_FILTER.accept(indexDir, fileName);
+ }
}