You are viewing a plain text version of this content. The canonical link for it is here.

Posted to notifications@asterixdb.apache.org by "Murtadha Hubail (Code Review)" <do...@asterixdb.incubator.apache.org> on 2018/02/03 03:03:22 UTC

Change in asterixdb[master]: [NO ISSUE][REPL] Add API to Release Partition

Murtadha Hubail has uploaded a new change for review.

  https://asterix-gerrit.ics.uci.edu/2349

Change subject: [NO ISSUE][REPL] Add API to Release Partition
......................................................................

[NO ISSUE][REPL] Add API to Release Partition

- user model changes: no
- storage format changes: no
- interface changes: yes

Details:
- Add API to allow an NC to release a partition
  by flushing all of its resources and unregistering
  all of its replicas. This API can be used to allow
  graceful failover of a cluster partition.
- Add test case.

Change-Id: I7c8bbfc123904f13aae9f640dbefc7d17df69a42
---
M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java
M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/MetadataNodeResponseMessage.java
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.1.sto.cmd
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.2.pollget.http
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.3.post.http
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.4.get.http
M asterixdb/asterix-app/src/test/resources/runtimets/replication.xml
A asterixdb/asterix-app/src/test/resources/runtimets/results/replication/release_partition/release_partition.2.adm
A asterixdb/asterix-app/src/test/resources/runtimets/results/replication/release_partition/release_partition.3.adm
A asterixdb/asterix-app/src/test/resources/runtimets/results/replication/release_partition/release_partition.4.adm
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IReplicaManager.java
12 files changed, 142 insertions(+), 2 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/49/2349/1

diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java
index e770bb3..142b99b 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java
@@ -96,6 +96,9 @@
             case "/promote":
                 processPromote(request, response);
                 break;
+            case "/release":
+                processRelease(request, response);
+                break;
             default:
                 sendError(response, HttpResponseStatus.NOT_FOUND);
                 break;
@@ -175,4 +178,14 @@
         appCtx.getReplicaManager().promote(Integer.valueOf(partition));
         response.setStatus(HttpResponseStatus.OK);
     }
+
+    private void processRelease(IServletRequest request, IServletResponse response) throws HyracksDataException {
+        final String partition = request.getParameter("partition");
+        if (partition == null) {
+            response.setStatus(HttpResponseStatus.BAD_REQUEST);
+            return;
+        }
+        appCtx.getReplicaManager().release(Integer.valueOf(partition));
+        response.setStatus(HttpResponseStatus.OK);
+    }
 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
index 4edae69..8c733fb 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
@@ -27,6 +27,7 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import org.apache.asterix.common.api.IDatasetLifecycleManager;
 import org.apache.asterix.common.api.INcApplicationContext;
 import org.apache.asterix.common.replication.IPartitionReplica;
 import org.apache.asterix.common.storage.IReplicaManager;
@@ -35,6 +36,7 @@
 import org.apache.asterix.replication.api.PartitionReplica;
 import org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.common.LocalResource;
 
 public class ReplicaManager implements IReplicaManager {
 
@@ -93,4 +95,29 @@
         recoveryManager.replayReplicaPartitionLogs(Stream.of(partition).collect(Collectors.toSet()), true);
         partitions.add(partition);
     }
+
+    @Override
+    public void release(int partition) throws HyracksDataException {
+        if (!partitions.contains(partition)) {
+            return;
+        }
+        final IDatasetLifecycleManager datasetLifecycleManager = appCtx.getDatasetLifecycleManager();
+        datasetLifecycleManager.flushDataset(appCtx.getReplicationManager().getReplicationStrategy());
+        closePartitionResources(partition);
+        final List<IPartitionReplica> partitionReplicas = getReplicas(partition);
+        for (IPartitionReplica replica : partitionReplicas) {
+            appCtx.getReplicationManager().unregister(replica);
+        }
+        partitions.remove(partition);
+    }
+
+    private void closePartitionResources(int partition) throws HyracksDataException {
+        final PersistentLocalResourceRepository resourceRepository =
+                (PersistentLocalResourceRepository) appCtx.getLocalResourceRepository();
+        final Map<Long, LocalResource> partitionResources = resourceRepository.getPartitionResources(partition);
+        final IDatasetLifecycleManager datasetLifecycleManager = appCtx.getDatasetLifecycleManager();
+        for (LocalResource resource : partitionResources.values()) {
+            datasetLifecycleManager.close(resource.getPath());
+        }
+    }
 }
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/MetadataNodeResponseMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/MetadataNodeResponseMessage.java
index 815d878..1d4a0a9 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/MetadataNodeResponseMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/MetadataNodeResponseMessage.java
@@ -40,8 +40,8 @@
     }
 
     @Override
-    public void handle(ICcApplicationContext appCtx) throws HyracksDataException, InterruptedException {
-        ((CcApplicationContext) appCtx).getNcLifecycleCoordinator().process(this);
+    public void handle(ICcApplicationContext appCtx) throws HyracksDataException {
+        appCtx.getNcLifecycleCoordinator().process(this);
     }
 
     @Override
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.1.sto.cmd b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.1.sto.cmd
new file mode 100644
index 0000000..7ddaa20
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.1.sto.cmd
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+nc:asterix_nc1 /addReplica 0 asterix_nc2
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.2.pollget.http b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.2.pollget.http
new file mode 100644
index 0000000..6867a5d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.2.pollget.http
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+//polltimeoutsecs=30
+
+nc:asterix_nc1 /admin/storage/partition/0
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.3.post.http b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.3.post.http
new file mode 100644
index 0000000..bb942d8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.3.post.http
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+nc:asterix_nc1 /admin/storage/release?partition=0
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.4.get.http b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.4.get.http
new file mode 100644
index 0000000..d5cf952
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.4.get.http
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+nc:asterix_nc1 /admin/storage/partition/0
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/replication.xml b/asterixdb/asterix-app/src/test/resources/runtimets/replication.xml
index 79c9f5c..469446d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/replication.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/replication.xml
@@ -44,5 +44,10 @@
         <output-dir compare="Text">flushed_component</output-dir>
       </compilation-unit>
     </test-case>
+    <test-case FilePath="replication">
+      <compilation-unit name="release_partition">
+        <output-dir compare="Text">release_partition</output-dir>
+      </compilation-unit>
+    </test-case>
   </test-group>
 </test-suite>
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/release_partition/release_partition.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/release_partition/release_partition.2.adm
new file mode 100644
index 0000000..e48cbcf
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/release_partition/release_partition.2.adm
@@ -0,0 +1,7 @@
+[ {
+  "partition" : 0,
+  "replicas" : [ {
+    "location" : "127.0.0.1:2002",
+    "status" : "IN_SYNC"
+  } ]
+} ]
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/release_partition/release_partition.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/release_partition/release_partition.3.adm
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/release_partition/release_partition.3.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/release_partition/release_partition.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/release_partition/release_partition.4.adm
new file mode 100644
index 0000000..8878e54
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/release_partition/release_partition.4.adm
@@ -0,0 +1 @@
+[ ]
\ No newline at end of file
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IReplicaManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IReplicaManager.java
index 19eee8f..b2deb1e 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IReplicaManager.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IReplicaManager.java
@@ -62,4 +62,13 @@
      * @param partition
      */
     void promote(int partition) throws HyracksDataException;
+
+    /**
+     * Releases a partition by flushing all its resources to disk
+     * then removing all partition replicas.
+     *
+     * @param partition
+     * @throws HyracksDataException
+     */
+    void release(int partition) throws HyracksDataException;
 }

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/2349
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I7c8bbfc123904f13aae9f640dbefc7d17df69a42
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>