You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2015/03/24 09:32:14 UTC
[1/9] hbase git commit: HBASE-13315 BufferedMutator should be
@InterfaceAudience.Public (Solomon Duskis)
Repository: hbase
Updated Branches:
refs/heads/hbase-12439 65d378189 -> 94fb511c4
HBASE-13315 BufferedMutator should be @InterfaceAudience.Public (Solomon Duskis)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1428a59c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1428a59c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1428a59c
Branch: refs/heads/hbase-12439
Commit: 1428a59caf422b43eb3e0d215d2c538c9ec8463f
Parents: 65d3781
Author: Nick Dimiduk <nd...@apache.org>
Authored: Mon Mar 23 12:16:29 2015 -0700
Committer: Nick Dimiduk <nd...@apache.org>
Committed: Mon Mar 23 12:16:29 2015 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hbase/client/BufferedMutator.java | 2 +-
.../java/org/apache/hadoop/hbase/client/BufferedMutatorImpl.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/1428a59c/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BufferedMutator.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BufferedMutator.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BufferedMutator.java
index 3b91078..8d3d33c 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BufferedMutator.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BufferedMutator.java
@@ -60,7 +60,7 @@ import java.util.List;
* @see Connection
* @since 1.0.0
*/
-@InterfaceAudience.Private
+@InterfaceAudience.Public
@InterfaceStability.Evolving
public interface BufferedMutator extends Closeable {
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/1428a59c/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BufferedMutatorImpl.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BufferedMutatorImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BufferedMutatorImpl.java
index 54e7ccd..249edec 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BufferedMutatorImpl.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BufferedMutatorImpl.java
@@ -42,7 +42,7 @@ import java.util.concurrent.TimeUnit;
* @since 1.0.0
*/
@InterfaceAudience.Private
-@InterfaceStability.Stable
+@InterfaceStability.Evolving
public class BufferedMutatorImpl implements BufferedMutator {
private static final Log LOG = LogFactory.getLog(BufferedMutatorImpl.class);
[8/9] hbase git commit: HBASE-13202 Procedure v2 - core framework
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureRunnableSet.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureRunnableSet.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureRunnableSet.java
new file mode 100644
index 0000000..126ec19
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureRunnableSet.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * Keep track of the runnable procedures
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface ProcedureRunnableSet {
+ /**
+ * Inserts the specified element at the front of this queue.
+ * @param proc the Procedure to add
+ */
+ void addFront(Procedure proc);
+
+ /**
+ * Inserts the specified element at the end of this queue.
+ * @param proc the Procedure to add
+ */
+ void addBack(Procedure proc);
+
+ /**
+ * The procedure can't run at the moment.
+ * add it back to the queue, giving priority to someone else.
+ * @param proc the Procedure to add back to the list
+ */
+ void yield(Procedure proc);
+
+ /**
+ * Fetch one Procedure from the queue
+ * @return the Procedure ID to execute, or null if nothing present.
+ */
+ Long poll();
+
+ /**
+ * In case the class is blocking on poll() waiting for items to be added,
+ * this method should awake poll() and poll() should return.
+ */
+ void signalAll();
+
+ /**
+ * Returns the number of elements in this collection.
+ * @return the number of elements in this collection.
+ */
+ int size();
+
+ /**
+ * Removes all of the elements from this collection.
+ */
+ void clear();
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureSimpleRunQueue.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureSimpleRunQueue.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureSimpleRunQueue.java
new file mode 100644
index 0000000..0a00ac5
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureSimpleRunQueue.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * Simple runqueue for the procedures
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class ProcedureSimpleRunQueue implements ProcedureRunnableSet {
+ private final Deque<Long> runnables = new ArrayDeque<Long>();
+ private final ReentrantLock lock = new ReentrantLock();
+ private final Condition waitCond = lock.newCondition();
+
+ @Override
+ public void addFront(final Procedure proc) {
+ lock.lock();
+ try {
+ runnables.addFirst(proc.getProcId());
+ waitCond.signal();
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ @Override
+ public void addBack(final Procedure proc) {
+ lock.lock();
+ try {
+ runnables.addLast(proc.getProcId());
+ waitCond.signal();
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ @Override
+ public void yield(final Procedure proc) {
+ addBack(proc);
+ }
+
+ @Override
+ @edu.umd.cs.findbugs.annotations.SuppressWarnings("WA_AWAIT_NOT_IN_LOOP")
+ public Long poll() {
+ lock.lock();
+ try {
+ if (runnables.isEmpty()) {
+ waitCond.await();
+ if (!runnables.isEmpty()) {
+ return runnables.pop();
+ }
+ } else {
+ return runnables.pop();
+ }
+ } catch (InterruptedException e) {
+ return null;
+ } finally {
+ lock.unlock();
+ }
+ return null;
+ }
+
+ @Override
+ public void signalAll() {
+ lock.lock();
+ try {
+ waitCond.signalAll();
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ @Override
+ public void clear() {
+ lock.lock();
+ try {
+ runnables.clear();
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ @Override
+ public int size() {
+ lock.lock();
+ try {
+ return runnables.size();
+ } finally {
+ lock.unlock();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureYieldException.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureYieldException.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureYieldException.java
new file mode 100644
index 0000000..177ff5b
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureYieldException.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+// TODO: Not used yet
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class ProcedureYieldException extends ProcedureException {
+ /** default constructor */
+ public ProcedureYieldException() {
+ super();
+ }
+
+ /**
+ * Constructor
+ * @param s message
+ */
+ public ProcedureYieldException(String s) {
+ super(s);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureException.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureException.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureException.java
new file mode 100644
index 0000000..6be512d
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureException.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure2;
+
+import java.io.IOException;
+
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage;
+import org.apache.hadoop.hbase.util.ForeignExceptionUtil;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+
+/**
+ * A RemoteProcedureException is an exception from another thread or process.
+ * <p>
+ * RemoteProcedureExceptions are sent to 'remote' peers to signal an abort in the face of failures.
+ * When serialized for transmission we encode using Protobufs to ensure version compatibility.
+ * <p>
+ * RemoteProcedureException exceptions contain a Throwable as its cause.
+ * This can be a "regular" exception generated locally or a ProxyThrowable that is a representation
+ * of the original exception created on original 'remote' source. These ProxyThrowables have their
+ * their stacks traces and messages overridden to reflect the original 'remote' exception.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+@SuppressWarnings("serial")
+public class RemoteProcedureException extends ProcedureException {
+
+ /**
+ * Name of the throwable's source such as a host or thread name. Must be non-null.
+ */
+ private final String source;
+
+ /**
+ * Create a new RemoteProcedureException that can be serialized.
+ * It is assumed that this came form a local source.
+ * @param source
+ * @param cause
+ */
+ public RemoteProcedureException(String source, Throwable cause) {
+ super(cause);
+ assert source != null;
+ assert cause != null;
+ this.source = source;
+ }
+
+ public String getSource() {
+ return source;
+ }
+
+ public IOException unwrapRemoteException() {
+ if (getCause() instanceof RemoteException) {
+ return ((RemoteException)getCause()).unwrapRemoteException();
+ }
+ if (getCause() instanceof IOException) {
+ return (IOException)getCause();
+ }
+ return new IOException(getCause());
+ }
+
+ @Override
+ public String toString() {
+ String className = getCause().getClass().getName();
+ return className + " via " + getSource() + ":" + getLocalizedMessage();
+ }
+
+ /**
+ * Converts a RemoteProcedureException to an array of bytes.
+ * @param source the name of the external exception source
+ * @param t the "local" external exception (local)
+ * @return protobuf serialized version of RemoteProcedureException
+ */
+ public static byte[] serialize(String source, Throwable t) {
+ return toProto(source, t).toByteArray();
+ }
+
+ /**
+ * Takes a series of bytes and tries to generate an RemoteProcedureException instance for it.
+ * @param bytes
+ * @return the ForeignExcpetion instance
+ * @throws InvalidProtocolBufferException if there was deserialization problem this is thrown.
+ */
+ public static RemoteProcedureException deserialize(byte[] bytes)
+ throws InvalidProtocolBufferException {
+ return fromProto(ForeignExceptionMessage.parseFrom(bytes));
+ }
+
+ public ForeignExceptionMessage convert() {
+ return ForeignExceptionUtil.toProtoForeignException(getSource(), getCause());
+ }
+
+ public static ForeignExceptionMessage toProto(String source, Throwable t) {
+ return ForeignExceptionUtil.toProtoForeignException(source, t);
+ }
+
+ public static RemoteProcedureException fromProto(final ForeignExceptionMessage eem) {
+ return new RemoteProcedureException(eem.getSource(), ForeignExceptionUtil.toIOException(eem));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RootProcedureState.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RootProcedureState.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RootProcedureState.java
new file mode 100644
index 0000000..2f9c007
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RootProcedureState.java
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState;
+
+/**
+ * Internal state of the ProcedureExecutor that describes the state of a "Root Procedure".
+ * A "Root Procedure" is a Procedure without parent, each subprocedure will be
+ * added to the "Root Procedure" stack (or rollback-stack).
+ *
+ * RootProcedureState is used and managed only by the ProcedureExecutor.
+ * Long rootProcId = getRootProcedureId(proc);
+ * rollbackStack.get(rootProcId).acquire(proc)
+ * rollbackStack.get(rootProcId).release(proc)
+ * ...
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+class RootProcedureState {
+ private static final Log LOG = LogFactory.getLog(RootProcedureState.class);
+
+ private enum State {
+ RUNNING, // The Procedure is running or ready to run
+ FAILED, // The Procedure failed, waiting for the rollback executing
+ ROLLINGBACK, // The Procedure failed and the execution was rolledback
+ }
+
+ private ArrayList<Procedure> subprocedures = null;
+ private State state = State.RUNNING;
+ private int running = 0;
+
+ public synchronized boolean isFailed() {
+ switch (state) {
+ case ROLLINGBACK:
+ case FAILED:
+ return true;
+ default:
+ break;
+ }
+ return false;
+ }
+
+ public synchronized boolean isRollingback() {
+ return state == State.ROLLINGBACK;
+ }
+
+ /**
+ * Called by the ProcedureExecutor to mark rollback execution
+ */
+ protected synchronized boolean setRollback() {
+ if (running == 0 && state == State.FAILED) {
+ state = State.ROLLINGBACK;
+ return true;
+ }
+ return false;
+ }
+
+ protected synchronized List<Procedure> getSubprocedures() {
+ return subprocedures;
+ }
+
+ protected synchronized RemoteProcedureException getException() {
+ if (subprocedures != null) {
+ for (Procedure proc: subprocedures) {
+ if (proc.hasException()) {
+ return proc.getException();
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Called by the ProcedureExecutor to mark the procedure step as running.
+ */
+ protected synchronized boolean acquire(final Procedure proc) {
+ if (state != State.RUNNING) return false;
+
+ running++;
+ return true;
+ }
+
+ /**
+ * Called by the ProcedureExecutor to mark the procedure step as finished.
+ */
+ protected synchronized void release(final Procedure proc) {
+ running--;
+ }
+
+ protected synchronized void abort() {
+ if (state == State.RUNNING) {
+ state = State.FAILED;
+ }
+ }
+
+ /**
+ * Called by the ProcedureExecutor after the procedure step is completed,
+ * to add the step to the rollback list (or procedure stack)
+ */
+ protected synchronized void addRollbackStep(final Procedure proc) {
+ if (proc.isFailed()) {
+ state = State.FAILED;
+ }
+ if (subprocedures == null) {
+ subprocedures = new ArrayList<Procedure>();
+ }
+ proc.addStackIndex(subprocedures.size());
+ subprocedures.add(proc);
+ }
+
+ /**
+ * Called on store load by the ProcedureExecutor to load part of the stack.
+ *
+ * Each procedure has its own stack-positions. Which means we have to write
+ * to the store only the Procedure we executed, and nothing else.
+ * on load we recreate the full stack by aggregating each procedure stack-positions.
+ */
+ protected synchronized void loadStack(final Procedure proc) {
+ int[] stackIndexes = proc.getStackIndexes();
+ if (stackIndexes != null) {
+ if (subprocedures == null) {
+ subprocedures = new ArrayList<Procedure>();
+ }
+ int diff = (1 + stackIndexes[stackIndexes.length - 1]) - subprocedures.size();
+ if (diff > 0) {
+ subprocedures.ensureCapacity(1 + stackIndexes[stackIndexes.length - 1]);
+ while (diff-- > 0) subprocedures.add(null);
+ }
+ for (int i = 0; i < stackIndexes.length; ++i) {
+ subprocedures.set(stackIndexes[i], proc);
+ }
+ }
+ if (proc.getState() == ProcedureState.ROLLEDBACK) {
+ state = State.ROLLINGBACK;
+ } else if (proc.isFailed()) {
+ state = State.FAILED;
+ }
+ }
+
+ /**
+ * Called on store load by the ProcedureExecutor to validate the procedure stack.
+ */
+ protected synchronized boolean isValid() {
+ if (subprocedures != null) {
+ for (Procedure proc: subprocedures) {
+ if (proc == null) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/SequentialProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/SequentialProcedure.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/SequentialProcedure.java
new file mode 100644
index 0000000..7bf959a
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/SequentialProcedure.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData;
+
+/**
+ * A SequentialProcedure describes one step in a procedure chain.
+ * -> Step 1 -> Step 2 -> Step 3
+ *
+ * The main difference from a base Procedure is that the execute() of a
+ * SequentialProcedure will be called only once, there will be no second
+ * execute() call once the child are finished. which means once the child
+ * of a SequentialProcedure are completed the SequentialProcedure is completed too.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public abstract class SequentialProcedure<TEnvironment> extends Procedure<TEnvironment> {
+ private boolean executed = false;
+
+ @Override
+ protected Procedure[] doExecute(final TEnvironment env)
+ throws ProcedureYieldException {
+ updateTimestamp();
+ try {
+ executed = !executed;
+ return executed ? execute(env) : null;
+ } finally {
+ updateTimestamp();
+ }
+ }
+
+ @Override
+ protected void doRollback(final TEnvironment env) {
+ updateTimestamp();
+ executed = !executed;
+ if (!executed) {
+ try {
+ rollback(env);
+ } finally {
+ updateTimestamp();
+ }
+ }
+ }
+
+ @Override
+ protected void serializeStateData(final OutputStream stream) throws IOException {
+ SequentialProcedureData.Builder data = SequentialProcedureData.newBuilder();
+ data.setExecuted(executed);
+ data.build().writeDelimitedTo(stream);
+ }
+
+ @Override
+ protected void deserializeStateData(final InputStream stream) throws IOException {
+ SequentialProcedureData data = SequentialProcedureData.parseDelimitedFrom(stream);
+ executed = data.getExecuted();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
new file mode 100644
index 0000000..f0a3765
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
@@ -0,0 +1,140 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.StateMachineProcedureData;
+
+/**
+ * Procedure described by a series of steps.
+ *
+ * The procedure implementor must have an enum of 'states', describing
+ * the various step of the procedure.
+ * Once the procedure is running, the procedure-framework will call executeFromState()
+ * using the 'state' provided by the user. The first call to executeFromState()
+ * will be performed with 'state = null'. The implementor can jump between
+ * states using setNextState(MyStateEnum.ordinal()).
+ * The rollback will call rollbackState() for each state that was executed, in reverse order.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public abstract class StateMachineProcedure<TEnvironment, TState>
+ extends Procedure<TEnvironment> {
+ private int stateCount = 0;
+ private int[] states = null;
+
+ protected enum Flow {
+ HAS_MORE_STATE,
+ NO_MORE_STATE,
+ }
+
+ /**
+ * called to perform a single step of the specified 'state' of the procedure
+ * @param state state to execute
+ * @return Flow.NO_MORE_STATE if the procedure is completed,
+ * Flow.HAS_MORE_STATE if there is another step.
+ */
+ protected abstract Flow executeFromState(TEnvironment env, TState state)
+ throws ProcedureYieldException;
+
+ /**
+ * called to perform the rollback of the specified state
+ * @param state state to rollback
+ */
+ protected abstract void rollbackState(TEnvironment env, TState state);
+
+ /**
+ * Convert an ordinal (or state id) to an Enum (or more descriptive) state object.
+ * @param stateId the ordinal() of the state enum (or state id)
+ * @return the state enum object
+ */
+ protected abstract TState getState(int stateId);
+
+ @Override
+ protected Procedure[] execute(final TEnvironment env)
+ throws ProcedureYieldException {
+ updateTimestamp();
+ try {
+ TState state = stateCount > 0 ? getState(states[stateCount-1]) : null;
+ if (executeFromState(env, state) == Flow.NO_MORE_STATE) {
+ // completed
+ return null;
+ }
+ return (isWaiting() || isFailed()) ? null : new Procedure[] {this};
+ } finally {
+ updateTimestamp();
+ }
+ }
+
+ @Override
+ protected void rollback(final TEnvironment env) {
+ try {
+ assert stateCount > 0 : "called rollback with no steps executed proc=" + this;
+ updateTimestamp();
+ rollbackState(env, getState(states[--stateCount]));
+ } finally {
+ updateTimestamp();
+ }
+ }
+
+ /**
+ * Set the next state for the procedure.
+ * @param stateId the ordinal() of the state enum (or state id)
+ */
+ protected void setNextState(final int stateId) {
+ if (states == null || states.length == stateCount) {
+ int newCapacity = stateCount + 8;
+ if (states != null) {
+ states = Arrays.copyOf(states, newCapacity);
+ } else {
+ states = new int[newCapacity];
+ }
+ }
+ states[stateCount++] = stateId;
+ }
+
+ @Override
+ protected void serializeStateData(final OutputStream stream) throws IOException {
+ StateMachineProcedureData.Builder data = StateMachineProcedureData.newBuilder();
+ for (int i = 0; i < stateCount; ++i) {
+ data.addState(states[i]);
+ }
+ data.build().writeDelimitedTo(stream);
+ }
+
+ @Override
+ protected void deserializeStateData(final InputStream stream) throws IOException {
+ StateMachineProcedureData data = StateMachineProcedureData.parseDelimitedFrom(stream);
+ stateCount = data.getStateCount();
+ if (stateCount > 0) {
+ states = new int[stateCount];
+ for (int i = 0; i < stateCount; ++i) {
+ states[i] = data.getState(i);
+ }
+ } else {
+ states = null;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/TwoPhaseProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/TwoPhaseProcedure.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/TwoPhaseProcedure.java
new file mode 100644
index 0000000..cd6b0a7
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/TwoPhaseProcedure.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public abstract class TwoPhaseProcedure<TEnvironment> extends Procedure<TEnvironment> {
+ // TODO (e.g. used by ACLs/VisibilityTags updates)
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/ProcedureStore.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/ProcedureStore.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/ProcedureStore.java
new file mode 100644
index 0000000..b440580
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/ProcedureStore.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2.store;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.procedure2.Procedure;
+
+/**
+ * The ProcedureStore is used by the executor to persist the state of each procedure execution.
+ * This allows to resume the execution of pending/in-progress procedures in case
+ * of machine failure or service shutdown.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public interface ProcedureStore {
+ /**
+ * Store listener interface.
+ * The main process should register a listener and respond to the store events.
+ */
+ public interface ProcedureStoreListener {
+ /**
+ * triggered when the store is not able to write out data.
+ * the main process should abort.
+ */
+ void abortProcess();
+ }
+
+ /**
+ * Add the listener to the notification list.
+ * @param listener The AssignmentListener to register
+ */
+ void registerListener(ProcedureStoreListener listener);
+
+ /**
+ * Remove the listener from the notification list.
+ * @param listener The AssignmentListener to unregister
+ * @return true if the listner was in the list and it was removed, otherwise false.
+ */
+ boolean unregisterListener(ProcedureStoreListener listener);
+
+ /**
+ * Start/Open the procedure store
+ * @param numThreads
+ */
+ void start(int numThreads) throws IOException;
+
+ /**
+ * Stop/Close the procedure store
+ */
+ void stop();
+
+ /**
+ * Acquire the lease for the procedure store.
+ */
+ void recoverLease() throws IOException;
+
+ /**
+ * Load the Procedures in the store.
+ * @return the set of procedures present in the store
+ */
+ Iterator<Procedure> load() throws IOException;
+
+ /**
+ * When a procedure is submitted to the executor insert(proc, null) will be called.
+ * 'proc' has a 'RUNNABLE' state and the initial information required to start up.
+ *
+ * When a procedure is executed and it returns children insert(proc, subprocs) will be called.
+ * 'proc' has a 'WAITING' state and an update state.
+ * 'subprocs' are the children in 'RUNNABLE' state with the initial information.
+ *
+ * @param proc the procedure to serialize and write to the store.
+ * @param subprocs the newly created child of the proc.
+ */
+ void insert(Procedure proc, Procedure[] subprocs);
+
+ /**
+ * The specified procedure was executed,
+ * and the new state should be written to the store.
+ * @param proc the procedure to serialize and write to the store.
+ */
+ void update(Procedure proc);
+
+ /**
+ * The specified procId was removed from the executor,
+ * due to completion, abort or failure.
+ * The store implementor should remove all the information about the specified procId.
+ * @param procId the ID of the procedure to remove.
+ */
+ void delete(long procId);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.java
new file mode 100644
index 0000000..4e4653a
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.java
@@ -0,0 +1,540 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2.store;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos;
+
+/**
+ * Keeps track of live procedures.
+ *
+ * It can be used by the ProcedureStore to identify which procedures are already
+ * deleted/completed to avoid the deserialization step on restart.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class ProcedureStoreTracker {
+ private final TreeMap<Long, BitSetNode> map = new TreeMap<Long, BitSetNode>();
+
+ private boolean keepDeletes = false;
+ private boolean partial = false;
+
+ public enum DeleteState { YES, NO, MAYBE }
+
+ public static class BitSetNode {
+ private final static long WORD_MASK = 0xffffffffffffffffL;
+ private final static int ADDRESS_BITS_PER_WORD = 6;
+ private final static int BITS_PER_WORD = 1 << ADDRESS_BITS_PER_WORD;
+ private final static int MAX_NODE_SIZE = 4 << ADDRESS_BITS_PER_WORD;
+
+ private long[] updated;
+ private long[] deleted;
+ private long start;
+
+ public void dump() {
+ System.out.printf("%06d:%06d min=%d max=%d%n", getStart(), getEnd(),
+ getMinProcId(), getMaxProcId());
+ System.out.println("Update:");
+ for (int i = 0; i < updated.length; ++i) {
+ for (int j = 0; j < BITS_PER_WORD; ++j) {
+ System.out.print((updated[i] & (1L << j)) != 0 ? "1" : "0");
+ }
+ System.out.println(" " + i);
+ }
+ System.out.println();
+ System.out.println("Delete:");
+ for (int i = 0; i < deleted.length; ++i) {
+ for (int j = 0; j < BITS_PER_WORD; ++j) {
+ System.out.print((deleted[i] & (1L << j)) != 0 ? "1" : "0");
+ }
+ System.out.println(" " + i);
+ }
+ System.out.println();
+ }
+
+ public BitSetNode(final long procId, final boolean partial) {
+ start = alignDown(procId);
+
+ int count = 2;
+ updated = new long[count];
+ deleted = new long[count];
+ for (int i = 0; i < count; ++i) {
+ updated[i] = 0;
+ deleted[i] = partial ? 0 : WORD_MASK;
+ }
+
+ updateState(procId, false);
+ }
+
+ protected BitSetNode(final long start, final long[] updated, final long[] deleted) {
+ this.start = start;
+ this.updated = updated;
+ this.deleted = deleted;
+ }
+
+ public void update(final long procId) {
+ updateState(procId, false);
+ }
+
+ public void delete(final long procId) {
+ updateState(procId, true);
+ }
+
+ public Long getStart() {
+ return start;
+ }
+
+ public Long getEnd() {
+ return start + (updated.length << ADDRESS_BITS_PER_WORD) - 1;
+ }
+
+ public boolean contains(final long procId) {
+ return start <= procId && procId <= getEnd();
+ }
+
+ public DeleteState isDeleted(final long procId) {
+ int bitmapIndex = getBitmapIndex(procId);
+ int wordIndex = bitmapIndex >> ADDRESS_BITS_PER_WORD;
+ if (wordIndex >= deleted.length) {
+ return DeleteState.MAYBE;
+ }
+ return (deleted[wordIndex] & (1L << bitmapIndex)) != 0 ? DeleteState.YES : DeleteState.NO;
+ }
+
+ private boolean isUpdated(final long procId) {
+ int bitmapIndex = getBitmapIndex(procId);
+ int wordIndex = bitmapIndex >> ADDRESS_BITS_PER_WORD;
+ if (wordIndex >= updated.length) {
+ return false;
+ }
+ return (updated[wordIndex] & (1L << bitmapIndex)) != 0;
+ }
+
+ public boolean isUpdated() {
+ // TODO: cache the value
+ for (int i = 0; i < updated.length; ++i) {
+ long deleteMask = ~deleted[i];
+ if ((updated[i] & deleteMask) != (WORD_MASK & deleteMask)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public boolean isEmpty() {
+ // TODO: cache the value
+ for (int i = 0; i < deleted.length; ++i) {
+ if (deleted[i] != WORD_MASK) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public void resetUpdates() {
+ for (int i = 0; i < updated.length; ++i) {
+ updated[i] = 0;
+ }
+ }
+
+ public void undeleteAll() {
+ for (int i = 0; i < updated.length; ++i) {
+ deleted[i] = 0;
+ }
+ }
+
+ public ProcedureProtos.ProcedureStoreTracker.TrackerNode convert() {
+ ProcedureProtos.ProcedureStoreTracker.TrackerNode.Builder builder =
+ ProcedureProtos.ProcedureStoreTracker.TrackerNode.newBuilder();
+ builder.setStartId(start);
+ for (int i = 0; i < updated.length; ++i) {
+ builder.addUpdated(updated[i]);
+ builder.addDeleted(deleted[i]);
+ }
+ return builder.build();
+ }
+
+ public static BitSetNode convert(ProcedureProtos.ProcedureStoreTracker.TrackerNode data) {
+ long start = data.getStartId();
+ int size = data.getUpdatedCount();
+ long[] updated = new long[size];
+ long[] deleted = new long[size];
+ for (int i = 0; i < size; ++i) {
+ updated[i] = data.getUpdated(i);
+ deleted[i] = data.getDeleted(i);
+ }
+ return new BitSetNode(start, updated, deleted);
+ }
+
+ // ========================================================================
+ // Grow/Merge Helpers
+ // ========================================================================
+ public boolean canGrow(final long procId) {
+ return (procId - start) < MAX_NODE_SIZE;
+ }
+
+ public boolean canMerge(final BitSetNode rightNode) {
+ return (start + rightNode.getEnd()) < MAX_NODE_SIZE;
+ }
+
+ public void grow(final long procId) {
+ int delta, offset;
+
+ if (procId < start) {
+ // add to head
+ long newStart = alignDown(procId);
+ delta = (int)(start - newStart) >> ADDRESS_BITS_PER_WORD;
+ offset = delta;
+ } else {
+ // Add to tail
+ long newEnd = alignUp(procId + 1);
+ delta = (int)(newEnd - getEnd()) >> ADDRESS_BITS_PER_WORD;
+ offset = 0;
+ }
+
+ long[] newBitmap;
+ int oldSize = updated.length;
+
+ newBitmap = new long[oldSize + delta];
+ System.arraycopy(updated, 0, newBitmap, offset, oldSize);
+ updated = newBitmap;
+
+ newBitmap = new long[deleted.length + delta];
+ System.arraycopy(deleted, 0, newBitmap, offset, oldSize);
+ deleted = newBitmap;
+
+ for (int i = 0; i < delta; ++i) {
+ updated[oldSize + i] = 0;
+ deleted[oldSize + i] = WORD_MASK;
+ }
+ }
+
+ public void merge(final BitSetNode rightNode) {
+ int delta = (int)(rightNode.getEnd() - getEnd()) >> ADDRESS_BITS_PER_WORD;
+
+ long[] newBitmap;
+ int oldSize = updated.length;
+ int newSize = (delta - rightNode.updated.length);
+ int offset = oldSize + newSize;
+
+ newBitmap = new long[oldSize + delta];
+ System.arraycopy(updated, 0, newBitmap, 0, oldSize);
+ System.arraycopy(rightNode.updated, 0, newBitmap, offset, rightNode.updated.length);
+ updated = newBitmap;
+
+ newBitmap = new long[oldSize + delta];
+ System.arraycopy(deleted, 0, newBitmap, 0, oldSize);
+ System.arraycopy(rightNode.deleted, 0, newBitmap, offset, rightNode.deleted.length);
+ deleted = newBitmap;
+
+ for (int i = 0; i < newSize; ++i) {
+ updated[offset + i] = 0;
+ deleted[offset + i] = WORD_MASK;
+ }
+ }
+
+ // ========================================================================
+ // Min/Max Helpers
+ // ========================================================================
+ public long getMinProcId() {
+ long minProcId = start;
+ for (int i = 0; i < deleted.length; ++i) {
+ if (deleted[i] == 0) {
+ return(minProcId);
+ }
+
+ if (deleted[i] != WORD_MASK) {
+ for (int j = 0; j < BITS_PER_WORD; ++j) {
+ if ((deleted[i] & (1L << j)) != 0) {
+ return minProcId + j;
+ }
+ }
+ }
+
+ minProcId += BITS_PER_WORD;
+ }
+ return minProcId;
+ }
+
+ public long getMaxProcId() {
+ long maxProcId = getEnd();
+ for (int i = deleted.length - 1; i >= 0; --i) {
+ if (deleted[i] == 0) {
+ return maxProcId;
+ }
+
+ if (deleted[i] != WORD_MASK) {
+ for (int j = BITS_PER_WORD - 1; j >= 0; --j) {
+ if ((deleted[i] & (1L << j)) == 0) {
+ return maxProcId - (BITS_PER_WORD - 1 - j);
+ }
+ }
+ }
+ maxProcId -= BITS_PER_WORD;
+ }
+ return maxProcId;
+ }
+
+ // ========================================================================
+ // Bitmap Helpers
+ // ========================================================================
+ private int getBitmapIndex(final long procId) {
+ return (int)(procId - start);
+ }
+
+ private void updateState(final long procId, final boolean isDeleted) {
+ int bitmapIndex = getBitmapIndex(procId);
+ int wordIndex = bitmapIndex >> ADDRESS_BITS_PER_WORD;
+ long value = (1L << bitmapIndex);
+
+ if (isDeleted) {
+ updated[wordIndex] |= value;
+ deleted[wordIndex] |= value;
+ } else {
+ updated[wordIndex] |= value;
+ deleted[wordIndex] &= ~value;
+ }
+ }
+
+ // ========================================================================
+ // Helpers
+ // ========================================================================
+ private static long alignUp(final long x) {
+ return (x + (BITS_PER_WORD - 1)) & -BITS_PER_WORD;
+ }
+
+ private static long alignDown(final long x) {
+ return x & -BITS_PER_WORD;
+ }
+ }
+
+ public void insert(final Procedure proc, final Procedure[] subprocs) {
+ insert(proc.getProcId());
+ if (subprocs != null) {
+ for (int i = 0; i < subprocs.length; ++i) {
+ insert(subprocs[i].getProcId());
+ }
+ }
+ }
+
+ public void update(final Procedure proc) {
+ update(proc.getProcId());
+ }
+
+ public void insert(long procId) {
+ BitSetNode node = getOrCreateNode(procId);
+ node.update(procId);
+ }
+
+ public void update(long procId) {
+ Map.Entry<Long, BitSetNode> entry = map.floorEntry(procId);
+ assert entry != null : "expected node to update procId=" + procId;
+
+ BitSetNode node = entry.getValue();
+ assert node.contains(procId);
+ node.update(procId);
+ }
+
+ public void delete(long procId) {
+ Map.Entry<Long, BitSetNode> entry = map.floorEntry(procId);
+ assert entry != null : "expected node to delete procId=" + procId;
+
+ BitSetNode node = entry.getValue();
+ assert node.contains(procId) : "expected procId in the node";
+ node.delete(procId);
+
+ if (!keepDeletes && node.isEmpty()) {
+ // TODO: RESET if (map.size() == 1)
+ map.remove(entry.getKey());
+ }
+ }
+
+ @InterfaceAudience.Private
+ public void setDeleted(final long procId, final boolean isDeleted) {
+ BitSetNode node = getOrCreateNode(procId);
+ node.updateState(procId, isDeleted);
+ }
+
+ public void clear() {
+ this.map.clear();
+ }
+
+ public DeleteState isDeleted(long procId) {
+ Map.Entry<Long, BitSetNode> entry = map.floorEntry(procId);
+ if (entry != null) {
+ BitSetNode node = entry.getValue();
+ DeleteState state = node.isDeleted(procId);
+ return partial && !node.isUpdated(procId) ? DeleteState.MAYBE : state;
+ }
+ return partial ? DeleteState.MAYBE : DeleteState.YES;
+ }
+
+ public long getMinProcId() {
+ // TODO: Cache?
+ Map.Entry<Long, BitSetNode> entry = map.firstEntry();
+ return entry == null ? 0 : entry.getValue().getMinProcId();
+ }
+
+ public void setKeepDeletes(boolean keepDeletes) {
+ this.keepDeletes = keepDeletes;
+ if (!keepDeletes) {
+ Iterator<Map.Entry<Long, BitSetNode>> it = map.entrySet().iterator();
+ while (it.hasNext()) {
+ Map.Entry<Long, BitSetNode> entry = it.next();
+ if (entry.getValue().isEmpty()) {
+ it.remove();
+ }
+ }
+ }
+ }
+
+ public void setPartialFlag(boolean isPartial) {
+ this.partial = isPartial;
+ }
+
+ public boolean isEmpty() {
+ for (Map.Entry<Long, BitSetNode> entry : map.entrySet()) {
+ if (entry.getValue().isEmpty() == false) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public boolean isUpdated() {
+ for (Map.Entry<Long, BitSetNode> entry : map.entrySet()) {
+ if (entry.getValue().isUpdated() == false) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public void resetUpdates() {
+ for (Map.Entry<Long, BitSetNode> entry : map.entrySet()) {
+ entry.getValue().resetUpdates();
+ }
+ }
+
+ public void undeleteAll() {
+ for (Map.Entry<Long, BitSetNode> entry : map.entrySet()) {
+ entry.getValue().undeleteAll();
+ }
+ }
+
+ private BitSetNode getOrCreateNode(final long procId) {
+ // can procId fit in the left node?
+ BitSetNode leftNode = null;
+ boolean leftCanGrow = false;
+ Map.Entry<Long, BitSetNode> leftEntry = map.floorEntry(procId);
+ if (leftEntry != null) {
+ leftNode = leftEntry.getValue();
+ if (leftNode.contains(procId)) {
+ return leftNode;
+ }
+ leftCanGrow = leftNode.canGrow(procId);
+ }
+
+ BitSetNode rightNode = null;
+ boolean rightCanGrow = false;
+ Map.Entry<Long, BitSetNode> rightEntry = map.ceilingEntry(procId);
+ if (rightEntry != null) {
+ rightNode = rightEntry.getValue();
+ rightCanGrow = rightNode.canGrow(procId);
+ if (leftNode != null) {
+ if (leftNode.canMerge(rightNode)) {
+ // merge left and right node
+ return mergeNodes(leftNode, rightNode);
+ }
+
+ if (leftCanGrow && rightCanGrow) {
+ if ((procId - leftNode.getEnd()) <= (rightNode.getStart() - procId)) {
+ // grow the left node
+ return growNode(leftNode, procId);
+ }
+ // grow the right node
+ return growNode(rightNode, procId);
+ }
+ }
+ }
+
+ // grow the left node
+ if (leftCanGrow) {
+ return growNode(leftNode, procId);
+ }
+
+ // grow the right node
+ if (rightCanGrow) {
+ return growNode(rightNode, procId);
+ }
+
+ // add new node
+ BitSetNode node = new BitSetNode(procId, partial);
+ map.put(node.getStart(), node);
+ return node;
+ }
+
+ private BitSetNode growNode(BitSetNode node, long procId) {
+ map.remove(node.getStart());
+ node.grow(procId);
+ map.put(node.getStart(), node);
+ return node;
+ }
+
+ private BitSetNode mergeNodes(BitSetNode leftNode, BitSetNode rightNode) {
+ leftNode.merge(rightNode);
+ map.remove(rightNode.getStart());
+ return leftNode;
+ }
+
+ public void dump() {
+ System.out.println("map " + map.size());
+ for (Map.Entry<Long, BitSetNode> entry : map.entrySet()) {
+ entry.getValue().dump();
+ }
+ }
+
+ public void writeTo(final OutputStream stream) throws IOException {
+ ProcedureProtos.ProcedureStoreTracker.Builder builder =
+ ProcedureProtos.ProcedureStoreTracker.newBuilder();
+ for (Map.Entry<Long, BitSetNode> entry : map.entrySet()) {
+ builder.addNode(entry.getValue().convert());
+ }
+ builder.build().writeDelimitedTo(stream);
+ }
+
+ public void readFrom(final InputStream stream) throws IOException {
+ ProcedureProtos.ProcedureStoreTracker data =
+ ProcedureProtos.ProcedureStoreTracker.parseDelimitedFrom(stream);
+ map.clear();
+ for (ProcedureProtos.ProcedureStoreTracker.TrackerNode protoNode: data.getNodeList()) {
+ BitSetNode node = BitSetNode.convert(protoNode);
+ map.put(node.getStart(), node);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/CorruptedWALProcedureStoreException.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/CorruptedWALProcedureStoreException.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/CorruptedWALProcedureStoreException.java
new file mode 100644
index 0000000..29db3bf
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/CorruptedWALProcedureStoreException.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2.store.wal;
+
+import org.apache.hadoop.hbase.HBaseIOException;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * Thrown when a procedure WAL is corrupted
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class CorruptedWALProcedureStoreException extends HBaseIOException {
+ /** default constructor */
+ public CorruptedWALProcedureStoreException() {
+ super();
+ }
+
+ /**
+ * Constructor
+ * @param s message
+ */
+ public CorruptedWALProcedureStoreException(String s) {
+ super(s);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFile.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFile.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFile.java
new file mode 100644
index 0000000..859b3cb
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFile.java
@@ -0,0 +1,152 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2.store.wal;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureWALHeader;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureWALTrailer;
+
+/**
+ * Describes a WAL File
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class ProcedureWALFile implements Comparable<ProcedureWALFile> {
+ private static final Log LOG = LogFactory.getLog(ProcedureWALFile.class);
+
+ private ProcedureWALHeader header;
+ private FSDataInputStream stream;
+ private FileStatus logStatus;
+ private FileSystem fs;
+ private Path logFile;
+ private long startPos;
+
+ public ProcedureWALFile(final FileSystem fs, final FileStatus logStatus) {
+ this.fs = fs;
+ this.logStatus = logStatus;
+ this.logFile = logStatus.getPath();
+ }
+
+ public ProcedureWALFile(FileSystem fs, Path logFile, ProcedureWALHeader header, long startPos) {
+ this.fs = fs;
+ this.logFile = logFile;
+ this.header = header;
+ this.startPos = startPos;
+ }
+
+ public void open() throws IOException {
+ if (stream == null) {
+ stream = fs.open(logFile);
+ }
+
+ if (header == null) {
+ header = ProcedureWALFormat.readHeader(stream);
+ startPos = stream.getPos();
+ } else {
+ stream.seek(startPos);
+ }
+ }
+
+ public ProcedureWALTrailer readTrailer() throws IOException {
+ try {
+ return ProcedureWALFormat.readTrailer(stream, startPos, logStatus.getLen());
+ } finally {
+ stream.seek(startPos);
+ }
+ }
+
+ public void readTracker(ProcedureStoreTracker tracker) throws IOException {
+ ProcedureWALTrailer trailer = readTrailer();
+ try {
+ stream.seek(trailer.getTrackerPos());
+ tracker.readFrom(stream);
+ } finally {
+ stream.seek(startPos);
+ }
+ }
+
+ public void close() {
+ if (stream == null) return;
+ try {
+ stream.close();
+ } catch (IOException e) {
+ LOG.warn("unable to close the wal file: " + logFile, e);
+ } finally {
+ stream = null;
+ }
+ }
+
+ public FSDataInputStream getStream() {
+ return stream;
+ }
+
+ public ProcedureWALHeader getHeader() {
+ return header;
+ }
+
+ public boolean isCompacted() {
+ return header.getType() == ProcedureWALFormat.LOG_TYPE_COMPACTED;
+ }
+
+ public long getLogId() {
+ return header.getLogId();
+ }
+
+ public long getSize() {
+ return logStatus.getLen();
+ }
+
+ public void removeFile() throws IOException {
+ close();
+ fs.delete(logFile, false);
+ }
+
+ @Override
+ public int compareTo(final ProcedureWALFile other) {
+ long diff = header.getLogId() - other.header.getLogId();
+ return (diff < 0) ? -1 : (diff > 0) ? 1 : 0;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof ProcedureWALFile)) return false;
+ return compareTo((ProcedureWALFile)o) == 0;
+ }
+
+ @Override
+ public int hashCode() {
+ return logFile.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return logFile.toString();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.java
new file mode 100644
index 0000000..b6b4622
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.java
@@ -0,0 +1,212 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2.store.wal;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Iterator;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.hbase.io.util.StreamUtils;
+import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker;
+import org.apache.hadoop.hbase.procedure2.util.ByteSlot;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureWALEntry;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureWALHeader;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureWALTrailer;
+
+/**
+ * Helper class that contains the WAL serialization utils.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public final class ProcedureWALFormat {
+ static final byte LOG_TYPE_STREAM = 0;
+ static final byte LOG_TYPE_COMPACTED = 1;
+ static final byte LOG_TYPE_MAX_VALID = 1;
+
+ static final byte HEADER_VERSION = 1;
+ static final byte TRAILER_VERSION = 1;
+ static final long HEADER_MAGIC = 0x31764c4157637250L;
+ static final long TRAILER_MAGIC = 0x50726357414c7631L;
+
+ interface Loader {
+ void removeLog(ProcedureWALFile log);
+ void markCorruptedWAL(ProcedureWALFile log, IOException e);
+ }
+
+ private ProcedureWALFormat() {}
+
+ public static Iterator<Procedure> load(final Iterator<ProcedureWALFile> logs,
+ final ProcedureStoreTracker tracker, final Loader loader) throws IOException {
+ ProcedureWALFormatReader reader = new ProcedureWALFormatReader(tracker);
+ tracker.setKeepDeletes(true);
+ try {
+ while (logs.hasNext()) {
+ ProcedureWALFile log = logs.next();
+ log.open();
+ try {
+ reader.read(log, loader);
+ } finally {
+ log.close();
+ }
+ }
+ // The tracker is now updated with all the procedures read from the logs
+ tracker.setPartialFlag(false);
+ tracker.resetUpdates();
+ } finally {
+ tracker.setKeepDeletes(false);
+ }
+ // TODO: Write compacted version?
+ return reader.getProcedures();
+ }
+
+ public static void writeHeader(OutputStream stream, ProcedureWALHeader header)
+ throws IOException {
+ header.writeDelimitedTo(stream);
+ }
+
+ /*
+ * +-----------------+
+ * | END OF WAL DATA | <---+
+ * +-----------------+ |
+ * | | |
+ * | Tracker | |
+ * | | |
+ * +-----------------+ |
+ * | version | |
+ * +-----------------+ |
+ * | TRAILER_MAGIC | |
+ * +-----------------+ |
+ * | offset |-----+
+ * +-----------------+
+ */
+ public static void writeTrailer(FSDataOutputStream stream, ProcedureStoreTracker tracker)
+ throws IOException {
+ long offset = stream.getPos();
+
+ // Write EOF Entry
+ ProcedureWALEntry.newBuilder()
+ .setType(ProcedureWALEntry.Type.EOF)
+ .build().writeDelimitedTo(stream);
+
+ // Write Tracker
+ tracker.writeTo(stream);
+
+ stream.write(TRAILER_VERSION);
+ StreamUtils.writeLong(stream, TRAILER_MAGIC);
+ StreamUtils.writeLong(stream, offset);
+ }
+
+ public static ProcedureWALHeader readHeader(InputStream stream)
+ throws IOException {
+ ProcedureWALHeader header = ProcedureWALHeader.parseDelimitedFrom(stream);
+
+ if (header.getVersion() < 0 || header.getVersion() != HEADER_VERSION) {
+ throw new IOException("Invalid Header version. got " + header.getVersion() +
+ " expected " + HEADER_VERSION);
+ }
+
+ if (header.getType() < 0 || header.getType() > LOG_TYPE_MAX_VALID) {
+ throw new IOException("Invalid header type. got " + header.getType());
+ }
+
+ return header;
+ }
+
+ public static ProcedureWALTrailer readTrailer(FSDataInputStream stream, long startPos, long size)
+ throws IOException {
+ long trailerPos = size - 17; // Beginning of the Trailer Jump
+
+ if (trailerPos < startPos) {
+ throw new IOException("Missing trailer: size=" + size + " startPos=" + startPos);
+ }
+
+ stream.seek(trailerPos);
+ int version = stream.read();
+ if (version != TRAILER_VERSION) {
+ throw new IOException("Invalid Trailer version. got " + version +
+ " expected " + TRAILER_VERSION);
+ }
+
+ long magic = StreamUtils.readLong(stream);
+ if (magic != TRAILER_MAGIC) {
+ throw new IOException("Invalid Trailer magic. got " + magic +
+ " expected " + TRAILER_MAGIC);
+ }
+
+ long trailerOffset = StreamUtils.readLong(stream);
+ stream.seek(trailerOffset);
+
+ ProcedureWALEntry entry = readEntry(stream);
+ if (entry.getType() != ProcedureWALEntry.Type.EOF) {
+ throw new IOException("Invalid Trailer begin");
+ }
+
+ ProcedureWALTrailer trailer = ProcedureWALTrailer.newBuilder()
+ .setVersion(version)
+ .setTrackerPos(stream.getPos())
+ .build();
+ return trailer;
+ }
+
+ public static ProcedureWALEntry readEntry(InputStream stream) throws IOException {
+ return ProcedureWALEntry.parseDelimitedFrom(stream);
+ }
+
+ public static void writeEntry(ByteSlot slot, ProcedureWALEntry.Type type,
+ Procedure proc, Procedure[] subprocs) throws IOException {
+ ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();
+ builder.setType(type);
+ builder.addProcedure(Procedure.convert(proc));
+ if (subprocs != null) {
+ for (int i = 0; i < subprocs.length; ++i) {
+ builder.addProcedure(Procedure.convert(subprocs[i]));
+ }
+ }
+ builder.build().writeDelimitedTo(slot);
+ }
+
+ public static void writeInsert(ByteSlot slot, Procedure proc)
+ throws IOException {
+ writeEntry(slot, ProcedureWALEntry.Type.INIT, proc, null);
+ }
+
+ public static void writeInsert(ByteSlot slot, Procedure proc, Procedure[] subprocs)
+ throws IOException {
+ writeEntry(slot, ProcedureWALEntry.Type.INSERT, proc, subprocs);
+ }
+
+ public static void writeUpdate(ByteSlot slot, Procedure proc)
+ throws IOException {
+ writeEntry(slot, ProcedureWALEntry.Type.UPDATE, proc, null);
+ }
+
+ public static void writeDelete(ByteSlot slot, long procId)
+ throws IOException {
+ ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();
+ builder.setType(ProcedureWALEntry.Type.DELETE);
+ builder.setProcId(procId);
+ builder.build().writeDelimitedTo(slot);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.java
new file mode 100644
index 0000000..66c374e
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.java
@@ -0,0 +1,161 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2.store.wal;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.HashMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureWALEntry;
+
+/**
+ * Helper class that loads the procedures stored in a WAL
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class ProcedureWALFormatReader {
+ private static final Log LOG = LogFactory.getLog(ProcedureWALFormatReader.class);
+
+ private final ProcedureStoreTracker tracker;
+ //private final long compactionLogId;
+
+ private final Map<Long, Procedure> procedures = new HashMap<Long, Procedure>();
+ private final Map<Long, ProcedureProtos.Procedure> localProcedures =
+ new HashMap<Long, ProcedureProtos.Procedure>();
+
+ private long maxProcId = 0;
+
+ public ProcedureWALFormatReader(final ProcedureStoreTracker tracker) {
+ this.tracker = tracker;
+ }
+
+ public void read(ProcedureWALFile log, ProcedureWALFormat.Loader loader) throws IOException {
+ FSDataInputStream stream = log.getStream();
+ try {
+ boolean hasMore = true;
+ while (hasMore) {
+ ProcedureWALEntry entry = ProcedureWALFormat.readEntry(stream);
+ switch (entry.getType()) {
+ case INIT:
+ readInitEntry(entry);
+ break;
+ case INSERT:
+ readInsertEntry(entry);
+ break;
+ case UPDATE:
+ case COMPACT:
+ readUpdateEntry(entry);
+ break;
+ case DELETE:
+ readDeleteEntry(entry);
+ break;
+ case EOF:
+ hasMore = false;
+ break;
+ default:
+ throw new CorruptedWALProcedureStoreException("Invalid entry: " + entry);
+ }
+ }
+ } catch (IOException e) {
+ LOG.error("got an exception while reading the procedure WAL: " + log, e);
+ loader.markCorruptedWAL(log, e);
+ }
+
+ if (localProcedures.isEmpty()) {
+ LOG.info("No active entry found in state log " + log + ". removing it");
+ loader.removeLog(log);
+ } else {
+ Iterator<Map.Entry<Long, ProcedureProtos.Procedure>> itd =
+ localProcedures.entrySet().iterator();
+ while (itd.hasNext()) {
+ Map.Entry<Long, ProcedureProtos.Procedure> entry = itd.next();
+ itd.remove();
+
+ // Deserialize the procedure
+ Procedure proc = Procedure.convert(entry.getValue());
+ procedures.put(entry.getKey(), proc);
+ }
+
+ // TODO: Some procedure may be already runnables (see readInitEntry())
+ // (we can also check the "update map" in the log trackers)
+ }
+ }
+
+ public Iterator<Procedure> getProcedures() {
+ return procedures.values().iterator();
+ }
+
+ private void loadEntries(final ProcedureWALEntry entry) {
+ for (ProcedureProtos.Procedure proc: entry.getProcedureList()) {
+ maxProcId = Math.max(maxProcId, proc.getProcId());
+ if (isRequired(proc.getProcId())) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("read " + entry.getType() + " entry " + proc.getProcId());
+ }
+ localProcedures.put(proc.getProcId(), proc);
+ tracker.setDeleted(proc.getProcId(), false);
+ }
+ }
+ }
+
+ private void readInitEntry(final ProcedureWALEntry entry)
+ throws IOException {
+ assert entry.getProcedureCount() == 1 : "Expected only one procedure";
+ // TODO: Make it runnable, before reading other files
+ loadEntries(entry);
+ }
+
+ private void readInsertEntry(final ProcedureWALEntry entry) throws IOException {
+ assert entry.getProcedureCount() >= 1 : "Expected one or more procedures";
+ loadEntries(entry);
+ }
+
+ private void readUpdateEntry(final ProcedureWALEntry entry) throws IOException {
+ assert entry.getProcedureCount() == 1 : "Expected only one procedure";
+ loadEntries(entry);
+ }
+
+ private void readDeleteEntry(final ProcedureWALEntry entry) throws IOException {
+ assert entry.getProcedureCount() == 0 : "Expected no procedures";
+ assert entry.hasProcId() : "expected ProcID";
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("read delete entry " + entry.getProcId());
+ }
+ maxProcId = Math.max(maxProcId, entry.getProcId());
+ localProcedures.remove(entry.getProcId());
+ tracker.setDeleted(entry.getProcId(), true);
+ }
+
+ private boolean isDeleted(final long procId) {
+ return tracker.isDeleted(procId) == ProcedureStoreTracker.DeleteState.YES;
+ }
+
+ private boolean isRequired(final long procId) {
+ return !isDeleted(procId) && !procedures.containsKey(procId);
+ }
+}
\ No newline at end of file
[3/9] hbase git commit: HBASE-13315 BufferedMutator should be
@InterfaceAudience.Public (addendum)
Posted by mb...@apache.org.
HBASE-13315 BufferedMutator should be @InterfaceAudience.Public (addendum)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5459008c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5459008c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5459008c
Branch: refs/heads/hbase-12439
Commit: 5459008ccdc83f09e6d8d7ec90144dc120e1a084
Parents: 6e9ded5
Author: Nick Dimiduk <nd...@apache.org>
Authored: Mon Mar 23 16:30:25 2015 -0700
Committer: Nick Dimiduk <nd...@apache.org>
Committed: Mon Mar 23 16:38:32 2015 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hbase/client/BufferedMutator.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/5459008c/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BufferedMutator.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BufferedMutator.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BufferedMutator.java
index 8d3d33c..4424cec 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BufferedMutator.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BufferedMutator.java
@@ -122,6 +122,8 @@ public interface BufferedMutator extends Closeable {
/**
* Listens for asynchronous exceptions on a {@link BufferedMutator}.
*/
+ @InterfaceAudience.Public
+ @InterfaceStability.Evolving
interface ExceptionListener {
public void onException(RetriesExhaustedWithDetailsException exception,
BufferedMutator mutator) throws RetriesExhaustedWithDetailsException;
[4/9] hbase git commit: HBASE-13202 Procedure v2 - core framework
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-protocol/src/main/protobuf/Procedure.proto
----------------------------------------------------------------------
diff --git a/hbase-protocol/src/main/protobuf/Procedure.proto b/hbase-protocol/src/main/protobuf/Procedure.proto
new file mode 100644
index 0000000..e48afa9
--- /dev/null
+++ b/hbase-protocol/src/main/protobuf/Procedure.proto
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "ProcedureProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "ErrorHandling.proto";
+
+enum ProcedureState {
+ INITIALIZING = 1; // Procedure in construction, not yet added to the executor
+ RUNNABLE = 2; // Procedure added to the executor, and ready to be executed
+ WAITING = 3; // The procedure is waiting on children to be completed
+ WAITING_TIMEOUT = 4; // The procedure is waiting a timout or an external event
+ ROLLEDBACK = 5; // The procedure failed and was rolledback
+ FINISHED = 6; // The procedure completed successfully
+}
+
+/**
+ * Procedure metadata, serialized by the ProcedureStore to be able to recover the old state.
+ */
+message Procedure {
+ // internal "static" state
+ required string class_name = 1; // full classname to be able to instantiate the procedure
+ optional uint64 parent_id = 2; // parent if not a root-procedure otherwise not set
+ required uint64 proc_id = 3;
+ required uint64 start_time = 4;
+ optional string owner = 5;
+
+ // internal "runtime" state
+ required ProcedureState state = 6;
+ repeated uint32 stack_id = 7; // stack indices in case the procedure was running
+ required uint64 last_update = 8;
+ optional uint32 timeout = 9;
+
+ // user state/results
+ optional ForeignExceptionMessage exception = 10;
+ optional bytes result = 11; // opaque (user) result structure
+ optional bytes state_data = 12; // opaque (user) procedure internal-state
+}
+
+/**
+ * SequentialProcedure data
+ */
+message SequentialProcedureData {
+ required bool executed = 1;
+}
+
+/**
+ * StateMachineProcedure data
+ */
+message StateMachineProcedureData {
+ repeated uint32 state = 1;
+}
+
+/**
+ * Procedure WAL header
+ */
+message ProcedureWALHeader {
+ required uint32 version = 1;
+ required uint32 type = 2;
+ required uint64 log_id = 3;
+ required uint64 min_proc_id = 4;
+}
+
+/**
+ * Procedure WAL trailer
+ */
+message ProcedureWALTrailer {
+ required uint32 version = 1;
+ required uint64 tracker_pos = 2;
+}
+
+message ProcedureStoreTracker {
+ message TrackerNode {
+ required uint64 start_id = 1;
+ repeated uint64 updated = 2;
+ repeated uint64 deleted = 3;
+ }
+
+ repeated TrackerNode node = 1;
+}
+
+message ProcedureWALEntry {
+ enum Type {
+ EOF = 1;
+ INIT = 2;
+ INSERT = 3;
+ UPDATE = 4;
+ DELETE = 5;
+ COMPACT = 6;
+ }
+
+ required Type type = 1;
+ repeated Procedure procedure = 2;
+ optional uint64 proc_id = 3;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 3f94abc..69f31ee 100644
--- a/pom.xml
+++ b/pom.xml
@@ -56,6 +56,7 @@
<module>hbase-client</module>
<module>hbase-hadoop-compat</module>
<module>hbase-common</module>
+ <module>hbase-procedure</module>
<module>hbase-it</module>
<module>hbase-examples</module>
<module>hbase-prefix-tree</module>
@@ -872,7 +873,7 @@
</fileMapper>
</fileMappers>
<outputDir>${basedir}/target/asciidoc</outputDir>
- </transformationSet>
+ </transformationSet>
</transformationSets>
</configuration>
</plugin>
@@ -1015,7 +1016,7 @@
<plugin>
<groupId>org.asciidoctor</groupId>
<artifactId>asciidoctor-maven-plugin</artifactId>
- <version>1.5.2</version>
+ <version>1.5.2</version>
<inherited>false</inherited>
<dependencies>
<dependency>
@@ -1035,10 +1036,10 @@
</configuration>
<executions>
<execution>
- <id>output-html</id>
+ <id>output-html</id>
<phase>site</phase>
<goals>
- <goal>process-asciidoc</goal>
+ <goal>process-asciidoc</goal>
</goals>
<configuration>
<attributes>
@@ -1192,6 +1193,7 @@
Modules are pretty heavy-weight things, so doing this work isn't too bad. -->
<server.test.jar>hbase-server-${project.version}-tests.jar</server.test.jar>
<common.test.jar>hbase-common-${project.version}-tests.jar</common.test.jar>
+ <procedure.test.jar>hbase-procedure-${project.version}-tests.jar</procedure.test.jar>
<it.test.jar>hbase-it-${project.version}-tests.jar</it.test.jar>
<annotations.test.jar>hbase-annotations-${project.version}-tests.jar</annotations.test.jar>
<surefire.version>2.18</surefire.version>
@@ -1263,6 +1265,17 @@
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-procedure</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-procedure</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
<artifactId>hbase-hadoop-compat</artifactId>
<version>${project.version}</version>
</dependency>
[7/9] hbase git commit: HBASE-13202 Procedure v2 - core framework
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java
new file mode 100644
index 0000000..c2a5571
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java
@@ -0,0 +1,683 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2.store.wal;
+
+import java.io.IOException;
+import java.io.FileNotFoundException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureStore;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker;
+import org.apache.hadoop.hbase.procedure2.util.ByteSlot;
+import org.apache.hadoop.hbase.procedure2.util.StringUtils;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureWALHeader;
+
+/**
+ * WAL implementation of the ProcedureStore.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class WALProcedureStore implements ProcedureStore {
+ private static final Log LOG = LogFactory.getLog(WALProcedureStore.class);
+
+ public interface LeaseRecovery {
+ void recoverFileLease(FileSystem fs, Path path) throws IOException;
+ }
+
+ private static final int MAX_RETRIES_BEFORE_ABORT = 3;
+
+ private static final String SYNC_WAIT_MSEC_CONF_KEY = "hbase.procedure.store.wal.sync.wait.msec";
+ private static final int DEFAULT_SYNC_WAIT_MSEC = 50;
+
+ private final CopyOnWriteArrayList<ProcedureStoreListener> listeners =
+ new CopyOnWriteArrayList<ProcedureStoreListener>();
+
+ private final LinkedList<ProcedureWALFile> logs = new LinkedList<ProcedureWALFile>();
+ private final ProcedureStoreTracker storeTracker = new ProcedureStoreTracker();
+ private final AtomicBoolean running = new AtomicBoolean(false);
+ private final ReentrantLock lock = new ReentrantLock();
+ private final Condition waitCond = lock.newCondition();
+ private final Condition slotCond = lock.newCondition();
+ private final Condition syncCond = lock.newCondition();
+
+ private final LeaseRecovery leaseRecovery;
+ private final Configuration conf;
+ private final FileSystem fs;
+ private final Path logDir;
+
+ private AtomicBoolean inSync = new AtomicBoolean(false);
+ private ArrayBlockingQueue<ByteSlot> slotsCache = null;
+ private Set<ProcedureWALFile> corruptedLogs = null;
+ private FSDataOutputStream stream = null;
+ private long totalSynced = 0;
+ private long flushLogId = 0;
+ private int slotIndex = 0;
+ private Thread syncThread;
+ private ByteSlot[] slots;
+ private int syncWaitMsec;
+
+ public WALProcedureStore(final Configuration conf, final FileSystem fs, final Path logDir,
+ final LeaseRecovery leaseRecovery) {
+ this.fs = fs;
+ this.conf = conf;
+ this.logDir = logDir;
+ this.leaseRecovery = leaseRecovery;
+ }
+
+ @Override
+ public void start(int numSlots) throws IOException {
+ if (running.getAndSet(true)) {
+ return;
+ }
+
+ // Init buffer slots
+ slots = new ByteSlot[numSlots];
+ slotsCache = new ArrayBlockingQueue(numSlots, true);
+ while (slotsCache.remainingCapacity() > 0) {
+ slotsCache.offer(new ByteSlot());
+ }
+
+ // Tunings
+ syncWaitMsec = conf.getInt(SYNC_WAIT_MSEC_CONF_KEY, DEFAULT_SYNC_WAIT_MSEC);
+
+ // Init sync thread
+ syncThread = new Thread() {
+ @Override
+ public void run() {
+ while (running.get()) {
+ try {
+ syncLoop();
+ } catch (IOException e) {
+ LOG.error("got an exception from the sync-loop", e);
+ sendAbortProcessSignal();
+ }
+ }
+ }
+ };
+ syncThread.start();
+ }
+
+ @Override
+ public void stop() {
+ if (!running.getAndSet(false)) {
+ return;
+ }
+
+ LOG.info("Stopping the WAL Procedure Store");
+
+ lock.lock();
+ try {
+ waitCond.signalAll();
+ } finally {
+ lock.unlock();
+ }
+
+ try {
+ syncThread.join();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+
+ // Close the writer
+ closeStream();
+
+ // Close the old logs
+ // they should be already closed, this is just in case the load fails
+ // and we call start() and then stop()
+ for (ProcedureWALFile log: logs) {
+ log.close();
+ }
+ logs.clear();
+ }
+
+ @Override
+ public void registerListener(ProcedureStoreListener listener) {
+ this.listeners.add(listener);
+ }
+
+ @Override
+ public boolean unregisterListener(ProcedureStoreListener listener) {
+ return this.listeners.remove(listener);
+ }
+
+ @Override
+ public void recoverLease() throws IOException {
+ LOG.info("Starting WAL Procedure Store lease recovery");
+ FileStatus[] oldLogs = getLogFiles();
+ while (running.get()) {
+ // Get Log-MaxID and recover lease on old logs
+ flushLogId = initOldLogs(oldLogs) + 1;
+
+ // Create new state-log
+ if (!rollWriter(flushLogId)) {
+ // someone else has already created this log
+ continue;
+ }
+
+ // We have the lease on the log
+ oldLogs = getLogFiles();
+ if (getMaxLogId(oldLogs) > flushLogId) {
+ // Someone else created new logs
+ logs.getLast().removeFile();
+ continue;
+ }
+
+ LOG.info("lease acquired flushLogId=" + flushLogId);
+ break;
+ }
+ }
+
+ @Override
+ public Iterator<Procedure> load() throws IOException {
+ if (logs.isEmpty()) {
+ throw new RuntimeException("recoverLease() must be called before loading data");
+ }
+
+ // Nothing to do, If we have only the current log.
+ if (logs.size() == 1) {
+ LOG.debug("No state logs to replay");
+ return null;
+ }
+
+ // Load the old logs
+ final ArrayList<ProcedureWALFile> toRemove = new ArrayList<ProcedureWALFile>();
+ Iterator<ProcedureWALFile> it = logs.descendingIterator();
+ it.next(); // Skip the current log
+ try {
+ return ProcedureWALFormat.load(it, storeTracker, new ProcedureWALFormat.Loader() {
+ @Override
+ public void removeLog(ProcedureWALFile log) {
+ toRemove.add(log);
+ }
+
+ @Override
+ public void markCorruptedWAL(ProcedureWALFile log, IOException e) {
+ if (corruptedLogs == null) {
+ corruptedLogs = new HashSet<ProcedureWALFile>();
+ }
+ corruptedLogs.add(log);
+ // TODO: sideline corrupted log
+ }
+ });
+ } finally {
+ if (!toRemove.isEmpty()) {
+ for (ProcedureWALFile log: toRemove) {
+ removeLogFile(log);
+ }
+ }
+ }
+ }
+
+ @Override
+ public void insert(final Procedure proc, final Procedure[] subprocs) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("insert " + proc + " subproc=" + Arrays.toString(subprocs));
+ }
+
+ ByteSlot slot = acquireSlot();
+ long logId = -1;
+ try {
+ // Serialize the insert
+ if (subprocs != null) {
+ ProcedureWALFormat.writeInsert(slot, proc, subprocs);
+ } else {
+ assert !proc.hasParent();
+ ProcedureWALFormat.writeInsert(slot, proc);
+ }
+
+ // Push the transaction data and wait until it is persisted
+ logId = pushData(slot);
+ } catch (IOException e) {
+ // We are not able to serialize the procedure.
+ // this is a code error, and we are not able to go on.
+ LOG.fatal("Unable to serialize one of the procedure: proc=" + proc +
+ " subprocs=" + Arrays.toString(subprocs), e);
+ throw new RuntimeException(e);
+ } finally {
+ releaseSlot(slot);
+ }
+
+ // Update the store tracker
+ synchronized (storeTracker) {
+ if (logId == flushLogId) {
+ storeTracker.insert(proc, subprocs);
+ }
+ }
+ }
+
+ @Override
+ public void update(final Procedure proc) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("update " + proc);
+ }
+
+ ByteSlot slot = acquireSlot();
+ long logId = -1;
+ try {
+ // Serialize the update
+ ProcedureWALFormat.writeUpdate(slot, proc);
+
+ // Push the transaction data and wait until it is persisted
+ logId = pushData(slot);
+ } catch (IOException e) {
+ // We are not able to serialize the procedure.
+ // this is a code error, and we are not able to go on.
+ LOG.fatal("Unable to serialize the procedure: " + proc, e);
+ throw new RuntimeException(e);
+ } finally {
+ releaseSlot(slot);
+ }
+
+ // Update the store tracker
+ boolean removeOldLogs = false;
+ synchronized (storeTracker) {
+ if (logId == flushLogId) {
+ storeTracker.update(proc);
+ removeOldLogs = storeTracker.isUpdated();
+ }
+ }
+
+ if (removeOldLogs) {
+ removeAllLogs(logId - 1);
+ }
+ }
+
+ @Override
+ public void delete(final long procId) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("delete " + procId);
+ }
+
+ ByteSlot slot = acquireSlot();
+ long logId = -1;
+ try {
+ // Serialize the delete
+ ProcedureWALFormat.writeDelete(slot, procId);
+
+ // Push the transaction data and wait until it is persisted
+ logId = pushData(slot);
+ } catch (IOException e) {
+ // We are not able to serialize the procedure.
+ // this is a code error, and we are not able to go on.
+ LOG.fatal("Unable to serialize the procedure: " + procId, e);
+ throw new RuntimeException(e);
+ } finally {
+ releaseSlot(slot);
+ }
+
+ boolean removeOldLogs = false;
+ synchronized (storeTracker) {
+ if (logId == flushLogId) {
+ storeTracker.delete(procId);
+ if (storeTracker.isEmpty()) {
+ removeOldLogs = rollWriterOrDie(logId + 1);
+ }
+ }
+ }
+
+ if (removeOldLogs) {
+ removeAllLogs(logId);
+ }
+ }
+
+ private ByteSlot acquireSlot() {
+ ByteSlot slot = slotsCache.poll();
+ return slot != null ? slot : new ByteSlot();
+ }
+
+ private void releaseSlot(final ByteSlot slot) {
+ slot.reset();
+ slotsCache.offer(slot);
+ }
+
+ private long pushData(final ByteSlot slot) {
+ assert !logs.isEmpty() : "recoverLease() must be called before inserting data";
+ long logId = -1;
+
+ lock.lock();
+ try {
+ // Wait for the sync to be completed
+ while (true) {
+ if (inSync.get()) {
+ syncCond.await();
+ } else if (slotIndex == slots.length) {
+ slotCond.signal();
+ syncCond.await();
+ } else {
+ break;
+ }
+ }
+
+ slots[slotIndex++] = slot;
+ logId = flushLogId;
+
+ // Notify that there is new data
+ if (slotIndex == 1) {
+ waitCond.signal();
+ }
+
+ // Notify that the slots are full
+ if (slotIndex == slots.length) {
+ slotCond.signal();
+ }
+ syncCond.await();
+ } catch (InterruptedException e) {
+ sendAbortProcessSignal();
+ } finally {
+ lock.unlock();
+ }
+ return logId;
+ }
+
+ private void syncLoop() throws IOException {
+ inSync.set(false);
+ while (running.get()) {
+ lock.lock();
+ try {
+ // Wait until new data is available
+ if (slotIndex == 0) {
+ LOG.debug("Waiting for data. flushed=" + StringUtils.humanSize(totalSynced));
+ waitCond.await();
+ if (slotIndex == 0) {
+ // no data.. probably a stop()
+ continue;
+ }
+ }
+
+ // Wait SYNC_WAIT_MSEC or the signal of "slots full" before flushing
+ slotCond.await(syncWaitMsec, TimeUnit.MILLISECONDS);
+
+ inSync.set(true);
+ totalSynced += syncSlots();
+ slotIndex = 0;
+ inSync.set(false);
+ syncCond.signalAll();
+ } catch (InterruptedException e) {
+ sendAbortProcessSignal();
+ } finally {
+ lock.unlock();
+ }
+ }
+ }
+
+ private long syncSlots() {
+ int retry = 0;
+ long totalSynced = 0;
+ do {
+ try {
+ totalSynced = syncSlots(stream, slots, 0, slotIndex);
+ break;
+ } catch (IOException e) {
+ if (++retry == MAX_RETRIES_BEFORE_ABORT) {
+ LOG.error("sync slot failed, abort.", e);
+ sendAbortProcessSignal();
+ }
+ }
+ } while (running.get());
+ return totalSynced;
+ }
+
+ protected long syncSlots(FSDataOutputStream stream, ByteSlot[] slots, int offset, int count)
+ throws IOException {
+ long totalSynced = 0;
+ for (int i = 0; i < count; ++i) {
+ ByteSlot data = slots[offset + i];
+ data.writeTo(stream);
+ totalSynced += data.size();
+ }
+ stream.hsync();
+ return totalSynced;
+ }
+
+ private void sendAbortProcessSignal() {
+ if (!this.listeners.isEmpty()) {
+ for (ProcedureStoreListener listener : this.listeners) {
+ listener.abortProcess();
+ }
+ }
+ }
+
+ private boolean rollWriterOrDie(final long logId) {
+ try {
+ return rollWriter(logId);
+ } catch (IOException e) {
+ LOG.warn("Unable to roll the log", e);
+ sendAbortProcessSignal();
+ return false;
+ }
+ }
+
+ private boolean rollWriter(final long logId) throws IOException {
+ ProcedureWALHeader header = ProcedureWALHeader.newBuilder()
+ .setVersion(ProcedureWALFormat.HEADER_VERSION)
+ .setType(ProcedureWALFormat.LOG_TYPE_STREAM)
+ .setMinProcId(storeTracker.getMinProcId())
+ .setLogId(logId)
+ .build();
+
+ FSDataOutputStream newStream = null;
+ Path newLogFile = null;
+ long startPos = -1;
+ try {
+ newLogFile = getLogFilePath(logId);
+ newStream = fs.create(newLogFile, false);
+ ProcedureWALFormat.writeHeader(newStream, header);
+ startPos = newStream.getPos();
+ } catch (FileAlreadyExistsException e) {
+ LOG.error("Log file with id=" + logId + " already exists", e);
+ return false;
+ }
+ lock.lock();
+ try {
+ closeStream();
+ synchronized (storeTracker) {
+ storeTracker.resetUpdates();
+ }
+ stream = newStream;
+ flushLogId = logId;
+ totalSynced = 0;
+ logs.add(new ProcedureWALFile(fs, newLogFile, header, startPos));
+ } finally {
+ lock.unlock();
+ }
+ LOG.info("Roll new state log: " + logId);
+ return true;
+ }
+
+ private void closeStream() {
+ try {
+ if (stream != null) {
+ try {
+ ProcedureWALFormat.writeTrailer(stream, storeTracker);
+ } catch (IOException e) {
+ LOG.error("Unable to write the trailer", e);
+ }
+ stream.close();
+ }
+ } catch (IOException e) {
+ LOG.error("Unable to close the stream", e);
+ } finally {
+ stream = null;
+ }
+ }
+
+ private void removeAllLogs(long lastLogId) {
+ LOG.info("Remove all state logs with ID less then " + lastLogId);
+ while (!logs.isEmpty()) {
+ ProcedureWALFile log = logs.getFirst();
+ if (lastLogId < log.getLogId()) {
+ break;
+ }
+
+ removeLogFile(log);
+ }
+ }
+
+ private boolean removeLogFile(final ProcedureWALFile log) {
+ try {
+ LOG.debug("remove log: " + log);
+ log.removeFile();
+ logs.remove(log);
+ } catch (IOException e) {
+ LOG.error("unable to remove log " + log, e);
+ return false;
+ }
+ return true;
+ }
+
+ public Set<ProcedureWALFile> getCorruptedLogs() {
+ return corruptedLogs;
+ }
+
+ // ==========================================================================
+ // FileSystem Log Files helpers
+ // ==========================================================================
+
+ protected Path getLogFilePath(final long logId) throws IOException {
+ return new Path(logDir, String.format("state-%020d.log", logId));
+ }
+
+ private static long getLogIdFromName(final String name) {
+ int end = name.lastIndexOf(".log");
+ int start = name.lastIndexOf('-') + 1;
+ while (start < end) {
+ if (name.charAt(start) != '0')
+ break;
+ start++;
+ }
+ return Long.parseLong(name.substring(start, end));
+ }
+
+ private FileStatus[] getLogFiles() throws IOException {
+ try {
+ return fs.listStatus(logDir, new PathFilter() {
+ @Override
+ public boolean accept(Path path) {
+ String name = path.getName();
+ return name.startsWith("state-") && name.endsWith(".log");
+ }
+ });
+ } catch (FileNotFoundException e) {
+ LOG.warn("log directory not found: " + e.getMessage());
+ return null;
+ }
+ }
+
+ private long getMaxLogId(final FileStatus[] logFiles) {
+ long maxLogId = 0;
+ if (logFiles != null && logFiles.length > 0) {
+ for (int i = 0; i < logFiles.length; ++i) {
+ maxLogId = Math.max(maxLogId, getLogIdFromName(logFiles[i].getPath().getName()));
+ }
+ }
+ return maxLogId;
+ }
+
+ /**
+ * @return Max-LogID of the specified log file set
+ */
+ private long initOldLogs(final FileStatus[] logFiles) throws IOException {
+ this.logs.clear();
+
+ long maxLogId = 0;
+ if (logFiles != null && logFiles.length > 0) {
+ for (int i = 0; i < logFiles.length; ++i) {
+ final Path logPath = logFiles[i].getPath();
+ leaseRecovery.recoverFileLease(fs, logPath);
+ maxLogId = Math.max(maxLogId, getLogIdFromName(logPath.getName()));
+
+ ProcedureWALFile log = initOldLog(logFiles[i]);
+ if (log != null) {
+ this.logs.add(log);
+ }
+ }
+ Collections.sort(this.logs);
+ initTrackerFromOldLogs();
+ }
+ return maxLogId;
+ }
+
+ private void initTrackerFromOldLogs() {
+ // TODO: Load the most recent tracker available
+ if (!logs.isEmpty()) {
+ ProcedureWALFile log = logs.getLast();
+ try {
+ log.readTracker(storeTracker);
+ } catch (IOException e) {
+ LOG.error("Unable to read tracker for " + log, e);
+ // try the next one...
+ storeTracker.clear();
+ storeTracker.setPartialFlag(true);
+ }
+ }
+ }
+
+ private ProcedureWALFile initOldLog(final FileStatus logFile) throws IOException {
+ ProcedureWALFile log = new ProcedureWALFile(fs, logFile);
+ try {
+ log.open();
+ LOG.debug("opening state-log: " + log);
+ if (log.isCompacted()) {
+ try {
+ log.readTrailer();
+ } catch (IOException e) {
+ // unfinished compacted log throw it away
+ LOG.warn("Unfinished compacted log " + logFile, e);
+ log.removeFile();
+ return null;
+ }
+ }
+ } catch (IOException e) {
+ if (logFile.getLen() == 0) {
+ LOG.warn("Remove uninitialized log " + logFile, e);
+ log.removeFile();
+ return null;
+ }
+
+ String msg = "Unable to read state log " + logFile;
+ LOG.error(msg, e);
+ throw new IOException(msg, e);
+ }
+ return log;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/ByteSlot.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/ByteSlot.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/ByteSlot.java
new file mode 100644
index 0000000..8904116
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/ByteSlot.java
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2.util;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Arrays;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * Similar to the ByteArrayOutputStream, with the exception that we can prepend an header.
+ * e.g. you write some data and you want to prepend an header that contains the data len or cksum.
+ * <code>
+ * ByteSlot slot = new ByteSlot();
+ * // write data
+ * slot.write(...);
+ * slot.write(...);
+ * // write header with the size of the written data
+ * slot.markHead();
+ * slot.write(Bytes.toBytes(slot.size()));
+ * // flush to stream as [header, data]
+ * slot.writeTo(stream);
+ * </code>
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class ByteSlot extends OutputStream {
+ private static final int DOUBLE_GROW_LIMIT = 1 << 20;
+ private static final int GROW_ALIGN = 128;
+
+ private byte[] buf;
+ private int head;
+ private int size;
+
+ public void reset() {
+ head = 0;
+ size = 0;
+ }
+
+ public void markHead() {
+ head = size;
+ }
+
+ public int getHead() {
+ return head;
+ }
+
+ public int size() {
+ return size;
+ }
+
+ public byte[] getBuffer() {
+ return buf;
+ }
+
+ public void writeAt(int offset, int b) {
+ head = Math.min(head, offset);
+ buf[offset] = (byte)b;
+ }
+
+ public void write(int b) {
+ ensureCapacity(size + 1);
+ buf[size++] = (byte)b;
+ }
+
+ public void write(byte[] b, int off, int len) {
+ ensureCapacity(size + len);
+ System.arraycopy(b, off, buf, size, len);
+ size += len;
+ }
+
+ public void writeTo(final OutputStream stream) throws IOException {
+ if (head != 0) {
+ stream.write(buf, head, size - head);
+ stream.write(buf, 0, head);
+ } else {
+ stream.write(buf, 0, size);
+ }
+ }
+
+ private void ensureCapacity(int minCapacity) {
+ minCapacity = (minCapacity + (GROW_ALIGN - 1)) & -GROW_ALIGN;
+ if (buf == null) {
+ buf = new byte[minCapacity];
+ } else if (minCapacity > buf.length) {
+ int newCapacity = buf.length << 1;
+ if (minCapacity > newCapacity || newCapacity > DOUBLE_GROW_LIMIT) {
+ newCapacity = minCapacity;
+ }
+ buf = Arrays.copyOf(buf, newCapacity);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/StringUtils.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/StringUtils.java
new file mode 100644
index 0000000..97134c2
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/StringUtils.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2.util;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public final class StringUtils {
+ private StringUtils() {}
+
+ public static String humanTimeDiff(long timeDiff) {
+ StringBuilder buf = new StringBuilder();
+ long hours = timeDiff / (60*60*1000);
+ long rem = (timeDiff % (60*60*1000));
+ long minutes = rem / (60*1000);
+ rem = rem % (60*1000);
+ float seconds = rem / 1000.0f;
+
+ if (hours != 0){
+ buf.append(hours);
+ buf.append("hrs, ");
+ }
+ if (minutes != 0){
+ buf.append(minutes);
+ buf.append("mins, ");
+ }
+ if (hours > 0 || minutes > 0) {
+ buf.append(seconds);
+ buf.append("sec");
+ } else {
+ buf.append(String.format("%.4fsec", seconds));
+ }
+ return buf.toString();
+ }
+
+ public static String humanSize(double size) {
+ if (size >= (1L << 40)) return String.format("%.1fT", size / (1L << 40));
+ if (size >= (1L << 30)) return String.format("%.1fG", size / (1L << 30));
+ if (size >= (1L << 20)) return String.format("%.1fM", size / (1L << 20));
+ if (size >= (1L << 10)) return String.format("%.1fK", size / (1L << 10));
+ return String.format("%.0f", size);
+ }
+
+ public static boolean isEmpty(final String input) {
+ return input == null || input.length() == 0;
+ }
+
+ public static String buildString(final String... parts) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < parts.length; ++i) {
+ sb.append(parts[i]);
+ }
+ return sb.toString();
+ }
+
+ public static StringBuilder appendStrings(final StringBuilder sb, final String... parts) {
+ for (int i = 0; i < parts.length; ++i) {
+ sb.append(parts[i]);
+ }
+ return sb;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/TimeoutBlockingQueue.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/TimeoutBlockingQueue.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/TimeoutBlockingQueue.java
new file mode 100644
index 0000000..368d26f
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/TimeoutBlockingQueue.java
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2.util;
+
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class TimeoutBlockingQueue<E> {
+ public static interface TimeoutRetriever<T> {
+ long getTimeout(T object);
+ TimeUnit getTimeUnit(T object);
+ }
+
+ private final ReentrantLock lock = new ReentrantLock();
+ private final Condition waitCond = lock.newCondition();
+ private final TimeoutRetriever<? super E> timeoutRetriever;
+
+ private E[] objects;
+ private int head = 0;
+ private int tail = 0;
+
+ public TimeoutBlockingQueue(TimeoutRetriever<? super E> timeoutRetriever) {
+ this(32, timeoutRetriever);
+ }
+
+ @SuppressWarnings("unchecked")
+ public TimeoutBlockingQueue(int capacity, TimeoutRetriever<? super E> timeoutRetriever) {
+ this.objects = (E[])new Object[capacity];
+ this.timeoutRetriever = timeoutRetriever;
+ }
+
+ public void dump() {
+ for (int i = 0; i < objects.length; ++i) {
+ if (i == head) {
+ System.out.print("[" + objects[i] + "] ");
+ } else if (i == tail) {
+ System.out.print("]" + objects[i] + "[ ");
+ } else {
+ System.out.print(objects[i] + " ");
+ }
+ }
+ System.out.println();
+ }
+
+ public void clear() {
+ lock.lock();
+ try {
+ if (head != tail) {
+ for (int i = head; i < tail; ++i) {
+ objects[i] = null;
+ }
+ head = 0;
+ tail = 0;
+ waitCond.signal();
+ }
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ public void add(E e) {
+ if (e == null) throw new NullPointerException();
+
+ lock.lock();
+ try {
+ addElement(e);
+ waitCond.signal();
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ @edu.umd.cs.findbugs.annotations.SuppressWarnings("WA_AWAIT_NOT_IN_LOOP")
+ public E poll() {
+ lock.lock();
+ try {
+ if (isEmpty()) {
+ waitCond.await();
+ return null;
+ }
+
+ E elem = objects[head];
+ long nanos = getNanosTimeout(elem);
+ nanos = waitCond.awaitNanos(nanos);
+ return nanos > 0 ? null : removeFirst();
+ } catch (InterruptedException e) {
+ return null;
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ public int size() {
+ return tail - head;
+ }
+
+ public boolean isEmpty() {
+ return (tail - head) == 0;
+ }
+
+ public void signalAll() {
+ lock.lock();
+ try {
+ waitCond.signalAll();
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ private void addElement(E elem) {
+ int size = (tail - head);
+ if ((objects.length - size) == 0) {
+ int capacity = size + ((size < 64) ? (size + 2) : (size >> 1));
+ E[] newObjects = (E[])new Object[capacity];
+
+ if (compareTimeouts(objects[tail - 1], elem) <= 0) {
+ // Append
+ System.arraycopy(objects, head, newObjects, 0, tail);
+ tail -= head;
+ newObjects[tail++] = elem;
+ } else if (compareTimeouts(objects[head], elem) > 0) {
+ // Prepend
+ System.arraycopy(objects, head, newObjects, 1, tail);
+ newObjects[0] = elem;
+ tail -= (head - 1);
+ } else {
+ // Insert in the middle
+ int index = upperBound(head, tail - 1, elem);
+ int newIndex = (index - head);
+ System.arraycopy(objects, head, newObjects, 0, newIndex);
+ newObjects[newIndex] = elem;
+ System.arraycopy(objects, index, newObjects, newIndex + 1, tail - index);
+ tail -= (head - 1);
+ }
+ head = 0;
+ objects = newObjects;
+ } else {
+ if (tail == objects.length) {
+ // shift down |-----AAAAAAA|
+ tail -= head;
+ System.arraycopy(objects, head, objects, 0, tail);
+ head = 0;
+ }
+
+ if (tail == head || compareTimeouts(objects[tail - 1], elem) <= 0) {
+ // Append
+ objects[tail++] = elem;
+ } else if (head > 0 && compareTimeouts(objects[head], elem) > 0) {
+ // Prepend
+ objects[--head] = elem;
+ } else {
+ // Insert in the middle
+ int index = upperBound(head, tail - 1, elem);
+ System.arraycopy(objects, index, objects, index + 1, tail - index);
+ objects[index] = elem;
+ tail++;
+ }
+ }
+ }
+
+ private E removeFirst() {
+ E elem = objects[head];
+ objects[head] = null;
+ head = (head + 1) % objects.length;
+ if (head == 0) tail = 0;
+ return elem;
+ }
+
+ private int upperBound(int start, int end, E key) {
+ while (start < end) {
+ int mid = (start + end) >>> 1;
+ E mitem = objects[mid];
+ int cmp = compareTimeouts(mitem, key);
+ if (cmp > 0) {
+ end = mid;
+ } else {
+ start = mid + 1;
+ }
+ }
+ return start;
+ }
+
+ private int compareTimeouts(final E a, final E b) {
+ long t1 = getNanosTimeout(a);
+ long t2 = getNanosTimeout(b);
+ return (t1 < t2) ? -1 : (t1 > t2) ? 1 : 0;
+ }
+
+ private long getNanosTimeout(final E obj) {
+ TimeUnit unit = timeoutRetriever.getTimeUnit(obj);
+ long timeout = timeoutRetriever.getTimeout(obj);
+ return unit.toNanos(timeout);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java
new file mode 100644
index 0000000..c73300d
--- /dev/null
+++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureStore;
+import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class ProcedureTestingUtility {
+ private static final Log LOG = LogFactory.getLog(ProcedureTestingUtility.class);
+
+ private ProcedureTestingUtility() {
+ }
+
+ public static ProcedureStore createStore(final Configuration conf, final FileSystem fs,
+ final Path baseDir) throws IOException {
+ return createWalStore(conf, fs, baseDir);
+ }
+
+ public static WALProcedureStore createWalStore(final Configuration conf, final FileSystem fs,
+ final Path logDir) throws IOException {
+ return new WALProcedureStore(conf, fs, logDir, new WALProcedureStore.LeaseRecovery() {
+ @Override
+ public void recoverFileLease(FileSystem fs, Path path) throws IOException {
+ // no-op
+ }
+ });
+ }
+
+ public static <TEnv> void restart(ProcedureExecutor<TEnv> procExecutor)
+ throws Exception {
+ ProcedureStore procStore = procExecutor.getStore();
+ int nthreads = procExecutor.getNumThreads();
+ procExecutor.stop();
+ procStore.stop();
+ procExecutor.join();
+ procStore.start(nthreads);
+ procExecutor.start(nthreads);
+ }
+
+ public static <TEnv> void setKillBeforeStoreUpdate(ProcedureExecutor<TEnv> procExecutor,
+ boolean value) {
+ if (procExecutor.testing == null) {
+ procExecutor.testing = new ProcedureExecutor.Testing();
+ }
+ procExecutor.testing.killBeforeStoreUpdate = value;
+ LOG.warn("Set Kill before store update to: " + procExecutor.testing.killBeforeStoreUpdate);
+ }
+
+ public static <TEnv> void setToggleKillBeforeStoreUpdate(ProcedureExecutor<TEnv> procExecutor,
+ boolean value) {
+ if (procExecutor.testing == null) {
+ procExecutor.testing = new ProcedureExecutor.Testing();
+ }
+ procExecutor.testing.toggleKillBeforeStoreUpdate = value;
+ }
+
+ public static <TEnv> void toggleKillBeforeStoreUpdate(ProcedureExecutor<TEnv> procExecutor) {
+ if (procExecutor.testing == null) {
+ procExecutor.testing = new ProcedureExecutor.Testing();
+ }
+ procExecutor.testing.killBeforeStoreUpdate = !procExecutor.testing.killBeforeStoreUpdate;
+ LOG.warn("Set Kill before store update to: " + procExecutor.testing.killBeforeStoreUpdate);
+ }
+
+ public static <TEnv> long submitAndWait(ProcedureExecutor<TEnv> procExecutor, Procedure proc) {
+ long procId = procExecutor.submitProcedure(proc);
+ waitProcedure(procExecutor, procId);
+ return procId;
+ }
+
+ public static <TEnv> void waitProcedure(ProcedureExecutor<TEnv> procExecutor, long procId) {
+ while (!procExecutor.isFinished(procId) && procExecutor.isRunning()) {
+ Threads.sleepWithoutInterrupt(250);
+ }
+ }
+
+ public static <TEnv> void assertProcNotYetCompleted(ProcedureExecutor<TEnv> procExecutor,
+ long procId) {
+ assertFalse(procExecutor.isFinished(procId));
+ assertEquals(null, procExecutor.getResult(procId));
+ }
+
+ public static <TEnv> void assertProcNotFailed(ProcedureExecutor<TEnv> procExecutor,
+ long procId) {
+ ProcedureResult result = procExecutor.getResult(procId);
+ assertTrue(result != null);
+ assertProcNotFailed(result);
+ }
+
+ public static void assertProcNotFailed(final ProcedureResult result) {
+ Exception exception = result.getException();
+ String msg = exception != null ? exception.toString() : "no exception found";
+ assertFalse(msg, result.isFailed());
+ }
+
+ public static void assertIsAbortException(final ProcedureResult result) {
+ LOG.info(result.getException());
+ assertTrue(result.isFailed());
+ assertTrue(result.getException().getCause() instanceof ProcedureAbortedException);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureExecution.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureExecution.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureExecution.java
new file mode 100644
index 0000000..43c5362
--- /dev/null
+++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureExecution.java
@@ -0,0 +1,300 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureStore;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+@Category({MasterTests.class, SmallTests.class})
+public class TestProcedureExecution {
+ private static final Log LOG = LogFactory.getLog(TestProcedureExecution.class);
+
+ private static final int PROCEDURE_EXECUTOR_SLOTS = 1;
+ private static final Procedure NULL_PROC = null;
+
+ private ProcedureExecutor<Void> procExecutor;
+ private ProcedureStore procStore;
+
+ private HBaseCommonTestingUtility htu;
+ private FileSystem fs;
+ private Path testDir;
+ private Path logDir;
+
+ @Before
+ public void setUp() throws IOException {
+ htu = new HBaseCommonTestingUtility();
+ testDir = htu.getDataTestDir();
+ fs = testDir.getFileSystem(htu.getConfiguration());
+ assertTrue(testDir.depth() > 1);
+
+ logDir = new Path(testDir, "proc-logs");
+ procStore = ProcedureTestingUtility.createWalStore(htu.getConfiguration(), fs, logDir);
+ procExecutor = new ProcedureExecutor(htu.getConfiguration(), null, procStore);
+ procStore.start(PROCEDURE_EXECUTOR_SLOTS);
+ procExecutor.start(PROCEDURE_EXECUTOR_SLOTS);
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ procExecutor.stop();
+ procStore.stop();
+ fs.delete(logDir, true);
+ }
+
+ private static class TestProcedureException extends Exception {
+ public TestProcedureException(String msg) { super(msg); }
+ }
+
+ public static class TestSequentialProcedure extends SequentialProcedure<Void> {
+ private final Procedure[] subProcs;
+ private final List<String> state;
+ private final Exception failure;
+ private final String name;
+
+ public TestSequentialProcedure() {
+ throw new UnsupportedOperationException("recovery should not be triggered here");
+ }
+
+ public TestSequentialProcedure(String name, List<String> state, Procedure... subProcs) {
+ this.state = state;
+ this.subProcs = subProcs;
+ this.name = name;
+ this.failure = null;
+ }
+
+ public TestSequentialProcedure(String name, List<String> state, Exception failure) {
+ this.state = state;
+ this.subProcs = null;
+ this.name = name;
+ this.failure = failure;
+ }
+
+ @Override
+ protected Procedure[] execute(Void env) {
+ state.add(name + "-execute");
+ if (failure != null) {
+ setFailure(new RemoteProcedureException(name + "-failure", failure));
+ return null;
+ }
+ return subProcs;
+ }
+
+ @Override
+ protected void rollback(Void env) {
+ state.add(name + "-rollback");
+ }
+
+ @Override
+ protected void abort(Void env) {
+ state.add(name + "-abort");
+ }
+ }
+
+ @Test(timeout=30000)
+ public void testBadSubprocList() {
+ List<String> state = new ArrayList<String>();
+ Procedure subProc2 = new TestSequentialProcedure("subProc2", state);
+ Procedure subProc1 = new TestSequentialProcedure("subProc1", state, subProc2, NULL_PROC);
+ Procedure rootProc = new TestSequentialProcedure("rootProc", state, subProc1);
+ long rootId = ProcedureTestingUtility.submitAndWait(procExecutor, rootProc);
+
+ // subProc1 has a "null" subprocedure which is catched as InvalidArgument
+ // failed state with 2 execute and 2 rollback
+ LOG.info(state);
+ ProcedureResult result = procExecutor.getResult(rootId);
+ LOG.info(result.getException());
+ assertTrue(state.toString(), result.isFailed());
+ assertTrue(result.getException().toString(),
+ result.getException().getCause() instanceof IllegalArgumentException);
+
+ assertEquals(state.toString(), 4, state.size());
+ assertEquals("rootProc-execute", state.get(0));
+ assertEquals("subProc1-execute", state.get(1));
+ assertEquals("subProc1-rollback", state.get(2));
+ assertEquals("rootProc-rollback", state.get(3));
+ }
+
+ @Test(timeout=30000)
+ public void testSingleSequentialProc() {
+ List<String> state = new ArrayList<String>();
+ Procedure subProc2 = new TestSequentialProcedure("subProc2", state);
+ Procedure subProc1 = new TestSequentialProcedure("subProc1", state, subProc2);
+ Procedure rootProc = new TestSequentialProcedure("rootProc", state, subProc1);
+ long rootId = ProcedureTestingUtility.submitAndWait(procExecutor, rootProc);
+
+ // successful state, with 3 execute
+ LOG.info(state);
+ ProcedureResult result = procExecutor.getResult(rootId);
+ ProcedureTestingUtility.assertProcNotFailed(result);
+ assertEquals(state.toString(), 3, state.size());
+ }
+
+ @Test(timeout=30000)
+ public void testSingleSequentialProcRollback() {
+ List<String> state = new ArrayList<String>();
+ Procedure subProc2 = new TestSequentialProcedure("subProc2", state,
+ new TestProcedureException("fail test"));
+ Procedure subProc1 = new TestSequentialProcedure("subProc1", state, subProc2);
+ Procedure rootProc = new TestSequentialProcedure("rootProc", state, subProc1);
+ long rootId = ProcedureTestingUtility.submitAndWait(procExecutor, rootProc);
+
+ // the 3rd proc fail, rollback after 2 successful execution
+ LOG.info(state);
+ ProcedureResult result = procExecutor.getResult(rootId);
+ LOG.info(result.getException());
+ assertTrue(state.toString(), result.isFailed());
+ assertTrue(result.getException().toString(),
+ result.getException().getCause() instanceof TestProcedureException);
+
+ assertEquals(state.toString(), 6, state.size());
+ assertEquals("rootProc-execute", state.get(0));
+ assertEquals("subProc1-execute", state.get(1));
+ assertEquals("subProc2-execute", state.get(2));
+ assertEquals("subProc2-rollback", state.get(3));
+ assertEquals("subProc1-rollback", state.get(4));
+ assertEquals("rootProc-rollback", state.get(5));
+ }
+
+ public static class TestWaitingProcedure extends SequentialProcedure<Void> {
+ private final List<String> state;
+ private final boolean hasChild;
+ private final String name;
+
+ public TestWaitingProcedure() {
+ throw new UnsupportedOperationException("recovery should not be triggered here");
+ }
+
+ public TestWaitingProcedure(String name, List<String> state, boolean hasChild) {
+ this.hasChild = hasChild;
+ this.state = state;
+ this.name = name;
+ }
+
+ @Override
+ protected Procedure[] execute(Void env) {
+ state.add(name + "-execute");
+ setState(ProcedureState.WAITING_TIMEOUT);
+ return hasChild ? new Procedure[] { new TestWaitChild(name, state) } : null;
+ }
+
+ @Override
+ protected void rollback(Void env) {
+ state.add(name + "-rollback");
+ }
+
+ @Override
+ protected void abort(Void env) {
+ state.add(name + "-abort");
+ }
+
+ public static class TestWaitChild extends SequentialProcedure<Void> {
+ private final List<String> state;
+ private final String name;
+
+ public TestWaitChild() {
+ throw new UnsupportedOperationException("recovery should not be triggered here");
+ }
+
+ public TestWaitChild(String name, List<String> state) {
+ this.name = name;
+ this.state = state;
+ }
+
+ @Override
+ protected Procedure[] execute(Void env) {
+ state.add(name + "-child-execute");
+ return null;
+ }
+
+ @Override
+ protected void rollback(Void env) {
+ state.add(name + "-child-rollback");
+ }
+
+ @Override
+ protected void abort(Void env) {
+ state.add(name + "-child-abort");
+ }
+ }
+ }
+
+ @Test(timeout=30000)
+ public void testAbortTimeout() {
+ final int PROC_TIMEOUT_MSEC = 2500;
+ List<String> state = new ArrayList<String>();
+ Procedure proc = new TestWaitingProcedure("wproc", state, false);
+ proc.setTimeout(PROC_TIMEOUT_MSEC);
+ long startTime = EnvironmentEdgeManager.currentTime();
+ long rootId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
+ long execTime = EnvironmentEdgeManager.currentTime() - startTime;
+ LOG.info(state);
+ assertTrue("we didn't wait enough execTime=" + execTime, execTime >= PROC_TIMEOUT_MSEC);
+ ProcedureResult result = procExecutor.getResult(rootId);
+ LOG.info(result.getException());
+ assertTrue(state.toString(), result.isFailed());
+ assertTrue(result.getException().toString(),
+ result.getException().getCause() instanceof TimeoutException);
+ assertEquals(state.toString(), 2, state.size());
+ assertEquals("wproc-execute", state.get(0));
+ assertEquals("wproc-rollback", state.get(1));
+ }
+
+ @Test(timeout=30000)
+ public void testAbortTimeoutWithChildren() {
+ List<String> state = new ArrayList<String>();
+ Procedure proc = new TestWaitingProcedure("wproc", state, true);
+ proc.setTimeout(2500);
+ long rootId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
+ LOG.info(state);
+ ProcedureResult result = procExecutor.getResult(rootId);
+ LOG.info(result.getException());
+ assertTrue(state.toString(), result.isFailed());
+ assertTrue(result.getException().toString(),
+ result.getException().getCause() instanceof TimeoutException);
+ assertEquals(state.toString(), 4, state.size());
+ assertEquals("wproc-execute", state.get(0));
+ assertEquals("wproc-child-execute", state.get(1));
+ assertEquals("wproc-child-rollback", state.get(2));
+ assertEquals("wproc-rollback", state.get(3));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureRecovery.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureRecovery.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureRecovery.java
new file mode 100644
index 0000000..ffc401c
--- /dev/null
+++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureRecovery.java
@@ -0,0 +1,486 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureStore;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.Threads;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+@Category({MasterTests.class, SmallTests.class})
+public class TestProcedureRecovery {
+ private static final Log LOG = LogFactory.getLog(TestProcedureRecovery.class);
+
+ private static final int PROCEDURE_EXECUTOR_SLOTS = 1;
+ private static final Procedure NULL_PROC = null;
+
+ private static ProcedureExecutor<Void> procExecutor;
+ private static ProcedureStore procStore;
+ private static int procSleepInterval;
+
+ private HBaseCommonTestingUtility htu;
+ private FileSystem fs;
+ private Path testDir;
+ private Path logDir;
+
+ @Before
+ public void setUp() throws IOException {
+ htu = new HBaseCommonTestingUtility();
+ testDir = htu.getDataTestDir();
+ fs = testDir.getFileSystem(htu.getConfiguration());
+ assertTrue(testDir.depth() > 1);
+
+ logDir = new Path(testDir, "proc-logs");
+ procStore = ProcedureTestingUtility.createStore(htu.getConfiguration(), fs, logDir);
+ procExecutor = new ProcedureExecutor(htu.getConfiguration(), null, procStore);
+ procExecutor.testing = new ProcedureExecutor.Testing();
+ procStore.start(PROCEDURE_EXECUTOR_SLOTS);
+ procExecutor.start(PROCEDURE_EXECUTOR_SLOTS);
+ procSleepInterval = 0;
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ procExecutor.stop();
+ procStore.stop();
+ fs.delete(logDir, true);
+ }
+
+ private void restart() throws Exception {
+ dumpLogDirState();
+ ProcedureTestingUtility.restart(procExecutor);
+ dumpLogDirState();
+ }
+
+ public static class TestSingleStepProcedure extends SequentialProcedure<Void> {
+ private int step = 0;
+
+ public TestSingleStepProcedure() { }
+
+ @Override
+ protected Procedure[] execute(Void env) {
+ LOG.debug("execute procedure " + this + " step=" + step);
+ step++;
+ setResult(Bytes.toBytes(step));
+ return null;
+ }
+
+ @Override
+ protected void rollback(Void env) { }
+
+ @Override
+ protected void abort(Void env) { }
+ }
+
+ public static class BaseTestStepProcedure extends SequentialProcedure<Void> {
+ private AtomicBoolean abort = new AtomicBoolean(false);
+ private int step = 0;
+
+ @Override
+ protected Procedure[] execute(Void env) {
+ LOG.debug("execute procedure " + this + " step=" + step);
+ ProcedureTestingUtility.toggleKillBeforeStoreUpdate(procExecutor);
+ step++;
+ Threads.sleepWithoutInterrupt(procSleepInterval);
+ if (isAborted()) {
+ setFailure(new RemoteProcedureException(getClass().getName(),
+ new ProcedureAbortedException(
+ "got an abort at " + getClass().getName() + " step=" + step)));
+ return null;
+ }
+ return null;
+ }
+
+ @Override
+ protected void rollback(Void env) {
+ LOG.debug("rollback procedure " + this + " step=" + step);
+ ProcedureTestingUtility.toggleKillBeforeStoreUpdate(procExecutor);
+ step++;
+ }
+
+ @Override
+ protected void abort(Void env) {
+ abort.set(true);
+ }
+
+ private boolean isAborted() {
+ boolean aborted = abort.get();
+ BaseTestStepProcedure proc = this;
+ while (proc.hasParent() && !aborted) {
+ proc = (BaseTestStepProcedure)procExecutor.getProcedure(proc.getParentProcId());
+ aborted = proc.isAborted();
+ }
+ return aborted;
+ }
+ }
+
+ public static class TestMultiStepProcedure extends BaseTestStepProcedure {
+ public TestMultiStepProcedure() { }
+
+ @Override
+ public Procedure[] execute(Void env) {
+ super.execute(env);
+ return isFailed() ? null : new Procedure[] { new Step1Procedure() };
+ }
+
+ public static class Step1Procedure extends BaseTestStepProcedure {
+ public Step1Procedure() { }
+
+ @Override
+ protected Procedure[] execute(Void env) {
+ super.execute(env);
+ return isFailed() ? null : new Procedure[] { new Step2Procedure() };
+ }
+ }
+
+ public static class Step2Procedure extends BaseTestStepProcedure {
+ public Step2Procedure() { }
+ }
+ }
+
+ @Test
+ public void testNoopLoad() throws Exception {
+ restart();
+ }
+
+ @Test(timeout=30000)
+ public void testSingleStepProcRecovery() throws Exception {
+ Procedure proc = new TestSingleStepProcedure();
+ procExecutor.testing.killBeforeStoreUpdate = true;
+ long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
+ assertFalse(procExecutor.isRunning());
+ procExecutor.testing.killBeforeStoreUpdate = false;
+
+ // Restart and verify that the procedures restart
+ long restartTs = EnvironmentEdgeManager.currentTime();
+ restart();
+ waitProcedure(procId);
+ ProcedureResult result = procExecutor.getResult(procId);
+ assertTrue(result.getLastUpdate() > restartTs);
+ ProcedureTestingUtility.assertProcNotFailed(result);
+ assertEquals(1, Bytes.toInt(result.getResult()));
+ long resultTs = result.getLastUpdate();
+
+ // Verify that after another restart the result is still there
+ restart();
+ result = procExecutor.getResult(procId);
+ ProcedureTestingUtility.assertProcNotFailed(result);
+ assertEquals(resultTs, result.getLastUpdate());
+ assertEquals(1, Bytes.toInt(result.getResult()));
+ }
+
+ @Test(timeout=30000)
+ public void testMultiStepProcRecovery() throws Exception {
+ // Step 0 - kill
+ Procedure proc = new TestMultiStepProcedure();
+ long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
+ assertFalse(procExecutor.isRunning());
+
+ // Step 0 exec && Step 1 - kill
+ restart();
+ waitProcedure(procId);
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
+ assertFalse(procExecutor.isRunning());
+
+ // Step 1 exec && step 2 - kill
+ restart();
+ waitProcedure(procId);
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
+ assertFalse(procExecutor.isRunning());
+
+ // Step 2 exec
+ restart();
+ waitProcedure(procId);
+ assertTrue(procExecutor.isRunning());
+
+ // The procedure is completed
+ ProcedureResult result = procExecutor.getResult(procId);
+ ProcedureTestingUtility.assertProcNotFailed(result);
+ }
+
+ @Test(timeout=30000)
+ public void testMultiStepRollbackRecovery() throws Exception {
+ // Step 0 - kill
+ Procedure proc = new TestMultiStepProcedure();
+ long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
+ assertFalse(procExecutor.isRunning());
+
+ // Step 0 exec && Step 1 - kill
+ restart();
+ waitProcedure(procId);
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
+ assertFalse(procExecutor.isRunning());
+
+ // Step 1 exec && step 2 - kill
+ restart();
+ waitProcedure(procId);
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
+ assertFalse(procExecutor.isRunning());
+
+ // Step 2 exec - rollback - kill
+ procSleepInterval = 2500;
+ restart();
+ assertTrue(procExecutor.abort(procId));
+ waitProcedure(procId);
+ assertFalse(procExecutor.isRunning());
+
+ // rollback - kill
+ restart();
+ waitProcedure(procId);
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
+ assertFalse(procExecutor.isRunning());
+
+ // rollback - complete
+ restart();
+ waitProcedure(procId);
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
+ assertFalse(procExecutor.isRunning());
+
+ // Restart the executor and get the result
+ restart();
+ waitProcedure(procId);
+
+ // The procedure is completed
+ ProcedureResult result = procExecutor.getResult(procId);
+ ProcedureTestingUtility.assertIsAbortException(result);
+ }
+
+ public static class TestStateMachineProcedure
+ extends StateMachineProcedure<Void, TestStateMachineProcedure.State> {
+ enum State { STATE_1, STATE_2, STATE_3, DONE }
+
+ public TestStateMachineProcedure() {}
+
+ private AtomicBoolean aborted = new AtomicBoolean(false);
+ private int iResult = 0;
+
+ @Override
+ protected StateMachineProcedure.Flow executeFromState(Void env, State state) {
+ if (state == null) {
+ LOG.info("Initializing " + this);
+ state = State.STATE_1;
+ setNextState(state);
+ }
+
+ switch (state) {
+ case STATE_1:
+ LOG.info("execute step 1 " + this);
+ setNextState(State.STATE_2);
+ iResult += 3;
+ break;
+ case STATE_2:
+ LOG.info("execute step 2 " + this);
+ setNextState(State.STATE_3);
+ iResult += 5;
+ break;
+ case STATE_3:
+ LOG.info("execute step 3 " + this);
+ Threads.sleepWithoutInterrupt(procSleepInterval);
+ if (aborted.get()) {
+ LOG.info("aborted step 3 " + this);
+ setAbortFailure("test", "aborted");
+ break;
+ }
+ setNextState(State.DONE);
+ iResult += 7;
+ setResult(Bytes.toBytes(iResult));
+ return Flow.NO_MORE_STATE;
+ default:
+ throw new UnsupportedOperationException();
+ }
+ return Flow.HAS_MORE_STATE;
+ }
+
+ @Override
+ protected void rollbackState(Void env, final State state) {
+ switch (state) {
+ case STATE_1:
+ LOG.info("rollback step 1 " + this);
+ break;
+ case STATE_2:
+ LOG.info("rollback step 2 " + this);
+ break;
+ case STATE_3:
+ LOG.info("rollback step 3 " + this);
+ break;
+ default:
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ @Override
+ protected State getState(final int stateId) {
+ return State.values()[stateId];
+ }
+
+ private void setNextState(final State state) {
+ setNextState(state.ordinal());
+ }
+
+ @Override
+ protected void abort(Void env) {
+ aborted.set(true);
+ }
+
+ @Override
+ protected void serializeStateData(final OutputStream stream) throws IOException {
+ super.serializeStateData(stream);
+ stream.write(Bytes.toBytes(iResult));
+ }
+
+ @Override
+ protected void deserializeStateData(final InputStream stream) throws IOException {
+ super.deserializeStateData(stream);
+ byte[] data = new byte[4];
+ stream.read(data);
+ iResult = Bytes.toInt(data);
+ }
+ }
+
+ @Test(timeout=30000)
+ public void testStateMachineRecovery() throws Exception {
+ ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExecutor, true);
+ ProcedureTestingUtility.setKillBeforeStoreUpdate(procExecutor, true);
+
+ // Step 1 - kill
+ Procedure proc = new TestStateMachineProcedure();
+ long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
+ assertFalse(procExecutor.isRunning());
+
+ // Step 1 exec && Step 2 - kill
+ restart();
+ waitProcedure(procId);
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
+ assertFalse(procExecutor.isRunning());
+
+ // Step 2 exec && step 3 - kill
+ restart();
+ waitProcedure(procId);
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
+ assertFalse(procExecutor.isRunning());
+
+ // Step 3 exec
+ restart();
+ waitProcedure(procId);
+ assertTrue(procExecutor.isRunning());
+
+ // The procedure is completed
+ ProcedureResult result = procExecutor.getResult(procId);
+ ProcedureTestingUtility.assertProcNotFailed(result);
+ assertEquals(15, Bytes.toInt(result.getResult()));
+ }
+
+ @Test(timeout=30000)
+ public void testStateMachineRollbackRecovery() throws Exception {
+ ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExecutor, true);
+ ProcedureTestingUtility.setKillBeforeStoreUpdate(procExecutor, true);
+
+ // Step 1 - kill
+ Procedure proc = new TestStateMachineProcedure();
+ long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
+ assertFalse(procExecutor.isRunning());
+
+ // Step 1 exec && Step 2 - kill
+ restart();
+ waitProcedure(procId);
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
+ assertFalse(procExecutor.isRunning());
+
+ // Step 2 exec && step 3 - kill
+ restart();
+ waitProcedure(procId);
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
+ assertFalse(procExecutor.isRunning());
+
+ // Step 3 exec - rollback step 3 - kill
+ procSleepInterval = 2500;
+ restart();
+ assertTrue(procExecutor.abort(procId));
+ waitProcedure(procId);
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
+ assertFalse(procExecutor.isRunning());
+
+ // Rollback step 3 - rollback step 2 - kill
+ restart();
+ waitProcedure(procId);
+ assertFalse(procExecutor.isRunning());
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
+
+ // Rollback step 2 - step 1 - kill
+ restart();
+ waitProcedure(procId);
+ assertFalse(procExecutor.isRunning());
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
+
+ // Rollback step 1 - complete
+ restart();
+ waitProcedure(procId);
+ assertTrue(procExecutor.isRunning());
+
+ // The procedure is completed
+ ProcedureResult result = procExecutor.getResult(procId);
+ ProcedureTestingUtility.assertIsAbortException(result);
+ }
+
+ private void waitProcedure(final long procId) {
+ ProcedureTestingUtility.waitProcedure(procExecutor, procId);
+ dumpLogDirState();
+ }
+
+ private void dumpLogDirState() {
+ try {
+ FileStatus[] files = fs.listStatus(logDir);
+ if (files != null && files.length > 0) {
+ for (FileStatus file: files) {
+ assertTrue(file.toString(), file.isFile());
+ LOG.debug("log file " + file.getPath() + " size=" + file.getLen());
+ }
+ } else {
+ LOG.debug("no files under: " + logDir);
+ }
+ } catch (IOException e) {
+ LOG.warn("Unable to dump " + logDir, e);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureRunQueues.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureRunQueues.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureRunQueues.java
new file mode 100644
index 0000000..c040be6
--- /dev/null
+++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureRunQueues.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.junit.Assert.assertEquals;
+
+@Category({MasterTests.class, SmallTests.class})
+public class TestProcedureRunQueues {
+ class TestRunQueue implements ProcedureFairRunQueues.FairObject {
+ private final String name;
+ private final int priority;
+
+ public TestRunQueue(String name, int priority) {
+ this.name = name;
+ this.priority = priority;
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ @Override
+ public boolean isAvailable() {
+ return true;
+ }
+
+ @Override
+ public int getPriority() {
+ return priority;
+ }
+ }
+
+ @Test
+ public void testFairQueues() throws Exception {
+ ProcedureFairRunQueues<String, TestRunQueue> fairq
+ = new ProcedureFairRunQueues<String, TestRunQueue>(1);
+ TestRunQueue a = fairq.add("A", new TestRunQueue("A", 1));
+ TestRunQueue b = fairq.add("B", new TestRunQueue("B", 1));
+ TestRunQueue m = fairq.add("M", new TestRunQueue("M", 2));
+
+ for (int i = 0; i < 3; ++i) {
+ assertEquals(a, fairq.poll());
+ assertEquals(b, fairq.poll());
+ assertEquals(m, fairq.poll());
+ assertEquals(m, fairq.poll());
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/TestProcedureStoreTracker.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/TestProcedureStoreTracker.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/TestProcedureStoreTracker.java
new file mode 100644
index 0000000..2c51782
--- /dev/null
+++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/TestProcedureStoreTracker.java
@@ -0,0 +1,168 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2.store;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+@Category({MasterTests.class, SmallTests.class})
+public class TestProcedureStoreTracker {
+ private static final Log LOG = LogFactory.getLog(TestProcedureStoreTracker.class);
+
+ static class TestProcedure extends Procedure<Void> {
+ public TestProcedure(long procId) {
+ setProcId(procId);
+ }
+
+ @Override
+ protected Procedure[] execute(Void env) { return null; }
+
+ @Override
+ protected void rollback(Void env) { /* no-op */ }
+
+ @Override
+ protected void abort(Void env) { /* no-op */ }
+
+ @Override
+ protected void serializeStateData(final OutputStream stream) { /* no-op */ }
+
+ @Override
+ protected void deserializeStateData(final InputStream stream) { /* no-op */ }
+ }
+
+ @Test
+ public void testSeqInsertAndDelete() {
+ ProcedureStoreTracker tracker = new ProcedureStoreTracker();
+ assertTrue(tracker.isEmpty());
+
+ final int MIN_PROC = 1;
+ final int MAX_PROC = 1 << 10;
+
+ // sequential insert
+ for (int i = MIN_PROC; i < MAX_PROC; ++i) {
+ tracker.insert(i);
+
+ // All the proc that we inserted should not be deleted
+ for (int j = MIN_PROC; j <= i; ++j) {
+ assertEquals(ProcedureStoreTracker.DeleteState.NO, tracker.isDeleted(j));
+ }
+ // All the proc that are not yet inserted should be result as deleted
+ for (int j = i + 1; j < MAX_PROC; ++j) {
+ assertTrue(tracker.isDeleted(j) != ProcedureStoreTracker.DeleteState.NO);
+ }
+ }
+
+ // sequential delete
+ for (int i = MIN_PROC; i < MAX_PROC; ++i) {
+ tracker.delete(i);
+
+ // All the proc that we deleted should be deleted
+ for (int j = MIN_PROC; j <= i; ++j) {
+ assertEquals(ProcedureStoreTracker.DeleteState.YES, tracker.isDeleted(j));
+ }
+ // All the proc that are not yet deleted should be result as not deleted
+ for (int j = i + 1; j < MAX_PROC; ++j) {
+ assertEquals(ProcedureStoreTracker.DeleteState.NO, tracker.isDeleted(j));
+ }
+ }
+ assertTrue(tracker.isEmpty());
+ }
+
+ @Test
+ public void testPartialTracker() {
+ ProcedureStoreTracker tracker = new ProcedureStoreTracker();
+ tracker.setPartialFlag(true);
+
+ // nothing in the tracker, the state is unknown
+ assertTrue(tracker.isEmpty());
+ assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(1));
+ assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(579));
+
+ // Mark 1 as deleted, now that is a known state
+ tracker.setDeleted(1, true);
+ tracker.dump();
+ assertEquals(ProcedureStoreTracker.DeleteState.YES, tracker.isDeleted(1));
+ assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(2));
+ assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(579));
+
+ // Mark 579 as non-deleted, now that is a known state
+ tracker.setDeleted(579, false);
+ assertEquals(ProcedureStoreTracker.DeleteState.YES, tracker.isDeleted(1));
+ assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(2));
+ assertEquals(ProcedureStoreTracker.DeleteState.NO, tracker.isDeleted(579));
+ assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(577));
+ assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(580));
+ }
+
+ @Test
+ public void testBasicCRUD() {
+ ProcedureStoreTracker tracker = new ProcedureStoreTracker();
+ assertTrue(tracker.isEmpty());
+
+ Procedure[] procs = new TestProcedure[] {
+ new TestProcedure(1), new TestProcedure(2), new TestProcedure(3),
+ new TestProcedure(4), new TestProcedure(5), new TestProcedure(6),
+ };
+
+ tracker.insert(procs[0], null);
+ tracker.insert(procs[1], new Procedure[] { procs[2], procs[3], procs[4] });
+ assertFalse(tracker.isEmpty());
+ assertTrue(tracker.isUpdated());
+
+ tracker.resetUpdates();
+ assertFalse(tracker.isUpdated());
+
+ for (int i = 0; i < 4; ++i) {
+ tracker.update(procs[i]);
+ assertFalse(tracker.isEmpty());
+ assertFalse(tracker.isUpdated());
+ }
+
+ tracker.update(procs[4]);
+ assertFalse(tracker.isEmpty());
+ assertTrue(tracker.isUpdated());
+
+ tracker.update(procs[5]);
+ assertFalse(tracker.isEmpty());
+ assertTrue(tracker.isUpdated());
+
+ for (int i = 0; i < 5; ++i) {
+ tracker.delete(procs[i].getProcId());
+ assertFalse(tracker.isEmpty());
+ assertTrue(tracker.isUpdated());
+ }
+ tracker.delete(procs[5].getProcId());
+ assertTrue(tracker.isEmpty());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestWALProcedureStore.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestWALProcedureStore.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestWALProcedureStore.java
new file mode 100644
index 0000000..71899ad
--- /dev/null
+++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestWALProcedureStore.java
@@ -0,0 +1,267 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2.store.wal;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Iterator;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
+import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
+import org.apache.hadoop.hbase.procedure2.SequentialProcedure;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.IOUtils;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+@Category({MasterTests.class, SmallTests.class})
+public class TestWALProcedureStore {
+ private static final Log LOG = LogFactory.getLog(TestWALProcedureStore.class);
+
+ private static final int PROCEDURE_STORE_SLOTS = 1;
+ private static final Procedure NULL_PROC = null;
+
+ private WALProcedureStore procStore;
+
+ private HBaseCommonTestingUtility htu;
+ private FileSystem fs;
+ private Path testDir;
+ private Path logDir;
+
+ @Before
+ public void setUp() throws IOException {
+ htu = new HBaseCommonTestingUtility();
+ testDir = htu.getDataTestDir();
+ fs = testDir.getFileSystem(htu.getConfiguration());
+ assertTrue(testDir.depth() > 1);
+
+ logDir = new Path(testDir, "proc-logs");
+ procStore = ProcedureTestingUtility.createWalStore(htu.getConfiguration(), fs, logDir);
+ procStore.start(PROCEDURE_STORE_SLOTS);
+ procStore.recoverLease();
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ procStore.stop();
+ fs.delete(logDir, true);
+ }
+
+ private Iterator<Procedure> storeRestart() throws Exception {
+ procStore.stop();
+ procStore.start(PROCEDURE_STORE_SLOTS);
+ procStore.recoverLease();
+ return procStore.load();
+ }
+
+ @Test
+ public void testEmptyLogLoad() throws Exception {
+ Iterator<Procedure> loader = storeRestart();
+ assertEquals(0, countProcedures(loader));
+ }
+
+ @Test
+ public void testLoad() throws Exception {
+ Set<Long> procIds = new HashSet<>();
+
+ // Insert something in the log
+ Procedure proc1 = new TestSequentialProcedure();
+ procIds.add(proc1.getProcId());
+ procStore.insert(proc1, null);
+
+ Procedure proc2 = new TestSequentialProcedure();
+ Procedure[] child2 = new Procedure[2];
+ child2[0] = new TestSequentialProcedure();
+ child2[1] = new TestSequentialProcedure();
+
+ procIds.add(proc2.getProcId());
+ procIds.add(child2[0].getProcId());
+ procIds.add(child2[1].getProcId());
+ procStore.insert(proc2, child2);
+
+ // Verify that everything is there
+ verifyProcIdsOnRestart(procIds);
+
+ // Update and delete something
+ procStore.update(proc1);
+ procStore.update(child2[1]);
+ procStore.delete(child2[1].getProcId());
+ procIds.remove(child2[1].getProcId());
+
+ // Verify that everything is there
+ verifyProcIdsOnRestart(procIds);
+
+ // Remove 4 byte from the trailers
+ procStore.stop();
+ FileStatus[] logs = fs.listStatus(logDir);
+ assertEquals(3, logs.length);
+ for (int i = 0; i < logs.length; ++i) {
+ corruptLog(logs[i], 4);
+ }
+ verifyProcIdsOnRestart(procIds);
+ }
+
+ @Test
+ public void testCorruptedTrailer() throws Exception {
+ // Insert something
+ for (int i = 0; i < 100; ++i) {
+ procStore.insert(new TestSequentialProcedure(), null);
+ }
+
+ // Stop the store
+ procStore.stop();
+
+ // Remove 4 byte from the trailer
+ FileStatus[] logs = fs.listStatus(logDir);
+ assertEquals(1, logs.length);
+ corruptLog(logs[0], 4);
+
+ int count = countProcedures(storeRestart());
+ assertEquals(100, count);
+ }
+
+ @Test
+ public void testCorruptedEntries() throws Exception {
+ // Insert something
+ for (int i = 0; i < 100; ++i) {
+ procStore.insert(new TestSequentialProcedure(), null);
+ }
+
+ // Stop the store
+ procStore.stop();
+
+ // Remove some byte from the log
+ // (enough to cut the trailer and corrupt some entries)
+ FileStatus[] logs = fs.listStatus(logDir);
+ assertEquals(1, logs.length);
+ corruptLog(logs[0], 1823);
+
+ int count = countProcedures(storeRestart());
+ assertTrue(procStore.getCorruptedLogs() != null);
+ assertEquals(1, procStore.getCorruptedLogs().size());
+ assertEquals(85, count);
+ }
+
+ private void corruptLog(final FileStatus logFile, final long dropBytes)
+ throws IOException {
+ assertTrue(logFile.getLen() > dropBytes);
+ LOG.debug("corrupt log " + logFile.getPath() +
+ " size=" + logFile.getLen() + " drop=" + dropBytes);
+ Path tmpPath = new Path(testDir, "corrupted.log");
+ InputStream in = fs.open(logFile.getPath());
+ OutputStream out = fs.create(tmpPath);
+ IOUtils.copyBytes(in, out, logFile.getLen() - dropBytes, true);
+ fs.rename(tmpPath, logFile.getPath());
+ }
+
+ private void verifyProcIdsOnRestart(final Set<Long> procIds) throws Exception {
+ int count = 0;
+ Iterator<Procedure> loader = storeRestart();
+ while (loader.hasNext()) {
+ Procedure proc = loader.next();
+ LOG.debug("loading procId=" + proc.getProcId());
+ assertTrue("procId=" + proc.getProcId() + " unexpected", procIds.contains(proc.getProcId()));
+ count++;
+ }
+ assertEquals(procIds.size(), count);
+ }
+
+ private void assertIsEmpty(Iterator<Procedure> iterator) {
+ assertEquals(0, countProcedures(iterator));
+ }
+
+ private int countProcedures(Iterator<Procedure> iterator) {
+ int count = 0;
+ while (iterator.hasNext()) {
+ Procedure proc = iterator.next();
+ LOG.trace("loading procId=" + proc.getProcId());
+ count++;
+ }
+ return count;
+ }
+
+ private void assertEmptyLogDir() {
+ try {
+ FileStatus[] status = fs.listStatus(logDir);
+ assertTrue("expected empty state-log dir", status == null || status.length == 0);
+ } catch (FileNotFoundException e) {
+ fail("expected the state-log dir to be present: " + logDir);
+ } catch (IOException e) {
+ fail("got en exception on state-log dir list: " + e.getMessage());
+ }
+ }
+
+ public static class TestSequentialProcedure extends SequentialProcedure<Void> {
+ private static long seqid = 0;
+
+ public TestSequentialProcedure() {
+ setProcId(++seqid);
+ }
+
+ @Override
+ protected Procedure[] execute(Void env) { return null; }
+
+ @Override
+ protected void rollback(Void env) { }
+
+ @Override
+ protected void abort(Void env) { }
+
+ @Override
+ protected void serializeStateData(final OutputStream stream) throws IOException {
+ long procId = getProcId();
+ if (procId % 2 == 0) {
+ stream.write(Bytes.toBytes(procId));
+ }
+ }
+
+ @Override
+ protected void deserializeStateData(InputStream stream) throws IOException {
+ long procId = getProcId();
+ if (procId % 2 == 0) {
+ byte[] bProcId = new byte[8];
+ assertEquals(8, stream.read(bProcId));
+ assertEquals(procId, Bytes.toLong(bProcId));
+ } else {
+ assertEquals(0, stream.available());
+ }
+ }
+ }
+}
\ No newline at end of file
[2/9] hbase git commit: HBASE-13310 Fix high priority findbugs
warnings
Posted by mb...@apache.org.
HBASE-13310 Fix high priority findbugs warnings
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6e9ded51
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6e9ded51
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6e9ded51
Branch: refs/heads/hbase-12439
Commit: 6e9ded51fb2fc4c3e63fb0da5030246f3466db71
Parents: 1428a59
Author: zhangduo <zh...@wandoujia.com>
Authored: Sun Mar 22 20:41:43 2015 +0800
Committer: zhangduo <zh...@wandoujia.com>
Committed: Tue Mar 24 06:50:04 2015 +0800
----------------------------------------------------------------------
.../apache/hadoop/hbase/HColumnDescriptor.java | 26 +++---
.../apache/hadoop/hbase/util/PrettyPrinter.java | 2 +-
.../org/apache/hadoop/hbase/util/Threads.java | 92 ++++++++++++++------
.../apache/hadoop/hbase/rest/RESTServer.java | 6 +-
.../apache/hadoop/hbase/rest/RowResource.java | 4 +-
.../org/apache/hadoop/hbase/rest/RowSpec.java | 8 +-
.../hbase/rest/model/ColumnSchemaModel.java | 10 +--
.../hadoop/hbase/rest/model/ScannerModel.java | 2 +-
.../rest/model/StorageClusterStatusModel.java | 4 +-
.../hbase/rest/model/TableRegionModel.java | 2 +-
.../hbase/rest/model/TableSchemaModel.java | 16 ++--
.../hadoop/hbase/io/hfile/HFileBlock.java | 13 +++
.../hadoop/hbase/master/HMasterCommandLine.java | 8 +-
.../regionserver/RegionCoprocessorHost.java | 5 +-
.../hbase/security/access/TableAuthManager.java | 11 +--
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 4 +-
.../hbase/util/hbck/OfflineMetaRepair.java | 2 +-
.../apache/hadoop/hbase/util/TestHBaseFsck.java | 16 ++--
.../hadoop/hbase/util/hbck/HbckTestingUtil.java | 2 +-
19 files changed, 141 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e9ded51/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
index 5335bef..d4d8ee4 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
@@ -734,7 +734,7 @@ public class HColumnDescriptor implements Comparable<HColumnDescriptor> {
String compressTagsStr = getValue(COMPRESS_TAGS);
boolean compressTags = DEFAULT_COMPRESS_TAGS;
if (compressTagsStr != null) {
- compressTags = Boolean.valueOf(compressTagsStr);
+ compressTags = Boolean.parseBoolean(compressTagsStr);
}
return compressTags;
}
@@ -747,7 +747,7 @@ public class HColumnDescriptor implements Comparable<HColumnDescriptor> {
String compressTagsStr = getValue(COMPRESS_TAGS);
boolean compressTags = DEFAULT_COMPRESS_TAGS;
if (compressTagsStr != null) {
- compressTags = Boolean.valueOf(compressTagsStr);
+ compressTags = Boolean.parseBoolean(compressTagsStr);
}
return compressTags;
}
@@ -778,8 +778,9 @@ public class HColumnDescriptor implements Comparable<HColumnDescriptor> {
*/
public boolean isInMemory() {
String value = getValue(HConstants.IN_MEMORY);
- if (value != null)
- return Boolean.valueOf(value).booleanValue();
+ if (value != null) {
+ return Boolean.parseBoolean(value);
+ }
return DEFAULT_IN_MEMORY;
}
@@ -827,7 +828,7 @@ public class HColumnDescriptor implements Comparable<HColumnDescriptor> {
*/
public int getTimeToLive() {
String value = getValue(TTL);
- return (value != null)? Integer.valueOf(value).intValue(): DEFAULT_TTL;
+ return (value != null)? Integer.parseInt(value) : DEFAULT_TTL;
}
/**
@@ -843,7 +844,7 @@ public class HColumnDescriptor implements Comparable<HColumnDescriptor> {
*/
public int getMinVersions() {
String value = getValue(MIN_VERSIONS);
- return (value != null)? Integer.valueOf(value).intValue(): 0;
+ return (value != null)? Integer.parseInt(value) : 0;
}
/**
@@ -861,8 +862,9 @@ public class HColumnDescriptor implements Comparable<HColumnDescriptor> {
*/
public boolean isBlockCacheEnabled() {
String value = getValue(BLOCKCACHE);
- if (value != null)
- return Boolean.valueOf(value).booleanValue();
+ if (value != null) {
+ return Boolean.parseBoolean(value);
+ }
return DEFAULT_BLOCKCACHE;
}
@@ -900,7 +902,7 @@ public class HColumnDescriptor implements Comparable<HColumnDescriptor> {
public int getScope() {
byte[] value = getValue(REPLICATION_SCOPE_BYTES);
if (value != null) {
- return Integer.valueOf(Bytes.toString(value));
+ return Integer.parseInt(Bytes.toString(value));
}
return DEFAULT_REPLICATION_SCOPE;
}
@@ -966,7 +968,9 @@ public class HColumnDescriptor implements Comparable<HColumnDescriptor> {
private boolean setAndGetBoolean(final String key, final boolean defaultSetting) {
String value = getValue(key);
- if (value != null) return Boolean.valueOf(value).booleanValue();
+ if (value != null) {
+ return Boolean.parseBoolean(value);
+ }
return defaultSetting;
}
@@ -1201,7 +1205,7 @@ public class HColumnDescriptor implements Comparable<HColumnDescriptor> {
@Override
public int hashCode() {
int result = Bytes.hashCode(this.name);
- result ^= Byte.valueOf(COLUMN_DESCRIPTOR_VERSION).hashCode();
+ result ^= (int) COLUMN_DESCRIPTOR_VERSION;
result ^= values.hashCode();
result ^= configuration.hashCode();
return result;
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e9ded51/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java
index 8c8f618..7728112 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/PrettyPrinter.java
@@ -34,7 +34,7 @@ public class PrettyPrinter {
StringBuilder human = new StringBuilder();
switch (unit) {
case TIME_INTERVAL:
- human.append(humanReadableTTL(Long.valueOf(value)));
+ human.append(humanReadableTTL(Long.parseLong(value)));
break;
default:
human.append(value);
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e9ded51/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Threads.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Threads.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Threads.java
index 81178c4..51a506b 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Threads.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Threads.java
@@ -35,6 +35,8 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
+import com.google.common.base.Preconditions;
+
/**
* Thread Utility
*/
@@ -266,43 +268,75 @@ public class Threads {
t.setUncaughtExceptionHandler(LOGGING_EXCEPTION_HANDLER);
}
- private static Method printThreadInfoMethod = null;
- private static boolean printThreadInfoMethodWithPrintStream = true;
+ private static interface PrintThreadInfoHelper {
- /**
- * Print all of the thread's information and stack traces. Wrapper around Hadoop's method.
- *
- * @param stream the stream to
- * @param title a string title for the stack trace
- */
- public static void printThreadInfo(PrintStream stream, String title) {
+ void printThreadInfo(PrintStream stream, String title);
- if (printThreadInfoMethod == null) {
+ }
+
+ private static class PrintThreadInfoLazyHolder {
+
+ public static final PrintThreadInfoHelper HELPER = initHelper();
+
+ private static PrintThreadInfoHelper initHelper() {
+ Method method = null;
try {
// Hadoop 2.7+ declares printThreadInfo(PrintStream, String)
- printThreadInfoMethod = ReflectionUtils.class.getMethod("printThreadInfo",
- PrintStream.class, String.class);
+ method = ReflectionUtils.class.getMethod("printThreadInfo", PrintStream.class,
+ String.class);
+ method.setAccessible(true);
+ final Method hadoop27Method = method;
+ return new PrintThreadInfoHelper() {
+
+ @Override
+ public void printThreadInfo(PrintStream stream, String title) {
+ try {
+ hadoop27Method.invoke(null, stream, title);
+ } catch (IllegalAccessException | IllegalArgumentException e) {
+ throw new RuntimeException(e);
+ } catch (InvocationTargetException e) {
+ throw new RuntimeException(e.getCause());
+ }
+ }
+ };
} catch (NoSuchMethodException e) {
- // Hadoop 2.6 and earlier declares printThreadInfo(PrintWriter, String)
- printThreadInfoMethodWithPrintStream = false;
- try {
- printThreadInfoMethod = ReflectionUtils.class.getMethod("printThreadInfo",
- PrintWriter.class, String.class);
- } catch (NoSuchMethodException e1) {
- throw new RuntimeException("Cannot find method. Check hadoop jars linked", e1);
- }
+ LOG.info(
+ "Can not find hadoop 2.7+ printThreadInfo method, try hadoop hadoop 2.6 and earlier", e);
}
- printThreadInfoMethod.setAccessible(true);
- }
+ try {
+ // Hadoop 2.6 and earlier declares printThreadInfo(PrintWriter, String)
+ method = ReflectionUtils.class.getMethod("printThreadInfo", PrintWriter.class,
+ String.class);
+ method.setAccessible(true);
+ final Method hadoop26Method = method;
+ return new PrintThreadInfoHelper() {
- try {
- if (printThreadInfoMethodWithPrintStream) {
- printThreadInfoMethod.invoke(null, stream, title);
- } else {
- printThreadInfoMethod.invoke(null, new PrintWriter(stream), title);
+ @Override
+ public void printThreadInfo(PrintStream stream, String title) {
+ try {
+ hadoop26Method.invoke(null, new PrintWriter(stream), title);
+ } catch (IllegalAccessException | IllegalArgumentException e) {
+ throw new RuntimeException(e);
+ } catch (InvocationTargetException e) {
+ throw new RuntimeException(e.getCause());
+ }
+ }
+ };
+ } catch (NoSuchMethodException e) {
+ LOG.warn("Cannot find printThreadInfo method. Check hadoop jars linked", e);
}
- } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
- throw new RuntimeException(e.getCause());
+ return null;
}
}
+
+ /**
+ * Print all of the thread's information and stack traces. Wrapper around Hadoop's method.
+ *
+ * @param stream the stream to
+ * @param title a string title for the stack trace
+ */
+ public static void printThreadInfo(PrintStream stream, String title) {
+ Preconditions.checkNotNull(PrintThreadInfoLazyHolder.HELPER,
+ "Cannot find method. Check hadoop jars linked").printThreadInfo(stream, title);
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e9ded51/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
----------------------------------------------------------------------
diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
index bb52fdb..370a083 100644
--- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java
@@ -127,8 +127,7 @@ public class RESTServer implements Constants {
// check for user-defined port setting, if so override the conf
if (commandLine != null && commandLine.hasOption("port")) {
String val = commandLine.getOptionValue("port");
- servlet.getConfiguration()
- .setInt("hbase.rest.port", Integer.valueOf(val));
+ servlet.getConfiguration().setInt("hbase.rest.port", Integer.parseInt(val));
LOG.debug("port set to " + val);
}
@@ -141,8 +140,7 @@ public class RESTServer implements Constants {
// check for user-defined info server port setting, if so override the conf
if (commandLine != null && commandLine.hasOption("infoport")) {
String val = commandLine.getOptionValue("infoport");
- servlet.getConfiguration()
- .setInt("hbase.rest.info.port", Integer.valueOf(val));
+ servlet.getConfiguration().setInt("hbase.rest.info.port", Integer.parseInt(val));
LOG.debug("Web UI port set to " + val);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e9ded51/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
----------------------------------------------------------------------
diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
index dad5a32..ff1345c 100644
--- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java
@@ -76,7 +76,7 @@ public class RowResource extends ResourceBase {
this.tableResource = tableResource;
this.rowspec = new RowSpec(rowspec);
if (versions != null) {
- this.rowspec.setMaxVersions(Integer.valueOf(versions));
+ this.rowspec.setMaxVersions(Integer.parseInt(versions));
}
this.check = check;
}
@@ -271,7 +271,7 @@ public class RowResource extends ResourceBase {
}
vals = headers.getRequestHeader("X-Timestamp");
if (vals != null && !vals.isEmpty()) {
- timestamp = Long.valueOf(vals.get(0));
+ timestamp = Long.parseLong(vals.get(0));
}
if (column == null) {
servlet.getMetrics().incrementFailedPutRequests(1);
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e9ded51/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RowSpec.java
----------------------------------------------------------------------
diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RowSpec.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RowSpec.java
index b6c1ca8..cc51c85 100644
--- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RowSpec.java
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RowSpec.java
@@ -164,7 +164,7 @@ public class RowSpec {
i++;
}
try {
- time0 = Long.valueOf(URLDecoder.decode(stamp.toString(),
+ time0 = Long.parseLong(URLDecoder.decode(stamp.toString(),
HConstants.UTF8_ENCODING));
} catch (NumberFormatException e) {
throw new IllegalArgumentException(e);
@@ -177,7 +177,7 @@ public class RowSpec {
i++;
}
try {
- time1 = Long.valueOf(URLDecoder.decode(stamp.toString(),
+ time1 = Long.parseLong(URLDecoder.decode(stamp.toString(),
HConstants.UTF8_ENCODING));
} catch (NumberFormatException e) {
throw new IllegalArgumentException(e);
@@ -245,7 +245,7 @@ public class RowSpec {
}
sb.append(c);
}
- maxVersions = Integer.valueOf(sb.toString());
+ maxVersions = Integer.parseInt(sb.toString());
} break;
case 'n': {
StringBuilder sb = new StringBuilder();
@@ -257,7 +257,7 @@ public class RowSpec {
}
sb.append(c);
}
- maxValues = Integer.valueOf(sb.toString());
+ maxValues = Integer.parseInt(sb.toString());
} break;
default:
throw new IllegalArgumentException("unknown parameter '" + c + "'");
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e9ded51/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ColumnSchemaModel.java
----------------------------------------------------------------------
diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ColumnSchemaModel.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ColumnSchemaModel.java
index ba0eed8..8562cde 100644
--- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ColumnSchemaModel.java
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ColumnSchemaModel.java
@@ -138,7 +138,7 @@ public class ColumnSchemaModel implements Serializable {
public boolean __getBlockcache() {
Object o = attrs.get(BLOCKCACHE);
return o != null ?
- Boolean.valueOf(o.toString()) : HColumnDescriptor.DEFAULT_BLOCKCACHE;
+ Boolean.parseBoolean(o.toString()) : HColumnDescriptor.DEFAULT_BLOCKCACHE;
}
/**
@@ -147,7 +147,7 @@ public class ColumnSchemaModel implements Serializable {
public int __getBlocksize() {
Object o = attrs.get(BLOCKSIZE);
return o != null ?
- Integer.valueOf(o.toString()) : HColumnDescriptor.DEFAULT_BLOCKSIZE;
+ Integer.parseInt(o.toString()) : HColumnDescriptor.DEFAULT_BLOCKSIZE;
}
/**
@@ -172,7 +172,7 @@ public class ColumnSchemaModel implements Serializable {
public boolean __getInMemory() {
Object o = attrs.get(IN_MEMORY);
return o != null ?
- Boolean.valueOf(o.toString()) : HColumnDescriptor.DEFAULT_IN_MEMORY;
+ Boolean.parseBoolean(o.toString()) : HColumnDescriptor.DEFAULT_IN_MEMORY;
}
/**
@@ -181,7 +181,7 @@ public class ColumnSchemaModel implements Serializable {
public int __getTTL() {
Object o = attrs.get(TTL);
return o != null ?
- Integer.valueOf(o.toString()) : HColumnDescriptor.DEFAULT_TTL;
+ Integer.parseInt(o.toString()) : HColumnDescriptor.DEFAULT_TTL;
}
/**
@@ -190,7 +190,7 @@ public class ColumnSchemaModel implements Serializable {
public int __getVersions() {
Object o = attrs.get(VERSIONS);
return o != null ?
- Integer.valueOf(o.toString()) : HColumnDescriptor.DEFAULT_VERSIONS;
+ Integer.parseInt(o.toString()) : HColumnDescriptor.DEFAULT_VERSIONS;
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e9ded51/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java
----------------------------------------------------------------------
diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java
index 784f7e6..25a6de3 100644
--- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java
@@ -413,7 +413,7 @@ public class ScannerModel implements ProtobufMessageHandler, Serializable {
}
} break;
case PageFilter:
- filter = new PageFilter(Long.valueOf(value));
+ filter = new PageFilter(Long.parseLong(value));
break;
case PrefixFilter:
filter = new PrefixFilter(Base64.decode(value));
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e9ded51/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterStatusModel.java
----------------------------------------------------------------------
diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterStatusModel.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterStatusModel.java
index 3b044e7..2caec66 100644
--- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterStatusModel.java
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterStatusModel.java
@@ -106,7 +106,9 @@ public class StorageClusterStatusModel
/**
* Represents a region hosted on a region server.
*/
- public static class Region {
+ public static class Region implements Serializable {
+ private static final long serialVersionUID = -1326683840086398193L;
+
private byte[] name;
private int stores;
private int storefiles;
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e9ded51/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableRegionModel.java
----------------------------------------------------------------------
diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableRegionModel.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableRegionModel.java
index d9b2b65..89fe12c 100644
--- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableRegionModel.java
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableRegionModel.java
@@ -142,7 +142,7 @@ public class TableRegionModel implements Serializable {
this.startKey = Bytes.toBytes(split[1]);
String tail = split[2];
split = tail.split("\\.");
- id = Long.valueOf(split[0]);
+ id = Long.parseLong(split[0]);
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e9ded51/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableSchemaModel.java
----------------------------------------------------------------------
diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableSchemaModel.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableSchemaModel.java
index 9e9fe47..593c3ab 100644
--- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableSchemaModel.java
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableSchemaModel.java
@@ -221,7 +221,7 @@ public class TableSchemaModel implements Serializable, ProtobufMessageHandler {
*/
public boolean __getIsMeta() {
Object o = attrs.get(IS_META);
- return o != null ? Boolean.valueOf(o.toString()) : false;
+ return o != null ? Boolean.parseBoolean(o.toString()) : false;
}
/**
@@ -229,7 +229,7 @@ public class TableSchemaModel implements Serializable, ProtobufMessageHandler {
*/
public boolean __getIsRoot() {
Object o = attrs.get(IS_ROOT);
- return o != null ? Boolean.valueOf(o.toString()) : false;
+ return o != null ? Boolean.parseBoolean(o.toString()) : false;
}
/**
@@ -237,8 +237,7 @@ public class TableSchemaModel implements Serializable, ProtobufMessageHandler {
*/
public boolean __getReadOnly() {
Object o = attrs.get(READONLY);
- return o != null ?
- Boolean.valueOf(o.toString()) : HTableDescriptor.DEFAULT_READONLY;
+ return o != null ? Boolean.parseBoolean(o.toString()) : HTableDescriptor.DEFAULT_READONLY;
}
/**
@@ -285,12 +284,10 @@ public class TableSchemaModel implements Serializable, ProtobufMessageHandler {
familyBuilder.addAttrs(attrBuilder);
}
if (familyAttrs.containsKey(TTL)) {
- familyBuilder.setTtl(
- Integer.valueOf(familyAttrs.get(TTL).toString()));
+ familyBuilder.setTtl(Integer.parseInt(familyAttrs.get(TTL).toString()));
}
if (familyAttrs.containsKey(VERSIONS)) {
- familyBuilder.setMaxVersions(
- Integer.valueOf(familyAttrs.get(VERSIONS).toString()));
+ familyBuilder.setMaxVersions(Integer.parseInt(familyAttrs.get(VERSIONS).toString()));
}
if (familyAttrs.containsKey(COMPRESSION)) {
familyBuilder.setCompression(familyAttrs.get(COMPRESSION).toString());
@@ -298,8 +295,7 @@ public class TableSchemaModel implements Serializable, ProtobufMessageHandler {
builder.addColumns(familyBuilder);
}
if (attrs.containsKey(READONLY)) {
- builder.setReadOnly(
- Boolean.valueOf(attrs.get(READONLY).toString()));
+ builder.setReadOnly(Boolean.parseBoolean(attrs.get(READONLY).toString()));
}
return builder.build().toByteArray();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e9ded51/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
index b096185..8f82a63 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
@@ -1749,6 +1749,19 @@ public class HFileBlock implements Cacheable {
}
@Override
+ public int hashCode() {
+ int result = 1;
+ result = result * 31 + blockType.hashCode();
+ result = result * 31 + nextBlockOnDiskSizeWithHeader;
+ result = result * 31 + (int) (offset ^ (offset >>> 32));
+ result = result * 31 + onDiskSizeWithoutHeader;
+ result = result * 31 + (int) (prevBlockOffset ^ (prevBlockOffset >>> 32));
+ result = result * 31 + uncompressedSizeWithoutHeader;
+ result = result * 31 + buf.hashCode();
+ return result;
+ }
+
+ @Override
public boolean equals(Object comparison) {
if (this == comparison) {
return true;
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e9ded51/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
index 6ac681e..3aeee40 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java
@@ -95,7 +95,7 @@ public class HMasterCommandLine extends ServerCommandLine {
if (cmd.hasOption("minRegionServers")) {
String val = cmd.getOptionValue("minRegionServers");
getConf().setInt("hbase.regions.server.count.min",
- Integer.valueOf(val));
+ Integer.parseInt(val));
LOG.debug("minRegionServers set to " + val);
}
@@ -103,7 +103,7 @@ public class HMasterCommandLine extends ServerCommandLine {
if (cmd.hasOption("minServers")) {
String val = cmd.getOptionValue("minServers");
getConf().setInt("hbase.regions.server.count.min",
- Integer.valueOf(val));
+ Integer.parseInt(val));
LOG.debug("minServers set to " + val);
}
@@ -116,13 +116,13 @@ public class HMasterCommandLine extends ServerCommandLine {
// master when we are in local/standalone mode. Useful testing)
if (cmd.hasOption("localRegionServers")) {
String val = cmd.getOptionValue("localRegionServers");
- getConf().setInt("hbase.regionservers", Integer.valueOf(val));
+ getConf().setInt("hbase.regionservers", Integer.parseInt(val));
LOG.debug("localRegionServers set to " + val);
}
// How many masters to startup inside this process; useful testing
if (cmd.hasOption("masters")) {
String val = cmd.getOptionValue("masters");
- getConf().setInt("hbase.masters", Integer.valueOf(val));
+ getConf().setInt("hbase.masters", Integer.parseInt(val));
LOG.debug("masters set to " + val);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e9ded51/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
index d10141c..d81d96c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
@@ -258,8 +258,9 @@ public class RegionCoprocessorHost
key + ", spec: " + spec);
continue;
}
- int priority = matcher.group(3).trim().isEmpty() ?
- Coprocessor.PRIORITY_USER : Integer.valueOf(matcher.group(3));
+ String priorityStr = matcher.group(3).trim();
+ int priority = priorityStr.isEmpty() ?
+ Coprocessor.PRIORITY_USER : Integer.parseInt(priorityStr);
String cfgSpec = null;
try {
cfgSpec = matcher.group(4);
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e9ded51/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java
index e73b23c..d043735 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java
@@ -23,6 +23,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -105,7 +106,7 @@ public class TableAuthManager {
private Configuration conf;
private ZKPermissionWatcher zkperms;
- private volatile long mtime;
+ private final AtomicLong mtime = new AtomicLong(0L);
private TableAuthManager(ZooKeeperWatcher watcher, Configuration conf)
throws IOException {
@@ -212,7 +213,7 @@ public class TableAuthManager {
}
}
globalCache = newCache;
- mtime++;
+ mtime.incrementAndGet();
} catch (IOException e) {
// Never happens
LOG.error("Error occured while updating the global cache", e);
@@ -240,7 +241,7 @@ public class TableAuthManager {
}
tableCache.put(table, newTablePerms);
- mtime++;
+ mtime.incrementAndGet();
}
/**
@@ -264,7 +265,7 @@ public class TableAuthManager {
}
nsCache.put(namespace, newTablePerms);
- mtime++;
+ mtime.incrementAndGet();
}
private PermissionCache<TablePermission> getTablePermissions(TableName table) {
@@ -741,7 +742,7 @@ public class TableAuthManager {
}
public long getMTime() {
- return mtime;
+ return mtime.get();
}
static Map<ZooKeeperWatcher,TableAuthManager> managerMap =
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e9ded51/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 28bae6a..1fb64a2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -3985,7 +3985,7 @@ public class HBaseFsck extends Configured implements Closeable {
* Display the full report from fsck. This displays all live and dead region
* servers, and all known regions.
*/
- public void setDisplayFullReport() {
+ public static void setDisplayFullReport() {
details = true;
}
@@ -3993,7 +3993,7 @@ public class HBaseFsck extends Configured implements Closeable {
* Set summary mode.
* Print only summary of the tables and status (OK or INCONSISTENT)
*/
- void setSummary() {
+ static void setSummary() {
summary = true;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e9ded51/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java
index efc141a..8cda518 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/OfflineMetaRepair.java
@@ -81,7 +81,7 @@ public class OfflineMetaRepair {
for (int i = 0; i < args.length; i++) {
String cmd = args[i];
if (cmd.equals("-details")) {
- fsck.setDisplayFullReport();
+ HBaseFsck.setDisplayFullReport();
} else if (cmd.equals("-base")) {
if (i == args.length - 1) {
System.err.println("OfflineMetaRepair: -base needs an HDFS path.");
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e9ded51/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index ac2d906..ac79656 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -1001,7 +1001,7 @@ public class TestHBaseFsck {
// fix the problem.
HBaseFsck fsck = new HBaseFsck(conf, hbfsckExecutorService);
fsck.connect();
- fsck.setDisplayFullReport(); // i.e. -details
+ HBaseFsck.setDisplayFullReport(); // i.e. -details
fsck.setTimeLag(0);
fsck.setFixAssignments(true);
fsck.setFixMeta(true);
@@ -1673,7 +1673,7 @@ public class TestHBaseFsck {
// fix lingering split parent
hbck = new HBaseFsck(conf, hbfsckExecutorService);
hbck.connect();
- hbck.setDisplayFullReport(); // i.e. -details
+ HBaseFsck.setDisplayFullReport(); // i.e. -details
hbck.setTimeLag(0);
hbck.setFixSplitParents(true);
hbck.onlineHbck();
@@ -1926,7 +1926,7 @@ public class TestHBaseFsck {
// verify that noHdfsChecking report the same errors
HBaseFsck fsck = new HBaseFsck(conf, hbfsckExecutorService);
fsck.connect();
- fsck.setDisplayFullReport(); // i.e. -details
+ HBaseFsck.setDisplayFullReport(); // i.e. -details
fsck.setTimeLag(0);
fsck.setCheckHdfs(false);
fsck.onlineHbck();
@@ -1937,7 +1937,7 @@ public class TestHBaseFsck {
// verify that fixAssignments works fine with noHdfsChecking
fsck = new HBaseFsck(conf, hbfsckExecutorService);
fsck.connect();
- fsck.setDisplayFullReport(); // i.e. -details
+ HBaseFsck.setDisplayFullReport(); // i.e. -details
fsck.setTimeLag(0);
fsck.setCheckHdfs(false);
fsck.setFixAssignments(true);
@@ -1979,7 +1979,7 @@ public class TestHBaseFsck {
// verify that noHdfsChecking report the same errors
HBaseFsck fsck = new HBaseFsck(conf, hbfsckExecutorService);
fsck.connect();
- fsck.setDisplayFullReport(); // i.e. -details
+ HBaseFsck.setDisplayFullReport(); // i.e. -details
fsck.setTimeLag(0);
fsck.setCheckHdfs(false);
fsck.onlineHbck();
@@ -1990,7 +1990,7 @@ public class TestHBaseFsck {
// verify that fixMeta doesn't work with noHdfsChecking
fsck = new HBaseFsck(conf, hbfsckExecutorService);
fsck.connect();
- fsck.setDisplayFullReport(); // i.e. -details
+ HBaseFsck.setDisplayFullReport(); // i.e. -details
fsck.setTimeLag(0);
fsck.setCheckHdfs(false);
fsck.setFixAssignments(true);
@@ -2045,7 +2045,7 @@ public class TestHBaseFsck {
// verify that noHdfsChecking can't detect ORPHAN_HDFS_REGION
HBaseFsck fsck = new HBaseFsck(conf, hbfsckExecutorService);
fsck.connect();
- fsck.setDisplayFullReport(); // i.e. -details
+ HBaseFsck.setDisplayFullReport(); // i.e. -details
fsck.setTimeLag(0);
fsck.setCheckHdfs(false);
fsck.onlineHbck();
@@ -2056,7 +2056,7 @@ public class TestHBaseFsck {
// verify that fixHdfsHoles doesn't work with noHdfsChecking
fsck = new HBaseFsck(conf, hbfsckExecutorService);
fsck.connect();
- fsck.setDisplayFullReport(); // i.e. -details
+ HBaseFsck.setDisplayFullReport(); // i.e. -details
fsck.setTimeLag(0);
fsck.setCheckHdfs(false);
fsck.setFixHdfsHoles(true);
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e9ded51/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
index bce8938..3691cd7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
@@ -51,7 +51,7 @@ public class HbckTestingUtil {
HBaseFsck fsck = new HBaseFsck(conf, exec);
try {
fsck.connect();
- fsck.setDisplayFullReport(); // i.e. -details
+ HBaseFsck.setDisplayFullReport(); // i.e. -details
fsck.setTimeLag(0);
fsck.setFixAssignments(fixAssignments);
fsck.setFixMeta(fixMeta);
[6/9] hbase git commit: HBASE-13202 Procedure v2 - core framework
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/util/TestTimeoutBlockingQueue.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/util/TestTimeoutBlockingQueue.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/util/TestTimeoutBlockingQueue.java
new file mode 100644
index 0000000..aff536a
--- /dev/null
+++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/util/TestTimeoutBlockingQueue.java
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2.util;
+
+
+import java.util.Arrays;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.procedure2.util.TimeoutBlockingQueue.TimeoutRetriever;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+@Category({MasterTests.class, SmallTests.class})
+public class TestTimeoutBlockingQueue {
+ private static final Log LOG = LogFactory.getLog(TestTimeoutBlockingQueue.class);
+
+ static class TestObject {
+ private long timeout;
+ private int seqId;
+
+ public TestObject(int seqId, long timeout) {
+ this.timeout = timeout;
+ this.seqId = seqId;
+ }
+
+ public long getTimeout() {
+ return timeout;
+ }
+
+ public String toString() {
+ return String.format("(%03d, %03d)", seqId, timeout);
+ }
+ }
+
+ static class TestObjectTimeoutRetriever implements TimeoutRetriever<TestObject> {
+ @Override
+ public long getTimeout(TestObject obj) {
+ return obj.getTimeout();
+ }
+
+ @Override
+ public TimeUnit getTimeUnit(TestObject obj) {
+ return TimeUnit.MILLISECONDS;
+ }
+ }
+
+ @Test
+ public void testOrder() {
+ TimeoutBlockingQueue<TestObject> queue =
+ new TimeoutBlockingQueue<TestObject>(8, new TestObjectTimeoutRetriever());
+
+ long[] timeouts = new long[] {500, 200, 700, 300, 600, 600, 200, 800, 500};
+
+ for (int i = 0; i < timeouts.length; ++i) {
+ for (int j = 0; j <= i; ++j) {
+ queue.add(new TestObject(j, timeouts[j]));
+ queue.dump();
+ }
+
+ long prev = 0;
+ for (int j = 0; j <= i; ++j) {
+ TestObject obj = queue.poll();
+ assertTrue(obj.getTimeout() >= prev);
+ prev = obj.getTimeout();
+ queue.dump();
+ }
+ }
+ }
+
+ @Test
+ public void testTimeoutBlockingQueue() {
+ TimeoutBlockingQueue<TestObject> queue;
+
+ int[][] testArray = new int[][] {
+ {200, 400, 600}, // append
+ {200, 400, 100}, // prepend
+ {200, 400, 300}, // insert
+ };
+
+ for (int i = 0; i < testArray.length; ++i) {
+ int[] sortedArray = Arrays.copyOf(testArray[i], testArray[i].length);
+ Arrays.sort(sortedArray);
+
+ // test with head == 0
+ queue = new TimeoutBlockingQueue<TestObject>(2, new TestObjectTimeoutRetriever());
+ for (int j = 0; j < testArray[i].length; ++j) {
+ queue.add(new TestObject(j, testArray[i][j]));
+ queue.dump();
+ }
+
+ for (int j = 0; !queue.isEmpty(); ++j) {
+ assertEquals(sortedArray[j], queue.poll().getTimeout());
+ }
+
+ queue = new TimeoutBlockingQueue<TestObject>(2, new TestObjectTimeoutRetriever());
+ queue.add(new TestObject(0, 50));
+ assertEquals(50, queue.poll().getTimeout());
+
+ // test with head > 0
+ for (int j = 0; j < testArray[i].length; ++j) {
+ queue.add(new TestObject(j, testArray[i][j]));
+ queue.dump();
+ }
+
+ for (int j = 0; !queue.isEmpty(); ++j) {
+ assertEquals(sortedArray[j], queue.poll().getTimeout());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-protocol/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-protocol/pom.xml b/hbase-protocol/pom.xml
index 7787c52..0d33332 100644
--- a/hbase-protocol/pom.xml
+++ b/hbase-protocol/pom.xml
@@ -176,6 +176,7 @@
<include>MapReduce.proto</include>
<include>Master.proto</include>
<include>MultiRowMutation.proto</include>
+ <include>Procedure.proto</include>
<include>Quota.proto</include>
<include>RegionServerStatus.proto</include>
<include>RowProcessor.proto</include>
[9/9] hbase git commit: HBASE-13202 Procedure v2 - core framework
Posted by mb...@apache.org.
HBASE-13202 Procedure v2 - core framework
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/94fb511c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/94fb511c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/94fb511c
Branch: refs/heads/hbase-12439
Commit: 94fb511c4a3696a3f2ef0e6f2f2c8d547716ccad
Parents: 5459008
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Mon Mar 23 22:40:42 2015 +0000
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Tue Mar 24 08:06:23 2015 +0000
----------------------------------------------------------------------
.../hadoop/hbase/io/util/StreamUtils.java | 12 +-
.../hadoop/hbase/util/ForeignExceptionUtil.java | 105 +
hbase-procedure/pom.xml | 200 +
.../hbase/procedure2/OnePhaseProcedure.java | 28 +
.../hadoop/hbase/procedure2/Procedure.java | 659 ++
.../procedure2/ProcedureAbortedException.java | 42 +
.../hbase/procedure2/ProcedureException.java | 45 +
.../hbase/procedure2/ProcedureExecutor.java | 1007 +++
.../procedure2/ProcedureFairRunQueues.java | 146 +
.../hbase/procedure2/ProcedureResult.java | 95 +
.../hbase/procedure2/ProcedureRunnableSet.java | 71 +
.../procedure2/ProcedureSimpleRunQueue.java | 116 +
.../procedure2/ProcedureYieldException.java | 40 +
.../procedure2/RemoteProcedureException.java | 116 +
.../hbase/procedure2/RootProcedureState.java | 177 +
.../hbase/procedure2/SequentialProcedure.java | 80 +
.../hbase/procedure2/StateMachineProcedure.java | 140 +
.../hbase/procedure2/TwoPhaseProcedure.java | 28 +
.../hbase/procedure2/store/ProcedureStore.java | 110 +
.../procedure2/store/ProcedureStoreTracker.java | 540 ++
.../CorruptedWALProcedureStoreException.java | 43 +
.../procedure2/store/wal/ProcedureWALFile.java | 152 +
.../store/wal/ProcedureWALFormat.java | 212 +
.../store/wal/ProcedureWALFormatReader.java | 161 +
.../procedure2/store/wal/WALProcedureStore.java | 683 ++
.../hadoop/hbase/procedure2/util/ByteSlot.java | 111 +
.../hbase/procedure2/util/StringUtils.java | 80 +
.../procedure2/util/TimeoutBlockingQueue.java | 216 +
.../procedure2/ProcedureTestingUtility.java | 130 +
.../procedure2/TestProcedureExecution.java | 300 +
.../hbase/procedure2/TestProcedureRecovery.java | 486 ++
.../procedure2/TestProcedureRunQueues.java | 72 +
.../store/TestProcedureStoreTracker.java | 168 +
.../store/wal/TestWALProcedureStore.java | 267 +
.../util/TestTimeoutBlockingQueue.java | 137 +
hbase-protocol/pom.xml | 1 +
.../protobuf/generated/ProcedureProtos.java | 7219 ++++++++++++++++++
.../src/main/protobuf/Procedure.proto | 114 +
pom.xml | 21 +-
39 files changed, 14325 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/StreamUtils.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/StreamUtils.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/StreamUtils.java
index 314ed2b..0b442a5 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/StreamUtils.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/StreamUtils.java
@@ -120,7 +120,7 @@ public class StreamUtils {
/**
* Reads a varInt value stored in an array.
- *
+ *
* @param input
* Input array where the varInt is available
* @param offset
@@ -198,4 +198,14 @@ public class StreamUtils {
out.write((byte) (0xff & (v >> 8)));
out.write((byte) (0xff & v));
}
+
+ public static long readLong(InputStream in) throws IOException {
+ long result = 0;
+ for (int shift = 56; shift >= 0; shift -= 8) {
+ long x = in.read();
+ if (x < 0) throw new IOException("EOF");
+ result |= (x << shift);
+ }
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ForeignExceptionUtil.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ForeignExceptionUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ForeignExceptionUtil.java
new file mode 100644
index 0000000..94a8c8c
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ForeignExceptionUtil.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage;
+import org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage;
+import org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage;
+
+/**
+ * Helper to convert Exceptions and StackTraces from/to protobuf.
+ * (see ErrorHandling.proto for the internal of the proto messages)
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public final class ForeignExceptionUtil {
+ private ForeignExceptionUtil() { }
+
+ public static IOException toIOException(final ForeignExceptionMessage eem) {
+ GenericExceptionMessage gem = eem.getGenericException();
+ StackTraceElement[] trace = toStackTrace(gem.getTraceList());
+ RemoteException re = new RemoteException(gem.getClassName(), gem.getMessage());
+ re.setStackTrace(trace);
+ return re.unwrapRemoteException();
+ }
+
+ public static ForeignExceptionMessage toProtoForeignException(String source, Throwable t) {
+ GenericExceptionMessage.Builder gemBuilder = GenericExceptionMessage.newBuilder();
+ gemBuilder.setClassName(t.getClass().getName());
+ if (t.getMessage() != null) {
+ gemBuilder.setMessage(t.getMessage());
+ }
+ // set the stack trace, if there is one
+ List<StackTraceElementMessage> stack = toProtoStackTraceElement(t.getStackTrace());
+ if (stack != null) {
+ gemBuilder.addAllTrace(stack);
+ }
+ GenericExceptionMessage payload = gemBuilder.build();
+ ForeignExceptionMessage.Builder exception = ForeignExceptionMessage.newBuilder();
+ exception.setGenericException(payload).setSource(source);
+ return exception.build();
+ }
+
+ /**
+ * Convert a stack trace to list of {@link StackTraceElement}.
+ * @param trace the stack trace to convert to protobuf message
+ * @return <tt>null</tt> if the passed stack is <tt>null</tt>.
+ */
+ public static List<StackTraceElementMessage> toProtoStackTraceElement(StackTraceElement[] trace) {
+ // if there is no stack trace, ignore it and just return the message
+ if (trace == null) return null;
+ // build the stack trace for the message
+ List<StackTraceElementMessage> pbTrace = new ArrayList<StackTraceElementMessage>(trace.length);
+ for (StackTraceElement elem : trace) {
+ StackTraceElementMessage.Builder stackBuilder = StackTraceElementMessage.newBuilder();
+ stackBuilder.setDeclaringClass(elem.getClassName());
+ stackBuilder.setFileName(elem.getFileName());
+ stackBuilder.setLineNumber(elem.getLineNumber());
+ stackBuilder.setMethodName(elem.getMethodName());
+ pbTrace.add(stackBuilder.build());
+ }
+ return pbTrace;
+ }
+
+ /**
+ * Unwind a serialized array of {@link StackTraceElementMessage}s to a
+ * {@link StackTraceElement}s.
+ * @param traceList list that was serialized
+ * @return the deserialized list or <tt>null</tt> if it couldn't be unwound (e.g. wasn't set on
+ * the sender).
+ */
+ public static StackTraceElement[] toStackTrace(List<StackTraceElementMessage> traceList) {
+ if (traceList == null || traceList.size() == 0) {
+ return new StackTraceElement[0]; // empty array
+ }
+ StackTraceElement[] trace = new StackTraceElement[traceList.size()];
+ for (int i = 0; i < traceList.size(); i++) {
+ StackTraceElementMessage elem = traceList.get(i);
+ trace[i] = new StackTraceElement(
+ elem.getDeclaringClass(), elem.getMethodName(), elem.getFileName(), elem.getLineNumber());
+ }
+ return trace;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-procedure/pom.xml b/hbase-procedure/pom.xml
new file mode 100644
index 0000000..fce3102
--- /dev/null
+++ b/hbase-procedure/pom.xml
@@ -0,0 +1,200 @@
+<?xml version="1.0"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>hbase</artifactId>
+ <groupId>org.apache.hbase</groupId>
+ <version>2.0.0-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>hbase-procedure</artifactId>
+ <name>HBase - Procedure</name>
+ <description>Procedure Framework</description>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>default-compile</id>
+ <configuration>
+ <compilerId>javac-with-errorprone</compilerId>
+ <forceJavacCompilerUse>true</forceJavacCompilerUse>
+ </configuration>
+ </execution>
+ <execution>
+ <id>default-testCompile</id>
+ <configuration>
+ <compilerId>javac-with-errorprone</compilerId>
+ <forceJavacCompilerUse>true</forceJavacCompilerUse>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ <!-- Make a jar and put the sources in the jar -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-source-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <!--Make it so assembly:single does nothing in here-->
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>${maven.assembly.version}</version>
+ <configuration>
+ <skipAssembly>true</skipAssembly>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <!-- Always skip the second part executions, since we only run
+ simple unit tests in this module. -->
+ <executions>
+ <execution>
+ <id>secondPartTestsExecution</id>
+ <phase>test</phase>
+ <goals>
+ <goal>test</goal>
+ </goals>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <version>${project.version}</version>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-annotations</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-protocol</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </dependency>
+ </dependencies>
+
+ <profiles>
+ <!-- Profiles for building against different hadoop versions -->
+ <profile>
+ <id>hadoop-1.1</id>
+ <activation>
+ <property>
+ <!--Below formatting for dev-support/generate-hadoopX-poms.sh-->
+ <!--h1--><name>hadoop.profile</name><value>1.1</value>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+ <profile>
+ <id>hadoop-1.0</id>
+ <activation>
+ <property>
+ <name>hadoop.profile</name>
+ <value>1.0</value>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+ <!--
+ profile for building against Hadoop 2.0.0-alpha. Activate using:
+ mvn -Dhadoop.profile=2.0
+ -->
+ <profile>
+ <id>hadoop-2.0</id>
+ <activation>
+ <property>
+ <!--Below formatting for dev-support/generate-hadoopX-poms.sh-->
+ <!--h2--><name>!hadoop.profile</name>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+ <!--
+ profile for building against Hadoop 3.0.x. Activate using:
+ mvn -Dhadoop.profile=3.0
+ -->
+ <profile>
+ <id>hadoop-3.0</id>
+ <activation>
+ <property>
+ <name>hadoop.profile</name>
+ <value>3.0</value>
+ </property>
+ </activation>
+ <properties>
+ <hadoop.version>3.0-SNAPSHOT</hadoop.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+</project>
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/OnePhaseProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/OnePhaseProcedure.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/OnePhaseProcedure.java
new file mode 100644
index 0000000..1c3be2d
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/OnePhaseProcedure.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public abstract class OnePhaseProcedure<TEnvironment> extends Procedure<TEnvironment> {
+ // TODO (e.g. used by online snapshots)
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
new file mode 100644
index 0000000..7135aa0
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
@@ -0,0 +1,659 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Modifier;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.procedure2.util.StringUtils;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState;
+import org.apache.hadoop.hbase.util.ByteStringer;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.protobuf.ByteString;
+
+/**
+ * Base Procedure class responsible to handle the Procedure Metadata
+ * e.g. state, startTime, lastUpdate, stack-indexes, ...
+ *
+ * execute() is called each time the procedure is executed.
+ * it may be called multiple times in case of failure and restart, so the
+ * code must be idempotent.
+ * the return is a set of sub-procedures or null in case the procedure doesn't
+ * have sub-procedures. Once the sub-procedures are successfully completed
+ * the execute() method is called again, you should think at it as a stack:
+ * -> step 1
+ * ---> step 2
+ * -> step 1
+ *
+ * rollback() is called when the procedure or one of the sub-procedures is failed.
+ * the rollback step is supposed to cleanup the resources created during the
+ * execute() step. in case of failure and restart rollback() may be called
+ * multiple times, so the code must be idempotent.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public abstract class Procedure<TEnvironment> implements Comparable<Procedure> {
+ // unchanged after initialization
+ private String owner = null;
+ private Long parentProcId = null;
+ private Long procId = null;
+ private long startTime;
+
+ // runtime state, updated every operation
+ private ProcedureState state = ProcedureState.INITIALIZING;
+ private Integer timeout = null;
+ private int[] stackIndexes = null;
+ private int childrenLatch = 0;
+ private long lastUpdate;
+
+ private RemoteProcedureException exception = null;
+ private byte[] result = null;
+
+ /**
+ * The main code of the procedure. It must be idempotent since execute()
+ * may be called multiple time in case of machine failure in the middle
+ * of the execution.
+ * @return a set of sub-procedures or null if there is nothing else to execute.
+ */
+ protected abstract Procedure[] execute(TEnvironment env)
+ throws ProcedureYieldException;
+
+ /**
+ * The code to undo what done by the execute() code.
+ * It is called when the procedure or one of the sub-procedure failed or an
+ * abort was requested. It should cleanup all the resources created by
+ * the execute() call. The implementation must be idempotent since rollback()
+ * may be called multiple time in case of machine failure in the middle
+ * of the execution.
+ */
+ protected abstract void rollback(TEnvironment env);
+
+ /**
+ * The abort() call is asynchronous and each procedure must decide how to deal
+ * with that, if they want to be abortable. The simplest implementation
+ * is to have an AtomicBoolean set in the abort() method and then the execute()
+ * will check if the abort flag is set or not.
+ * abort() may be called multiple times from the client, so the implementation
+ * must be idempotent.
+ *
+ * NOTE: abort() is not like Thread.interrupt() it is just a notification
+ * that allows the procedure implementor where to abort to avoid leak and
+ * have a better control on what was executed and what not.
+ */
+ protected abstract void abort(TEnvironment env);
+
+ /**
+ * The user-level code of the procedure may have some state to
+ * persist (e.g. input arguments) to be able to resume on failure.
+ * @param stream the stream that will contain the user serialized data
+ */
+ protected abstract void serializeStateData(final OutputStream stream)
+ throws IOException;
+
+ /**
+ * Called on store load to allow the user to decode the previously serialized
+ * state.
+ * @param stream the stream that contains the user serialized data
+ */
+ protected abstract void deserializeStateData(final InputStream stream)
+ throws IOException;
+
+ /**
+ * The user should override this method, and try to take a lock if necessary.
+ * A lock can be anything, and it is up to the implementor.
+ * Example: in our Master we can execute request in parallel for different tables
+ * create t1 and create t2 can be executed at the same time.
+ * anything else on t1/t2 is queued waiting that specific table create to happen.
+ *
+ * @return true if the lock was acquired and false otherwise
+ */
+ protected boolean acquireLock(final TEnvironment env) {
+ return true;
+ }
+
+ /**
+ * The user should override this method, and release lock if necessary.
+ */
+ protected void releaseLock(final TEnvironment env) {
+ // no-op
+ }
+
+ /**
+ * Called when the procedure is marked as completed (success or rollback).
+ * The procedure implementor may use this method to cleanup in-memory states.
+ * This operation will not be retried on failure.
+ */
+ protected void completionCleanup(final TEnvironment env) {
+ // no-op
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(getClass().getName());
+
+ if (procId != null) {
+ sb.append(" id=");
+ sb.append(getProcId());
+ }
+
+ if (hasParent()) {
+ sb.append(" parent=");
+ sb.append(getParentProcId());
+ }
+
+ if (hasOwner()) {
+ sb.append(" owner=");
+ sb.append(getOwner());
+ }
+
+ sb.append(" state=");
+ sb.append(getState());
+ return sb.toString();
+ }
+
+ /**
+ * @return the serialized result if any, otherwise null
+ */
+ public byte[] getResult() {
+ return result;
+ }
+
+ /**
+ * The procedure may leave a "result" on completion.
+ * @param result the serialized result that will be passed to the client
+ */
+ protected void setResult(final byte[] result) {
+ this.result = result;
+ }
+
+ public long getProcId() {
+ return procId;
+ }
+
+ public boolean hasParent() {
+ return parentProcId != null;
+ }
+
+ public boolean hasException() {
+ return exception != null;
+ }
+
+ public boolean hasTimeout() {
+ return timeout != null;
+ }
+
+ public long getParentProcId() {
+ return parentProcId;
+ }
+
+ /**
+ * @return true if the procedure has failed.
+ * true may mean failed but not yet rolledback or failed and rolledback.
+ */
+ public synchronized boolean isFailed() {
+ return exception != null || state == ProcedureState.ROLLEDBACK;
+ }
+
+ /**
+ * @return true if the procedure is finished successfully.
+ */
+ public synchronized boolean isSuccess() {
+ return state == ProcedureState.FINISHED && exception == null;
+ }
+
+ /**
+ * @return true if the procedure is finished. The Procedure may be completed
+ * successfuly or failed and rolledback.
+ */
+ public synchronized boolean isFinished() {
+ switch (state) {
+ case ROLLEDBACK:
+ return true;
+ case FINISHED:
+ return exception == null;
+ default:
+ break;
+ }
+ return false;
+ }
+
+ /**
+ * @return true if the procedure is waiting for a child to finish or for an external event.
+ */
+ public synchronized boolean isWaiting() {
+ switch (state) {
+ case WAITING:
+ case WAITING_TIMEOUT:
+ return true;
+ default:
+ break;
+ }
+ return false;
+ }
+
+ public synchronized RemoteProcedureException getException() {
+ return exception;
+ }
+
+ public long getStartTime() {
+ return startTime;
+ }
+
+ public synchronized long getLastUpdate() {
+ return lastUpdate;
+ }
+
+ public synchronized long elapsedTime() {
+ return lastUpdate - startTime;
+ }
+
+ /**
+ * @param timeout timeout in msec
+ */
+ protected void setTimeout(final int timeout) {
+ this.timeout = timeout;
+ }
+
+ /**
+ * @return the timeout in msec
+ */
+ public int getTimeout() {
+ return timeout;
+ }
+
+ /**
+ * @return the remaining time before the timeout
+ */
+ public long getTimeRemaining() {
+ return Math.max(0, timeout - (EnvironmentEdgeManager.currentTime() - startTime));
+ }
+
+ protected void setOwner(final String owner) {
+ this.owner = StringUtils.isEmpty(owner) ? null : owner;
+ }
+
+ public String getOwner() {
+ return owner;
+ }
+
+ public boolean hasOwner() {
+ return owner != null;
+ }
+
+ @VisibleForTesting
+ @InterfaceAudience.Private
+ protected synchronized void setState(final ProcedureState state) {
+ this.state = state;
+ updateTimestamp();
+ }
+
+ @InterfaceAudience.Private
+ protected synchronized ProcedureState getState() {
+ return state;
+ }
+
+ protected void setFailure(final String source, final Throwable cause) {
+ setFailure(new RemoteProcedureException(source, cause));
+ }
+
+ protected synchronized void setFailure(final RemoteProcedureException exception) {
+ this.exception = exception;
+ if (!isFinished()) {
+ setState(ProcedureState.FINISHED);
+ }
+ }
+
+ protected void setAbortFailure(final String source, final String msg) {
+ setFailure(source, new ProcedureAbortedException(msg));
+ }
+
+ @InterfaceAudience.Private
+ protected synchronized boolean setTimeoutFailure() {
+ if (state == ProcedureState.WAITING_TIMEOUT) {
+ long timeDiff = EnvironmentEdgeManager.currentTime() - lastUpdate;
+ setFailure("ProcedureExecutor", new TimeoutException(
+ "Operation timed out after " + StringUtils.humanTimeDiff(timeDiff)));
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Called by the ProcedureExecutor to assign the ID to the newly created procedure.
+ */
+ @VisibleForTesting
+ @InterfaceAudience.Private
+ protected void setProcId(final long procId) {
+ this.procId = procId;
+ this.startTime = EnvironmentEdgeManager.currentTime();
+ setState(ProcedureState.RUNNABLE);
+ }
+
+ /**
+ * Called by the ProcedureExecutor to assign the parent to the newly created procedure.
+ */
+ @InterfaceAudience.Private
+ protected void setParentProcId(final long parentProcId) {
+ this.parentProcId = parentProcId;
+ }
+
+ /**
+ * Internal method called by the ProcedureExecutor that starts the
+ * user-level code execute().
+ */
+ @InterfaceAudience.Private
+ protected Procedure[] doExecute(final TEnvironment env)
+ throws ProcedureYieldException {
+ try {
+ updateTimestamp();
+ return execute(env);
+ } finally {
+ updateTimestamp();
+ }
+ }
+
+ /**
+ * Internal method called by the ProcedureExecutor that starts the
+ * user-level code rollback().
+ */
+ @InterfaceAudience.Private
+ protected void doRollback(final TEnvironment env) {
+ try {
+ updateTimestamp();
+ rollback(env);
+ } finally {
+ updateTimestamp();
+ }
+ }
+
+ /**
+ * Called on store load to initialize the Procedure internals after
+ * the creation/deserialization.
+ */
+ @InterfaceAudience.Private
+ protected void setStartTime(final long startTime) {
+ this.startTime = startTime;
+ }
+
+ /**
+ * Called on store load to initialize the Procedure internals after
+ * the creation/deserialization.
+ */
+ private synchronized void setLastUpdate(final long lastUpdate) {
+ this.lastUpdate = lastUpdate;
+ }
+
+ protected synchronized void updateTimestamp() {
+ this.lastUpdate = EnvironmentEdgeManager.currentTime();
+ }
+
+ /**
+ * Called by the ProcedureExecutor on procedure-load to restore the latch state
+ */
+ @InterfaceAudience.Private
+ protected synchronized void setChildrenLatch(final int numChildren) {
+ this.childrenLatch = numChildren;
+ }
+
+ /**
+ * Called by the ProcedureExecutor on procedure-load to restore the latch state
+ */
+ @InterfaceAudience.Private
+ protected synchronized void incChildrenLatch() {
+ // TODO: can this be inferred from the stack? I think so...
+ this.childrenLatch++;
+ }
+
+ /**
+ * Called by the ProcedureExecutor to notify that one of the sub-procedures
+ * has completed.
+ */
+ @InterfaceAudience.Private
+ protected synchronized boolean childrenCountDown() {
+ assert childrenLatch > 0;
+ return --childrenLatch == 0;
+ }
+
+ /**
+ * Called by the RootProcedureState on procedure execution.
+ * Each procedure store its stack-index positions.
+ */
+ @InterfaceAudience.Private
+ protected void addStackIndex(final int index) {
+ if (stackIndexes == null) {
+ stackIndexes = new int[] { index };
+ } else {
+ int count = stackIndexes.length;
+ stackIndexes = Arrays.copyOf(stackIndexes, count + 1);
+ stackIndexes[count] = index;
+ }
+ }
+
+ @InterfaceAudience.Private
+ protected boolean removeStackIndex() {
+ if (stackIndexes.length > 1) {
+ stackIndexes = Arrays.copyOf(stackIndexes, stackIndexes.length - 1);
+ return false;
+ } else {
+ stackIndexes = null;
+ return true;
+ }
+ }
+
+ /**
+ * Called on store load to initialize the Procedure internals after
+ * the creation/deserialization.
+ */
+ @InterfaceAudience.Private
+ protected void setStackIndexes(final List<Integer> stackIndexes) {
+ this.stackIndexes = new int[stackIndexes.size()];
+ for (int i = 0; i < this.stackIndexes.length; ++i) {
+ this.stackIndexes[i] = stackIndexes.get(i);
+ }
+ }
+
+ @InterfaceAudience.Private
+ protected boolean wasExecuted() {
+ return stackIndexes != null;
+ }
+
+ @InterfaceAudience.Private
+ protected int[] getStackIndexes() {
+ return stackIndexes;
+ }
+
+ @Override
+ public int compareTo(final Procedure other) {
+ long diff = getProcId() - other.getProcId();
+ return (diff < 0) ? -1 : (diff > 0) ? 1 : 0;
+ }
+
+ /*
+ * Helper to lookup the root Procedure ID given a specified procedure.
+ */
+ @InterfaceAudience.Private
+ protected static Long getRootProcedureId(final Map<Long, Procedure> procedures, Procedure proc) {
+ while (proc.hasParent()) {
+ proc = procedures.get(proc.getParentProcId());
+ if (proc == null) return null;
+ }
+ return proc.getProcId();
+ }
+
+ protected static Procedure newInstance(final String className) throws IOException {
+ try {
+ Class<?> clazz = Class.forName(className);
+ if (!Modifier.isPublic(clazz.getModifiers())) {
+ throw new Exception("the " + clazz + " class is not public");
+ }
+
+ Constructor<?> ctor = clazz.getConstructor();
+ assert ctor != null : "no constructor found";
+ if (!Modifier.isPublic(ctor.getModifiers())) {
+ throw new Exception("the " + clazz + " constructor is not public");
+ }
+ return (Procedure)ctor.newInstance();
+ } catch (Exception e) {
+ throw new IOException("The procedure class " + className +
+ " must be accessible and have an empty constructor", e);
+ }
+ }
+
+ protected static void validateClass(final Procedure proc) throws IOException {
+ try {
+ Class<?> clazz = proc.getClass();
+ if (!Modifier.isPublic(clazz.getModifiers())) {
+ throw new Exception("the " + clazz + " class is not public");
+ }
+
+ Constructor<?> ctor = clazz.getConstructor();
+ assert ctor != null;
+ if (!Modifier.isPublic(ctor.getModifiers())) {
+ throw new Exception("the " + clazz + " constructor is not public");
+ }
+ } catch (Exception e) {
+ throw new IOException("The procedure class " + proc.getClass().getName() +
+ " must be accessible and have an empty constructor", e);
+ }
+ }
+
+ /**
+ * Helper to convert the procedure to protobuf.
+ * Used by ProcedureStore implementations.
+ */
+ @InterfaceAudience.Private
+ public static ProcedureProtos.Procedure convert(final Procedure proc)
+ throws IOException {
+ Preconditions.checkArgument(proc != null);
+ validateClass(proc);
+
+ ProcedureProtos.Procedure.Builder builder = ProcedureProtos.Procedure.newBuilder()
+ .setClassName(proc.getClass().getName())
+ .setProcId(proc.getProcId())
+ .setState(proc.getState())
+ .setStartTime(proc.getStartTime())
+ .setLastUpdate(proc.getLastUpdate());
+
+ if (proc.hasParent()) {
+ builder.setParentId(proc.getParentProcId());
+ }
+
+ if (proc.hasTimeout()) {
+ builder.setTimeout(proc.getTimeout());
+ }
+
+ if (proc.hasOwner()) {
+ builder.setOwner(proc.getOwner());
+ }
+
+ int[] stackIds = proc.getStackIndexes();
+ if (stackIds != null) {
+ for (int i = 0; i < stackIds.length; ++i) {
+ builder.addStackId(stackIds[i]);
+ }
+ }
+
+ if (proc.hasException()) {
+ RemoteProcedureException exception = proc.getException();
+ builder.setException(
+ RemoteProcedureException.toProto(exception.getSource(), exception.getCause()));
+ }
+
+ byte[] result = proc.getResult();
+ if (result != null) {
+ builder.setResult(ByteStringer.wrap(result));
+ }
+
+ ByteString.Output stateStream = ByteString.newOutput();
+ proc.serializeStateData(stateStream);
+ if (stateStream.size() > 0) {
+ builder.setStateData(stateStream.toByteString());
+ }
+
+ return builder.build();
+ }
+
+ /**
+ * Helper to convert the protobuf procedure.
+ * Used by ProcedureStore implementations.
+ *
+ * TODO: OPTIMIZATION: some of the field never change during the execution
+ * (e.g. className, procId, parentId, ...).
+ * We can split in 'data' and 'state', and the store
+ * may take advantage of it by storing the data only on insert().
+ */
+ @InterfaceAudience.Private
+ public static Procedure convert(final ProcedureProtos.Procedure proto)
+ throws IOException {
+ // Procedure from class name
+ Procedure proc = Procedure.newInstance(proto.getClassName());
+
+ // set fields
+ proc.setProcId(proto.getProcId());
+ proc.setState(proto.getState());
+ proc.setStartTime(proto.getStartTime());
+ proc.setLastUpdate(proto.getLastUpdate());
+
+ if (proto.hasParentId()) {
+ proc.setParentProcId(proto.getParentId());
+ }
+
+ if (proto.hasOwner()) {
+ proc.setOwner(proto.getOwner());
+ }
+
+ if (proto.hasTimeout()) {
+ proc.setTimeout(proto.getTimeout());
+ }
+
+ if (proto.getStackIdCount() > 0) {
+ proc.setStackIndexes(proto.getStackIdList());
+ }
+
+ if (proto.hasException()) {
+ assert proc.getState() == ProcedureState.FINISHED ||
+ proc.getState() == ProcedureState.ROLLEDBACK :
+ "The procedure must be failed (waiting to rollback) or rolledback";
+ proc.setFailure(RemoteProcedureException.fromProto(proto.getException()));
+ }
+
+ if (proto.hasResult()) {
+ proc.setResult(proto.getResult().toByteArray());
+ }
+
+ // we want to call deserialize even when the stream is empty, mainly for testing.
+ proc.deserializeStateData(proto.getStateData().newInput());
+
+ return proc;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureAbortedException.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureAbortedException.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureAbortedException.java
new file mode 100644
index 0000000..2e409cf
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureAbortedException.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * Thrown when a procedure is aborted
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class ProcedureAbortedException extends ProcedureException {
+ /** default constructor */
+ public ProcedureAbortedException() {
+ super();
+ }
+
+ /**
+ * Constructor
+ * @param s message
+ */
+ public ProcedureAbortedException(String s) {
+ super(s);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureException.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureException.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureException.java
new file mode 100644
index 0000000..9f922b1
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureException.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class ProcedureException extends IOException {
+ /** default constructor */
+ public ProcedureException() {
+ super();
+ }
+
+ /**
+ * Constructor
+ * @param s message
+ */
+ public ProcedureException(String s) {
+ super(s);
+ }
+
+ public ProcedureException(Throwable t) {
+ super(t);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
new file mode 100644
index 0000000..c477b9e
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
@@ -0,0 +1,1007 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.HashSet;
+import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureStore;
+import org.apache.hadoop.hbase.procedure2.util.TimeoutBlockingQueue;
+import org.apache.hadoop.hbase.procedure2.util.TimeoutBlockingQueue.TimeoutRetriever;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.procedure2.util.StringUtils;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Thread Pool that executes the submitted procedures.
+ * The executor has a ProcedureStore associated.
+ * Each operation is logged and on restart the pending procedures are resumed.
+ *
+ * Unless the Procedure code throws an error (e.g. invalid user input)
+ * the procedure will complete (at some point in time), On restart the pending
+ * procedures are resumed and the once failed will be rolledback.
+ *
+ * The user can add procedures to the executor via submitProcedure(proc)
+ * check for the finished state via isFinished(procId)
+ * and get the result via getResult(procId)
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class ProcedureExecutor<TEnvironment> {
+ private static final Log LOG = LogFactory.getLog(ProcedureExecutor.class);
+
+ Testing testing = null;
+ public static class Testing {
+ protected boolean killBeforeStoreUpdate = false;
+ protected boolean toggleKillBeforeStoreUpdate = false;
+
+ protected boolean shouldKillBeforeStoreUpdate() {
+ final boolean kill = this.killBeforeStoreUpdate;
+ if (this.toggleKillBeforeStoreUpdate) {
+ this.killBeforeStoreUpdate = !kill;
+ LOG.warn("Toggle Kill before store update to: " + this.killBeforeStoreUpdate);
+ }
+ return kill;
+ }
+ }
+
+ public interface ProcedureExecutorListener {
+ void procedureLoaded(long procId);
+ void procedureAdded(long procId);
+ void procedureFinished(long procId);
+ }
+
+ /**
+ * Used by the TimeoutBlockingQueue to get the timeout interval of the procedure
+ */
+ private static class ProcedureTimeoutRetriever implements TimeoutRetriever<Procedure> {
+ @Override
+ public long getTimeout(Procedure proc) {
+ long delta = (EnvironmentEdgeManager.currentTime() - proc.getLastUpdate());
+ return Math.max(0, proc.getTimeout() - delta);
+ }
+
+ @Override
+ public TimeUnit getTimeUnit(Procedure proc) {
+ return TimeUnit.MILLISECONDS;
+ }
+ }
+
+ /**
+ * Internal cleaner that removes the completed procedure results after a TTL.
+ * NOTE: This is a special case handled in timeoutLoop().
+ *
+ * Since the client code looks more or less like:
+ * procId = master.doOperation()
+ * while (master.getProcResult(procId) == ProcInProgress);
+ * The master should not throw away the proc result as soon as the procedure is done
+ * but should wait a result request from the client (see executor.removeResult(procId))
+ * The client will call something like master.isProcDone() or master.getProcResult()
+ * which will return the result/state to the client, and it will mark the completed
+ * proc as ready to delete. note that the client may not receive the response from
+ * the master (e.g. master failover) so, if we delay a bit the real deletion of
+ * the proc result the client will be able to get the result the next try.
+ */
+ private static class CompletedProcedureCleaner<TEnvironment> extends Procedure<TEnvironment> {
+ private static final Log LOG = LogFactory.getLog(CompletedProcedureCleaner.class);
+
+ private static final String CLEANER_INTERVAL_CONF_KEY = "hbase.procedure.cleaner.interval";
+ private static final int DEFAULT_CLEANER_INTERVAL = 30 * 1000; // 30sec
+
+ private static final String EVICT_TTL_CONF_KEY = "hbase.procedure.cleaner.evict.ttl";
+ private static final int DEFAULT_EVICT_TTL = 15 * 60000; // 15min
+
+ private static final String EVICT_ACKED_TTL_CONF_KEY ="hbase.procedure.cleaner.acked.evict.ttl";
+ private static final int DEFAULT_ACKED_EVICT_TTL = 5 * 60000; // 5min
+
+ private final Map<Long, ProcedureResult> completed;
+ private final ProcedureStore store;
+ private final Configuration conf;
+
+ public CompletedProcedureCleaner(final Configuration conf, final ProcedureStore store,
+ final Map<Long, ProcedureResult> completedMap) {
+ // set the timeout interval that triggers the periodic-procedure
+ setTimeout(conf.getInt(CLEANER_INTERVAL_CONF_KEY, DEFAULT_CLEANER_INTERVAL));
+ this.completed = completedMap;
+ this.store = store;
+ this.conf = conf;
+ }
+
+ public void periodicExecute(final TEnvironment env) {
+ if (completed.isEmpty()) {
+ LOG.debug("no completed procedures to cleanup");
+ return;
+ }
+
+ final long evictTtl = conf.getInt(EVICT_TTL_CONF_KEY, DEFAULT_EVICT_TTL);
+ final long evictAckTtl = conf.getInt(EVICT_ACKED_TTL_CONF_KEY, DEFAULT_ACKED_EVICT_TTL);
+
+ LOG.debug("processing " + completed.size() + " completed procedures");
+ long now = EnvironmentEdgeManager.currentTime();
+ Iterator<Map.Entry<Long, ProcedureResult>> it = completed.entrySet().iterator();
+ while (it.hasNext()) {
+ Map.Entry<Long, ProcedureResult> entry = it.next();
+ ProcedureResult result = entry.getValue();
+
+ // TODO: Select TTL based on Procedure type
+ if ((result.hasClientAckTime() && (now - result.getClientAckTime()) >= evictAckTtl) ||
+ (now - result.getLastUpdate()) >= evictTtl) {
+ LOG.debug("Evict completed procedure " + entry.getKey());
+ store.delete(entry.getKey());
+ it.remove();
+ }
+ }
+ }
+
+ @Override
+ protected Procedure[] execute(final TEnvironment env) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected void rollback(final TEnvironment env) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected void abort(final TEnvironment env) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void serializeStateData(final OutputStream stream) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void deserializeStateData(final InputStream stream) {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ /**
+ * Map the the procId returned by submitProcedure(), the Root-ProcID, to the ProcedureResult.
+ * Once a Root-Procedure completes (success or failure), the result will be added to this map.
+ * The user of ProcedureExecutor should call getResult(procId) to get the result.
+ */
+ private final ConcurrentHashMap<Long, ProcedureResult> completed =
+ new ConcurrentHashMap<Long, ProcedureResult>();
+
+ /**
+ * Map the the procId returned by submitProcedure(), the Root-ProcID, to the RootProcedureState.
+ * The RootProcedureState contains the execution stack of the Root-Procedure,
+ * It is added to the map by submitProcedure() and removed on procedure completion.
+ */
+ private final ConcurrentHashMap<Long, RootProcedureState> rollbackStack =
+ new ConcurrentHashMap<Long, RootProcedureState>();
+
+ /**
+ * Helper map to lookup the live procedures by ID.
+ * This map contains every procedure. root-procedures and subprocedures.
+ */
+ private final ConcurrentHashMap<Long, Procedure> procedures =
+ new ConcurrentHashMap<Long, Procedure>();
+
+ /**
+ * Timeout Queue that contains Procedures in a WAITING_TIMEOUT state
+ * or periodic procedures.
+ */
+ private final TimeoutBlockingQueue<Procedure> waitingTimeout =
+ new TimeoutBlockingQueue<Procedure>(new ProcedureTimeoutRetriever());
+
+ /**
+ * Queue that contains runnable procedures.
+ */
+ private final ProcedureRunnableSet runnables;
+
+ private final CopyOnWriteArrayList<ProcedureExecutorListener> listeners =
+ new CopyOnWriteArrayList<ProcedureExecutorListener>();
+
+ private final AtomicInteger activeExecutorCount = new AtomicInteger(0);
+ private final AtomicBoolean running = new AtomicBoolean(false);
+ private final AtomicLong lastProcId = new AtomicLong(-1);
+ private final TEnvironment environment;
+ private final ProcedureStore store;
+ private final Configuration conf;
+
+ private Thread[] threads;
+
+ public ProcedureExecutor(final Configuration conf, final TEnvironment environment,
+ final ProcedureStore store) {
+ this(conf, environment, store, new ProcedureSimpleRunQueue());
+ }
+
+ public ProcedureExecutor(final Configuration conf, final TEnvironment environment,
+ final ProcedureStore store, final ProcedureRunnableSet runqueue) {
+ this.environment = environment;
+ this.runnables = runqueue;
+ this.store = store;
+ this.conf = conf;
+ }
+
+ private List<Map.Entry<Long, RootProcedureState>> load() throws IOException {
+ Preconditions.checkArgument(completed.isEmpty());
+ Preconditions.checkArgument(rollbackStack.isEmpty());
+ Preconditions.checkArgument(procedures.isEmpty());
+ Preconditions.checkArgument(waitingTimeout.isEmpty());
+ Preconditions.checkArgument(runnables.size() == 0);
+
+ // 1. Load the procedures
+ Iterator<Procedure> loader = store.load();
+ if (loader == null) {
+ lastProcId.set(0);
+ return null;
+ }
+
+ long logMaxProcId = 0;
+ int runnablesCount = 0;
+ while (loader.hasNext()) {
+ Procedure proc = loader.next();
+ procedures.put(proc.getProcId(), proc);
+ logMaxProcId = Math.max(logMaxProcId, proc.getProcId());
+ LOG.debug("Loading procedure " + proc +
+ " state=" + proc.getState() + " isFailed=" + proc.hasException());
+ if (!proc.hasParent() && !proc.isFinished()) {
+ rollbackStack.put(proc.getProcId(), new RootProcedureState());
+ }
+ if (proc.getState() == ProcedureState.RUNNABLE) {
+ runnablesCount++;
+ }
+ }
+ assert lastProcId.get() < 0;
+ lastProcId.set(logMaxProcId);
+
+ // 2. Initialize the stacks
+ TreeSet<Procedure> runnableSet = null;
+ HashSet<Procedure> waitingSet = null;
+ for (final Procedure proc: procedures.values()) {
+ Long rootProcId = getRootProcedureId(proc);
+ if (rootProcId == null) {
+ // The 'proc' was ready to run but the root procedure was rolledback?
+ runnables.addBack(proc);
+ continue;
+ }
+
+ if (!proc.hasParent() && proc.isFinished()) {
+ LOG.debug("The procedure is completed " + proc +
+ " state=" + proc.getState() + " isFailed=" + proc.hasException());
+ assert !rollbackStack.containsKey(proc.getProcId());
+ completed.put(proc.getProcId(), newResultFromProcedure(proc));
+ continue;
+ }
+
+ if (proc.hasParent() && !proc.isFinished()) {
+ Procedure parent = procedures.get(proc.getParentProcId());
+ // corrupted procedures are handled later at step 3
+ if (parent != null) {
+ parent.incChildrenLatch();
+ }
+ }
+
+ RootProcedureState procStack = rollbackStack.get(rootProcId);
+ procStack.loadStack(proc);
+
+ switch (proc.getState()) {
+ case RUNNABLE:
+ if (runnableSet == null) {
+ runnableSet = new TreeSet<Procedure>();
+ }
+ runnableSet.add(proc);
+ break;
+ case WAITING_TIMEOUT:
+ if (waitingSet == null) {
+ waitingSet = new HashSet<Procedure>();
+ }
+ waitingSet.add(proc);
+ break;
+ case FINISHED:
+ if (proc.hasException()) {
+ // add the proc to the runnables to perform the rollback
+ runnables.addBack(proc);
+ break;
+ }
+ case ROLLEDBACK:
+ case INITIALIZING:
+ String msg = "Unexpected " + proc.getState() + " state for " + proc;
+ LOG.error(msg);
+ throw new UnsupportedOperationException(msg);
+ default:
+ break;
+ }
+ }
+
+ // 3. Validate the stacks
+ List<Map.Entry<Long, RootProcedureState>> corrupted = null;
+ Iterator<Map.Entry<Long, RootProcedureState>> itStack = rollbackStack.entrySet().iterator();
+ while (itStack.hasNext()) {
+ Map.Entry<Long, RootProcedureState> entry = itStack.next();
+ RootProcedureState procStack = entry.getValue();
+ if (procStack.isValid()) continue;
+
+ for (Procedure proc: procStack.getSubprocedures()) {
+ procedures.remove(proc.getProcId());
+ if (runnableSet != null) runnableSet.remove(proc);
+ if (waitingSet != null) waitingSet.remove(proc);
+ }
+ itStack.remove();
+ if (corrupted == null) {
+ corrupted = new ArrayList<Map.Entry<Long, RootProcedureState>>();
+ }
+ corrupted.add(entry);
+ }
+
+ // 4. Push the runnables
+ if (runnableSet != null) {
+ // TODO: See ProcedureWALFormatReader.readInitEntry() some procedure
+ // may be started way before this stuff.
+ for (Procedure proc: runnableSet) {
+ if (!proc.hasParent()) {
+ sendProcedureLoadedNotification(proc.getProcId());
+ }
+ runnables.addBack(proc);
+ }
+ }
+ return corrupted;
+ }
+
+ public void start(int numThreads) throws IOException {
+ if (running.getAndSet(true)) {
+ LOG.warn("Already running");
+ return;
+ }
+
+ // We have numThreads executor + one timer thread used for timing out
+ // procedures and triggering periodic procedures.
+ threads = new Thread[numThreads + 1];
+ LOG.info("Starting procedure executor threads=" + threads.length);
+
+ // Initialize procedures executor
+ for (int i = 0; i < numThreads; ++i) {
+ threads[i] = new Thread() {
+ @Override
+ public void run() {
+ execLoop();
+ }
+ };
+ }
+
+ // Initialize procedures timeout handler (this is the +1 thread)
+ threads[numThreads] = new Thread() {
+ @Override
+ public void run() {
+ timeoutLoop();
+ }
+ };
+
+ // Acquire the store lease.
+ store.recoverLease();
+
+ // TODO: Split in two steps.
+ // TODO: Handle corrupted procedure returned (probably just a WARN)
+ // The first one will make sure that we have the latest id,
+ // so we can start the threads and accept new procedures.
+ // The second step will do the actual load of old procedures.
+ load();
+
+ // Start the executors. Here we must have the lastProcId set.
+ for (int i = 0; i < threads.length; ++i) {
+ threads[i].start();
+ }
+
+ // Add completed cleaner
+ waitingTimeout.add(new CompletedProcedureCleaner(conf, store, completed));
+ }
+
+ public void stop() {
+ if (!running.getAndSet(false)) {
+ return;
+ }
+
+ LOG.info("Stopping the procedure executor");
+ runnables.signalAll();
+ waitingTimeout.signalAll();
+ }
+
+ public void join() {
+ boolean interrupted = false;
+
+ for (int i = 0; i < threads.length; ++i) {
+ try {
+ threads[i].join();
+ } catch (InterruptedException ex) {
+ interrupted = true;
+ }
+ }
+
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+
+ completed.clear();
+ rollbackStack.clear();
+ procedures.clear();
+ waitingTimeout.clear();
+ runnables.clear();
+ lastProcId.set(-1);
+ }
+
+ public boolean isRunning() {
+ return running.get();
+ }
+
+ /**
+ * @return the number of execution threads.
+ */
+ public int getNumThreads() {
+ return threads == null ? 0 : (threads.length - 1);
+ }
+
+ public int getActiveExecutorCount() {
+ return activeExecutorCount.get();
+ }
+
+ public TEnvironment getEnvironment() {
+ return this.environment;
+ }
+
+ public ProcedureStore getStore() {
+ return this.store;
+ }
+
+ public void registerListener(ProcedureExecutorListener listener) {
+ this.listeners.add(listener);
+ }
+
+ public boolean unregisterListener(ProcedureExecutorListener listener) {
+ return this.listeners.remove(listener);
+ }
+
+ /**
+ * Add a new root-procedure to the executor.
+ * @param proc the new procedure to execute.
+ * @return the procedure id, that can be used to monitor the operation
+ */
+ public long submitProcedure(final Procedure proc) {
+ Preconditions.checkArgument(proc.getState() == ProcedureState.INITIALIZING);
+ Preconditions.checkArgument(isRunning());
+ Preconditions.checkArgument(lastProcId.get() >= 0);
+ Preconditions.checkArgument(!proc.hasParent());
+
+ // Initialize the Procedure ID
+ proc.setProcId(nextProcId());
+
+ // Commit the transaction
+ store.insert(proc, null);
+ LOG.debug("procedure " + proc + " added to the store");
+
+ // Create the rollback stack for the procedure
+ RootProcedureState stack = new RootProcedureState();
+ rollbackStack.put(proc.getProcId(), stack);
+
+ // Submit the new subprocedures
+ assert !procedures.containsKey(proc.getProcId());
+ procedures.put(proc.getProcId(), proc);
+ sendProcedureAddedNotification(proc.getProcId());
+ runnables.addBack(proc);
+ return proc.getProcId();
+ }
+
+ public ProcedureResult getResult(final long procId) {
+ return completed.get(procId);
+ }
+
+ /**
+ * Return true if the procedure is finished.
+ * The state may be "completed successfully" or "failed and rolledback".
+ * Use getResult() to check the state or get the result data.
+ * @param procId the ID of the procedure to check
+ * @return true if the procedure execution is finished, otherwise false.
+ */
+ public boolean isFinished(final long procId) {
+ return completed.containsKey(procId);
+ }
+
+ /**
+ * Mark the specified completed procedure, as ready to remove.
+ * @param procId the ID of the procedure to remove
+ */
+ public void removeResult(final long procId) {
+ ProcedureResult result = completed.get(procId);
+ if (result == null) {
+ assert !procedures.containsKey(procId) : "procId=" + procId + " is still running";
+ LOG.debug("Procedure procId=" + procId + " already removed by the cleaner");
+ return;
+ }
+
+ // The CompletedProcedureCleaner will take care of deletion, once the TTL is expired.
+ result.setClientAckTime(EnvironmentEdgeManager.currentTime());
+ }
+
+ /**
+ * Send an abort notification the specified procedure.
+ * Depending on the procedure implementation the abort can be considered or ignored.
+ * @param procId the procedure to abort
+ * @return true if the procedure exist and has received the abort, otherwise false.
+ */
+ public boolean abort(final long procId) {
+ Procedure proc = procedures.get(procId);
+ if (proc != null) {
+ proc.abort(getEnvironment());
+ return true;
+ }
+ return false;
+ }
+
+ public Map<Long, ProcedureResult> getResults() {
+ return Collections.unmodifiableMap(completed);
+ }
+
+ public Procedure getProcedure(final long procId) {
+ return procedures.get(procId);
+ }
+
+ /**
+ * Execution loop (N threads)
+ * while the executor is in a running state,
+ * fetch a procedure from the runnables queue and start the execution.
+ */
+ private void execLoop() {
+ while (running.get()) {
+ Long procId = runnables.poll();
+ Procedure proc = procId != null ? procedures.get(procId) : null;
+ if (proc == null) continue;
+
+ try {
+ activeExecutorCount.incrementAndGet();
+ execLoop(proc);
+ } finally {
+ activeExecutorCount.decrementAndGet();
+ }
+ }
+ }
+
+ private void execLoop(Procedure proc) {
+ LOG.debug("starting the execution of " + proc);
+
+ Long rootProcId = getRootProcedureId(proc);
+ if (rootProcId == null) {
+ // The 'proc' was ready to run but the root procedure was rolledback
+ executeRollback(proc);
+ return;
+ }
+
+ RootProcedureState procStack = rollbackStack.get(rootProcId);
+ if (procStack == null) return;
+
+ do {
+ // Try to acquire the execution
+ if (!procStack.acquire(proc)) {
+ if (procStack.setRollback()) {
+ // we have the 'rollback-lock' we can start rollingback
+ executeRollback(rootProcId, procStack);
+ } else {
+ // if we can't rollback means that some child is still running.
+ // the rollback will be executed after all the children are done.
+ // If the procedure was never executed, remove and mark it as rolledback.
+ if (!proc.wasExecuted()) {
+ executeRollback(proc);
+ }
+ }
+ break;
+ }
+
+ // Execute the procedure
+ assert proc.getState() == ProcedureState.RUNNABLE;
+ if (proc.acquireLock(getEnvironment())) {
+ execProcedure(procStack, proc);
+ proc.releaseLock(getEnvironment());
+ } else {
+ runnables.yield(proc);
+ }
+ procStack.release(proc);
+
+ // allows to kill the executor before something is stored to the wal.
+ // useful to test the procedure recovery.
+ if (testing != null && !running.get()) {
+ break;
+ }
+
+ if (proc.getProcId() == rootProcId && proc.isSuccess()) {
+ // Finalize the procedure state
+ LOG.info("Procedure completed in " +
+ StringUtils.humanTimeDiff(proc.elapsedTime()) + ": " + proc);
+ procedureFinished(proc);
+ break;
+ }
+ } while (procStack.isFailed());
+ }
+
+ private void timeoutLoop() {
+ while (running.get()) {
+ Procedure proc = waitingTimeout.poll();
+ if (proc == null) continue;
+
+ if (proc.getTimeRemaining() > 100) {
+ // got an early wake, maybe a stop?
+ // re-enqueue the task in case was not a stop or just a signal
+ waitingTimeout.add(proc);
+ continue;
+ }
+
+ // ----------------------------------------------------------------------------
+ // TODO-MAYBE: Should we provide a notification to the store with the
+ // full set of procedures pending and completed to write a compacted
+ // version of the log (in case is a log)?
+ // In theory no, procedures are have a short life, so at some point the store
+ // will have the tracker saying everything is in the last log.
+ // ----------------------------------------------------------------------------
+
+ // The CompletedProcedureCleaner is a special case, and it acts as a chore.
+ // instead of bringing the Chore class in, we reuse this timeout thread for
+ // this special case.
+ if (proc instanceof CompletedProcedureCleaner) {
+ try {
+ ((CompletedProcedureCleaner)proc).periodicExecute(getEnvironment());
+ } catch (Throwable e) {
+ LOG.error("ignoring CompletedProcedureCleaner exception: " + e.getMessage(), e);
+ }
+ proc.setStartTime(EnvironmentEdgeManager.currentTime());
+ waitingTimeout.add(proc);
+ continue;
+ }
+
+ // The procedure received an "abort-timeout", call abort() and
+ // add the procedure back in the queue for rollback.
+ if (proc.setTimeoutFailure()) {
+ long rootProcId = Procedure.getRootProcedureId(procedures, proc);
+ RootProcedureState procStack = rollbackStack.get(rootProcId);
+ procStack.abort();
+ store.update(proc);
+ runnables.addFront(proc);
+ continue;
+ }
+ }
+ }
+
+ /**
+ * Execute the rollback of the full procedure stack.
+ * Once the procedure is rolledback, the root-procedure will be visible as
+ * finished to user, and the result will be the fatal exception.
+ */
+ private boolean executeRollback(final long rootProcId, final RootProcedureState procStack) {
+ Procedure rootProc = procedures.get(rootProcId);
+ RemoteProcedureException exception = rootProc.getException();
+ if (exception == null) {
+ exception = procStack.getException();
+ rootProc.setFailure(exception);
+ store.update(rootProc);
+ }
+
+ List<Procedure> subprocStack = procStack.getSubprocedures();
+ assert subprocStack != null : "called rollback with no steps executed rootProc=" + rootProc;
+
+ int stackTail = subprocStack.size();
+ boolean reuseLock = false;
+ while (stackTail --> 0) {
+ final Procedure proc = subprocStack.remove(stackTail);
+
+ if (!reuseLock && !proc.acquireLock(getEnvironment())) {
+ // can't take a lock on the procedure, add the root-proc back on the
+ // queue waiting for the lock availability
+ runnables.yield(rootProc);
+ return false;
+ }
+
+ executeRollback(proc);
+ boolean abortRollback = !running.get();
+
+ // If the next procedure is the same to this one
+ // (e.g. StateMachineProcedure reuse the same instance)
+ // we can avoid to lock/unlock each step
+ reuseLock = stackTail > 0 && (subprocStack.get(stackTail - 1) == proc) && !abortRollback;
+ if (!reuseLock) {
+ proc.releaseLock(getEnvironment());
+ }
+
+ // allows to kill the executor before something is stored to the wal.
+ // useful to test the procedure recovery.
+ if (abortRollback) {
+ return false;
+ }
+ }
+
+ // Finalize the procedure state
+ LOG.info("Rolledback procedure " + rootProc +
+ " exec-time=" + StringUtils.humanTimeDiff(rootProc.elapsedTime()) +
+ " exception=" + exception.getMessage());
+ procedureFinished(rootProc);
+ return true;
+ }
+
+ /**
+ * Execute the rollback of the procedure step.
+ * It updates the store with the new state (stack index)
+ * or will remove completly the procedure in case it is a child.
+ */
+ private void executeRollback(final Procedure proc) {
+ try {
+ proc.doRollback(getEnvironment());
+ } catch (Throwable e) {
+ // Catch NullPointerExceptions or similar errors...
+ LOG.fatal("CODE-BUG: uncatched runtime exception for procedure: " + proc, e);
+ }
+
+ // allows to kill the executor before something is stored to the wal.
+ // useful to test the procedure recovery.
+ if (testing != null && testing.shouldKillBeforeStoreUpdate()) {
+ stop();
+ return;
+ }
+
+ if (proc.removeStackIndex()) {
+ proc.setState(ProcedureState.ROLLEDBACK);
+ if (proc.hasParent()) {
+ store.delete(proc.getProcId());
+ procedures.remove(proc.getProcId());
+ } else {
+ store.update(proc);
+ }
+ } else {
+ store.update(proc);
+ }
+ }
+
+ /**
+ * Executes the specified procedure
+ * - calls the doExecute() of the procedure
+ * - if the procedure execution didn't fail (e.g. invalid user input)
+ * - ...and returned subprocedures
+ * - the subprocedures are initialized.
+ * - the subprocedures are added to the store
+ * - the subprocedures are added to the runnable queue
+ * - the procedure is now in a WAITING state, waiting for the subprocedures to complete
+ * - ...if there are no subprocedure
+ * - the procedure completed successfully
+ * - if there is a parent (WAITING)
+ * - the parent state will be set to RUNNABLE
+ * - in case of failure
+ * - the store is updated with the new state
+ * - the executor (caller of this method) will start the rollback of the procedure
+ */
+ private void execProcedure(final RootProcedureState procStack, final Procedure procedure) {
+ Preconditions.checkArgument(procedure.getState() == ProcedureState.RUNNABLE);
+
+ // Execute the procedure
+ boolean reExecute = false;
+ Procedure[] subprocs = null;
+ do {
+ reExecute = false;
+ try {
+ subprocs = procedure.doExecute(getEnvironment());
+ if (subprocs != null && subprocs.length == 0) {
+ subprocs = null;
+ }
+ } catch (ProcedureYieldException e) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("yield procedure: " + procedure);
+ }
+ runnables.yield(procedure);
+ return;
+ } catch (Throwable e) {
+ // Catch NullPointerExceptions or similar errors...
+ String msg = "CODE-BUG: uncatched runtime exception for procedure: " + procedure;
+ LOG.error(msg, e);
+ procedure.setFailure(new RemoteProcedureException(msg, e));
+ }
+
+ if (!procedure.isFailed()) {
+ if (subprocs != null) {
+ if (subprocs.length == 1 && subprocs[0] == procedure) {
+ // quick-shortcut for a state machine like procedure
+ subprocs = null;
+ reExecute = true;
+ } else {
+ // yield the current procedure, and make the subprocedure runnable
+ for (int i = 0; i < subprocs.length; ++i) {
+ Procedure subproc = subprocs[i];
+ if (subproc == null) {
+ String msg = "subproc[" + i + "] is null, aborting the procedure";
+ procedure.setFailure(new RemoteProcedureException(msg,
+ new IllegalArgumentException(msg)));
+ subprocs = null;
+ break;
+ }
+
+ assert subproc.getState() == ProcedureState.INITIALIZING;
+ subproc.setParentProcId(procedure.getProcId());
+ subproc.setProcId(nextProcId());
+ }
+
+ if (!procedure.isFailed()) {
+ procedure.setChildrenLatch(subprocs.length);
+ switch (procedure.getState()) {
+ case RUNNABLE:
+ procedure.setState(ProcedureState.WAITING);
+ break;
+ case WAITING_TIMEOUT:
+ waitingTimeout.add(procedure);
+ break;
+ default:
+ break;
+ }
+ }
+ }
+ } else if (procedure.getState() == ProcedureState.WAITING_TIMEOUT) {
+ waitingTimeout.add(procedure);
+ } else {
+ // No subtask, so we are done
+ procedure.setState(ProcedureState.FINISHED);
+ }
+ }
+
+ // Add the procedure to the stack
+ procStack.addRollbackStep(procedure);
+
+ // allows to kill the executor before something is stored to the wal.
+ // useful to test the procedure recovery.
+ if (testing != null && testing.shouldKillBeforeStoreUpdate()) {
+ stop();
+ return;
+ }
+
+ // Commit the transaction
+ if (subprocs != null && !procedure.isFailed()) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("store add " + procedure + " children " + Arrays.toString(subprocs));
+ }
+ store.insert(procedure, subprocs);
+ } else {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("store update " + procedure);
+ }
+ store.update(procedure);
+ }
+
+ assert (reExecute && subprocs == null) || !reExecute;
+ } while (reExecute);
+
+ // Submit the new subprocedures
+ if (subprocs != null && !procedure.isFailed()) {
+ for (int i = 0; i < subprocs.length; ++i) {
+ Procedure subproc = subprocs[i];
+ assert !procedures.containsKey(subproc.getProcId());
+ procedures.put(subproc.getProcId(), subproc);
+ runnables.addFront(subproc);
+ }
+ }
+
+ if (procedure.isFinished() && procedure.hasParent()) {
+ Procedure parent = procedures.get(procedure.getParentProcId());
+ if (parent == null) {
+ assert procStack.isRollingback();
+ return;
+ }
+
+ // If this procedure is the last child awake the parent procedure
+ if (LOG.isTraceEnabled()) {
+ LOG.trace(parent + " child is done: " + procedure);
+ }
+ if (parent.childrenCountDown() && parent.getState() == ProcedureState.WAITING) {
+ parent.setState(ProcedureState.RUNNABLE);
+ store.update(parent);
+ runnables.addFront(parent);
+ if (LOG.isTraceEnabled()) {
+ LOG.trace(parent + " all the children finished their work, resume.");
+ }
+ return;
+ }
+ }
+ }
+
+ private void sendProcedureLoadedNotification(final long procId) {
+ if (!this.listeners.isEmpty()) {
+ for (ProcedureExecutorListener listener: this.listeners) {
+ try {
+ listener.procedureLoaded(procId);
+ } catch (Throwable e) {
+ LOG.error("the listener " + listener + " had an error: " + e.getMessage(), e);
+ }
+ }
+ }
+ }
+
+ private void sendProcedureAddedNotification(final long procId) {
+ if (!this.listeners.isEmpty()) {
+ for (ProcedureExecutorListener listener: this.listeners) {
+ try {
+ listener.procedureAdded(procId);
+ } catch (Throwable e) {
+ LOG.error("the listener " + listener + " had an error: " + e.getMessage(), e);
+ }
+ }
+ }
+ }
+
+ private void sendProcedureFinishedNotification(final long procId) {
+ if (!this.listeners.isEmpty()) {
+ for (ProcedureExecutorListener listener: this.listeners) {
+ try {
+ listener.procedureFinished(procId);
+ } catch (Throwable e) {
+ LOG.error("the listener " + listener + " had an error: " + e.getMessage(), e);
+ }
+ }
+ }
+ }
+
+ private long nextProcId() {
+ long procId = lastProcId.incrementAndGet();
+ if (procId < 0) {
+ while (!lastProcId.compareAndSet(procId, 0)) {
+ procId = lastProcId.get();
+ if (procId >= 0)
+ break;
+ }
+ while (procedures.containsKey(procId)) {
+ procId = lastProcId.incrementAndGet();
+ }
+ }
+ return procId;
+ }
+
+ private Long getRootProcedureId(Procedure proc) {
+ return Procedure.getRootProcedureId(procedures, proc);
+ }
+
+ private void procedureFinished(final Procedure proc) {
+ proc.completionCleanup(getEnvironment());
+ completed.put(proc.getProcId(), newResultFromProcedure(proc));
+ rollbackStack.remove(proc.getProcId());
+ sendProcedureFinishedNotification(proc.getProcId());
+ }
+
+ private static ProcedureResult newResultFromProcedure(final Procedure proc) {
+ if (proc.isFailed()) {
+ return new ProcedureResult(proc.getStartTime(), proc.getLastUpdate(), proc.getException());
+ }
+ return new ProcedureResult(proc.getStartTime(), proc.getLastUpdate(), proc.getResult());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureFairRunQueues.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureFairRunQueues.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureFairRunQueues.java
new file mode 100644
index 0000000..d22df9d
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureFairRunQueues.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import java.util.Map;
+
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.ConcurrentSkipListMap;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * This class is a container of queues that allows to select a queue
+ * in a round robin fashion, considering priority of the queue.
+ *
+ * the quantum is just how many poll() will return the same object.
+ * e.g. if quantum is 1 and you have A and B as object you'll get: A B A B
+ * e.g. if quantum is 2 and you have A and B as object you'll get: A A B B A A B B
+ * then the object priority is just a priority * quantum
+ *
+ * Example:
+ * - three queues (A, B, C) with priorities (1, 1, 2)
+ * - The first poll() will return A
+ * - The second poll() will return B
+ * - The third and forth poll() will return C
+ * - and so on again and again.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class ProcedureFairRunQueues<TKey, TQueue extends ProcedureFairRunQueues.FairObject> {
+ private ConcurrentSkipListMap<TKey, TQueue> objMap =
+ new ConcurrentSkipListMap<TKey, TQueue>();
+
+ private final ReentrantLock lock = new ReentrantLock();
+ private final Condition waitCond = lock.newCondition();
+ private final int quantum;
+
+ private Map.Entry<TKey, TQueue> current;
+ private int currentQuantum = 0;
+
+ public interface FairObject {
+ boolean isAvailable();
+ int getPriority();
+ }
+
+ /**
+ * @param quantum how many poll() will return the same object.
+ */
+ public ProcedureFairRunQueues(final int quantum) {
+ this.quantum = quantum;
+ }
+
+ public TQueue get(final TKey key) {
+ return objMap.get(key);
+ }
+
+ public TQueue add(final TKey key, final TQueue queue) {
+ TQueue oldq = objMap.putIfAbsent(key, queue);
+ return oldq != null ? oldq : queue;
+ }
+
+ public TQueue remove(final TKey key) {
+ return objMap.remove(key);
+ }
+
+ public void clear() {
+ lock.lock();
+ try {
+ current = null;
+ objMap.clear();
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ public TQueue poll() {
+ lock.lock();
+ try {
+ TQueue queue;
+ if (currentQuantum == 0) {
+ if (nextObject() == null) {
+ // nothing here
+ return null;
+ }
+
+ queue = current.getValue();
+ currentQuantum = calculateQuantum(queue) - 1;
+ } else {
+ currentQuantum--;
+ queue = current.getValue();
+ }
+
+ if (!queue.isAvailable()) {
+ Map.Entry<TKey, TQueue> last = current;
+ // Try the next one
+ do {
+ if (nextObject() == null)
+ return null;
+ } while (current != last && !current.getValue().isAvailable());
+
+ queue = current.getValue();
+ currentQuantum = calculateQuantum(queue) - 1;
+ }
+
+ return queue;
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ private Map.Entry<TKey, TQueue> nextObject() {
+ Map.Entry<TKey, TQueue> next = null;
+
+ // If we have already a key, try the next one
+ if (current != null) {
+ next = objMap.higherEntry(current.getKey());
+ }
+
+ // if there is no higher key, go back to the first
+ current = (next != null) ? next : objMap.firstEntry();
+ return current;
+ }
+
+ private int calculateQuantum(final TQueue fairObject) {
+ // TODO
+ return Math.max(1, fairObject.getPriority() * quantum);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureResult.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureResult.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureResult.java
new file mode 100644
index 0000000..0aebd5a
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureResult.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * Once a Procedure completes the ProcedureExecutor takes all the useful
+ * information of the procedure (e.g. exception/result) and creates a ProcedureResult.
+ * The user of the Procedure framework will get the procedure result with
+ * procedureExecutor.getResult(procId)
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class ProcedureResult {
+ private final RemoteProcedureException exception;
+ private final long lastUpdate;
+ private final long startTime;
+ private final byte[] result;
+
+ private long clientAckTime = -1;
+
+ public ProcedureResult(final long startTime, final long lastUpdate,
+ final RemoteProcedureException exception) {
+ this.lastUpdate = lastUpdate;
+ this.startTime = startTime;
+ this.exception = exception;
+ this.result = null;
+ }
+
+ public ProcedureResult(final long startTime, final long lastUpdate, final byte[] result) {
+ this.lastUpdate = lastUpdate;
+ this.startTime = startTime;
+ this.exception = null;
+ this.result = result;
+ }
+
+ public boolean isFailed() {
+ return exception != null;
+ }
+
+ public RemoteProcedureException getException() {
+ return exception;
+ }
+
+ public boolean hasResultData() {
+ return result != null;
+ }
+
+ public byte[] getResult() {
+ return result;
+ }
+
+ public long getStartTime() {
+ return startTime;
+ }
+
+ public long getLastUpdate() {
+ return lastUpdate;
+ }
+
+ public long executionTime() {
+ return lastUpdate - startTime;
+ }
+
+ public boolean hasClientAckTime() {
+ return clientAckTime > 0;
+ }
+
+ public long getClientAckTime() {
+ return clientAckTime;
+ }
+
+ @InterfaceAudience.Private
+ protected void setClientAckTime(final long timestamp) {
+ this.clientAckTime = timestamp;
+ }
+}
\ No newline at end of file
[5/9] hbase git commit: HBASE-13202 Procedure v2 - core framework
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/94fb511c/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ProcedureProtos.java
----------------------------------------------------------------------
diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ProcedureProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ProcedureProtos.java
new file mode 100644
index 0000000..2826a0e
--- /dev/null
+++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ProcedureProtos.java
@@ -0,0 +1,7219 @@
+// Generated by the protocol buffer compiler. DO NOT EDIT!
+// source: Procedure.proto
+
+package org.apache.hadoop.hbase.protobuf.generated;
+
+public final class ProcedureProtos {
+ private ProcedureProtos() {}
+ public static void registerAllExtensions(
+ com.google.protobuf.ExtensionRegistry registry) {
+ }
+ /**
+ * Protobuf enum {@code ProcedureState}
+ */
+ public enum ProcedureState
+ implements com.google.protobuf.ProtocolMessageEnum {
+ /**
+ * <code>INITIALIZING = 1;</code>
+ *
+ * <pre>
+ * Procedure in construction, not yet added to the executor
+ * </pre>
+ */
+ INITIALIZING(0, 1),
+ /**
+ * <code>RUNNABLE = 2;</code>
+ *
+ * <pre>
+ * Procedure added to the executor, and ready to be executed
+ * </pre>
+ */
+ RUNNABLE(1, 2),
+ /**
+ * <code>WAITING = 3;</code>
+ *
+ * <pre>
+ * The procedure is waiting on children to be completed
+ * </pre>
+ */
+ WAITING(2, 3),
+ /**
+ * <code>WAITING_TIMEOUT = 4;</code>
+ *
+ * <pre>
+ * The procedure is waiting a timout or an external event
+ * </pre>
+ */
+ WAITING_TIMEOUT(3, 4),
+ /**
+ * <code>ROLLEDBACK = 5;</code>
+ *
+ * <pre>
+ * The procedure failed and was rolledback
+ * </pre>
+ */
+ ROLLEDBACK(4, 5),
+ /**
+ * <code>FINISHED = 6;</code>
+ *
+ * <pre>
+ * The procedure completed successfully
+ * </pre>
+ */
+ FINISHED(5, 6),
+ ;
+
+ /**
+ * <code>INITIALIZING = 1;</code>
+ *
+ * <pre>
+ * Procedure in construction, not yet added to the executor
+ * </pre>
+ */
+ public static final int INITIALIZING_VALUE = 1;
+ /**
+ * <code>RUNNABLE = 2;</code>
+ *
+ * <pre>
+ * Procedure added to the executor, and ready to be executed
+ * </pre>
+ */
+ public static final int RUNNABLE_VALUE = 2;
+ /**
+ * <code>WAITING = 3;</code>
+ *
+ * <pre>
+ * The procedure is waiting on children to be completed
+ * </pre>
+ */
+ public static final int WAITING_VALUE = 3;
+ /**
+ * <code>WAITING_TIMEOUT = 4;</code>
+ *
+ * <pre>
+ * The procedure is waiting a timout or an external event
+ * </pre>
+ */
+ public static final int WAITING_TIMEOUT_VALUE = 4;
+ /**
+ * <code>ROLLEDBACK = 5;</code>
+ *
+ * <pre>
+ * The procedure failed and was rolledback
+ * </pre>
+ */
+ public static final int ROLLEDBACK_VALUE = 5;
+ /**
+ * <code>FINISHED = 6;</code>
+ *
+ * <pre>
+ * The procedure completed successfully
+ * </pre>
+ */
+ public static final int FINISHED_VALUE = 6;
+
+
+ public final int getNumber() { return value; }
+
+ public static ProcedureState valueOf(int value) {
+ switch (value) {
+ case 1: return INITIALIZING;
+ case 2: return RUNNABLE;
+ case 3: return WAITING;
+ case 4: return WAITING_TIMEOUT;
+ case 5: return ROLLEDBACK;
+ case 6: return FINISHED;
+ default: return null;
+ }
+ }
+
+ public static com.google.protobuf.Internal.EnumLiteMap<ProcedureState>
+ internalGetValueMap() {
+ return internalValueMap;
+ }
+ private static com.google.protobuf.Internal.EnumLiteMap<ProcedureState>
+ internalValueMap =
+ new com.google.protobuf.Internal.EnumLiteMap<ProcedureState>() {
+ public ProcedureState findValueByNumber(int number) {
+ return ProcedureState.valueOf(number);
+ }
+ };
+
+ public final com.google.protobuf.Descriptors.EnumValueDescriptor
+ getValueDescriptor() {
+ return getDescriptor().getValues().get(index);
+ }
+ public final com.google.protobuf.Descriptors.EnumDescriptor
+ getDescriptorForType() {
+ return getDescriptor();
+ }
+ public static final com.google.protobuf.Descriptors.EnumDescriptor
+ getDescriptor() {
+ return org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.getDescriptor().getEnumTypes().get(0);
+ }
+
+ private static final ProcedureState[] VALUES = values();
+
+ public static ProcedureState valueOf(
+ com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+ if (desc.getType() != getDescriptor()) {
+ throw new java.lang.IllegalArgumentException(
+ "EnumValueDescriptor is not for this type.");
+ }
+ return VALUES[desc.getIndex()];
+ }
+
+ private final int index;
+ private final int value;
+
+ private ProcedureState(int index, int value) {
+ this.index = index;
+ this.value = value;
+ }
+
+ // @@protoc_insertion_point(enum_scope:ProcedureState)
+ }
+
+ public interface ProcedureOrBuilder
+ extends com.google.protobuf.MessageOrBuilder {
+
+ // required string class_name = 1;
+ /**
+ * <code>required string class_name = 1;</code>
+ *
+ * <pre>
+ * internal "static" state
+ * </pre>
+ */
+ boolean hasClassName();
+ /**
+ * <code>required string class_name = 1;</code>
+ *
+ * <pre>
+ * internal "static" state
+ * </pre>
+ */
+ java.lang.String getClassName();
+ /**
+ * <code>required string class_name = 1;</code>
+ *
+ * <pre>
+ * internal "static" state
+ * </pre>
+ */
+ com.google.protobuf.ByteString
+ getClassNameBytes();
+
+ // optional uint64 parent_id = 2;
+ /**
+ * <code>optional uint64 parent_id = 2;</code>
+ *
+ * <pre>
+ * parent if not a root-procedure otherwise not set
+ * </pre>
+ */
+ boolean hasParentId();
+ /**
+ * <code>optional uint64 parent_id = 2;</code>
+ *
+ * <pre>
+ * parent if not a root-procedure otherwise not set
+ * </pre>
+ */
+ long getParentId();
+
+ // required uint64 proc_id = 3;
+ /**
+ * <code>required uint64 proc_id = 3;</code>
+ */
+ boolean hasProcId();
+ /**
+ * <code>required uint64 proc_id = 3;</code>
+ */
+ long getProcId();
+
+ // required uint64 start_time = 4;
+ /**
+ * <code>required uint64 start_time = 4;</code>
+ */
+ boolean hasStartTime();
+ /**
+ * <code>required uint64 start_time = 4;</code>
+ */
+ long getStartTime();
+
+ // optional string owner = 5;
+ /**
+ * <code>optional string owner = 5;</code>
+ */
+ boolean hasOwner();
+ /**
+ * <code>optional string owner = 5;</code>
+ */
+ java.lang.String getOwner();
+ /**
+ * <code>optional string owner = 5;</code>
+ */
+ com.google.protobuf.ByteString
+ getOwnerBytes();
+
+ // required .ProcedureState state = 6;
+ /**
+ * <code>required .ProcedureState state = 6;</code>
+ *
+ * <pre>
+ * internal "runtime" state
+ * </pre>
+ */
+ boolean hasState();
+ /**
+ * <code>required .ProcedureState state = 6;</code>
+ *
+ * <pre>
+ * internal "runtime" state
+ * </pre>
+ */
+ org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState getState();
+
+ // repeated uint32 stack_id = 7;
+ /**
+ * <code>repeated uint32 stack_id = 7;</code>
+ *
+ * <pre>
+ * stack indices in case the procedure was running
+ * </pre>
+ */
+ java.util.List<java.lang.Integer> getStackIdList();
+ /**
+ * <code>repeated uint32 stack_id = 7;</code>
+ *
+ * <pre>
+ * stack indices in case the procedure was running
+ * </pre>
+ */
+ int getStackIdCount();
+ /**
+ * <code>repeated uint32 stack_id = 7;</code>
+ *
+ * <pre>
+ * stack indices in case the procedure was running
+ * </pre>
+ */
+ int getStackId(int index);
+
+ // required uint64 last_update = 8;
+ /**
+ * <code>required uint64 last_update = 8;</code>
+ */
+ boolean hasLastUpdate();
+ /**
+ * <code>required uint64 last_update = 8;</code>
+ */
+ long getLastUpdate();
+
+ // optional uint32 timeout = 9;
+ /**
+ * <code>optional uint32 timeout = 9;</code>
+ */
+ boolean hasTimeout();
+ /**
+ * <code>optional uint32 timeout = 9;</code>
+ */
+ int getTimeout();
+
+ // optional .ForeignExceptionMessage exception = 10;
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ boolean hasException();
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage getException();
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessageOrBuilder getExceptionOrBuilder();
+
+ // optional bytes result = 11;
+ /**
+ * <code>optional bytes result = 11;</code>
+ *
+ * <pre>
+ * opaque (user) result structure
+ * </pre>
+ */
+ boolean hasResult();
+ /**
+ * <code>optional bytes result = 11;</code>
+ *
+ * <pre>
+ * opaque (user) result structure
+ * </pre>
+ */
+ com.google.protobuf.ByteString getResult();
+
+ // optional bytes state_data = 12;
+ /**
+ * <code>optional bytes state_data = 12;</code>
+ *
+ * <pre>
+ * opaque (user) procedure internal-state
+ * </pre>
+ */
+ boolean hasStateData();
+ /**
+ * <code>optional bytes state_data = 12;</code>
+ *
+ * <pre>
+ * opaque (user) procedure internal-state
+ * </pre>
+ */
+ com.google.protobuf.ByteString getStateData();
+ }
+ /**
+ * Protobuf type {@code Procedure}
+ *
+ * <pre>
+ **
+ * Procedure metadata, serialized by the ProcedureStore to be able to recover the old state.
+ * </pre>
+ */
+ public static final class Procedure extends
+ com.google.protobuf.GeneratedMessage
+ implements ProcedureOrBuilder {
+ // Use Procedure.newBuilder() to construct.
+ private Procedure(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+ super(builder);
+ this.unknownFields = builder.getUnknownFields();
+ }
+ private Procedure(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+ private static final Procedure defaultInstance;
+ public static Procedure getDefaultInstance() {
+ return defaultInstance;
+ }
+
+ public Procedure getDefaultInstanceForType() {
+ return defaultInstance;
+ }
+
+ private final com.google.protobuf.UnknownFieldSet unknownFields;
+ @java.lang.Override
+ public final com.google.protobuf.UnknownFieldSet
+ getUnknownFields() {
+ return this.unknownFields;
+ }
+ private Procedure(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ initFields();
+ int mutable_bitField0_ = 0;
+ com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+ com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+ boolean done = false;
+ while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+ case 0:
+ done = true;
+ break;
+ default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+ done = true;
+ }
+ break;
+ }
+ case 10: {
+ bitField0_ |= 0x00000001;
+ className_ = input.readBytes();
+ break;
+ }
+ case 16: {
+ bitField0_ |= 0x00000002;
+ parentId_ = input.readUInt64();
+ break;
+ }
+ case 24: {
+ bitField0_ |= 0x00000004;
+ procId_ = input.readUInt64();
+ break;
+ }
+ case 32: {
+ bitField0_ |= 0x00000008;
+ startTime_ = input.readUInt64();
+ break;
+ }
+ case 42: {
+ bitField0_ |= 0x00000010;
+ owner_ = input.readBytes();
+ break;
+ }
+ case 48: {
+ int rawValue = input.readEnum();
+ org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState value = org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState.valueOf(rawValue);
+ if (value == null) {
+ unknownFields.mergeVarintField(6, rawValue);
+ } else {
+ bitField0_ |= 0x00000020;
+ state_ = value;
+ }
+ break;
+ }
+ case 56: {
+ if (!((mutable_bitField0_ & 0x00000040) == 0x00000040)) {
+ stackId_ = new java.util.ArrayList<java.lang.Integer>();
+ mutable_bitField0_ |= 0x00000040;
+ }
+ stackId_.add(input.readUInt32());
+ break;
+ }
+ case 58: {
+ int length = input.readRawVarint32();
+ int limit = input.pushLimit(length);
+ if (!((mutable_bitField0_ & 0x00000040) == 0x00000040) && input.getBytesUntilLimit() > 0) {
+ stackId_ = new java.util.ArrayList<java.lang.Integer>();
+ mutable_bitField0_ |= 0x00000040;
+ }
+ while (input.getBytesUntilLimit() > 0) {
+ stackId_.add(input.readUInt32());
+ }
+ input.popLimit(limit);
+ break;
+ }
+ case 64: {
+ bitField0_ |= 0x00000040;
+ lastUpdate_ = input.readUInt64();
+ break;
+ }
+ case 72: {
+ bitField0_ |= 0x00000080;
+ timeout_ = input.readUInt32();
+ break;
+ }
+ case 82: {
+ org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.Builder subBuilder = null;
+ if (((bitField0_ & 0x00000100) == 0x00000100)) {
+ subBuilder = exception_.toBuilder();
+ }
+ exception_ = input.readMessage(org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.PARSER, extensionRegistry);
+ if (subBuilder != null) {
+ subBuilder.mergeFrom(exception_);
+ exception_ = subBuilder.buildPartial();
+ }
+ bitField0_ |= 0x00000100;
+ break;
+ }
+ case 90: {
+ bitField0_ |= 0x00000200;
+ result_ = input.readBytes();
+ break;
+ }
+ case 98: {
+ bitField0_ |= 0x00000400;
+ stateData_ = input.readBytes();
+ break;
+ }
+ }
+ }
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+ throw new com.google.protobuf.InvalidProtocolBufferException(
+ e.getMessage()).setUnfinishedMessage(this);
+ } finally {
+ if (((mutable_bitField0_ & 0x00000040) == 0x00000040)) {
+ stackId_ = java.util.Collections.unmodifiableList(stackId_);
+ }
+ this.unknownFields = unknownFields.build();
+ makeExtensionsImmutable();
+ }
+ }
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.internal_static_Procedure_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.internal_static_Procedure_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure.class, org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure.Builder.class);
+ }
+
+ public static com.google.protobuf.Parser<Procedure> PARSER =
+ new com.google.protobuf.AbstractParser<Procedure>() {
+ public Procedure parsePartialFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return new Procedure(input, extensionRegistry);
+ }
+ };
+
+ @java.lang.Override
+ public com.google.protobuf.Parser<Procedure> getParserForType() {
+ return PARSER;
+ }
+
+ private int bitField0_;
+ // required string class_name = 1;
+ public static final int CLASS_NAME_FIELD_NUMBER = 1;
+ private java.lang.Object className_;
+ /**
+ * <code>required string class_name = 1;</code>
+ *
+ * <pre>
+ * internal "static" state
+ * </pre>
+ */
+ public boolean hasClassName() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>required string class_name = 1;</code>
+ *
+ * <pre>
+ * internal "static" state
+ * </pre>
+ */
+ public java.lang.String getClassName() {
+ java.lang.Object ref = className_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ if (bs.isValidUtf8()) {
+ className_ = s;
+ }
+ return s;
+ }
+ }
+ /**
+ * <code>required string class_name = 1;</code>
+ *
+ * <pre>
+ * internal "static" state
+ * </pre>
+ */
+ public com.google.protobuf.ByteString
+ getClassNameBytes() {
+ java.lang.Object ref = className_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ className_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
+ // optional uint64 parent_id = 2;
+ public static final int PARENT_ID_FIELD_NUMBER = 2;
+ private long parentId_;
+ /**
+ * <code>optional uint64 parent_id = 2;</code>
+ *
+ * <pre>
+ * parent if not a root-procedure otherwise not set
+ * </pre>
+ */
+ public boolean hasParentId() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>optional uint64 parent_id = 2;</code>
+ *
+ * <pre>
+ * parent if not a root-procedure otherwise not set
+ * </pre>
+ */
+ public long getParentId() {
+ return parentId_;
+ }
+
+ // required uint64 proc_id = 3;
+ public static final int PROC_ID_FIELD_NUMBER = 3;
+ private long procId_;
+ /**
+ * <code>required uint64 proc_id = 3;</code>
+ */
+ public boolean hasProcId() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ /**
+ * <code>required uint64 proc_id = 3;</code>
+ */
+ public long getProcId() {
+ return procId_;
+ }
+
+ // required uint64 start_time = 4;
+ public static final int START_TIME_FIELD_NUMBER = 4;
+ private long startTime_;
+ /**
+ * <code>required uint64 start_time = 4;</code>
+ */
+ public boolean hasStartTime() {
+ return ((bitField0_ & 0x00000008) == 0x00000008);
+ }
+ /**
+ * <code>required uint64 start_time = 4;</code>
+ */
+ public long getStartTime() {
+ return startTime_;
+ }
+
+ // optional string owner = 5;
+ public static final int OWNER_FIELD_NUMBER = 5;
+ private java.lang.Object owner_;
+ /**
+ * <code>optional string owner = 5;</code>
+ */
+ public boolean hasOwner() {
+ return ((bitField0_ & 0x00000010) == 0x00000010);
+ }
+ /**
+ * <code>optional string owner = 5;</code>
+ */
+ public java.lang.String getOwner() {
+ java.lang.Object ref = owner_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ if (bs.isValidUtf8()) {
+ owner_ = s;
+ }
+ return s;
+ }
+ }
+ /**
+ * <code>optional string owner = 5;</code>
+ */
+ public com.google.protobuf.ByteString
+ getOwnerBytes() {
+ java.lang.Object ref = owner_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ owner_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
+ // required .ProcedureState state = 6;
+ public static final int STATE_FIELD_NUMBER = 6;
+ private org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState state_;
+ /**
+ * <code>required .ProcedureState state = 6;</code>
+ *
+ * <pre>
+ * internal "runtime" state
+ * </pre>
+ */
+ public boolean hasState() {
+ return ((bitField0_ & 0x00000020) == 0x00000020);
+ }
+ /**
+ * <code>required .ProcedureState state = 6;</code>
+ *
+ * <pre>
+ * internal "runtime" state
+ * </pre>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState getState() {
+ return state_;
+ }
+
+ // repeated uint32 stack_id = 7;
+ public static final int STACK_ID_FIELD_NUMBER = 7;
+ private java.util.List<java.lang.Integer> stackId_;
+ /**
+ * <code>repeated uint32 stack_id = 7;</code>
+ *
+ * <pre>
+ * stack indices in case the procedure was running
+ * </pre>
+ */
+ public java.util.List<java.lang.Integer>
+ getStackIdList() {
+ return stackId_;
+ }
+ /**
+ * <code>repeated uint32 stack_id = 7;</code>
+ *
+ * <pre>
+ * stack indices in case the procedure was running
+ * </pre>
+ */
+ public int getStackIdCount() {
+ return stackId_.size();
+ }
+ /**
+ * <code>repeated uint32 stack_id = 7;</code>
+ *
+ * <pre>
+ * stack indices in case the procedure was running
+ * </pre>
+ */
+ public int getStackId(int index) {
+ return stackId_.get(index);
+ }
+
+ // required uint64 last_update = 8;
+ public static final int LAST_UPDATE_FIELD_NUMBER = 8;
+ private long lastUpdate_;
+ /**
+ * <code>required uint64 last_update = 8;</code>
+ */
+ public boolean hasLastUpdate() {
+ return ((bitField0_ & 0x00000040) == 0x00000040);
+ }
+ /**
+ * <code>required uint64 last_update = 8;</code>
+ */
+ public long getLastUpdate() {
+ return lastUpdate_;
+ }
+
+ // optional uint32 timeout = 9;
+ public static final int TIMEOUT_FIELD_NUMBER = 9;
+ private int timeout_;
+ /**
+ * <code>optional uint32 timeout = 9;</code>
+ */
+ public boolean hasTimeout() {
+ return ((bitField0_ & 0x00000080) == 0x00000080);
+ }
+ /**
+ * <code>optional uint32 timeout = 9;</code>
+ */
+ public int getTimeout() {
+ return timeout_;
+ }
+
+ // optional .ForeignExceptionMessage exception = 10;
+ public static final int EXCEPTION_FIELD_NUMBER = 10;
+ private org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage exception_;
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ public boolean hasException() {
+ return ((bitField0_ & 0x00000100) == 0x00000100);
+ }
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage getException() {
+ return exception_;
+ }
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessageOrBuilder getExceptionOrBuilder() {
+ return exception_;
+ }
+
+ // optional bytes result = 11;
+ public static final int RESULT_FIELD_NUMBER = 11;
+ private com.google.protobuf.ByteString result_;
+ /**
+ * <code>optional bytes result = 11;</code>
+ *
+ * <pre>
+ * opaque (user) result structure
+ * </pre>
+ */
+ public boolean hasResult() {
+ return ((bitField0_ & 0x00000200) == 0x00000200);
+ }
+ /**
+ * <code>optional bytes result = 11;</code>
+ *
+ * <pre>
+ * opaque (user) result structure
+ * </pre>
+ */
+ public com.google.protobuf.ByteString getResult() {
+ return result_;
+ }
+
+ // optional bytes state_data = 12;
+ public static final int STATE_DATA_FIELD_NUMBER = 12;
+ private com.google.protobuf.ByteString stateData_;
+ /**
+ * <code>optional bytes state_data = 12;</code>
+ *
+ * <pre>
+ * opaque (user) procedure internal-state
+ * </pre>
+ */
+ public boolean hasStateData() {
+ return ((bitField0_ & 0x00000400) == 0x00000400);
+ }
+ /**
+ * <code>optional bytes state_data = 12;</code>
+ *
+ * <pre>
+ * opaque (user) procedure internal-state
+ * </pre>
+ */
+ public com.google.protobuf.ByteString getStateData() {
+ return stateData_;
+ }
+
+ private void initFields() {
+ className_ = "";
+ parentId_ = 0L;
+ procId_ = 0L;
+ startTime_ = 0L;
+ owner_ = "";
+ state_ = org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState.INITIALIZING;
+ stackId_ = java.util.Collections.emptyList();
+ lastUpdate_ = 0L;
+ timeout_ = 0;
+ exception_ = org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.getDefaultInstance();
+ result_ = com.google.protobuf.ByteString.EMPTY;
+ stateData_ = com.google.protobuf.ByteString.EMPTY;
+ }
+ private byte memoizedIsInitialized = -1;
+ public final boolean isInitialized() {
+ byte isInitialized = memoizedIsInitialized;
+ if (isInitialized != -1) return isInitialized == 1;
+
+ if (!hasClassName()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasProcId()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasStartTime()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasState()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasLastUpdate()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ memoizedIsInitialized = 1;
+ return true;
+ }
+
+ public void writeTo(com.google.protobuf.CodedOutputStream output)
+ throws java.io.IOException {
+ getSerializedSize();
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ output.writeBytes(1, getClassNameBytes());
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ output.writeUInt64(2, parentId_);
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ output.writeUInt64(3, procId_);
+ }
+ if (((bitField0_ & 0x00000008) == 0x00000008)) {
+ output.writeUInt64(4, startTime_);
+ }
+ if (((bitField0_ & 0x00000010) == 0x00000010)) {
+ output.writeBytes(5, getOwnerBytes());
+ }
+ if (((bitField0_ & 0x00000020) == 0x00000020)) {
+ output.writeEnum(6, state_.getNumber());
+ }
+ for (int i = 0; i < stackId_.size(); i++) {
+ output.writeUInt32(7, stackId_.get(i));
+ }
+ if (((bitField0_ & 0x00000040) == 0x00000040)) {
+ output.writeUInt64(8, lastUpdate_);
+ }
+ if (((bitField0_ & 0x00000080) == 0x00000080)) {
+ output.writeUInt32(9, timeout_);
+ }
+ if (((bitField0_ & 0x00000100) == 0x00000100)) {
+ output.writeMessage(10, exception_);
+ }
+ if (((bitField0_ & 0x00000200) == 0x00000200)) {
+ output.writeBytes(11, result_);
+ }
+ if (((bitField0_ & 0x00000400) == 0x00000400)) {
+ output.writeBytes(12, stateData_);
+ }
+ getUnknownFields().writeTo(output);
+ }
+
+ private int memoizedSerializedSize = -1;
+ public int getSerializedSize() {
+ int size = memoizedSerializedSize;
+ if (size != -1) return size;
+
+ size = 0;
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(1, getClassNameBytes());
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeUInt64Size(2, parentId_);
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeUInt64Size(3, procId_);
+ }
+ if (((bitField0_ & 0x00000008) == 0x00000008)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeUInt64Size(4, startTime_);
+ }
+ if (((bitField0_ & 0x00000010) == 0x00000010)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(5, getOwnerBytes());
+ }
+ if (((bitField0_ & 0x00000020) == 0x00000020)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeEnumSize(6, state_.getNumber());
+ }
+ {
+ int dataSize = 0;
+ for (int i = 0; i < stackId_.size(); i++) {
+ dataSize += com.google.protobuf.CodedOutputStream
+ .computeUInt32SizeNoTag(stackId_.get(i));
+ }
+ size += dataSize;
+ size += 1 * getStackIdList().size();
+ }
+ if (((bitField0_ & 0x00000040) == 0x00000040)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeUInt64Size(8, lastUpdate_);
+ }
+ if (((bitField0_ & 0x00000080) == 0x00000080)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeUInt32Size(9, timeout_);
+ }
+ if (((bitField0_ & 0x00000100) == 0x00000100)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeMessageSize(10, exception_);
+ }
+ if (((bitField0_ & 0x00000200) == 0x00000200)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(11, result_);
+ }
+ if (((bitField0_ & 0x00000400) == 0x00000400)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(12, stateData_);
+ }
+ size += getUnknownFields().getSerializedSize();
+ memoizedSerializedSize = size;
+ return size;
+ }
+
+ private static final long serialVersionUID = 0L;
+ @java.lang.Override
+ protected java.lang.Object writeReplace()
+ throws java.io.ObjectStreamException {
+ return super.writeReplace();
+ }
+
+ @java.lang.Override
+ public boolean equals(final java.lang.Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure)) {
+ return super.equals(obj);
+ }
+ org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure other = (org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure) obj;
+
+ boolean result = true;
+ result = result && (hasClassName() == other.hasClassName());
+ if (hasClassName()) {
+ result = result && getClassName()
+ .equals(other.getClassName());
+ }
+ result = result && (hasParentId() == other.hasParentId());
+ if (hasParentId()) {
+ result = result && (getParentId()
+ == other.getParentId());
+ }
+ result = result && (hasProcId() == other.hasProcId());
+ if (hasProcId()) {
+ result = result && (getProcId()
+ == other.getProcId());
+ }
+ result = result && (hasStartTime() == other.hasStartTime());
+ if (hasStartTime()) {
+ result = result && (getStartTime()
+ == other.getStartTime());
+ }
+ result = result && (hasOwner() == other.hasOwner());
+ if (hasOwner()) {
+ result = result && getOwner()
+ .equals(other.getOwner());
+ }
+ result = result && (hasState() == other.hasState());
+ if (hasState()) {
+ result = result &&
+ (getState() == other.getState());
+ }
+ result = result && getStackIdList()
+ .equals(other.getStackIdList());
+ result = result && (hasLastUpdate() == other.hasLastUpdate());
+ if (hasLastUpdate()) {
+ result = result && (getLastUpdate()
+ == other.getLastUpdate());
+ }
+ result = result && (hasTimeout() == other.hasTimeout());
+ if (hasTimeout()) {
+ result = result && (getTimeout()
+ == other.getTimeout());
+ }
+ result = result && (hasException() == other.hasException());
+ if (hasException()) {
+ result = result && getException()
+ .equals(other.getException());
+ }
+ result = result && (hasResult() == other.hasResult());
+ if (hasResult()) {
+ result = result && getResult()
+ .equals(other.getResult());
+ }
+ result = result && (hasStateData() == other.hasStateData());
+ if (hasStateData()) {
+ result = result && getStateData()
+ .equals(other.getStateData());
+ }
+ result = result &&
+ getUnknownFields().equals(other.getUnknownFields());
+ return result;
+ }
+
+ private int memoizedHashCode = 0;
+ @java.lang.Override
+ public int hashCode() {
+ if (memoizedHashCode != 0) {
+ return memoizedHashCode;
+ }
+ int hash = 41;
+ hash = (19 * hash) + getDescriptorForType().hashCode();
+ if (hasClassName()) {
+ hash = (37 * hash) + CLASS_NAME_FIELD_NUMBER;
+ hash = (53 * hash) + getClassName().hashCode();
+ }
+ if (hasParentId()) {
+ hash = (37 * hash) + PARENT_ID_FIELD_NUMBER;
+ hash = (53 * hash) + hashLong(getParentId());
+ }
+ if (hasProcId()) {
+ hash = (37 * hash) + PROC_ID_FIELD_NUMBER;
+ hash = (53 * hash) + hashLong(getProcId());
+ }
+ if (hasStartTime()) {
+ hash = (37 * hash) + START_TIME_FIELD_NUMBER;
+ hash = (53 * hash) + hashLong(getStartTime());
+ }
+ if (hasOwner()) {
+ hash = (37 * hash) + OWNER_FIELD_NUMBER;
+ hash = (53 * hash) + getOwner().hashCode();
+ }
+ if (hasState()) {
+ hash = (37 * hash) + STATE_FIELD_NUMBER;
+ hash = (53 * hash) + hashEnum(getState());
+ }
+ if (getStackIdCount() > 0) {
+ hash = (37 * hash) + STACK_ID_FIELD_NUMBER;
+ hash = (53 * hash) + getStackIdList().hashCode();
+ }
+ if (hasLastUpdate()) {
+ hash = (37 * hash) + LAST_UPDATE_FIELD_NUMBER;
+ hash = (53 * hash) + hashLong(getLastUpdate());
+ }
+ if (hasTimeout()) {
+ hash = (37 * hash) + TIMEOUT_FIELD_NUMBER;
+ hash = (53 * hash) + getTimeout();
+ }
+ if (hasException()) {
+ hash = (37 * hash) + EXCEPTION_FIELD_NUMBER;
+ hash = (53 * hash) + getException().hashCode();
+ }
+ if (hasResult()) {
+ hash = (37 * hash) + RESULT_FIELD_NUMBER;
+ hash = (53 * hash) + getResult().hashCode();
+ }
+ if (hasStateData()) {
+ hash = (37 * hash) + STATE_DATA_FIELD_NUMBER;
+ hash = (53 * hash) + getStateData().hashCode();
+ }
+ hash = (29 * hash) + getUnknownFields().hashCode();
+ memoizedHashCode = hash;
+ return hash;
+ }
+
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure parseFrom(
+ com.google.protobuf.ByteString data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure parseFrom(
+ com.google.protobuf.ByteString data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure parseFrom(byte[] data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure parseFrom(
+ byte[] data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure parseFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure parseFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure parseDelimitedFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure parseDelimitedFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure parseFrom(
+ com.google.protobuf.CodedInputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure parseFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+
+ public static Builder newBuilder() { return Builder.create(); }
+ public Builder newBuilderForType() { return newBuilder(); }
+ public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure prototype) {
+ return newBuilder().mergeFrom(prototype);
+ }
+ public Builder toBuilder() { return newBuilder(this); }
+
+ @java.lang.Override
+ protected Builder newBuilderForType(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ Builder builder = new Builder(parent);
+ return builder;
+ }
+ /**
+ * Protobuf type {@code Procedure}
+ *
+ * <pre>
+ **
+ * Procedure metadata, serialized by the ProcedureStore to be able to recover the old state.
+ * </pre>
+ */
+ public static final class Builder extends
+ com.google.protobuf.GeneratedMessage.Builder<Builder>
+ implements org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureOrBuilder {
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.internal_static_Procedure_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.internal_static_Procedure_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure.class, org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure.Builder.class);
+ }
+
+ // Construct using org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure.newBuilder()
+ private Builder() {
+ maybeForceBuilderInitialization();
+ }
+
+ private Builder(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ super(parent);
+ maybeForceBuilderInitialization();
+ }
+ private void maybeForceBuilderInitialization() {
+ if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+ getExceptionFieldBuilder();
+ }
+ }
+ private static Builder create() {
+ return new Builder();
+ }
+
+ public Builder clear() {
+ super.clear();
+ className_ = "";
+ bitField0_ = (bitField0_ & ~0x00000001);
+ parentId_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000002);
+ procId_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000004);
+ startTime_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000008);
+ owner_ = "";
+ bitField0_ = (bitField0_ & ~0x00000010);
+ state_ = org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState.INITIALIZING;
+ bitField0_ = (bitField0_ & ~0x00000020);
+ stackId_ = java.util.Collections.emptyList();
+ bitField0_ = (bitField0_ & ~0x00000040);
+ lastUpdate_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000080);
+ timeout_ = 0;
+ bitField0_ = (bitField0_ & ~0x00000100);
+ if (exceptionBuilder_ == null) {
+ exception_ = org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.getDefaultInstance();
+ } else {
+ exceptionBuilder_.clear();
+ }
+ bitField0_ = (bitField0_ & ~0x00000200);
+ result_ = com.google.protobuf.ByteString.EMPTY;
+ bitField0_ = (bitField0_ & ~0x00000400);
+ stateData_ = com.google.protobuf.ByteString.EMPTY;
+ bitField0_ = (bitField0_ & ~0x00000800);
+ return this;
+ }
+
+ public Builder clone() {
+ return create().mergeFrom(buildPartial());
+ }
+
+ public com.google.protobuf.Descriptors.Descriptor
+ getDescriptorForType() {
+ return org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.internal_static_Procedure_descriptor;
+ }
+
+ public org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure getDefaultInstanceForType() {
+ return org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure.getDefaultInstance();
+ }
+
+ public org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure build() {
+ org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure result = buildPartial();
+ if (!result.isInitialized()) {
+ throw newUninitializedMessageException(result);
+ }
+ return result;
+ }
+
+ public org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure buildPartial() {
+ org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure result = new org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure(this);
+ int from_bitField0_ = bitField0_;
+ int to_bitField0_ = 0;
+ if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+ to_bitField0_ |= 0x00000001;
+ }
+ result.className_ = className_;
+ if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+ to_bitField0_ |= 0x00000002;
+ }
+ result.parentId_ = parentId_;
+ if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+ to_bitField0_ |= 0x00000004;
+ }
+ result.procId_ = procId_;
+ if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
+ to_bitField0_ |= 0x00000008;
+ }
+ result.startTime_ = startTime_;
+ if (((from_bitField0_ & 0x00000010) == 0x00000010)) {
+ to_bitField0_ |= 0x00000010;
+ }
+ result.owner_ = owner_;
+ if (((from_bitField0_ & 0x00000020) == 0x00000020)) {
+ to_bitField0_ |= 0x00000020;
+ }
+ result.state_ = state_;
+ if (((bitField0_ & 0x00000040) == 0x00000040)) {
+ stackId_ = java.util.Collections.unmodifiableList(stackId_);
+ bitField0_ = (bitField0_ & ~0x00000040);
+ }
+ result.stackId_ = stackId_;
+ if (((from_bitField0_ & 0x00000080) == 0x00000080)) {
+ to_bitField0_ |= 0x00000040;
+ }
+ result.lastUpdate_ = lastUpdate_;
+ if (((from_bitField0_ & 0x00000100) == 0x00000100)) {
+ to_bitField0_ |= 0x00000080;
+ }
+ result.timeout_ = timeout_;
+ if (((from_bitField0_ & 0x00000200) == 0x00000200)) {
+ to_bitField0_ |= 0x00000100;
+ }
+ if (exceptionBuilder_ == null) {
+ result.exception_ = exception_;
+ } else {
+ result.exception_ = exceptionBuilder_.build();
+ }
+ if (((from_bitField0_ & 0x00000400) == 0x00000400)) {
+ to_bitField0_ |= 0x00000200;
+ }
+ result.result_ = result_;
+ if (((from_bitField0_ & 0x00000800) == 0x00000800)) {
+ to_bitField0_ |= 0x00000400;
+ }
+ result.stateData_ = stateData_;
+ result.bitField0_ = to_bitField0_;
+ onBuilt();
+ return result;
+ }
+
+ public Builder mergeFrom(com.google.protobuf.Message other) {
+ if (other instanceof org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure) {
+ return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure)other);
+ } else {
+ super.mergeFrom(other);
+ return this;
+ }
+ }
+
+ public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure other) {
+ if (other == org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure.getDefaultInstance()) return this;
+ if (other.hasClassName()) {
+ bitField0_ |= 0x00000001;
+ className_ = other.className_;
+ onChanged();
+ }
+ if (other.hasParentId()) {
+ setParentId(other.getParentId());
+ }
+ if (other.hasProcId()) {
+ setProcId(other.getProcId());
+ }
+ if (other.hasStartTime()) {
+ setStartTime(other.getStartTime());
+ }
+ if (other.hasOwner()) {
+ bitField0_ |= 0x00000010;
+ owner_ = other.owner_;
+ onChanged();
+ }
+ if (other.hasState()) {
+ setState(other.getState());
+ }
+ if (!other.stackId_.isEmpty()) {
+ if (stackId_.isEmpty()) {
+ stackId_ = other.stackId_;
+ bitField0_ = (bitField0_ & ~0x00000040);
+ } else {
+ ensureStackIdIsMutable();
+ stackId_.addAll(other.stackId_);
+ }
+ onChanged();
+ }
+ if (other.hasLastUpdate()) {
+ setLastUpdate(other.getLastUpdate());
+ }
+ if (other.hasTimeout()) {
+ setTimeout(other.getTimeout());
+ }
+ if (other.hasException()) {
+ mergeException(other.getException());
+ }
+ if (other.hasResult()) {
+ setResult(other.getResult());
+ }
+ if (other.hasStateData()) {
+ setStateData(other.getStateData());
+ }
+ this.mergeUnknownFields(other.getUnknownFields());
+ return this;
+ }
+
+ public final boolean isInitialized() {
+ if (!hasClassName()) {
+
+ return false;
+ }
+ if (!hasProcId()) {
+
+ return false;
+ }
+ if (!hasStartTime()) {
+
+ return false;
+ }
+ if (!hasState()) {
+
+ return false;
+ }
+ if (!hasLastUpdate()) {
+
+ return false;
+ }
+ return true;
+ }
+
+ public Builder mergeFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure parsedMessage = null;
+ try {
+ parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ parsedMessage = (org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure) e.getUnfinishedMessage();
+ throw e;
+ } finally {
+ if (parsedMessage != null) {
+ mergeFrom(parsedMessage);
+ }
+ }
+ return this;
+ }
+ private int bitField0_;
+
+ // required string class_name = 1;
+ private java.lang.Object className_ = "";
+ /**
+ * <code>required string class_name = 1;</code>
+ *
+ * <pre>
+ * internal "static" state
+ * </pre>
+ */
+ public boolean hasClassName() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>required string class_name = 1;</code>
+ *
+ * <pre>
+ * internal "static" state
+ * </pre>
+ */
+ public java.lang.String getClassName() {
+ java.lang.Object ref = className_;
+ if (!(ref instanceof java.lang.String)) {
+ java.lang.String s = ((com.google.protobuf.ByteString) ref)
+ .toStringUtf8();
+ className_ = s;
+ return s;
+ } else {
+ return (java.lang.String) ref;
+ }
+ }
+ /**
+ * <code>required string class_name = 1;</code>
+ *
+ * <pre>
+ * internal "static" state
+ * </pre>
+ */
+ public com.google.protobuf.ByteString
+ getClassNameBytes() {
+ java.lang.Object ref = className_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ className_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+ /**
+ * <code>required string class_name = 1;</code>
+ *
+ * <pre>
+ * internal "static" state
+ * </pre>
+ */
+ public Builder setClassName(
+ java.lang.String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000001;
+ className_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required string class_name = 1;</code>
+ *
+ * <pre>
+ * internal "static" state
+ * </pre>
+ */
+ public Builder clearClassName() {
+ bitField0_ = (bitField0_ & ~0x00000001);
+ className_ = getDefaultInstance().getClassName();
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required string class_name = 1;</code>
+ *
+ * <pre>
+ * internal "static" state
+ * </pre>
+ */
+ public Builder setClassNameBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000001;
+ className_ = value;
+ onChanged();
+ return this;
+ }
+
+ // optional uint64 parent_id = 2;
+ private long parentId_ ;
+ /**
+ * <code>optional uint64 parent_id = 2;</code>
+ *
+ * <pre>
+ * parent if not a root-procedure otherwise not set
+ * </pre>
+ */
+ public boolean hasParentId() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>optional uint64 parent_id = 2;</code>
+ *
+ * <pre>
+ * parent if not a root-procedure otherwise not set
+ * </pre>
+ */
+ public long getParentId() {
+ return parentId_;
+ }
+ /**
+ * <code>optional uint64 parent_id = 2;</code>
+ *
+ * <pre>
+ * parent if not a root-procedure otherwise not set
+ * </pre>
+ */
+ public Builder setParentId(long value) {
+ bitField0_ |= 0x00000002;
+ parentId_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional uint64 parent_id = 2;</code>
+ *
+ * <pre>
+ * parent if not a root-procedure otherwise not set
+ * </pre>
+ */
+ public Builder clearParentId() {
+ bitField0_ = (bitField0_ & ~0x00000002);
+ parentId_ = 0L;
+ onChanged();
+ return this;
+ }
+
+ // required uint64 proc_id = 3;
+ private long procId_ ;
+ /**
+ * <code>required uint64 proc_id = 3;</code>
+ */
+ public boolean hasProcId() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ /**
+ * <code>required uint64 proc_id = 3;</code>
+ */
+ public long getProcId() {
+ return procId_;
+ }
+ /**
+ * <code>required uint64 proc_id = 3;</code>
+ */
+ public Builder setProcId(long value) {
+ bitField0_ |= 0x00000004;
+ procId_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required uint64 proc_id = 3;</code>
+ */
+ public Builder clearProcId() {
+ bitField0_ = (bitField0_ & ~0x00000004);
+ procId_ = 0L;
+ onChanged();
+ return this;
+ }
+
+ // required uint64 start_time = 4;
+ private long startTime_ ;
+ /**
+ * <code>required uint64 start_time = 4;</code>
+ */
+ public boolean hasStartTime() {
+ return ((bitField0_ & 0x00000008) == 0x00000008);
+ }
+ /**
+ * <code>required uint64 start_time = 4;</code>
+ */
+ public long getStartTime() {
+ return startTime_;
+ }
+ /**
+ * <code>required uint64 start_time = 4;</code>
+ */
+ public Builder setStartTime(long value) {
+ bitField0_ |= 0x00000008;
+ startTime_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required uint64 start_time = 4;</code>
+ */
+ public Builder clearStartTime() {
+ bitField0_ = (bitField0_ & ~0x00000008);
+ startTime_ = 0L;
+ onChanged();
+ return this;
+ }
+
+ // optional string owner = 5;
+ private java.lang.Object owner_ = "";
+ /**
+ * <code>optional string owner = 5;</code>
+ */
+ public boolean hasOwner() {
+ return ((bitField0_ & 0x00000010) == 0x00000010);
+ }
+ /**
+ * <code>optional string owner = 5;</code>
+ */
+ public java.lang.String getOwner() {
+ java.lang.Object ref = owner_;
+ if (!(ref instanceof java.lang.String)) {
+ java.lang.String s = ((com.google.protobuf.ByteString) ref)
+ .toStringUtf8();
+ owner_ = s;
+ return s;
+ } else {
+ return (java.lang.String) ref;
+ }
+ }
+ /**
+ * <code>optional string owner = 5;</code>
+ */
+ public com.google.protobuf.ByteString
+ getOwnerBytes() {
+ java.lang.Object ref = owner_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ owner_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+ /**
+ * <code>optional string owner = 5;</code>
+ */
+ public Builder setOwner(
+ java.lang.String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000010;
+ owner_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional string owner = 5;</code>
+ */
+ public Builder clearOwner() {
+ bitField0_ = (bitField0_ & ~0x00000010);
+ owner_ = getDefaultInstance().getOwner();
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional string owner = 5;</code>
+ */
+ public Builder setOwnerBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000010;
+ owner_ = value;
+ onChanged();
+ return this;
+ }
+
+ // required .ProcedureState state = 6;
+ private org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState state_ = org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState.INITIALIZING;
+ /**
+ * <code>required .ProcedureState state = 6;</code>
+ *
+ * <pre>
+ * internal "runtime" state
+ * </pre>
+ */
+ public boolean hasState() {
+ return ((bitField0_ & 0x00000020) == 0x00000020);
+ }
+ /**
+ * <code>required .ProcedureState state = 6;</code>
+ *
+ * <pre>
+ * internal "runtime" state
+ * </pre>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState getState() {
+ return state_;
+ }
+ /**
+ * <code>required .ProcedureState state = 6;</code>
+ *
+ * <pre>
+ * internal "runtime" state
+ * </pre>
+ */
+ public Builder setState(org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000020;
+ state_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required .ProcedureState state = 6;</code>
+ *
+ * <pre>
+ * internal "runtime" state
+ * </pre>
+ */
+ public Builder clearState() {
+ bitField0_ = (bitField0_ & ~0x00000020);
+ state_ = org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState.INITIALIZING;
+ onChanged();
+ return this;
+ }
+
+ // repeated uint32 stack_id = 7;
+ private java.util.List<java.lang.Integer> stackId_ = java.util.Collections.emptyList();
+ private void ensureStackIdIsMutable() {
+ if (!((bitField0_ & 0x00000040) == 0x00000040)) {
+ stackId_ = new java.util.ArrayList<java.lang.Integer>(stackId_);
+ bitField0_ |= 0x00000040;
+ }
+ }
+ /**
+ * <code>repeated uint32 stack_id = 7;</code>
+ *
+ * <pre>
+ * stack indices in case the procedure was running
+ * </pre>
+ */
+ public java.util.List<java.lang.Integer>
+ getStackIdList() {
+ return java.util.Collections.unmodifiableList(stackId_);
+ }
+ /**
+ * <code>repeated uint32 stack_id = 7;</code>
+ *
+ * <pre>
+ * stack indices in case the procedure was running
+ * </pre>
+ */
+ public int getStackIdCount() {
+ return stackId_.size();
+ }
+ /**
+ * <code>repeated uint32 stack_id = 7;</code>
+ *
+ * <pre>
+ * stack indices in case the procedure was running
+ * </pre>
+ */
+ public int getStackId(int index) {
+ return stackId_.get(index);
+ }
+ /**
+ * <code>repeated uint32 stack_id = 7;</code>
+ *
+ * <pre>
+ * stack indices in case the procedure was running
+ * </pre>
+ */
+ public Builder setStackId(
+ int index, int value) {
+ ensureStackIdIsMutable();
+ stackId_.set(index, value);
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>repeated uint32 stack_id = 7;</code>
+ *
+ * <pre>
+ * stack indices in case the procedure was running
+ * </pre>
+ */
+ public Builder addStackId(int value) {
+ ensureStackIdIsMutable();
+ stackId_.add(value);
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>repeated uint32 stack_id = 7;</code>
+ *
+ * <pre>
+ * stack indices in case the procedure was running
+ * </pre>
+ */
+ public Builder addAllStackId(
+ java.lang.Iterable<? extends java.lang.Integer> values) {
+ ensureStackIdIsMutable();
+ super.addAll(values, stackId_);
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>repeated uint32 stack_id = 7;</code>
+ *
+ * <pre>
+ * stack indices in case the procedure was running
+ * </pre>
+ */
+ public Builder clearStackId() {
+ stackId_ = java.util.Collections.emptyList();
+ bitField0_ = (bitField0_ & ~0x00000040);
+ onChanged();
+ return this;
+ }
+
+ // required uint64 last_update = 8;
+ private long lastUpdate_ ;
+ /**
+ * <code>required uint64 last_update = 8;</code>
+ */
+ public boolean hasLastUpdate() {
+ return ((bitField0_ & 0x00000080) == 0x00000080);
+ }
+ /**
+ * <code>required uint64 last_update = 8;</code>
+ */
+ public long getLastUpdate() {
+ return lastUpdate_;
+ }
+ /**
+ * <code>required uint64 last_update = 8;</code>
+ */
+ public Builder setLastUpdate(long value) {
+ bitField0_ |= 0x00000080;
+ lastUpdate_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required uint64 last_update = 8;</code>
+ */
+ public Builder clearLastUpdate() {
+ bitField0_ = (bitField0_ & ~0x00000080);
+ lastUpdate_ = 0L;
+ onChanged();
+ return this;
+ }
+
+ // optional uint32 timeout = 9;
+ private int timeout_ ;
+ /**
+ * <code>optional uint32 timeout = 9;</code>
+ */
+ public boolean hasTimeout() {
+ return ((bitField0_ & 0x00000100) == 0x00000100);
+ }
+ /**
+ * <code>optional uint32 timeout = 9;</code>
+ */
+ public int getTimeout() {
+ return timeout_;
+ }
+ /**
+ * <code>optional uint32 timeout = 9;</code>
+ */
+ public Builder setTimeout(int value) {
+ bitField0_ |= 0x00000100;
+ timeout_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional uint32 timeout = 9;</code>
+ */
+ public Builder clearTimeout() {
+ bitField0_ = (bitField0_ & ~0x00000100);
+ timeout_ = 0;
+ onChanged();
+ return this;
+ }
+
+ // optional .ForeignExceptionMessage exception = 10;
+ private org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage exception_ = org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.getDefaultInstance();
+ private com.google.protobuf.SingleFieldBuilder<
+ org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.Builder, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessageOrBuilder> exceptionBuilder_;
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ public boolean hasException() {
+ return ((bitField0_ & 0x00000200) == 0x00000200);
+ }
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage getException() {
+ if (exceptionBuilder_ == null) {
+ return exception_;
+ } else {
+ return exceptionBuilder_.getMessage();
+ }
+ }
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ public Builder setException(org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage value) {
+ if (exceptionBuilder_ == null) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ exception_ = value;
+ onChanged();
+ } else {
+ exceptionBuilder_.setMessage(value);
+ }
+ bitField0_ |= 0x00000200;
+ return this;
+ }
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ public Builder setException(
+ org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.Builder builderForValue) {
+ if (exceptionBuilder_ == null) {
+ exception_ = builderForValue.build();
+ onChanged();
+ } else {
+ exceptionBuilder_.setMessage(builderForValue.build());
+ }
+ bitField0_ |= 0x00000200;
+ return this;
+ }
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ public Builder mergeException(org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage value) {
+ if (exceptionBuilder_ == null) {
+ if (((bitField0_ & 0x00000200) == 0x00000200) &&
+ exception_ != org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.getDefaultInstance()) {
+ exception_ =
+ org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.newBuilder(exception_).mergeFrom(value).buildPartial();
+ } else {
+ exception_ = value;
+ }
+ onChanged();
+ } else {
+ exceptionBuilder_.mergeFrom(value);
+ }
+ bitField0_ |= 0x00000200;
+ return this;
+ }
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ public Builder clearException() {
+ if (exceptionBuilder_ == null) {
+ exception_ = org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.getDefaultInstance();
+ onChanged();
+ } else {
+ exceptionBuilder_.clear();
+ }
+ bitField0_ = (bitField0_ & ~0x00000200);
+ return this;
+ }
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.Builder getExceptionBuilder() {
+ bitField0_ |= 0x00000200;
+ onChanged();
+ return getExceptionFieldBuilder().getBuilder();
+ }
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessageOrBuilder getExceptionOrBuilder() {
+ if (exceptionBuilder_ != null) {
+ return exceptionBuilder_.getMessageOrBuilder();
+ } else {
+ return exception_;
+ }
+ }
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ private com.google.protobuf.SingleFieldBuilder<
+ org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.Builder, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessageOrBuilder>
+ getExceptionFieldBuilder() {
+ if (exceptionBuilder_ == null) {
+ exceptionBuilder_ = new com.google.protobuf.SingleFieldBuilder<
+ org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.Builder, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessageOrBuilder>(
+ exception_,
+ getParentForChildren(),
+ isClean());
+ exception_ = null;
+ }
+ return exceptionBuilder_;
+ }
+
+ // optional bytes result = 11;
+ private com.google.protobuf.ByteString result_ = com.google.protobuf.ByteString.EMPTY;
+ /**
+ * <code>optional bytes result = 11;</code>
+ *
+ * <pre>
+ * opaque (user) result structure
+ * </pre>
+ */
+ public boolean hasResult() {
+ return ((bitField0_ & 0x00000400) == 0x00000400);
+ }
+ /**
+ * <code>optional bytes result = 11;</code>
+ *
+ * <pre>
+ * opaque (user) result structure
+ * </pre>
+ */
+ public com.google.protobuf.ByteString getResult() {
+ return result_;
+ }
+ /**
+ * <code>optional bytes result = 11;</code>
+ *
+ * <pre>
+ * opaque (user) result structure
+ * </pre>
+ */
+ public Builder setResult(com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000400;
+ result_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional bytes result = 11;</code>
+ *
+ * <pre>
+ * opaque (user) result structure
+ * </pre>
+ */
+ public Builder clearResult() {
+ bitField0_ = (bitField0_ & ~0x00000400);
+ result_ = getDefaultInstance().getResult();
+ onChanged();
+ return this;
+ }
+
+ // optional bytes state_data = 12;
+ private com.google.protobuf.ByteString stateData_ = com.google.protobuf.ByteString.EMPTY;
+ /**
+ * <code>optional bytes state_data = 12;</code>
+ *
+ * <pre>
+ * opaque (user) procedure internal-state
+ * </pre>
+ */
+ public boolean hasStateData() {
+ return ((bitField0_ & 0x00000800) == 0x00000800);
+ }
+ /**
+ * <code>optional bytes state_data = 12;</code>
+ *
+ * <pre>
+ * opaque (user) procedure internal-state
+ * </pre>
+ */
+ public com.google.protobuf.ByteString getStateData() {
+ return stateData_;
+ }
+ /**
+ * <code>optional bytes state_data = 12;</code>
+ *
+ * <pre>
+ * opaque (user) procedure internal-state
+ * </pre>
+ */
+ public Builder setStateData(com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000800;
+ stateData_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional bytes state_data = 12;</code>
+ *
+ * <pre>
+ * opaque (user) procedure internal-state
+ * </pre>
+ */
+ public Builder clearStateData() {
+ bitField0_ = (bitField0_ & ~0x00000800);
+ stateData_ = getDefaultInstance().getStateData();
+ onChanged();
+ return this;
+ }
+
+ // @@protoc_insertion_point(builder_scope:Procedure)
+ }
+
+ static {
+ defaultInstance = new Procedure(true);
+ defaultInstance.initFields();
+ }
+
+ // @@protoc_insertion_point(class_scope:Procedure)
+ }
+
+ public interface SequentialProcedureDataOrBuilder
+ extends com.google.protobuf.MessageOrBuilder {
+
+ // required bool executed = 1;
+ /**
+ * <code>required bool executed = 1;</code>
+ */
+ boolean hasExecuted();
+ /**
+ * <code>required bool executed = 1;</code>
+ */
+ boolean getExecuted();
+ }
+ /**
+ * Protobuf type {@code SequentialProcedureData}
+ *
+ * <pre>
+ **
+ * SequentialProcedure data
+ * </pre>
+ */
+ public static final class SequentialProcedureData extends
+ com.google.protobuf.GeneratedMessage
+ implements SequentialProcedureDataOrBuilder {
+ // Use SequentialProcedureData.newBuilder() to construct.
+ private SequentialProcedureData(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+ super(builder);
+ this.unknownFields = builder.getUnknownFields();
+ }
+ private SequentialProcedureData(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+ private static final SequentialProcedureData defaultInstance;
+ public static SequentialProcedureData getDefaultInstance() {
+ return defaultInstance;
+ }
+
+ public SequentialProcedureData getDefaultInstanceForType() {
+ return defaultInstance;
+ }
+
+ private final com.google.protobuf.UnknownFieldSet unknownFields;
+ @java.lang.Override
+ public final com.google.protobuf.UnknownFieldSet
+ getUnknownFields() {
+ return this.unknownFields;
+ }
+ private SequentialProcedureData(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ initFields();
+ int mutable_bitField0_ = 0;
+ com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+ com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+ boolean done = false;
+ while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+ case 0:
+ done = true;
+ break;
+ default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+ done = true;
+ }
+ break;
+ }
+ case 8: {
+ bitField0_ |= 0x00000001;
+ executed_ = input.readBool();
+ break;
+ }
+ }
+ }
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+ throw new com.google.protobuf.InvalidProtocolBufferException(
+ e.getMessage()).setUnfinishedMessage(this);
+ } finally {
+ this.unknownFields = unknownFields.build();
+ makeExtensionsImmutable();
+ }
+ }
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.internal_static_SequentialProcedureData_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.internal_static_SequentialProcedureData_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData.class, org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData.Builder.class);
+ }
+
+ public static com.google.protobuf.Parser<SequentialProcedureData> PARSER =
+ new com.google.protobuf.AbstractParser<SequentialProcedureData>() {
+ public SequentialProcedureData parsePartialFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return new SequentialProcedureData(input, extensionRegistry);
+ }
+ };
+
+ @java.lang.Override
+ public com.google.protobuf.Parser<SequentialProcedureData> getParserForType() {
+ return PARSER;
+ }
+
+ private int bitField0_;
+ // required bool executed = 1;
+ public static final int EXECUTED_FIELD_NUMBER = 1;
+ private boolean executed_;
+ /**
+ * <code>required bool executed = 1;</code>
+ */
+ public boolean hasExecuted() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>required bool executed = 1;</code>
+ */
+ public boolean getExecuted() {
+ return executed_;
+ }
+
+ private void initFields() {
+ executed_ = false;
+ }
+ private byte memoizedIsInitialized = -1;
+ public final boolean isInitialized() {
+ byte isInitialized = memoizedIsInitialized;
+ if (isInitialized != -1) return isInitialized == 1;
+
+ if (!hasExecuted()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ memoizedIsInitialized = 1;
+ return true;
+ }
+
+ public void writeTo(com.google.protobuf.CodedOutputStream output)
+ throws java.io.IOException {
+ getSerializedSize();
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ output.writeBool(1, executed_);
+ }
+ getUnknownFields().writeTo(output);
+ }
+
+ private int memoizedSerializedSize = -1;
+ public int getSerializedSize() {
+ int size = memoizedSerializedSize;
+ if (size != -1) return size;
+
+ size = 0;
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBoolSize(1, executed_);
+ }
+ size += getUnknownFields().getSerializedSize();
+ memoizedSerializedSize = size;
+ return size;
+ }
+
+ private static final long serialVersionUID = 0L;
+ @java.lang.Override
+ protected java.lang.Object writeReplace()
+ throws java.io.ObjectStreamException {
+ return super.writeReplace();
+ }
+
+ @java.lang.Override
+ public boolean equals(final java.lang.Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData)) {
+ return super.equals(obj);
+ }
+ org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData other = (org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData) obj;
+
+ boolean result = true;
+ result = result && (hasExecuted() == other.hasExecuted());
+ if (hasExecuted()) {
+ result = result && (getExecuted()
+ == other.getExecuted());
+ }
+ result = result &&
+ getUnknownFields().equals(other.getUnknownFields());
+ return result;
+ }
+
+ private int memoizedHashCode = 0;
+ @java.lang.Override
+ public int hashCode() {
+ if (memoizedHashCode != 0) {
+ return memoizedHashCode;
+ }
+ int hash = 41;
+ hash = (19 * hash) + getDescriptorForType().hashCode();
+ if (hasExecuted()) {
+ hash = (37 * hash) + EXECUTED_FIELD_NUMBER;
+ hash = (53 * hash) + hashBoolean(getExecuted());
+ }
+ hash = (29 * hash) + getUnknownFields().hashCode();
+ memoizedHashCode = hash;
+ return hash;
+ }
+
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData parseFrom(
+ com.google.protobuf.ByteString data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData parseFrom(
+ com.google.protobuf.ByteString data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData parseFrom(byte[] data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData parseFrom(
+ byte[] data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData parseFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData parseFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData parseDelimitedFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData parseDelimitedFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData parseFrom(
+ com.google.protobuf.CodedInputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData parseFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+
+ public static Builder newBuilder() { return Builder.create(); }
+ public Builder newBuilderForType() { return newBuilder(); }
+ public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData prototype) {
+ return newBuilder().mergeFrom(prototype);
+ }
+ public Builder toBuilder() { return newBuilder(this); }
+
+ @java.lang.Override
+ protected Builder newBuilderForType(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ Builder builder = new Builder(parent);
+ return builder;
+ }
+ /**
+ * Protobuf type {@code SequentialProcedureData}
+ *
+ * <pre>
+ **
+ * SequentialProcedure data
+ * </pre>
+ */
+ public static final class Builder extends
+ com.google.protobuf.GeneratedMessage.Builder<Builder>
+ implements org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureDataOrBuilder {
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.internal_static_SequentialProcedureData_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.internal_static_SequentialProcedureData_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData.class, org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData.Builder.class);
+ }
+
+ // Construct using org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData.newBuilder()
+ private Builder() {
+ maybeForceBuilderInitialization();
+ }
+
+ private Builder(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ super(parent);
+ maybeForceBuilderInitialization();
+ }
+ private void maybeForceBuilderInitialization() {
+ if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+ }
+ }
+ private static Builder create() {
+ return new Builder();
+ }
+
+ public Builder clear() {
+ super.clear();
+ executed_ = false;
+ bitField0_ = (bitField0_ & ~0x00000001);
+ return this;
+ }
+
+ public Builder clone() {
+ return create().mergeFrom(buildPartial());
+ }
+
+ public com.google.protobuf.Descriptors.Descriptor
+ getDescriptorForType() {
+ return org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.internal_static_SequentialProcedureData_descriptor;
+ }
+
+ public org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData getDefaultInstanceForType() {
+ return org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData.getDefaultInstance();
+ }
+
+ public org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData build() {
+ org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData result = buildPartial();
+ if (!result.isInitialized()) {
+ throw newUninitializedMessageException(result);
+ }
+ return result;
+ }
+
+ public org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData buildPartial() {
+ org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData result = new org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData(this);
+ int from_bitField0_ = bitField0_;
+ int to_bitField0_ = 0;
+ if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+ to_bitField0_ |= 0x00000001;
+ }
+ result.executed_ = executed_;
+ result.bitField0_ = to_bitField0_;
+ onBuilt();
+ return result;
+ }
+
+ public Builder mergeFrom(com.google.protobuf.Message other) {
+ if (other instanceof org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData) {
+ return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData)other);
+ } else {
+ super.mergeFrom(other);
+ return this;
+ }
+ }
+
+ public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData other) {
+ if (other == org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData.getDefaultInstance()) return this;
+ if (other.hasExecuted()) {
+ setExecuted(other.getExecuted());
+ }
+ this.mergeUnknownFields(other.getUnknownFields());
+ return this;
+ }
+
+ public final boolean isInitialized() {
+ if (!hasExecuted()) {
+
+ return false;
+ }
+ return true;
+ }
+
+ public Builder mergeFrom(
+ com.g
<TRUNCATED>