You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2015/03/26 11:19:11 UTC
[01/15] hbase git commit: HBASE-13298 Clarify if
Table.{set|get}WriteBufferSize() is deprecated or not
Repository: hbase
Updated Branches:
refs/heads/hbase-12439 962c89a1d -> ddc1bcae9 (forced update)
HBASE-13298 Clarify if Table.{set|get}WriteBufferSize() is deprecated or not
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/50daed26
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/50daed26
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/50daed26
Branch: refs/heads/hbase-12439
Commit: 50daed268eb964e09a023d30fe8880441442495f
Parents: 5a58116
Author: stack <st...@apache.org>
Authored: Wed Mar 25 13:10:04 2015 -0700
Committer: stack <st...@apache.org>
Committed: Wed Mar 25 13:10:04 2015 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hbase/client/Table.java | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/50daed26/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Table.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Table.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Table.java
index 55fb1c4..498c587 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Table.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Table.java
@@ -498,7 +498,9 @@ public interface Table extends Closeable {
* The default value comes from the configuration parameter
* {@code hbase.client.write.buffer}.
* @return The size of the write buffer in bytes.
+ * @deprecated as of 1.0.1 (should not have been in 1.0.0). Replaced by {@link BufferedMutator#getWriteBufferSize()}
*/
+ @Deprecated
long getWriteBufferSize();
/**
@@ -508,7 +510,10 @@ public interface Table extends Closeable {
* write buffer, the buffer gets flushed.
* @param writeBufferSize The new write buffer size, in bytes.
* @throws IOException if a remote or network exception occurs.
+ * @deprecated as of 1.0.1 (should not have been in 1.0.0). Replaced by {@link BufferedMutator} and
+ * {@link BufferedMutatorParams#writeBufferSize(long)}
*/
+ @Deprecated
void setWriteBufferSize(long writeBufferSize) throws IOException;
/**
[07/15] hbase git commit: HBASE-13326 Disabled table can't be enabled
after HBase is restarted (addendum)
Posted by mb...@apache.org.
HBASE-13326 Disabled table can't be enabled after HBase is restarted (addendum)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bfeb067e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bfeb067e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bfeb067e
Branch: refs/heads/hbase-12439
Commit: bfeb067e3a50513cfc20bb8ee57c49f1d5c1f19f
Parents: 948a6a6
Author: Andrey Stepachev <oc...@gmail.com>
Authored: Thu Mar 26 08:13:42 2015 +0000
Committer: Andrey Stepachev <oc...@gmail.com>
Committed: Thu Mar 26 08:13:42 2015 +0000
----------------------------------------------------------------------
dev-support/test-patch.sh | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/bfeb067e/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index 64cf8a7..20d5f25 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -324,7 +324,7 @@ setup () {
echo ""
echo ""
echo "$MVN clean package checkstyle:checkstyle-aggregate findbugs:findbugs -DskipTests \
- -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavacWarnings.txt 2>&1"
+ -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavacWarnings.txt 2>&1"
export MAVEN_OPTS="${MAVEN_OPTS}"
# build core and tests
$MVN clean package checkstyle:checkstyle-aggregate findbugs:findbugs -DskipTests \
@@ -757,13 +757,13 @@ $JIRA_COMMENT_FOOTER"
JIRA_COMMENT="$JIRA_COMMENT
{color:red}-1 findbugs{color}. The patch appears to introduce $findbugsWarnings \
- new Findbugs (version ${findbugs_version}) warnings."
+ new Findbugs (version ${findbugs_version}) warnings."
return 1
fi
JIRA_COMMENT="$JIRA_COMMENT
{color:green}+1 findbugs{color}. The patch does not introduce any \
- new Findbugs (version ${findbugs_version}) warnings."
+ new Findbugs (version ${findbugs_version}) warnings."
return 0
}
[13/15] hbase git commit: HBASE-13202 Procedure v2 - core framework
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java
new file mode 100644
index 0000000..c2a5571
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java
@@ -0,0 +1,683 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2.store.wal;
+
+import java.io.IOException;
+import java.io.FileNotFoundException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureStore;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker;
+import org.apache.hadoop.hbase.procedure2.util.ByteSlot;
+import org.apache.hadoop.hbase.procedure2.util.StringUtils;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureWALHeader;
+
+/**
+ * WAL implementation of the ProcedureStore.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class WALProcedureStore implements ProcedureStore {
+ private static final Log LOG = LogFactory.getLog(WALProcedureStore.class);
+
+ public interface LeaseRecovery {
+ void recoverFileLease(FileSystem fs, Path path) throws IOException;
+ }
+
+ private static final int MAX_RETRIES_BEFORE_ABORT = 3;
+
+ private static final String SYNC_WAIT_MSEC_CONF_KEY = "hbase.procedure.store.wal.sync.wait.msec";
+ private static final int DEFAULT_SYNC_WAIT_MSEC = 50;
+
+ private final CopyOnWriteArrayList<ProcedureStoreListener> listeners =
+ new CopyOnWriteArrayList<ProcedureStoreListener>();
+
+ private final LinkedList<ProcedureWALFile> logs = new LinkedList<ProcedureWALFile>();
+ private final ProcedureStoreTracker storeTracker = new ProcedureStoreTracker();
+ private final AtomicBoolean running = new AtomicBoolean(false);
+ private final ReentrantLock lock = new ReentrantLock();
+ private final Condition waitCond = lock.newCondition();
+ private final Condition slotCond = lock.newCondition();
+ private final Condition syncCond = lock.newCondition();
+
+ private final LeaseRecovery leaseRecovery;
+ private final Configuration conf;
+ private final FileSystem fs;
+ private final Path logDir;
+
+ private AtomicBoolean inSync = new AtomicBoolean(false);
+ private ArrayBlockingQueue<ByteSlot> slotsCache = null;
+ private Set<ProcedureWALFile> corruptedLogs = null;
+ private FSDataOutputStream stream = null;
+ private long totalSynced = 0;
+ private long flushLogId = 0;
+ private int slotIndex = 0;
+ private Thread syncThread;
+ private ByteSlot[] slots;
+ private int syncWaitMsec;
+
+ public WALProcedureStore(final Configuration conf, final FileSystem fs, final Path logDir,
+ final LeaseRecovery leaseRecovery) {
+ this.fs = fs;
+ this.conf = conf;
+ this.logDir = logDir;
+ this.leaseRecovery = leaseRecovery;
+ }
+
+ @Override
+ public void start(int numSlots) throws IOException {
+ if (running.getAndSet(true)) {
+ return;
+ }
+
+ // Init buffer slots
+ slots = new ByteSlot[numSlots];
+ slotsCache = new ArrayBlockingQueue(numSlots, true);
+ while (slotsCache.remainingCapacity() > 0) {
+ slotsCache.offer(new ByteSlot());
+ }
+
+ // Tunings
+ syncWaitMsec = conf.getInt(SYNC_WAIT_MSEC_CONF_KEY, DEFAULT_SYNC_WAIT_MSEC);
+
+ // Init sync thread
+ syncThread = new Thread() {
+ @Override
+ public void run() {
+ while (running.get()) {
+ try {
+ syncLoop();
+ } catch (IOException e) {
+ LOG.error("got an exception from the sync-loop", e);
+ sendAbortProcessSignal();
+ }
+ }
+ }
+ };
+ syncThread.start();
+ }
+
+ @Override
+ public void stop() {
+ if (!running.getAndSet(false)) {
+ return;
+ }
+
+ LOG.info("Stopping the WAL Procedure Store");
+
+ lock.lock();
+ try {
+ waitCond.signalAll();
+ } finally {
+ lock.unlock();
+ }
+
+ try {
+ syncThread.join();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+
+ // Close the writer
+ closeStream();
+
+ // Close the old logs
+ // they should be already closed, this is just in case the load fails
+ // and we call start() and then stop()
+ for (ProcedureWALFile log: logs) {
+ log.close();
+ }
+ logs.clear();
+ }
+
+ @Override
+ public void registerListener(ProcedureStoreListener listener) {
+ this.listeners.add(listener);
+ }
+
+ @Override
+ public boolean unregisterListener(ProcedureStoreListener listener) {
+ return this.listeners.remove(listener);
+ }
+
+ @Override
+ public void recoverLease() throws IOException {
+ LOG.info("Starting WAL Procedure Store lease recovery");
+ FileStatus[] oldLogs = getLogFiles();
+ while (running.get()) {
+ // Get Log-MaxID and recover lease on old logs
+ flushLogId = initOldLogs(oldLogs) + 1;
+
+ // Create new state-log
+ if (!rollWriter(flushLogId)) {
+ // someone else has already created this log
+ continue;
+ }
+
+ // We have the lease on the log
+ oldLogs = getLogFiles();
+ if (getMaxLogId(oldLogs) > flushLogId) {
+ // Someone else created new logs
+ logs.getLast().removeFile();
+ continue;
+ }
+
+ LOG.info("lease acquired flushLogId=" + flushLogId);
+ break;
+ }
+ }
+
+ @Override
+ public Iterator<Procedure> load() throws IOException {
+ if (logs.isEmpty()) {
+ throw new RuntimeException("recoverLease() must be called before loading data");
+ }
+
+ // Nothing to do, If we have only the current log.
+ if (logs.size() == 1) {
+ LOG.debug("No state logs to replay");
+ return null;
+ }
+
+ // Load the old logs
+ final ArrayList<ProcedureWALFile> toRemove = new ArrayList<ProcedureWALFile>();
+ Iterator<ProcedureWALFile> it = logs.descendingIterator();
+ it.next(); // Skip the current log
+ try {
+ return ProcedureWALFormat.load(it, storeTracker, new ProcedureWALFormat.Loader() {
+ @Override
+ public void removeLog(ProcedureWALFile log) {
+ toRemove.add(log);
+ }
+
+ @Override
+ public void markCorruptedWAL(ProcedureWALFile log, IOException e) {
+ if (corruptedLogs == null) {
+ corruptedLogs = new HashSet<ProcedureWALFile>();
+ }
+ corruptedLogs.add(log);
+ // TODO: sideline corrupted log
+ }
+ });
+ } finally {
+ if (!toRemove.isEmpty()) {
+ for (ProcedureWALFile log: toRemove) {
+ removeLogFile(log);
+ }
+ }
+ }
+ }
+
+ @Override
+ public void insert(final Procedure proc, final Procedure[] subprocs) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("insert " + proc + " subproc=" + Arrays.toString(subprocs));
+ }
+
+ ByteSlot slot = acquireSlot();
+ long logId = -1;
+ try {
+ // Serialize the insert
+ if (subprocs != null) {
+ ProcedureWALFormat.writeInsert(slot, proc, subprocs);
+ } else {
+ assert !proc.hasParent();
+ ProcedureWALFormat.writeInsert(slot, proc);
+ }
+
+ // Push the transaction data and wait until it is persisted
+ logId = pushData(slot);
+ } catch (IOException e) {
+ // We are not able to serialize the procedure.
+ // this is a code error, and we are not able to go on.
+ LOG.fatal("Unable to serialize one of the procedure: proc=" + proc +
+ " subprocs=" + Arrays.toString(subprocs), e);
+ throw new RuntimeException(e);
+ } finally {
+ releaseSlot(slot);
+ }
+
+ // Update the store tracker
+ synchronized (storeTracker) {
+ if (logId == flushLogId) {
+ storeTracker.insert(proc, subprocs);
+ }
+ }
+ }
+
+ @Override
+ public void update(final Procedure proc) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("update " + proc);
+ }
+
+ ByteSlot slot = acquireSlot();
+ long logId = -1;
+ try {
+ // Serialize the update
+ ProcedureWALFormat.writeUpdate(slot, proc);
+
+ // Push the transaction data and wait until it is persisted
+ logId = pushData(slot);
+ } catch (IOException e) {
+ // We are not able to serialize the procedure.
+ // this is a code error, and we are not able to go on.
+ LOG.fatal("Unable to serialize the procedure: " + proc, e);
+ throw new RuntimeException(e);
+ } finally {
+ releaseSlot(slot);
+ }
+
+ // Update the store tracker
+ boolean removeOldLogs = false;
+ synchronized (storeTracker) {
+ if (logId == flushLogId) {
+ storeTracker.update(proc);
+ removeOldLogs = storeTracker.isUpdated();
+ }
+ }
+
+ if (removeOldLogs) {
+ removeAllLogs(logId - 1);
+ }
+ }
+
+ @Override
+ public void delete(final long procId) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("delete " + procId);
+ }
+
+ ByteSlot slot = acquireSlot();
+ long logId = -1;
+ try {
+ // Serialize the delete
+ ProcedureWALFormat.writeDelete(slot, procId);
+
+ // Push the transaction data and wait until it is persisted
+ logId = pushData(slot);
+ } catch (IOException e) {
+ // We are not able to serialize the procedure.
+ // this is a code error, and we are not able to go on.
+ LOG.fatal("Unable to serialize the procedure: " + procId, e);
+ throw new RuntimeException(e);
+ } finally {
+ releaseSlot(slot);
+ }
+
+ boolean removeOldLogs = false;
+ synchronized (storeTracker) {
+ if (logId == flushLogId) {
+ storeTracker.delete(procId);
+ if (storeTracker.isEmpty()) {
+ removeOldLogs = rollWriterOrDie(logId + 1);
+ }
+ }
+ }
+
+ if (removeOldLogs) {
+ removeAllLogs(logId);
+ }
+ }
+
+ private ByteSlot acquireSlot() {
+ ByteSlot slot = slotsCache.poll();
+ return slot != null ? slot : new ByteSlot();
+ }
+
+ private void releaseSlot(final ByteSlot slot) {
+ slot.reset();
+ slotsCache.offer(slot);
+ }
+
+ private long pushData(final ByteSlot slot) {
+ assert !logs.isEmpty() : "recoverLease() must be called before inserting data";
+ long logId = -1;
+
+ lock.lock();
+ try {
+ // Wait for the sync to be completed
+ while (true) {
+ if (inSync.get()) {
+ syncCond.await();
+ } else if (slotIndex == slots.length) {
+ slotCond.signal();
+ syncCond.await();
+ } else {
+ break;
+ }
+ }
+
+ slots[slotIndex++] = slot;
+ logId = flushLogId;
+
+ // Notify that there is new data
+ if (slotIndex == 1) {
+ waitCond.signal();
+ }
+
+ // Notify that the slots are full
+ if (slotIndex == slots.length) {
+ slotCond.signal();
+ }
+ syncCond.await();
+ } catch (InterruptedException e) {
+ sendAbortProcessSignal();
+ } finally {
+ lock.unlock();
+ }
+ return logId;
+ }
+
+ private void syncLoop() throws IOException {
+ inSync.set(false);
+ while (running.get()) {
+ lock.lock();
+ try {
+ // Wait until new data is available
+ if (slotIndex == 0) {
+ LOG.debug("Waiting for data. flushed=" + StringUtils.humanSize(totalSynced));
+ waitCond.await();
+ if (slotIndex == 0) {
+ // no data.. probably a stop()
+ continue;
+ }
+ }
+
+ // Wait SYNC_WAIT_MSEC or the signal of "slots full" before flushing
+ slotCond.await(syncWaitMsec, TimeUnit.MILLISECONDS);
+
+ inSync.set(true);
+ totalSynced += syncSlots();
+ slotIndex = 0;
+ inSync.set(false);
+ syncCond.signalAll();
+ } catch (InterruptedException e) {
+ sendAbortProcessSignal();
+ } finally {
+ lock.unlock();
+ }
+ }
+ }
+
+ private long syncSlots() {
+ int retry = 0;
+ long totalSynced = 0;
+ do {
+ try {
+ totalSynced = syncSlots(stream, slots, 0, slotIndex);
+ break;
+ } catch (IOException e) {
+ if (++retry == MAX_RETRIES_BEFORE_ABORT) {
+ LOG.error("sync slot failed, abort.", e);
+ sendAbortProcessSignal();
+ }
+ }
+ } while (running.get());
+ return totalSynced;
+ }
+
+ protected long syncSlots(FSDataOutputStream stream, ByteSlot[] slots, int offset, int count)
+ throws IOException {
+ long totalSynced = 0;
+ for (int i = 0; i < count; ++i) {
+ ByteSlot data = slots[offset + i];
+ data.writeTo(stream);
+ totalSynced += data.size();
+ }
+ stream.hsync();
+ return totalSynced;
+ }
+
+ private void sendAbortProcessSignal() {
+ if (!this.listeners.isEmpty()) {
+ for (ProcedureStoreListener listener : this.listeners) {
+ listener.abortProcess();
+ }
+ }
+ }
+
+ private boolean rollWriterOrDie(final long logId) {
+ try {
+ return rollWriter(logId);
+ } catch (IOException e) {
+ LOG.warn("Unable to roll the log", e);
+ sendAbortProcessSignal();
+ return false;
+ }
+ }
+
+ private boolean rollWriter(final long logId) throws IOException {
+ ProcedureWALHeader header = ProcedureWALHeader.newBuilder()
+ .setVersion(ProcedureWALFormat.HEADER_VERSION)
+ .setType(ProcedureWALFormat.LOG_TYPE_STREAM)
+ .setMinProcId(storeTracker.getMinProcId())
+ .setLogId(logId)
+ .build();
+
+ FSDataOutputStream newStream = null;
+ Path newLogFile = null;
+ long startPos = -1;
+ try {
+ newLogFile = getLogFilePath(logId);
+ newStream = fs.create(newLogFile, false);
+ ProcedureWALFormat.writeHeader(newStream, header);
+ startPos = newStream.getPos();
+ } catch (FileAlreadyExistsException e) {
+ LOG.error("Log file with id=" + logId + " already exists", e);
+ return false;
+ }
+ lock.lock();
+ try {
+ closeStream();
+ synchronized (storeTracker) {
+ storeTracker.resetUpdates();
+ }
+ stream = newStream;
+ flushLogId = logId;
+ totalSynced = 0;
+ logs.add(new ProcedureWALFile(fs, newLogFile, header, startPos));
+ } finally {
+ lock.unlock();
+ }
+ LOG.info("Roll new state log: " + logId);
+ return true;
+ }
+
+ private void closeStream() {
+ try {
+ if (stream != null) {
+ try {
+ ProcedureWALFormat.writeTrailer(stream, storeTracker);
+ } catch (IOException e) {
+ LOG.error("Unable to write the trailer", e);
+ }
+ stream.close();
+ }
+ } catch (IOException e) {
+ LOG.error("Unable to close the stream", e);
+ } finally {
+ stream = null;
+ }
+ }
+
+ private void removeAllLogs(long lastLogId) {
+ LOG.info("Remove all state logs with ID less then " + lastLogId);
+ while (!logs.isEmpty()) {
+ ProcedureWALFile log = logs.getFirst();
+ if (lastLogId < log.getLogId()) {
+ break;
+ }
+
+ removeLogFile(log);
+ }
+ }
+
+ private boolean removeLogFile(final ProcedureWALFile log) {
+ try {
+ LOG.debug("remove log: " + log);
+ log.removeFile();
+ logs.remove(log);
+ } catch (IOException e) {
+ LOG.error("unable to remove log " + log, e);
+ return false;
+ }
+ return true;
+ }
+
+ public Set<ProcedureWALFile> getCorruptedLogs() {
+ return corruptedLogs;
+ }
+
+ // ==========================================================================
+ // FileSystem Log Files helpers
+ // ==========================================================================
+
+ protected Path getLogFilePath(final long logId) throws IOException {
+ return new Path(logDir, String.format("state-%020d.log", logId));
+ }
+
+ private static long getLogIdFromName(final String name) {
+ int end = name.lastIndexOf(".log");
+ int start = name.lastIndexOf('-') + 1;
+ while (start < end) {
+ if (name.charAt(start) != '0')
+ break;
+ start++;
+ }
+ return Long.parseLong(name.substring(start, end));
+ }
+
+ private FileStatus[] getLogFiles() throws IOException {
+ try {
+ return fs.listStatus(logDir, new PathFilter() {
+ @Override
+ public boolean accept(Path path) {
+ String name = path.getName();
+ return name.startsWith("state-") && name.endsWith(".log");
+ }
+ });
+ } catch (FileNotFoundException e) {
+ LOG.warn("log directory not found: " + e.getMessage());
+ return null;
+ }
+ }
+
+ private long getMaxLogId(final FileStatus[] logFiles) {
+ long maxLogId = 0;
+ if (logFiles != null && logFiles.length > 0) {
+ for (int i = 0; i < logFiles.length; ++i) {
+ maxLogId = Math.max(maxLogId, getLogIdFromName(logFiles[i].getPath().getName()));
+ }
+ }
+ return maxLogId;
+ }
+
+ /**
+ * @return Max-LogID of the specified log file set
+ */
+ private long initOldLogs(final FileStatus[] logFiles) throws IOException {
+ this.logs.clear();
+
+ long maxLogId = 0;
+ if (logFiles != null && logFiles.length > 0) {
+ for (int i = 0; i < logFiles.length; ++i) {
+ final Path logPath = logFiles[i].getPath();
+ leaseRecovery.recoverFileLease(fs, logPath);
+ maxLogId = Math.max(maxLogId, getLogIdFromName(logPath.getName()));
+
+ ProcedureWALFile log = initOldLog(logFiles[i]);
+ if (log != null) {
+ this.logs.add(log);
+ }
+ }
+ Collections.sort(this.logs);
+ initTrackerFromOldLogs();
+ }
+ return maxLogId;
+ }
+
+ private void initTrackerFromOldLogs() {
+ // TODO: Load the most recent tracker available
+ if (!logs.isEmpty()) {
+ ProcedureWALFile log = logs.getLast();
+ try {
+ log.readTracker(storeTracker);
+ } catch (IOException e) {
+ LOG.error("Unable to read tracker for " + log, e);
+ // try the next one...
+ storeTracker.clear();
+ storeTracker.setPartialFlag(true);
+ }
+ }
+ }
+
+ private ProcedureWALFile initOldLog(final FileStatus logFile) throws IOException {
+ ProcedureWALFile log = new ProcedureWALFile(fs, logFile);
+ try {
+ log.open();
+ LOG.debug("opening state-log: " + log);
+ if (log.isCompacted()) {
+ try {
+ log.readTrailer();
+ } catch (IOException e) {
+ // unfinished compacted log throw it away
+ LOG.warn("Unfinished compacted log " + logFile, e);
+ log.removeFile();
+ return null;
+ }
+ }
+ } catch (IOException e) {
+ if (logFile.getLen() == 0) {
+ LOG.warn("Remove uninitialized log " + logFile, e);
+ log.removeFile();
+ return null;
+ }
+
+ String msg = "Unable to read state log " + logFile;
+ LOG.error(msg, e);
+ throw new IOException(msg, e);
+ }
+ return log;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/ByteSlot.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/ByteSlot.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/ByteSlot.java
new file mode 100644
index 0000000..8904116
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/ByteSlot.java
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2.util;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Arrays;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * Similar to the ByteArrayOutputStream, with the exception that we can prepend an header.
+ * e.g. you write some data and you want to prepend an header that contains the data len or cksum.
+ * <code>
+ * ByteSlot slot = new ByteSlot();
+ * // write data
+ * slot.write(...);
+ * slot.write(...);
+ * // write header with the size of the written data
+ * slot.markHead();
+ * slot.write(Bytes.toBytes(slot.size()));
+ * // flush to stream as [header, data]
+ * slot.writeTo(stream);
+ * </code>
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class ByteSlot extends OutputStream {
+ private static final int DOUBLE_GROW_LIMIT = 1 << 20;
+ private static final int GROW_ALIGN = 128;
+
+ private byte[] buf;
+ private int head;
+ private int size;
+
+ public void reset() {
+ head = 0;
+ size = 0;
+ }
+
+ public void markHead() {
+ head = size;
+ }
+
+ public int getHead() {
+ return head;
+ }
+
+ public int size() {
+ return size;
+ }
+
+ public byte[] getBuffer() {
+ return buf;
+ }
+
+ public void writeAt(int offset, int b) {
+ head = Math.min(head, offset);
+ buf[offset] = (byte)b;
+ }
+
+ public void write(int b) {
+ ensureCapacity(size + 1);
+ buf[size++] = (byte)b;
+ }
+
+ public void write(byte[] b, int off, int len) {
+ ensureCapacity(size + len);
+ System.arraycopy(b, off, buf, size, len);
+ size += len;
+ }
+
+ public void writeTo(final OutputStream stream) throws IOException {
+ if (head != 0) {
+ stream.write(buf, head, size - head);
+ stream.write(buf, 0, head);
+ } else {
+ stream.write(buf, 0, size);
+ }
+ }
+
+ private void ensureCapacity(int minCapacity) {
+ minCapacity = (minCapacity + (GROW_ALIGN - 1)) & -GROW_ALIGN;
+ if (buf == null) {
+ buf = new byte[minCapacity];
+ } else if (minCapacity > buf.length) {
+ int newCapacity = buf.length << 1;
+ if (minCapacity > newCapacity || newCapacity > DOUBLE_GROW_LIMIT) {
+ newCapacity = minCapacity;
+ }
+ buf = Arrays.copyOf(buf, newCapacity);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/StringUtils.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/StringUtils.java
new file mode 100644
index 0000000..97134c2
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/StringUtils.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2.util;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public final class StringUtils {
+ private StringUtils() {}
+
+ public static String humanTimeDiff(long timeDiff) {
+ StringBuilder buf = new StringBuilder();
+ long hours = timeDiff / (60*60*1000);
+ long rem = (timeDiff % (60*60*1000));
+ long minutes = rem / (60*1000);
+ rem = rem % (60*1000);
+ float seconds = rem / 1000.0f;
+
+ if (hours != 0){
+ buf.append(hours);
+ buf.append("hrs, ");
+ }
+ if (minutes != 0){
+ buf.append(minutes);
+ buf.append("mins, ");
+ }
+ if (hours > 0 || minutes > 0) {
+ buf.append(seconds);
+ buf.append("sec");
+ } else {
+ buf.append(String.format("%.4fsec", seconds));
+ }
+ return buf.toString();
+ }
+
+ public static String humanSize(double size) {
+ if (size >= (1L << 40)) return String.format("%.1fT", size / (1L << 40));
+ if (size >= (1L << 30)) return String.format("%.1fG", size / (1L << 30));
+ if (size >= (1L << 20)) return String.format("%.1fM", size / (1L << 20));
+ if (size >= (1L << 10)) return String.format("%.1fK", size / (1L << 10));
+ return String.format("%.0f", size);
+ }
+
+ public static boolean isEmpty(final String input) {
+ return input == null || input.length() == 0;
+ }
+
+ public static String buildString(final String... parts) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < parts.length; ++i) {
+ sb.append(parts[i]);
+ }
+ return sb.toString();
+ }
+
+ public static StringBuilder appendStrings(final StringBuilder sb, final String... parts) {
+ for (int i = 0; i < parts.length; ++i) {
+ sb.append(parts[i]);
+ }
+ return sb;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/TimeoutBlockingQueue.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/TimeoutBlockingQueue.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/TimeoutBlockingQueue.java
new file mode 100644
index 0000000..368d26f
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/TimeoutBlockingQueue.java
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2.util;
+
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class TimeoutBlockingQueue<E> {
+ public static interface TimeoutRetriever<T> {
+ long getTimeout(T object);
+ TimeUnit getTimeUnit(T object);
+ }
+
+ private final ReentrantLock lock = new ReentrantLock();
+ private final Condition waitCond = lock.newCondition();
+ private final TimeoutRetriever<? super E> timeoutRetriever;
+
+ private E[] objects;
+ private int head = 0;
+ private int tail = 0;
+
+ public TimeoutBlockingQueue(TimeoutRetriever<? super E> timeoutRetriever) {
+ this(32, timeoutRetriever);
+ }
+
+ @SuppressWarnings("unchecked")
+ public TimeoutBlockingQueue(int capacity, TimeoutRetriever<? super E> timeoutRetriever) {
+ this.objects = (E[])new Object[capacity];
+ this.timeoutRetriever = timeoutRetriever;
+ }
+
+ public void dump() {
+ for (int i = 0; i < objects.length; ++i) {
+ if (i == head) {
+ System.out.print("[" + objects[i] + "] ");
+ } else if (i == tail) {
+ System.out.print("]" + objects[i] + "[ ");
+ } else {
+ System.out.print(objects[i] + " ");
+ }
+ }
+ System.out.println();
+ }
+
+ public void clear() {
+ lock.lock();
+ try {
+ if (head != tail) {
+ for (int i = head; i < tail; ++i) {
+ objects[i] = null;
+ }
+ head = 0;
+ tail = 0;
+ waitCond.signal();
+ }
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ public void add(E e) {
+ if (e == null) throw new NullPointerException();
+
+ lock.lock();
+ try {
+ addElement(e);
+ waitCond.signal();
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ @edu.umd.cs.findbugs.annotations.SuppressWarnings("WA_AWAIT_NOT_IN_LOOP")
+ public E poll() {
+ lock.lock();
+ try {
+ if (isEmpty()) {
+ waitCond.await();
+ return null;
+ }
+
+ E elem = objects[head];
+ long nanos = getNanosTimeout(elem);
+ nanos = waitCond.awaitNanos(nanos);
+ return nanos > 0 ? null : removeFirst();
+ } catch (InterruptedException e) {
+ return null;
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ public int size() {
+ return tail - head;
+ }
+
+ public boolean isEmpty() {
+ return (tail - head) == 0;
+ }
+
+ public void signalAll() {
+ lock.lock();
+ try {
+ waitCond.signalAll();
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ private void addElement(E elem) {
+ int size = (tail - head);
+ if ((objects.length - size) == 0) {
+ int capacity = size + ((size < 64) ? (size + 2) : (size >> 1));
+ E[] newObjects = (E[])new Object[capacity];
+
+ if (compareTimeouts(objects[tail - 1], elem) <= 0) {
+ // Append
+ System.arraycopy(objects, head, newObjects, 0, tail);
+ tail -= head;
+ newObjects[tail++] = elem;
+ } else if (compareTimeouts(objects[head], elem) > 0) {
+ // Prepend
+ System.arraycopy(objects, head, newObjects, 1, tail);
+ newObjects[0] = elem;
+ tail -= (head - 1);
+ } else {
+ // Insert in the middle
+ int index = upperBound(head, tail - 1, elem);
+ int newIndex = (index - head);
+ System.arraycopy(objects, head, newObjects, 0, newIndex);
+ newObjects[newIndex] = elem;
+ System.arraycopy(objects, index, newObjects, newIndex + 1, tail - index);
+ tail -= (head - 1);
+ }
+ head = 0;
+ objects = newObjects;
+ } else {
+ if (tail == objects.length) {
+ // shift down |-----AAAAAAA|
+ tail -= head;
+ System.arraycopy(objects, head, objects, 0, tail);
+ head = 0;
+ }
+
+ if (tail == head || compareTimeouts(objects[tail - 1], elem) <= 0) {
+ // Append
+ objects[tail++] = elem;
+ } else if (head > 0 && compareTimeouts(objects[head], elem) > 0) {
+ // Prepend
+ objects[--head] = elem;
+ } else {
+ // Insert in the middle
+ int index = upperBound(head, tail - 1, elem);
+ System.arraycopy(objects, index, objects, index + 1, tail - index);
+ objects[index] = elem;
+ tail++;
+ }
+ }
+ }
+
+ private E removeFirst() {
+ E elem = objects[head];
+ objects[head] = null;
+ head = (head + 1) % objects.length;
+ if (head == 0) tail = 0;
+ return elem;
+ }
+
+ private int upperBound(int start, int end, E key) {
+ while (start < end) {
+ int mid = (start + end) >>> 1;
+ E mitem = objects[mid];
+ int cmp = compareTimeouts(mitem, key);
+ if (cmp > 0) {
+ end = mid;
+ } else {
+ start = mid + 1;
+ }
+ }
+ return start;
+ }
+
+ private int compareTimeouts(final E a, final E b) {
+ long t1 = getNanosTimeout(a);
+ long t2 = getNanosTimeout(b);
+ return (t1 < t2) ? -1 : (t1 > t2) ? 1 : 0;
+ }
+
+ private long getNanosTimeout(final E obj) {
+ TimeUnit unit = timeoutRetriever.getTimeUnit(obj);
+ long timeout = timeoutRetriever.getTimeout(obj);
+ return unit.toNanos(timeout);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java
new file mode 100644
index 0000000..c73300d
--- /dev/null
+++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureStore;
+import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class ProcedureTestingUtility {
+ private static final Log LOG = LogFactory.getLog(ProcedureTestingUtility.class);
+
+ private ProcedureTestingUtility() {
+ }
+
+ public static ProcedureStore createStore(final Configuration conf, final FileSystem fs,
+ final Path baseDir) throws IOException {
+ return createWalStore(conf, fs, baseDir);
+ }
+
+ public static WALProcedureStore createWalStore(final Configuration conf, final FileSystem fs,
+ final Path logDir) throws IOException {
+ return new WALProcedureStore(conf, fs, logDir, new WALProcedureStore.LeaseRecovery() {
+ @Override
+ public void recoverFileLease(FileSystem fs, Path path) throws IOException {
+ // no-op
+ }
+ });
+ }
+
+ public static <TEnv> void restart(ProcedureExecutor<TEnv> procExecutor)
+ throws Exception {
+ ProcedureStore procStore = procExecutor.getStore();
+ int nthreads = procExecutor.getNumThreads();
+ procExecutor.stop();
+ procStore.stop();
+ procExecutor.join();
+ procStore.start(nthreads);
+ procExecutor.start(nthreads);
+ }
+
+ public static <TEnv> void setKillBeforeStoreUpdate(ProcedureExecutor<TEnv> procExecutor,
+ boolean value) {
+ if (procExecutor.testing == null) {
+ procExecutor.testing = new ProcedureExecutor.Testing();
+ }
+ procExecutor.testing.killBeforeStoreUpdate = value;
+ LOG.warn("Set Kill before store update to: " + procExecutor.testing.killBeforeStoreUpdate);
+ }
+
+ public static <TEnv> void setToggleKillBeforeStoreUpdate(ProcedureExecutor<TEnv> procExecutor,
+ boolean value) {
+ if (procExecutor.testing == null) {
+ procExecutor.testing = new ProcedureExecutor.Testing();
+ }
+ procExecutor.testing.toggleKillBeforeStoreUpdate = value;
+ }
+
+ public static <TEnv> void toggleKillBeforeStoreUpdate(ProcedureExecutor<TEnv> procExecutor) {
+ if (procExecutor.testing == null) {
+ procExecutor.testing = new ProcedureExecutor.Testing();
+ }
+ procExecutor.testing.killBeforeStoreUpdate = !procExecutor.testing.killBeforeStoreUpdate;
+ LOG.warn("Set Kill before store update to: " + procExecutor.testing.killBeforeStoreUpdate);
+ }
+
+ public static <TEnv> long submitAndWait(ProcedureExecutor<TEnv> procExecutor, Procedure proc) {
+ long procId = procExecutor.submitProcedure(proc);
+ waitProcedure(procExecutor, procId);
+ return procId;
+ }
+
+ public static <TEnv> void waitProcedure(ProcedureExecutor<TEnv> procExecutor, long procId) {
+ while (!procExecutor.isFinished(procId) && procExecutor.isRunning()) {
+ Threads.sleepWithoutInterrupt(250);
+ }
+ }
+
+ public static <TEnv> void assertProcNotYetCompleted(ProcedureExecutor<TEnv> procExecutor,
+ long procId) {
+ assertFalse(procExecutor.isFinished(procId));
+ assertEquals(null, procExecutor.getResult(procId));
+ }
+
+ public static <TEnv> void assertProcNotFailed(ProcedureExecutor<TEnv> procExecutor,
+ long procId) {
+ ProcedureResult result = procExecutor.getResult(procId);
+ assertTrue(result != null);
+ assertProcNotFailed(result);
+ }
+
+ public static void assertProcNotFailed(final ProcedureResult result) {
+ Exception exception = result.getException();
+ String msg = exception != null ? exception.toString() : "no exception found";
+ assertFalse(msg, result.isFailed());
+ }
+
+ public static void assertIsAbortException(final ProcedureResult result) {
+ LOG.info(result.getException());
+ assertTrue(result.isFailed());
+ assertTrue(result.getException().getCause() instanceof ProcedureAbortedException);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureExecution.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureExecution.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureExecution.java
new file mode 100644
index 0000000..53f2dc6
--- /dev/null
+++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureExecution.java
@@ -0,0 +1,303 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureStore;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+@Category({MasterTests.class, SmallTests.class})
+public class TestProcedureExecution {
+ private static final Log LOG = LogFactory.getLog(TestProcedureExecution.class);
+
+ private static final int PROCEDURE_EXECUTOR_SLOTS = 1;
+ private static final Procedure NULL_PROC = null;
+
+ private ProcedureExecutor<Void> procExecutor;
+ private ProcedureStore procStore;
+
+ private HBaseCommonTestingUtility htu;
+ private FileSystem fs;
+ private Path testDir;
+ private Path logDir;
+
+ @Before
+ public void setUp() throws IOException {
+ htu = new HBaseCommonTestingUtility();
+ testDir = htu.getDataTestDir();
+ fs = testDir.getFileSystem(htu.getConfiguration());
+ assertTrue(testDir.depth() > 1);
+
+ logDir = new Path(testDir, "proc-logs");
+ procStore = ProcedureTestingUtility.createWalStore(htu.getConfiguration(), fs, logDir);
+ procExecutor = new ProcedureExecutor(htu.getConfiguration(), null, procStore);
+ procStore.start(PROCEDURE_EXECUTOR_SLOTS);
+ procExecutor.start(PROCEDURE_EXECUTOR_SLOTS);
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ procExecutor.stop();
+ procStore.stop();
+ fs.delete(logDir, true);
+ }
+
+ private static class TestProcedureException extends Exception {
+ public TestProcedureException(String msg) { super(msg); }
+ }
+
+ public static class TestSequentialProcedure extends SequentialProcedure<Void> {
+ private final Procedure[] subProcs;
+ private final List<String> state;
+ private final Exception failure;
+ private final String name;
+
+ public TestSequentialProcedure() {
+ throw new UnsupportedOperationException("recovery should not be triggered here");
+ }
+
+ public TestSequentialProcedure(String name, List<String> state, Procedure... subProcs) {
+ this.state = state;
+ this.subProcs = subProcs;
+ this.name = name;
+ this.failure = null;
+ }
+
+ public TestSequentialProcedure(String name, List<String> state, Exception failure) {
+ this.state = state;
+ this.subProcs = null;
+ this.name = name;
+ this.failure = failure;
+ }
+
+ @Override
+ protected Procedure[] execute(Void env) {
+ state.add(name + "-execute");
+ if (failure != null) {
+ setFailure(new RemoteProcedureException(name + "-failure", failure));
+ return null;
+ }
+ return subProcs;
+ }
+
+ @Override
+ protected void rollback(Void env) {
+ state.add(name + "-rollback");
+ }
+
+ @Override
+ protected boolean abort(Void env) {
+ state.add(name + "-abort");
+ return true;
+ }
+ }
+
+ @Test(timeout=30000)
+ public void testBadSubprocList() {
+ List<String> state = new ArrayList<String>();
+ Procedure subProc2 = new TestSequentialProcedure("subProc2", state);
+ Procedure subProc1 = new TestSequentialProcedure("subProc1", state, subProc2, NULL_PROC);
+ Procedure rootProc = new TestSequentialProcedure("rootProc", state, subProc1);
+ long rootId = ProcedureTestingUtility.submitAndWait(procExecutor, rootProc);
+
+ // subProc1 has a "null" subprocedure which is catched as InvalidArgument
+ // failed state with 2 execute and 2 rollback
+ LOG.info(state);
+ ProcedureResult result = procExecutor.getResult(rootId);
+ LOG.info(result.getException());
+ assertTrue(state.toString(), result.isFailed());
+ assertTrue(result.getException().toString(),
+ result.getException().getCause() instanceof IllegalArgumentException);
+
+ assertEquals(state.toString(), 4, state.size());
+ assertEquals("rootProc-execute", state.get(0));
+ assertEquals("subProc1-execute", state.get(1));
+ assertEquals("subProc1-rollback", state.get(2));
+ assertEquals("rootProc-rollback", state.get(3));
+ }
+
+ @Test(timeout=30000)
+ public void testSingleSequentialProc() {
+ List<String> state = new ArrayList<String>();
+ Procedure subProc2 = new TestSequentialProcedure("subProc2", state);
+ Procedure subProc1 = new TestSequentialProcedure("subProc1", state, subProc2);
+ Procedure rootProc = new TestSequentialProcedure("rootProc", state, subProc1);
+ long rootId = ProcedureTestingUtility.submitAndWait(procExecutor, rootProc);
+
+ // successful state, with 3 execute
+ LOG.info(state);
+ ProcedureResult result = procExecutor.getResult(rootId);
+ ProcedureTestingUtility.assertProcNotFailed(result);
+ assertEquals(state.toString(), 3, state.size());
+ }
+
+ @Test(timeout=30000)
+ public void testSingleSequentialProcRollback() {
+ List<String> state = new ArrayList<String>();
+ Procedure subProc2 = new TestSequentialProcedure("subProc2", state,
+ new TestProcedureException("fail test"));
+ Procedure subProc1 = new TestSequentialProcedure("subProc1", state, subProc2);
+ Procedure rootProc = new TestSequentialProcedure("rootProc", state, subProc1);
+ long rootId = ProcedureTestingUtility.submitAndWait(procExecutor, rootProc);
+
+ // the 3rd proc fail, rollback after 2 successful execution
+ LOG.info(state);
+ ProcedureResult result = procExecutor.getResult(rootId);
+ LOG.info(result.getException());
+ assertTrue(state.toString(), result.isFailed());
+ assertTrue(result.getException().toString(),
+ result.getException().getCause() instanceof TestProcedureException);
+
+ assertEquals(state.toString(), 6, state.size());
+ assertEquals("rootProc-execute", state.get(0));
+ assertEquals("subProc1-execute", state.get(1));
+ assertEquals("subProc2-execute", state.get(2));
+ assertEquals("subProc2-rollback", state.get(3));
+ assertEquals("subProc1-rollback", state.get(4));
+ assertEquals("rootProc-rollback", state.get(5));
+ }
+
+ public static class TestWaitingProcedure extends SequentialProcedure<Void> {
+ private final List<String> state;
+ private final boolean hasChild;
+ private final String name;
+
+ public TestWaitingProcedure() {
+ throw new UnsupportedOperationException("recovery should not be triggered here");
+ }
+
+ public TestWaitingProcedure(String name, List<String> state, boolean hasChild) {
+ this.hasChild = hasChild;
+ this.state = state;
+ this.name = name;
+ }
+
+ @Override
+ protected Procedure[] execute(Void env) {
+ state.add(name + "-execute");
+ setState(ProcedureState.WAITING_TIMEOUT);
+ return hasChild ? new Procedure[] { new TestWaitChild(name, state) } : null;
+ }
+
+ @Override
+ protected void rollback(Void env) {
+ state.add(name + "-rollback");
+ }
+
+ @Override
+ protected boolean abort(Void env) {
+ state.add(name + "-abort");
+ return true;
+ }
+
+ public static class TestWaitChild extends SequentialProcedure<Void> {
+ private final List<String> state;
+ private final String name;
+
+ public TestWaitChild() {
+ throw new UnsupportedOperationException("recovery should not be triggered here");
+ }
+
+ public TestWaitChild(String name, List<String> state) {
+ this.name = name;
+ this.state = state;
+ }
+
+ @Override
+ protected Procedure[] execute(Void env) {
+ state.add(name + "-child-execute");
+ return null;
+ }
+
+ @Override
+ protected void rollback(Void env) {
+ state.add(name + "-child-rollback");
+ }
+
+ @Override
+ protected boolean abort(Void env) {
+ state.add(name + "-child-abort");
+ return true;
+ }
+ }
+ }
+
+ @Test(timeout=30000)
+ public void testAbortTimeout() {
+ final int PROC_TIMEOUT_MSEC = 2500;
+ List<String> state = new ArrayList<String>();
+ Procedure proc = new TestWaitingProcedure("wproc", state, false);
+ proc.setTimeout(PROC_TIMEOUT_MSEC);
+ long startTime = EnvironmentEdgeManager.currentTime();
+ long rootId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
+ long execTime = EnvironmentEdgeManager.currentTime() - startTime;
+ LOG.info(state);
+ assertTrue("we didn't wait enough execTime=" + execTime, execTime >= PROC_TIMEOUT_MSEC);
+ ProcedureResult result = procExecutor.getResult(rootId);
+ LOG.info(result.getException());
+ assertTrue(state.toString(), result.isFailed());
+ assertTrue(result.getException().toString(),
+ result.getException().getCause() instanceof TimeoutException);
+ assertEquals(state.toString(), 2, state.size());
+ assertEquals("wproc-execute", state.get(0));
+ assertEquals("wproc-rollback", state.get(1));
+ }
+
+ @Test(timeout=30000)
+ public void testAbortTimeoutWithChildren() {
+ List<String> state = new ArrayList<String>();
+ Procedure proc = new TestWaitingProcedure("wproc", state, true);
+ proc.setTimeout(2500);
+ long rootId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
+ LOG.info(state);
+ ProcedureResult result = procExecutor.getResult(rootId);
+ LOG.info(result.getException());
+ assertTrue(state.toString(), result.isFailed());
+ assertTrue(result.getException().toString(),
+ result.getException().getCause() instanceof TimeoutException);
+ assertEquals(state.toString(), 4, state.size());
+ assertEquals("wproc-execute", state.get(0));
+ assertEquals("wproc-child-execute", state.get(1));
+ assertEquals("wproc-child-rollback", state.get(2));
+ assertEquals("wproc-rollback", state.get(3));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureRecovery.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureRecovery.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureRecovery.java
new file mode 100644
index 0000000..72c679f
--- /dev/null
+++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureRecovery.java
@@ -0,0 +1,488 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureStore;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.Threads;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+@Category({MasterTests.class, SmallTests.class})
+public class TestProcedureRecovery {
+ private static final Log LOG = LogFactory.getLog(TestProcedureRecovery.class);
+
+ private static final int PROCEDURE_EXECUTOR_SLOTS = 1;
+ private static final Procedure NULL_PROC = null;
+
+ private static ProcedureExecutor<Void> procExecutor;
+ private static ProcedureStore procStore;
+ private static int procSleepInterval;
+
+ private HBaseCommonTestingUtility htu;
+ private FileSystem fs;
+ private Path testDir;
+ private Path logDir;
+
+ @Before
+ public void setUp() throws IOException {
+ htu = new HBaseCommonTestingUtility();
+ testDir = htu.getDataTestDir();
+ fs = testDir.getFileSystem(htu.getConfiguration());
+ assertTrue(testDir.depth() > 1);
+
+ logDir = new Path(testDir, "proc-logs");
+ procStore = ProcedureTestingUtility.createStore(htu.getConfiguration(), fs, logDir);
+ procExecutor = new ProcedureExecutor(htu.getConfiguration(), null, procStore);
+ procExecutor.testing = new ProcedureExecutor.Testing();
+ procStore.start(PROCEDURE_EXECUTOR_SLOTS);
+ procExecutor.start(PROCEDURE_EXECUTOR_SLOTS);
+ procSleepInterval = 0;
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ procExecutor.stop();
+ procStore.stop();
+ fs.delete(logDir, true);
+ }
+
+ private void restart() throws Exception {
+ dumpLogDirState();
+ ProcedureTestingUtility.restart(procExecutor);
+ dumpLogDirState();
+ }
+
+ public static class TestSingleStepProcedure extends SequentialProcedure<Void> {
+ private int step = 0;
+
+ public TestSingleStepProcedure() { }
+
+ @Override
+ protected Procedure[] execute(Void env) {
+ LOG.debug("execute procedure " + this + " step=" + step);
+ step++;
+ setResult(Bytes.toBytes(step));
+ return null;
+ }
+
+ @Override
+ protected void rollback(Void env) { }
+
+ @Override
+ protected boolean abort(Void env) { return true; }
+ }
+
+ public static class BaseTestStepProcedure extends SequentialProcedure<Void> {
+ private AtomicBoolean abort = new AtomicBoolean(false);
+ private int step = 0;
+
+ @Override
+ protected Procedure[] execute(Void env) {
+ LOG.debug("execute procedure " + this + " step=" + step);
+ ProcedureTestingUtility.toggleKillBeforeStoreUpdate(procExecutor);
+ step++;
+ Threads.sleepWithoutInterrupt(procSleepInterval);
+ if (isAborted()) {
+ setFailure(new RemoteProcedureException(getClass().getName(),
+ new ProcedureAbortedException(
+ "got an abort at " + getClass().getName() + " step=" + step)));
+ return null;
+ }
+ return null;
+ }
+
+ @Override
+ protected void rollback(Void env) {
+ LOG.debug("rollback procedure " + this + " step=" + step);
+ ProcedureTestingUtility.toggleKillBeforeStoreUpdate(procExecutor);
+ step++;
+ }
+
+ @Override
+ protected boolean abort(Void env) {
+ abort.set(true);
+ return true;
+ }
+
+ private boolean isAborted() {
+ boolean aborted = abort.get();
+ BaseTestStepProcedure proc = this;
+ while (proc.hasParent() && !aborted) {
+ proc = (BaseTestStepProcedure)procExecutor.getProcedure(proc.getParentProcId());
+ aborted = proc.isAborted();
+ }
+ return aborted;
+ }
+ }
+
+ public static class TestMultiStepProcedure extends BaseTestStepProcedure {
+ public TestMultiStepProcedure() { }
+
+ @Override
+ public Procedure[] execute(Void env) {
+ super.execute(env);
+ return isFailed() ? null : new Procedure[] { new Step1Procedure() };
+ }
+
+ public static class Step1Procedure extends BaseTestStepProcedure {
+ public Step1Procedure() { }
+
+ @Override
+ protected Procedure[] execute(Void env) {
+ super.execute(env);
+ return isFailed() ? null : new Procedure[] { new Step2Procedure() };
+ }
+ }
+
+ public static class Step2Procedure extends BaseTestStepProcedure {
+ public Step2Procedure() { }
+ }
+ }
+
+ @Test
+ public void testNoopLoad() throws Exception {
+ restart();
+ }
+
+ @Test(timeout=30000)
+ public void testSingleStepProcRecovery() throws Exception {
+ Procedure proc = new TestSingleStepProcedure();
+ procExecutor.testing.killBeforeStoreUpdate = true;
+ long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
+ assertFalse(procExecutor.isRunning());
+ procExecutor.testing.killBeforeStoreUpdate = false;
+
+ // Restart and verify that the procedures restart
+ long restartTs = EnvironmentEdgeManager.currentTime();
+ restart();
+ waitProcedure(procId);
+ ProcedureResult result = procExecutor.getResult(procId);
+ assertTrue(result.getLastUpdate() > restartTs);
+ ProcedureTestingUtility.assertProcNotFailed(result);
+ assertEquals(1, Bytes.toInt(result.getResult()));
+ long resultTs = result.getLastUpdate();
+
+ // Verify that after another restart the result is still there
+ restart();
+ result = procExecutor.getResult(procId);
+ ProcedureTestingUtility.assertProcNotFailed(result);
+ assertEquals(resultTs, result.getLastUpdate());
+ assertEquals(1, Bytes.toInt(result.getResult()));
+ }
+
+ @Test(timeout=30000)
+ public void testMultiStepProcRecovery() throws Exception {
+ // Step 0 - kill
+ Procedure proc = new TestMultiStepProcedure();
+ long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
+ assertFalse(procExecutor.isRunning());
+
+ // Step 0 exec && Step 1 - kill
+ restart();
+ waitProcedure(procId);
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
+ assertFalse(procExecutor.isRunning());
+
+ // Step 1 exec && step 2 - kill
+ restart();
+ waitProcedure(procId);
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
+ assertFalse(procExecutor.isRunning());
+
+ // Step 2 exec
+ restart();
+ waitProcedure(procId);
+ assertTrue(procExecutor.isRunning());
+
+ // The procedure is completed
+ ProcedureResult result = procExecutor.getResult(procId);
+ ProcedureTestingUtility.assertProcNotFailed(result);
+ }
+
+ @Test(timeout=30000)
+ public void testMultiStepRollbackRecovery() throws Exception {
+ // Step 0 - kill
+ Procedure proc = new TestMultiStepProcedure();
+ long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
+ assertFalse(procExecutor.isRunning());
+
+ // Step 0 exec && Step 1 - kill
+ restart();
+ waitProcedure(procId);
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
+ assertFalse(procExecutor.isRunning());
+
+ // Step 1 exec && step 2 - kill
+ restart();
+ waitProcedure(procId);
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
+ assertFalse(procExecutor.isRunning());
+
+ // Step 2 exec - rollback - kill
+ procSleepInterval = 2500;
+ restart();
+ assertTrue(procExecutor.abort(procId));
+ waitProcedure(procId);
+ assertFalse(procExecutor.isRunning());
+
+ // rollback - kill
+ restart();
+ waitProcedure(procId);
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
+ assertFalse(procExecutor.isRunning());
+
+ // rollback - complete
+ restart();
+ waitProcedure(procId);
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
+ assertFalse(procExecutor.isRunning());
+
+ // Restart the executor and get the result
+ restart();
+ waitProcedure(procId);
+
+ // The procedure is completed
+ ProcedureResult result = procExecutor.getResult(procId);
+ ProcedureTestingUtility.assertIsAbortException(result);
+ }
+
+ public static class TestStateMachineProcedure
+ extends StateMachineProcedure<Void, TestStateMachineProcedure.State> {
+ enum State { STATE_1, STATE_2, STATE_3, DONE }
+
+ public TestStateMachineProcedure() {}
+
+ private AtomicBoolean aborted = new AtomicBoolean(false);
+ private int iResult = 0;
+
+ @Override
+ protected StateMachineProcedure.Flow executeFromState(Void env, State state) {
+ if (state == null) {
+ LOG.info("Initializing " + this);
+ state = State.STATE_1;
+ setNextState(state);
+ }
+
+ switch (state) {
+ case STATE_1:
+ LOG.info("execute step 1 " + this);
+ setNextState(State.STATE_2);
+ iResult += 3;
+ break;
+ case STATE_2:
+ LOG.info("execute step 2 " + this);
+ setNextState(State.STATE_3);
+ iResult += 5;
+ break;
+ case STATE_3:
+ LOG.info("execute step 3 " + this);
+ Threads.sleepWithoutInterrupt(procSleepInterval);
+ if (aborted.get()) {
+ LOG.info("aborted step 3 " + this);
+ setAbortFailure("test", "aborted");
+ break;
+ }
+ setNextState(State.DONE);
+ iResult += 7;
+ setResult(Bytes.toBytes(iResult));
+ return Flow.NO_MORE_STATE;
+ default:
+ throw new UnsupportedOperationException();
+ }
+ return Flow.HAS_MORE_STATE;
+ }
+
+ @Override
+ protected void rollbackState(Void env, final State state) {
+ switch (state) {
+ case STATE_1:
+ LOG.info("rollback step 1 " + this);
+ break;
+ case STATE_2:
+ LOG.info("rollback step 2 " + this);
+ break;
+ case STATE_3:
+ LOG.info("rollback step 3 " + this);
+ break;
+ default:
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ @Override
+ protected State getState(final int stateId) {
+ return State.values()[stateId];
+ }
+
+ private void setNextState(final State state) {
+ setNextState(state.ordinal());
+ }
+
+ @Override
+ protected boolean abort(Void env) {
+ aborted.set(true);
+ return true;
+ }
+
+ @Override
+ protected void serializeStateData(final OutputStream stream) throws IOException {
+ super.serializeStateData(stream);
+ stream.write(Bytes.toBytes(iResult));
+ }
+
+ @Override
+ protected void deserializeStateData(final InputStream stream) throws IOException {
+ super.deserializeStateData(stream);
+ byte[] data = new byte[4];
+ stream.read(data);
+ iResult = Bytes.toInt(data);
+ }
+ }
+
+ @Test(timeout=30000)
+ public void testStateMachineRecovery() throws Exception {
+ ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExecutor, true);
+ ProcedureTestingUtility.setKillBeforeStoreUpdate(procExecutor, true);
+
+ // Step 1 - kill
+ Procedure proc = new TestStateMachineProcedure();
+ long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
+ assertFalse(procExecutor.isRunning());
+
+ // Step 1 exec && Step 2 - kill
+ restart();
+ waitProcedure(procId);
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
+ assertFalse(procExecutor.isRunning());
+
+ // Step 2 exec && step 3 - kill
+ restart();
+ waitProcedure(procId);
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
+ assertFalse(procExecutor.isRunning());
+
+ // Step 3 exec
+ restart();
+ waitProcedure(procId);
+ assertTrue(procExecutor.isRunning());
+
+ // The procedure is completed
+ ProcedureResult result = procExecutor.getResult(procId);
+ ProcedureTestingUtility.assertProcNotFailed(result);
+ assertEquals(15, Bytes.toInt(result.getResult()));
+ }
+
+ @Test(timeout=30000)
+ public void testStateMachineRollbackRecovery() throws Exception {
+ ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExecutor, true);
+ ProcedureTestingUtility.setKillBeforeStoreUpdate(procExecutor, true);
+
+ // Step 1 - kill
+ Procedure proc = new TestStateMachineProcedure();
+ long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
+ assertFalse(procExecutor.isRunning());
+
+ // Step 1 exec && Step 2 - kill
+ restart();
+ waitProcedure(procId);
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
+ assertFalse(procExecutor.isRunning());
+
+ // Step 2 exec && step 3 - kill
+ restart();
+ waitProcedure(procId);
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
+ assertFalse(procExecutor.isRunning());
+
+ // Step 3 exec - rollback step 3 - kill
+ procSleepInterval = 2500;
+ restart();
+ assertTrue(procExecutor.abort(procId));
+ waitProcedure(procId);
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
+ assertFalse(procExecutor.isRunning());
+
+ // Rollback step 3 - rollback step 2 - kill
+ restart();
+ waitProcedure(procId);
+ assertFalse(procExecutor.isRunning());
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
+
+ // Rollback step 2 - step 1 - kill
+ restart();
+ waitProcedure(procId);
+ assertFalse(procExecutor.isRunning());
+ ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
+
+ // Rollback step 1 - complete
+ restart();
+ waitProcedure(procId);
+ assertTrue(procExecutor.isRunning());
+
+ // The procedure is completed
+ ProcedureResult result = procExecutor.getResult(procId);
+ ProcedureTestingUtility.assertIsAbortException(result);
+ }
+
+ private void waitProcedure(final long procId) {
+ ProcedureTestingUtility.waitProcedure(procExecutor, procId);
+ dumpLogDirState();
+ }
+
+ private void dumpLogDirState() {
+ try {
+ FileStatus[] files = fs.listStatus(logDir);
+ if (files != null && files.length > 0) {
+ for (FileStatus file: files) {
+ assertTrue(file.toString(), file.isFile());
+ LOG.debug("log file " + file.getPath() + " size=" + file.getLen());
+ }
+ } else {
+ LOG.debug("no files under: " + logDir);
+ }
+ } catch (IOException e) {
+ LOG.warn("Unable to dump " + logDir, e);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureRunQueues.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureRunQueues.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureRunQueues.java
new file mode 100644
index 0000000..c040be6
--- /dev/null
+++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureRunQueues.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.junit.Assert.assertEquals;
+
+@Category({MasterTests.class, SmallTests.class})
+public class TestProcedureRunQueues {
+ class TestRunQueue implements ProcedureFairRunQueues.FairObject {
+ private final String name;
+ private final int priority;
+
+ public TestRunQueue(String name, int priority) {
+ this.name = name;
+ this.priority = priority;
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ @Override
+ public boolean isAvailable() {
+ return true;
+ }
+
+ @Override
+ public int getPriority() {
+ return priority;
+ }
+ }
+
+ @Test
+ public void testFairQueues() throws Exception {
+ ProcedureFairRunQueues<String, TestRunQueue> fairq
+ = new ProcedureFairRunQueues<String, TestRunQueue>(1);
+ TestRunQueue a = fairq.add("A", new TestRunQueue("A", 1));
+ TestRunQueue b = fairq.add("B", new TestRunQueue("B", 1));
+ TestRunQueue m = fairq.add("M", new TestRunQueue("M", 2));
+
+ for (int i = 0; i < 3; ++i) {
+ assertEquals(a, fairq.poll());
+ assertEquals(b, fairq.poll());
+ assertEquals(m, fairq.poll());
+ assertEquals(m, fairq.poll());
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/TestProcedureStoreTracker.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/TestProcedureStoreTracker.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/TestProcedureStoreTracker.java
new file mode 100644
index 0000000..0669549
--- /dev/null
+++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/TestProcedureStoreTracker.java
@@ -0,0 +1,168 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2.store;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+@Category({MasterTests.class, SmallTests.class})
+public class TestProcedureStoreTracker {
+ private static final Log LOG = LogFactory.getLog(TestProcedureStoreTracker.class);
+
+ static class TestProcedure extends Procedure<Void> {
+ public TestProcedure(long procId) {
+ setProcId(procId);
+ }
+
+ @Override
+ protected Procedure[] execute(Void env) { return null; }
+
+ @Override
+ protected void rollback(Void env) { /* no-op */ }
+
+ @Override
+ protected boolean abort(Void env) { return false; }
+
+ @Override
+ protected void serializeStateData(final OutputStream stream) { /* no-op */ }
+
+ @Override
+ protected void deserializeStateData(final InputStream stream) { /* no-op */ }
+ }
+
+ @Test
+ public void testSeqInsertAndDelete() {
+ ProcedureStoreTracker tracker = new ProcedureStoreTracker();
+ assertTrue(tracker.isEmpty());
+
+ final int MIN_PROC = 1;
+ final int MAX_PROC = 1 << 10;
+
+ // sequential insert
+ for (int i = MIN_PROC; i < MAX_PROC; ++i) {
+ tracker.insert(i);
+
+ // All the proc that we inserted should not be deleted
+ for (int j = MIN_PROC; j <= i; ++j) {
+ assertEquals(ProcedureStoreTracker.DeleteState.NO, tracker.isDeleted(j));
+ }
+ // All the proc that are not yet inserted should be result as deleted
+ for (int j = i + 1; j < MAX_PROC; ++j) {
+ assertTrue(tracker.isDeleted(j) != ProcedureStoreTracker.DeleteState.NO);
+ }
+ }
+
+ // sequential delete
+ for (int i = MIN_PROC; i < MAX_PROC; ++i) {
+ tracker.delete(i);
+
+ // All the proc that we deleted should be deleted
+ for (int j = MIN_PROC; j <= i; ++j) {
+ assertEquals(ProcedureStoreTracker.DeleteState.YES, tracker.isDeleted(j));
+ }
+ // All the proc that are not yet deleted should be result as not deleted
+ for (int j = i + 1; j < MAX_PROC; ++j) {
+ assertEquals(ProcedureStoreTracker.DeleteState.NO, tracker.isDeleted(j));
+ }
+ }
+ assertTrue(tracker.isEmpty());
+ }
+
+ @Test
+ public void testPartialTracker() {
+ ProcedureStoreTracker tracker = new ProcedureStoreTracker();
+ tracker.setPartialFlag(true);
+
+ // nothing in the tracker, the state is unknown
+ assertTrue(tracker.isEmpty());
+ assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(1));
+ assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(579));
+
+ // Mark 1 as deleted, now that is a known state
+ tracker.setDeleted(1, true);
+ tracker.dump();
+ assertEquals(ProcedureStoreTracker.DeleteState.YES, tracker.isDeleted(1));
+ assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(2));
+ assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(579));
+
+ // Mark 579 as non-deleted, now that is a known state
+ tracker.setDeleted(579, false);
+ assertEquals(ProcedureStoreTracker.DeleteState.YES, tracker.isDeleted(1));
+ assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(2));
+ assertEquals(ProcedureStoreTracker.DeleteState.NO, tracker.isDeleted(579));
+ assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(577));
+ assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(580));
+ }
+
+ @Test
+ public void testBasicCRUD() {
+ ProcedureStoreTracker tracker = new ProcedureStoreTracker();
+ assertTrue(tracker.isEmpty());
+
+ Procedure[] procs = new TestProcedure[] {
+ new TestProcedure(1), new TestProcedure(2), new TestProcedure(3),
+ new TestProcedure(4), new TestProcedure(5), new TestProcedure(6),
+ };
+
+ tracker.insert(procs[0], null);
+ tracker.insert(procs[1], new Procedure[] { procs[2], procs[3], procs[4] });
+ assertFalse(tracker.isEmpty());
+ assertTrue(tracker.isUpdated());
+
+ tracker.resetUpdates();
+ assertFalse(tracker.isUpdated());
+
+ for (int i = 0; i < 4; ++i) {
+ tracker.update(procs[i]);
+ assertFalse(tracker.isEmpty());
+ assertFalse(tracker.isUpdated());
+ }
+
+ tracker.update(procs[4]);
+ assertFalse(tracker.isEmpty());
+ assertTrue(tracker.isUpdated());
+
+ tracker.update(procs[5]);
+ assertFalse(tracker.isEmpty());
+ assertTrue(tracker.isUpdated());
+
+ for (int i = 0; i < 5; ++i) {
+ tracker.delete(procs[i].getProcId());
+ assertFalse(tracker.isEmpty());
+ assertTrue(tracker.isUpdated());
+ }
+ tracker.delete(procs[5].getProcId());
+ assertTrue(tracker.isEmpty());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestWALProcedureStore.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestWALProcedureStore.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestWALProcedureStore.java
new file mode 100644
index 0000000..7106132
--- /dev/null
+++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestWALProcedureStore.java
@@ -0,0 +1,267 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2.store.wal;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Iterator;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
+import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
+import org.apache.hadoop.hbase.procedure2.SequentialProcedure;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.IOUtils;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+@Category({MasterTests.class, SmallTests.class})
+public class TestWALProcedureStore {
+ private static final Log LOG = LogFactory.getLog(TestWALProcedureStore.class);
+
+ private static final int PROCEDURE_STORE_SLOTS = 1;
+ private static final Procedure NULL_PROC = null;
+
+ private WALProcedureStore procStore;
+
+ private HBaseCommonTestingUtility htu;
+ private FileSystem fs;
+ private Path testDir;
+ private Path logDir;
+
+ @Before
+ public void setUp() throws IOException {
+ htu = new HBaseCommonTestingUtility();
+ testDir = htu.getDataTestDir();
+ fs = testDir.getFileSystem(htu.getConfiguration());
+ assertTrue(testDir.depth() > 1);
+
+ logDir = new Path(testDir, "proc-logs");
+ procStore = ProcedureTestingUtility.createWalStore(htu.getConfiguration(), fs, logDir);
+ procStore.start(PROCEDURE_STORE_SLOTS);
+ procStore.recoverLease();
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ procStore.stop();
+ fs.delete(logDir, true);
+ }
+
+ private Iterator<Procedure> storeRestart() throws Exception {
+ procStore.stop();
+ procStore.start(PROCEDURE_STORE_SLOTS);
+ procStore.recoverLease();
+ return procStore.load();
+ }
+
+ @Test
+ public void testEmptyLogLoad() throws Exception {
+ Iterator<Procedure> loader = storeRestart();
+ assertEquals(0, countProcedures(loader));
+ }
+
+ @Test
+ public void testLoad() throws Exception {
+ Set<Long> procIds = new HashSet<>();
+
+ // Insert something in the log
+ Procedure proc1 = new TestSequentialProcedure();
+ procIds.add(proc1.getProcId());
+ procStore.insert(proc1, null);
+
+ Procedure proc2 = new TestSequentialProcedure();
+ Procedure[] child2 = new Procedure[2];
+ child2[0] = new TestSequentialProcedure();
+ child2[1] = new TestSequentialProcedure();
+
+ procIds.add(proc2.getProcId());
+ procIds.add(child2[0].getProcId());
+ procIds.add(child2[1].getProcId());
+ procStore.insert(proc2, child2);
+
+ // Verify that everything is there
+ verifyProcIdsOnRestart(procIds);
+
+ // Update and delete something
+ procStore.update(proc1);
+ procStore.update(child2[1]);
+ procStore.delete(child2[1].getProcId());
+ procIds.remove(child2[1].getProcId());
+
+ // Verify that everything is there
+ verifyProcIdsOnRestart(procIds);
+
+ // Remove 4 byte from the trailers
+ procStore.stop();
+ FileStatus[] logs = fs.listStatus(logDir);
+ assertEquals(3, logs.length);
+ for (int i = 0; i < logs.length; ++i) {
+ corruptLog(logs[i], 4);
+ }
+ verifyProcIdsOnRestart(procIds);
+ }
+
+ @Test
+ public void testCorruptedTrailer() throws Exception {
+ // Insert something
+ for (int i = 0; i < 100; ++i) {
+ procStore.insert(new TestSequentialProcedure(), null);
+ }
+
+ // Stop the store
+ procStore.stop();
+
+ // Remove 4 byte from the trailer
+ FileStatus[] logs = fs.listStatus(logDir);
+ assertEquals(1, logs.length);
+ corruptLog(logs[0], 4);
+
+ int count = countProcedures(storeRestart());
+ assertEquals(100, count);
+ }
+
+ @Test
+ public void testCorruptedEntries() throws Exception {
+ // Insert something
+ for (int i = 0; i < 100; ++i) {
+ procStore.insert(new TestSequentialProcedure(), null);
+ }
+
+ // Stop the store
+ procStore.stop();
+
+ // Remove some byte from the log
+ // (enough to cut the trailer and corrupt some entries)
+ FileStatus[] logs = fs.listStatus(logDir);
+ assertEquals(1, logs.length);
+ corruptLog(logs[0], 1823);
+
+ int count = countProcedures(storeRestart());
+ assertTrue(procStore.getCorruptedLogs() != null);
+ assertEquals(1, procStore.getCorruptedLogs().size());
+ assertEquals(85, count);
+ }
+
+ private void corruptLog(final FileStatus logFile, final long dropBytes)
+ throws IOException {
+ assertTrue(logFile.getLen() > dropBytes);
+ LOG.debug("corrupt log " + logFile.getPath() +
+ " size=" + logFile.getLen() + " drop=" + dropBytes);
+ Path tmpPath = new Path(testDir, "corrupted.log");
+ InputStream in = fs.open(logFile.getPath());
+ OutputStream out = fs.create(tmpPath);
+ IOUtils.copyBytes(in, out, logFile.getLen() - dropBytes, true);
+ fs.rename(tmpPath, logFile.getPath());
+ }
+
+ private void verifyProcIdsOnRestart(final Set<Long> procIds) throws Exception {
+ int count = 0;
+ Iterator<Procedure> loader = storeRestart();
+ while (loader.hasNext()) {
+ Procedure proc = loader.next();
+ LOG.debug("loading procId=" + proc.getProcId());
+ assertTrue("procId=" + proc.getProcId() + " unexpected", procIds.contains(proc.getProcId()));
+ count++;
+ }
+ assertEquals(procIds.size(), count);
+ }
+
+ private void assertIsEmpty(Iterator<Procedure> iterator) {
+ assertEquals(0, countProcedures(iterator));
+ }
+
+ private int countProcedures(Iterator<Procedure> iterator) {
+ int count = 0;
+ while (iterator.hasNext()) {
+ Procedure proc = iterator.next();
+ LOG.trace("loading procId=" + proc.getProcId());
+ count++;
+ }
+ return count;
+ }
+
+ private void assertEmptyLogDir() {
+ try {
+ FileStatus[] status = fs.listStatus(logDir);
+ assertTrue("expected empty state-log dir", status == null || status.length == 0);
+ } catch (FileNotFoundException e) {
+ fail("expected the state-log dir to be present: " + logDir);
+ } catch (IOException e) {
+ fail("got en exception on state-log dir list: " + e.getMessage());
+ }
+ }
+
+ public static class TestSequentialProcedure extends SequentialProcedure<Void> {
+ private static long seqid = 0;
+
+ public TestSequentialProcedure() {
+ setProcId(++seqid);
+ }
+
+ @Override
+ protected Procedure[] execute(Void env) { return null; }
+
+ @Override
+ protected void rollback(Void env) { }
+
+ @Override
+ protected boolean abort(Void env) { return false; }
+
+ @Override
+ protected void serializeStateData(final OutputStream stream) throws IOException {
+ long procId = getProcId();
+ if (procId % 2 == 0) {
+ stream.write(Bytes.toBytes(procId));
+ }
+ }
+
+ @Override
+ protected void deserializeStateData(InputStream stream) throws IOException {
+ long procId = getProcId();
+ if (procId % 2 == 0) {
+ byte[] bProcId = new byte[8];
+ assertEquals(8, stream.read(bProcId));
+ assertEquals(procId, Bytes.toLong(bProcId));
+ } else {
+ assertEquals(0, stream.available());
+ }
+ }
+ }
+}
\ No newline at end of file
[11/15] hbase git commit: HBASE-13202 Procedure v2 - core framework
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ProcedureProtos.java
----------------------------------------------------------------------
diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ProcedureProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ProcedureProtos.java
new file mode 100644
index 0000000..3c7dcdb
--- /dev/null
+++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ProcedureProtos.java
@@ -0,0 +1,7219 @@
+// Generated by the protocol buffer compiler. DO NOT EDIT!
+// source: Procedure.proto
+
+package org.apache.hadoop.hbase.protobuf.generated;
+
+public final class ProcedureProtos {
+ private ProcedureProtos() {}
+ public static void registerAllExtensions(
+ com.google.protobuf.ExtensionRegistry registry) {
+ }
+ /**
+ * Protobuf enum {@code ProcedureState}
+ */
+ public enum ProcedureState
+ implements com.google.protobuf.ProtocolMessageEnum {
+ /**
+ * <code>INITIALIZING = 1;</code>
+ *
+ * <pre>
+ * Procedure in construction, not yet added to the executor
+ * </pre>
+ */
+ INITIALIZING(0, 1),
+ /**
+ * <code>RUNNABLE = 2;</code>
+ *
+ * <pre>
+ * Procedure added to the executor, and ready to be executed
+ * </pre>
+ */
+ RUNNABLE(1, 2),
+ /**
+ * <code>WAITING = 3;</code>
+ *
+ * <pre>
+ * The procedure is waiting on children to be completed
+ * </pre>
+ */
+ WAITING(2, 3),
+ /**
+ * <code>WAITING_TIMEOUT = 4;</code>
+ *
+ * <pre>
+ * The procedure is waiting a timout or an external event
+ * </pre>
+ */
+ WAITING_TIMEOUT(3, 4),
+ /**
+ * <code>ROLLEDBACK = 5;</code>
+ *
+ * <pre>
+ * The procedure failed and was rolledback
+ * </pre>
+ */
+ ROLLEDBACK(4, 5),
+ /**
+ * <code>FINISHED = 6;</code>
+ *
+ * <pre>
+ * The procedure execution is completed. may need a rollback if failed.
+ * </pre>
+ */
+ FINISHED(5, 6),
+ ;
+
+ /**
+ * <code>INITIALIZING = 1;</code>
+ *
+ * <pre>
+ * Procedure in construction, not yet added to the executor
+ * </pre>
+ */
+ public static final int INITIALIZING_VALUE = 1;
+ /**
+ * <code>RUNNABLE = 2;</code>
+ *
+ * <pre>
+ * Procedure added to the executor, and ready to be executed
+ * </pre>
+ */
+ public static final int RUNNABLE_VALUE = 2;
+ /**
+ * <code>WAITING = 3;</code>
+ *
+ * <pre>
+ * The procedure is waiting on children to be completed
+ * </pre>
+ */
+ public static final int WAITING_VALUE = 3;
+ /**
+ * <code>WAITING_TIMEOUT = 4;</code>
+ *
+ * <pre>
+ * The procedure is waiting a timout or an external event
+ * </pre>
+ */
+ public static final int WAITING_TIMEOUT_VALUE = 4;
+ /**
+ * <code>ROLLEDBACK = 5;</code>
+ *
+ * <pre>
+ * The procedure failed and was rolledback
+ * </pre>
+ */
+ public static final int ROLLEDBACK_VALUE = 5;
+ /**
+ * <code>FINISHED = 6;</code>
+ *
+ * <pre>
+ * The procedure execution is completed. may need a rollback if failed.
+ * </pre>
+ */
+ public static final int FINISHED_VALUE = 6;
+
+
+ public final int getNumber() { return value; }
+
+ public static ProcedureState valueOf(int value) {
+ switch (value) {
+ case 1: return INITIALIZING;
+ case 2: return RUNNABLE;
+ case 3: return WAITING;
+ case 4: return WAITING_TIMEOUT;
+ case 5: return ROLLEDBACK;
+ case 6: return FINISHED;
+ default: return null;
+ }
+ }
+
+ public static com.google.protobuf.Internal.EnumLiteMap<ProcedureState>
+ internalGetValueMap() {
+ return internalValueMap;
+ }
+ private static com.google.protobuf.Internal.EnumLiteMap<ProcedureState>
+ internalValueMap =
+ new com.google.protobuf.Internal.EnumLiteMap<ProcedureState>() {
+ public ProcedureState findValueByNumber(int number) {
+ return ProcedureState.valueOf(number);
+ }
+ };
+
+ public final com.google.protobuf.Descriptors.EnumValueDescriptor
+ getValueDescriptor() {
+ return getDescriptor().getValues().get(index);
+ }
+ public final com.google.protobuf.Descriptors.EnumDescriptor
+ getDescriptorForType() {
+ return getDescriptor();
+ }
+ public static final com.google.protobuf.Descriptors.EnumDescriptor
+ getDescriptor() {
+ return org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.getDescriptor().getEnumTypes().get(0);
+ }
+
+ private static final ProcedureState[] VALUES = values();
+
+ public static ProcedureState valueOf(
+ com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+ if (desc.getType() != getDescriptor()) {
+ throw new java.lang.IllegalArgumentException(
+ "EnumValueDescriptor is not for this type.");
+ }
+ return VALUES[desc.getIndex()];
+ }
+
+ private final int index;
+ private final int value;
+
+ private ProcedureState(int index, int value) {
+ this.index = index;
+ this.value = value;
+ }
+
+ // @@protoc_insertion_point(enum_scope:ProcedureState)
+ }
+
+ public interface ProcedureOrBuilder
+ extends com.google.protobuf.MessageOrBuilder {
+
+ // required string class_name = 1;
+ /**
+ * <code>required string class_name = 1;</code>
+ *
+ * <pre>
+ * internal "static" state
+ * </pre>
+ */
+ boolean hasClassName();
+ /**
+ * <code>required string class_name = 1;</code>
+ *
+ * <pre>
+ * internal "static" state
+ * </pre>
+ */
+ java.lang.String getClassName();
+ /**
+ * <code>required string class_name = 1;</code>
+ *
+ * <pre>
+ * internal "static" state
+ * </pre>
+ */
+ com.google.protobuf.ByteString
+ getClassNameBytes();
+
+ // optional uint64 parent_id = 2;
+ /**
+ * <code>optional uint64 parent_id = 2;</code>
+ *
+ * <pre>
+ * parent if not a root-procedure otherwise not set
+ * </pre>
+ */
+ boolean hasParentId();
+ /**
+ * <code>optional uint64 parent_id = 2;</code>
+ *
+ * <pre>
+ * parent if not a root-procedure otherwise not set
+ * </pre>
+ */
+ long getParentId();
+
+ // required uint64 proc_id = 3;
+ /**
+ * <code>required uint64 proc_id = 3;</code>
+ */
+ boolean hasProcId();
+ /**
+ * <code>required uint64 proc_id = 3;</code>
+ */
+ long getProcId();
+
+ // required uint64 start_time = 4;
+ /**
+ * <code>required uint64 start_time = 4;</code>
+ */
+ boolean hasStartTime();
+ /**
+ * <code>required uint64 start_time = 4;</code>
+ */
+ long getStartTime();
+
+ // optional string owner = 5;
+ /**
+ * <code>optional string owner = 5;</code>
+ */
+ boolean hasOwner();
+ /**
+ * <code>optional string owner = 5;</code>
+ */
+ java.lang.String getOwner();
+ /**
+ * <code>optional string owner = 5;</code>
+ */
+ com.google.protobuf.ByteString
+ getOwnerBytes();
+
+ // required .ProcedureState state = 6;
+ /**
+ * <code>required .ProcedureState state = 6;</code>
+ *
+ * <pre>
+ * internal "runtime" state
+ * </pre>
+ */
+ boolean hasState();
+ /**
+ * <code>required .ProcedureState state = 6;</code>
+ *
+ * <pre>
+ * internal "runtime" state
+ * </pre>
+ */
+ org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState getState();
+
+ // repeated uint32 stack_id = 7;
+ /**
+ * <code>repeated uint32 stack_id = 7;</code>
+ *
+ * <pre>
+ * stack indices in case the procedure was running
+ * </pre>
+ */
+ java.util.List<java.lang.Integer> getStackIdList();
+ /**
+ * <code>repeated uint32 stack_id = 7;</code>
+ *
+ * <pre>
+ * stack indices in case the procedure was running
+ * </pre>
+ */
+ int getStackIdCount();
+ /**
+ * <code>repeated uint32 stack_id = 7;</code>
+ *
+ * <pre>
+ * stack indices in case the procedure was running
+ * </pre>
+ */
+ int getStackId(int index);
+
+ // required uint64 last_update = 8;
+ /**
+ * <code>required uint64 last_update = 8;</code>
+ */
+ boolean hasLastUpdate();
+ /**
+ * <code>required uint64 last_update = 8;</code>
+ */
+ long getLastUpdate();
+
+ // optional uint32 timeout = 9;
+ /**
+ * <code>optional uint32 timeout = 9;</code>
+ */
+ boolean hasTimeout();
+ /**
+ * <code>optional uint32 timeout = 9;</code>
+ */
+ int getTimeout();
+
+ // optional .ForeignExceptionMessage exception = 10;
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ boolean hasException();
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage getException();
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessageOrBuilder getExceptionOrBuilder();
+
+ // optional bytes result = 11;
+ /**
+ * <code>optional bytes result = 11;</code>
+ *
+ * <pre>
+ * opaque (user) result structure
+ * </pre>
+ */
+ boolean hasResult();
+ /**
+ * <code>optional bytes result = 11;</code>
+ *
+ * <pre>
+ * opaque (user) result structure
+ * </pre>
+ */
+ com.google.protobuf.ByteString getResult();
+
+ // optional bytes state_data = 12;
+ /**
+ * <code>optional bytes state_data = 12;</code>
+ *
+ * <pre>
+ * opaque (user) procedure internal-state
+ * </pre>
+ */
+ boolean hasStateData();
+ /**
+ * <code>optional bytes state_data = 12;</code>
+ *
+ * <pre>
+ * opaque (user) procedure internal-state
+ * </pre>
+ */
+ com.google.protobuf.ByteString getStateData();
+ }
+ /**
+ * Protobuf type {@code Procedure}
+ *
+ * <pre>
+ **
+ * Procedure metadata, serialized by the ProcedureStore to be able to recover the old state.
+ * </pre>
+ */
+ public static final class Procedure extends
+ com.google.protobuf.GeneratedMessage
+ implements ProcedureOrBuilder {
+ // Use Procedure.newBuilder() to construct.
+ private Procedure(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+ super(builder);
+ this.unknownFields = builder.getUnknownFields();
+ }
+ private Procedure(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+ private static final Procedure defaultInstance;
+ public static Procedure getDefaultInstance() {
+ return defaultInstance;
+ }
+
+ public Procedure getDefaultInstanceForType() {
+ return defaultInstance;
+ }
+
+ private final com.google.protobuf.UnknownFieldSet unknownFields;
+ @java.lang.Override
+ public final com.google.protobuf.UnknownFieldSet
+ getUnknownFields() {
+ return this.unknownFields;
+ }
+ private Procedure(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ initFields();
+ int mutable_bitField0_ = 0;
+ com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+ com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+ boolean done = false;
+ while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+ case 0:
+ done = true;
+ break;
+ default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+ done = true;
+ }
+ break;
+ }
+ case 10: {
+ bitField0_ |= 0x00000001;
+ className_ = input.readBytes();
+ break;
+ }
+ case 16: {
+ bitField0_ |= 0x00000002;
+ parentId_ = input.readUInt64();
+ break;
+ }
+ case 24: {
+ bitField0_ |= 0x00000004;
+ procId_ = input.readUInt64();
+ break;
+ }
+ case 32: {
+ bitField0_ |= 0x00000008;
+ startTime_ = input.readUInt64();
+ break;
+ }
+ case 42: {
+ bitField0_ |= 0x00000010;
+ owner_ = input.readBytes();
+ break;
+ }
+ case 48: {
+ int rawValue = input.readEnum();
+ org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState value = org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState.valueOf(rawValue);
+ if (value == null) {
+ unknownFields.mergeVarintField(6, rawValue);
+ } else {
+ bitField0_ |= 0x00000020;
+ state_ = value;
+ }
+ break;
+ }
+ case 56: {
+ if (!((mutable_bitField0_ & 0x00000040) == 0x00000040)) {
+ stackId_ = new java.util.ArrayList<java.lang.Integer>();
+ mutable_bitField0_ |= 0x00000040;
+ }
+ stackId_.add(input.readUInt32());
+ break;
+ }
+ case 58: {
+ int length = input.readRawVarint32();
+ int limit = input.pushLimit(length);
+ if (!((mutable_bitField0_ & 0x00000040) == 0x00000040) && input.getBytesUntilLimit() > 0) {
+ stackId_ = new java.util.ArrayList<java.lang.Integer>();
+ mutable_bitField0_ |= 0x00000040;
+ }
+ while (input.getBytesUntilLimit() > 0) {
+ stackId_.add(input.readUInt32());
+ }
+ input.popLimit(limit);
+ break;
+ }
+ case 64: {
+ bitField0_ |= 0x00000040;
+ lastUpdate_ = input.readUInt64();
+ break;
+ }
+ case 72: {
+ bitField0_ |= 0x00000080;
+ timeout_ = input.readUInt32();
+ break;
+ }
+ case 82: {
+ org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.Builder subBuilder = null;
+ if (((bitField0_ & 0x00000100) == 0x00000100)) {
+ subBuilder = exception_.toBuilder();
+ }
+ exception_ = input.readMessage(org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.PARSER, extensionRegistry);
+ if (subBuilder != null) {
+ subBuilder.mergeFrom(exception_);
+ exception_ = subBuilder.buildPartial();
+ }
+ bitField0_ |= 0x00000100;
+ break;
+ }
+ case 90: {
+ bitField0_ |= 0x00000200;
+ result_ = input.readBytes();
+ break;
+ }
+ case 98: {
+ bitField0_ |= 0x00000400;
+ stateData_ = input.readBytes();
+ break;
+ }
+ }
+ }
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+ throw new com.google.protobuf.InvalidProtocolBufferException(
+ e.getMessage()).setUnfinishedMessage(this);
+ } finally {
+ if (((mutable_bitField0_ & 0x00000040) == 0x00000040)) {
+ stackId_ = java.util.Collections.unmodifiableList(stackId_);
+ }
+ this.unknownFields = unknownFields.build();
+ makeExtensionsImmutable();
+ }
+ }
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.internal_static_Procedure_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.internal_static_Procedure_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure.class, org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure.Builder.class);
+ }
+
+ public static com.google.protobuf.Parser<Procedure> PARSER =
+ new com.google.protobuf.AbstractParser<Procedure>() {
+ public Procedure parsePartialFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return new Procedure(input, extensionRegistry);
+ }
+ };
+
+ @java.lang.Override
+ public com.google.protobuf.Parser<Procedure> getParserForType() {
+ return PARSER;
+ }
+
+ private int bitField0_;
+ // required string class_name = 1;
+ public static final int CLASS_NAME_FIELD_NUMBER = 1;
+ private java.lang.Object className_;
+ /**
+ * <code>required string class_name = 1;</code>
+ *
+ * <pre>
+ * internal "static" state
+ * </pre>
+ */
+ public boolean hasClassName() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>required string class_name = 1;</code>
+ *
+ * <pre>
+ * internal "static" state
+ * </pre>
+ */
+ public java.lang.String getClassName() {
+ java.lang.Object ref = className_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ if (bs.isValidUtf8()) {
+ className_ = s;
+ }
+ return s;
+ }
+ }
+ /**
+ * <code>required string class_name = 1;</code>
+ *
+ * <pre>
+ * internal "static" state
+ * </pre>
+ */
+ public com.google.protobuf.ByteString
+ getClassNameBytes() {
+ java.lang.Object ref = className_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ className_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
+ // optional uint64 parent_id = 2;
+ public static final int PARENT_ID_FIELD_NUMBER = 2;
+ private long parentId_;
+ /**
+ * <code>optional uint64 parent_id = 2;</code>
+ *
+ * <pre>
+ * parent if not a root-procedure otherwise not set
+ * </pre>
+ */
+ public boolean hasParentId() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>optional uint64 parent_id = 2;</code>
+ *
+ * <pre>
+ * parent if not a root-procedure otherwise not set
+ * </pre>
+ */
+ public long getParentId() {
+ return parentId_;
+ }
+
+ // required uint64 proc_id = 3;
+ public static final int PROC_ID_FIELD_NUMBER = 3;
+ private long procId_;
+ /**
+ * <code>required uint64 proc_id = 3;</code>
+ */
+ public boolean hasProcId() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ /**
+ * <code>required uint64 proc_id = 3;</code>
+ */
+ public long getProcId() {
+ return procId_;
+ }
+
+ // required uint64 start_time = 4;
+ public static final int START_TIME_FIELD_NUMBER = 4;
+ private long startTime_;
+ /**
+ * <code>required uint64 start_time = 4;</code>
+ */
+ public boolean hasStartTime() {
+ return ((bitField0_ & 0x00000008) == 0x00000008);
+ }
+ /**
+ * <code>required uint64 start_time = 4;</code>
+ */
+ public long getStartTime() {
+ return startTime_;
+ }
+
+ // optional string owner = 5;
+ public static final int OWNER_FIELD_NUMBER = 5;
+ private java.lang.Object owner_;
+ /**
+ * <code>optional string owner = 5;</code>
+ */
+ public boolean hasOwner() {
+ return ((bitField0_ & 0x00000010) == 0x00000010);
+ }
+ /**
+ * <code>optional string owner = 5;</code>
+ */
+ public java.lang.String getOwner() {
+ java.lang.Object ref = owner_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ if (bs.isValidUtf8()) {
+ owner_ = s;
+ }
+ return s;
+ }
+ }
+ /**
+ * <code>optional string owner = 5;</code>
+ */
+ public com.google.protobuf.ByteString
+ getOwnerBytes() {
+ java.lang.Object ref = owner_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ owner_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
+ // required .ProcedureState state = 6;
+ public static final int STATE_FIELD_NUMBER = 6;
+ private org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState state_;
+ /**
+ * <code>required .ProcedureState state = 6;</code>
+ *
+ * <pre>
+ * internal "runtime" state
+ * </pre>
+ */
+ public boolean hasState() {
+ return ((bitField0_ & 0x00000020) == 0x00000020);
+ }
+ /**
+ * <code>required .ProcedureState state = 6;</code>
+ *
+ * <pre>
+ * internal "runtime" state
+ * </pre>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState getState() {
+ return state_;
+ }
+
+ // repeated uint32 stack_id = 7;
+ public static final int STACK_ID_FIELD_NUMBER = 7;
+ private java.util.List<java.lang.Integer> stackId_;
+ /**
+ * <code>repeated uint32 stack_id = 7;</code>
+ *
+ * <pre>
+ * stack indices in case the procedure was running
+ * </pre>
+ */
+ public java.util.List<java.lang.Integer>
+ getStackIdList() {
+ return stackId_;
+ }
+ /**
+ * <code>repeated uint32 stack_id = 7;</code>
+ *
+ * <pre>
+ * stack indices in case the procedure was running
+ * </pre>
+ */
+ public int getStackIdCount() {
+ return stackId_.size();
+ }
+ /**
+ * <code>repeated uint32 stack_id = 7;</code>
+ *
+ * <pre>
+ * stack indices in case the procedure was running
+ * </pre>
+ */
+ public int getStackId(int index) {
+ return stackId_.get(index);
+ }
+
+ // required uint64 last_update = 8;
+ public static final int LAST_UPDATE_FIELD_NUMBER = 8;
+ private long lastUpdate_;
+ /**
+ * <code>required uint64 last_update = 8;</code>
+ */
+ public boolean hasLastUpdate() {
+ return ((bitField0_ & 0x00000040) == 0x00000040);
+ }
+ /**
+ * <code>required uint64 last_update = 8;</code>
+ */
+ public long getLastUpdate() {
+ return lastUpdate_;
+ }
+
+ // optional uint32 timeout = 9;
+ public static final int TIMEOUT_FIELD_NUMBER = 9;
+ private int timeout_;
+ /**
+ * <code>optional uint32 timeout = 9;</code>
+ */
+ public boolean hasTimeout() {
+ return ((bitField0_ & 0x00000080) == 0x00000080);
+ }
+ /**
+ * <code>optional uint32 timeout = 9;</code>
+ */
+ public int getTimeout() {
+ return timeout_;
+ }
+
+ // optional .ForeignExceptionMessage exception = 10;
+ public static final int EXCEPTION_FIELD_NUMBER = 10;
+ private org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage exception_;
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ public boolean hasException() {
+ return ((bitField0_ & 0x00000100) == 0x00000100);
+ }
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage getException() {
+ return exception_;
+ }
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessageOrBuilder getExceptionOrBuilder() {
+ return exception_;
+ }
+
+ // optional bytes result = 11;
+ public static final int RESULT_FIELD_NUMBER = 11;
+ private com.google.protobuf.ByteString result_;
+ /**
+ * <code>optional bytes result = 11;</code>
+ *
+ * <pre>
+ * opaque (user) result structure
+ * </pre>
+ */
+ public boolean hasResult() {
+ return ((bitField0_ & 0x00000200) == 0x00000200);
+ }
+ /**
+ * <code>optional bytes result = 11;</code>
+ *
+ * <pre>
+ * opaque (user) result structure
+ * </pre>
+ */
+ public com.google.protobuf.ByteString getResult() {
+ return result_;
+ }
+
+ // optional bytes state_data = 12;
+ public static final int STATE_DATA_FIELD_NUMBER = 12;
+ private com.google.protobuf.ByteString stateData_;
+ /**
+ * <code>optional bytes state_data = 12;</code>
+ *
+ * <pre>
+ * opaque (user) procedure internal-state
+ * </pre>
+ */
+ public boolean hasStateData() {
+ return ((bitField0_ & 0x00000400) == 0x00000400);
+ }
+ /**
+ * <code>optional bytes state_data = 12;</code>
+ *
+ * <pre>
+ * opaque (user) procedure internal-state
+ * </pre>
+ */
+ public com.google.protobuf.ByteString getStateData() {
+ return stateData_;
+ }
+
+ private void initFields() {
+ className_ = "";
+ parentId_ = 0L;
+ procId_ = 0L;
+ startTime_ = 0L;
+ owner_ = "";
+ state_ = org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState.INITIALIZING;
+ stackId_ = java.util.Collections.emptyList();
+ lastUpdate_ = 0L;
+ timeout_ = 0;
+ exception_ = org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.getDefaultInstance();
+ result_ = com.google.protobuf.ByteString.EMPTY;
+ stateData_ = com.google.protobuf.ByteString.EMPTY;
+ }
+ private byte memoizedIsInitialized = -1;
+ public final boolean isInitialized() {
+ byte isInitialized = memoizedIsInitialized;
+ if (isInitialized != -1) return isInitialized == 1;
+
+ if (!hasClassName()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasProcId()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasStartTime()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasState()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasLastUpdate()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ memoizedIsInitialized = 1;
+ return true;
+ }
+
+ public void writeTo(com.google.protobuf.CodedOutputStream output)
+ throws java.io.IOException {
+ getSerializedSize();
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ output.writeBytes(1, getClassNameBytes());
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ output.writeUInt64(2, parentId_);
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ output.writeUInt64(3, procId_);
+ }
+ if (((bitField0_ & 0x00000008) == 0x00000008)) {
+ output.writeUInt64(4, startTime_);
+ }
+ if (((bitField0_ & 0x00000010) == 0x00000010)) {
+ output.writeBytes(5, getOwnerBytes());
+ }
+ if (((bitField0_ & 0x00000020) == 0x00000020)) {
+ output.writeEnum(6, state_.getNumber());
+ }
+ for (int i = 0; i < stackId_.size(); i++) {
+ output.writeUInt32(7, stackId_.get(i));
+ }
+ if (((bitField0_ & 0x00000040) == 0x00000040)) {
+ output.writeUInt64(8, lastUpdate_);
+ }
+ if (((bitField0_ & 0x00000080) == 0x00000080)) {
+ output.writeUInt32(9, timeout_);
+ }
+ if (((bitField0_ & 0x00000100) == 0x00000100)) {
+ output.writeMessage(10, exception_);
+ }
+ if (((bitField0_ & 0x00000200) == 0x00000200)) {
+ output.writeBytes(11, result_);
+ }
+ if (((bitField0_ & 0x00000400) == 0x00000400)) {
+ output.writeBytes(12, stateData_);
+ }
+ getUnknownFields().writeTo(output);
+ }
+
+ private int memoizedSerializedSize = -1;
+ public int getSerializedSize() {
+ int size = memoizedSerializedSize;
+ if (size != -1) return size;
+
+ size = 0;
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(1, getClassNameBytes());
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeUInt64Size(2, parentId_);
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeUInt64Size(3, procId_);
+ }
+ if (((bitField0_ & 0x00000008) == 0x00000008)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeUInt64Size(4, startTime_);
+ }
+ if (((bitField0_ & 0x00000010) == 0x00000010)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(5, getOwnerBytes());
+ }
+ if (((bitField0_ & 0x00000020) == 0x00000020)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeEnumSize(6, state_.getNumber());
+ }
+ {
+ int dataSize = 0;
+ for (int i = 0; i < stackId_.size(); i++) {
+ dataSize += com.google.protobuf.CodedOutputStream
+ .computeUInt32SizeNoTag(stackId_.get(i));
+ }
+ size += dataSize;
+ size += 1 * getStackIdList().size();
+ }
+ if (((bitField0_ & 0x00000040) == 0x00000040)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeUInt64Size(8, lastUpdate_);
+ }
+ if (((bitField0_ & 0x00000080) == 0x00000080)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeUInt32Size(9, timeout_);
+ }
+ if (((bitField0_ & 0x00000100) == 0x00000100)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeMessageSize(10, exception_);
+ }
+ if (((bitField0_ & 0x00000200) == 0x00000200)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(11, result_);
+ }
+ if (((bitField0_ & 0x00000400) == 0x00000400)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(12, stateData_);
+ }
+ size += getUnknownFields().getSerializedSize();
+ memoizedSerializedSize = size;
+ return size;
+ }
+
+ private static final long serialVersionUID = 0L;
+ @java.lang.Override
+ protected java.lang.Object writeReplace()
+ throws java.io.ObjectStreamException {
+ return super.writeReplace();
+ }
+
+ @java.lang.Override
+ public boolean equals(final java.lang.Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure)) {
+ return super.equals(obj);
+ }
+ org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure other = (org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure) obj;
+
+ boolean result = true;
+ result = result && (hasClassName() == other.hasClassName());
+ if (hasClassName()) {
+ result = result && getClassName()
+ .equals(other.getClassName());
+ }
+ result = result && (hasParentId() == other.hasParentId());
+ if (hasParentId()) {
+ result = result && (getParentId()
+ == other.getParentId());
+ }
+ result = result && (hasProcId() == other.hasProcId());
+ if (hasProcId()) {
+ result = result && (getProcId()
+ == other.getProcId());
+ }
+ result = result && (hasStartTime() == other.hasStartTime());
+ if (hasStartTime()) {
+ result = result && (getStartTime()
+ == other.getStartTime());
+ }
+ result = result && (hasOwner() == other.hasOwner());
+ if (hasOwner()) {
+ result = result && getOwner()
+ .equals(other.getOwner());
+ }
+ result = result && (hasState() == other.hasState());
+ if (hasState()) {
+ result = result &&
+ (getState() == other.getState());
+ }
+ result = result && getStackIdList()
+ .equals(other.getStackIdList());
+ result = result && (hasLastUpdate() == other.hasLastUpdate());
+ if (hasLastUpdate()) {
+ result = result && (getLastUpdate()
+ == other.getLastUpdate());
+ }
+ result = result && (hasTimeout() == other.hasTimeout());
+ if (hasTimeout()) {
+ result = result && (getTimeout()
+ == other.getTimeout());
+ }
+ result = result && (hasException() == other.hasException());
+ if (hasException()) {
+ result = result && getException()
+ .equals(other.getException());
+ }
+ result = result && (hasResult() == other.hasResult());
+ if (hasResult()) {
+ result = result && getResult()
+ .equals(other.getResult());
+ }
+ result = result && (hasStateData() == other.hasStateData());
+ if (hasStateData()) {
+ result = result && getStateData()
+ .equals(other.getStateData());
+ }
+ result = result &&
+ getUnknownFields().equals(other.getUnknownFields());
+ return result;
+ }
+
+ private int memoizedHashCode = 0;
+ @java.lang.Override
+ public int hashCode() {
+ if (memoizedHashCode != 0) {
+ return memoizedHashCode;
+ }
+ int hash = 41;
+ hash = (19 * hash) + getDescriptorForType().hashCode();
+ if (hasClassName()) {
+ hash = (37 * hash) + CLASS_NAME_FIELD_NUMBER;
+ hash = (53 * hash) + getClassName().hashCode();
+ }
+ if (hasParentId()) {
+ hash = (37 * hash) + PARENT_ID_FIELD_NUMBER;
+ hash = (53 * hash) + hashLong(getParentId());
+ }
+ if (hasProcId()) {
+ hash = (37 * hash) + PROC_ID_FIELD_NUMBER;
+ hash = (53 * hash) + hashLong(getProcId());
+ }
+ if (hasStartTime()) {
+ hash = (37 * hash) + START_TIME_FIELD_NUMBER;
+ hash = (53 * hash) + hashLong(getStartTime());
+ }
+ if (hasOwner()) {
+ hash = (37 * hash) + OWNER_FIELD_NUMBER;
+ hash = (53 * hash) + getOwner().hashCode();
+ }
+ if (hasState()) {
+ hash = (37 * hash) + STATE_FIELD_NUMBER;
+ hash = (53 * hash) + hashEnum(getState());
+ }
+ if (getStackIdCount() > 0) {
+ hash = (37 * hash) + STACK_ID_FIELD_NUMBER;
+ hash = (53 * hash) + getStackIdList().hashCode();
+ }
+ if (hasLastUpdate()) {
+ hash = (37 * hash) + LAST_UPDATE_FIELD_NUMBER;
+ hash = (53 * hash) + hashLong(getLastUpdate());
+ }
+ if (hasTimeout()) {
+ hash = (37 * hash) + TIMEOUT_FIELD_NUMBER;
+ hash = (53 * hash) + getTimeout();
+ }
+ if (hasException()) {
+ hash = (37 * hash) + EXCEPTION_FIELD_NUMBER;
+ hash = (53 * hash) + getException().hashCode();
+ }
+ if (hasResult()) {
+ hash = (37 * hash) + RESULT_FIELD_NUMBER;
+ hash = (53 * hash) + getResult().hashCode();
+ }
+ if (hasStateData()) {
+ hash = (37 * hash) + STATE_DATA_FIELD_NUMBER;
+ hash = (53 * hash) + getStateData().hashCode();
+ }
+ hash = (29 * hash) + getUnknownFields().hashCode();
+ memoizedHashCode = hash;
+ return hash;
+ }
+
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure parseFrom(
+ com.google.protobuf.ByteString data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure parseFrom(
+ com.google.protobuf.ByteString data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure parseFrom(byte[] data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure parseFrom(
+ byte[] data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure parseFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure parseFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure parseDelimitedFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure parseDelimitedFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure parseFrom(
+ com.google.protobuf.CodedInputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure parseFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+
+ public static Builder newBuilder() { return Builder.create(); }
+ public Builder newBuilderForType() { return newBuilder(); }
+ public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure prototype) {
+ return newBuilder().mergeFrom(prototype);
+ }
+ public Builder toBuilder() { return newBuilder(this); }
+
+ @java.lang.Override
+ protected Builder newBuilderForType(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ Builder builder = new Builder(parent);
+ return builder;
+ }
+ /**
+ * Protobuf type {@code Procedure}
+ *
+ * <pre>
+ **
+ * Procedure metadata, serialized by the ProcedureStore to be able to recover the old state.
+ * </pre>
+ */
+ public static final class Builder extends
+ com.google.protobuf.GeneratedMessage.Builder<Builder>
+ implements org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureOrBuilder {
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.internal_static_Procedure_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.internal_static_Procedure_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure.class, org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure.Builder.class);
+ }
+
+ // Construct using org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure.newBuilder()
+ private Builder() {
+ maybeForceBuilderInitialization();
+ }
+
+ private Builder(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ super(parent);
+ maybeForceBuilderInitialization();
+ }
+ private void maybeForceBuilderInitialization() {
+ if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+ getExceptionFieldBuilder();
+ }
+ }
+ private static Builder create() {
+ return new Builder();
+ }
+
+ public Builder clear() {
+ super.clear();
+ className_ = "";
+ bitField0_ = (bitField0_ & ~0x00000001);
+ parentId_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000002);
+ procId_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000004);
+ startTime_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000008);
+ owner_ = "";
+ bitField0_ = (bitField0_ & ~0x00000010);
+ state_ = org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState.INITIALIZING;
+ bitField0_ = (bitField0_ & ~0x00000020);
+ stackId_ = java.util.Collections.emptyList();
+ bitField0_ = (bitField0_ & ~0x00000040);
+ lastUpdate_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000080);
+ timeout_ = 0;
+ bitField0_ = (bitField0_ & ~0x00000100);
+ if (exceptionBuilder_ == null) {
+ exception_ = org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.getDefaultInstance();
+ } else {
+ exceptionBuilder_.clear();
+ }
+ bitField0_ = (bitField0_ & ~0x00000200);
+ result_ = com.google.protobuf.ByteString.EMPTY;
+ bitField0_ = (bitField0_ & ~0x00000400);
+ stateData_ = com.google.protobuf.ByteString.EMPTY;
+ bitField0_ = (bitField0_ & ~0x00000800);
+ return this;
+ }
+
+ public Builder clone() {
+ return create().mergeFrom(buildPartial());
+ }
+
+ public com.google.protobuf.Descriptors.Descriptor
+ getDescriptorForType() {
+ return org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.internal_static_Procedure_descriptor;
+ }
+
+ public org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure getDefaultInstanceForType() {
+ return org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure.getDefaultInstance();
+ }
+
+ public org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure build() {
+ org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure result = buildPartial();
+ if (!result.isInitialized()) {
+ throw newUninitializedMessageException(result);
+ }
+ return result;
+ }
+
+ public org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure buildPartial() {
+ org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure result = new org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure(this);
+ int from_bitField0_ = bitField0_;
+ int to_bitField0_ = 0;
+ if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+ to_bitField0_ |= 0x00000001;
+ }
+ result.className_ = className_;
+ if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+ to_bitField0_ |= 0x00000002;
+ }
+ result.parentId_ = parentId_;
+ if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+ to_bitField0_ |= 0x00000004;
+ }
+ result.procId_ = procId_;
+ if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
+ to_bitField0_ |= 0x00000008;
+ }
+ result.startTime_ = startTime_;
+ if (((from_bitField0_ & 0x00000010) == 0x00000010)) {
+ to_bitField0_ |= 0x00000010;
+ }
+ result.owner_ = owner_;
+ if (((from_bitField0_ & 0x00000020) == 0x00000020)) {
+ to_bitField0_ |= 0x00000020;
+ }
+ result.state_ = state_;
+ if (((bitField0_ & 0x00000040) == 0x00000040)) {
+ stackId_ = java.util.Collections.unmodifiableList(stackId_);
+ bitField0_ = (bitField0_ & ~0x00000040);
+ }
+ result.stackId_ = stackId_;
+ if (((from_bitField0_ & 0x00000080) == 0x00000080)) {
+ to_bitField0_ |= 0x00000040;
+ }
+ result.lastUpdate_ = lastUpdate_;
+ if (((from_bitField0_ & 0x00000100) == 0x00000100)) {
+ to_bitField0_ |= 0x00000080;
+ }
+ result.timeout_ = timeout_;
+ if (((from_bitField0_ & 0x00000200) == 0x00000200)) {
+ to_bitField0_ |= 0x00000100;
+ }
+ if (exceptionBuilder_ == null) {
+ result.exception_ = exception_;
+ } else {
+ result.exception_ = exceptionBuilder_.build();
+ }
+ if (((from_bitField0_ & 0x00000400) == 0x00000400)) {
+ to_bitField0_ |= 0x00000200;
+ }
+ result.result_ = result_;
+ if (((from_bitField0_ & 0x00000800) == 0x00000800)) {
+ to_bitField0_ |= 0x00000400;
+ }
+ result.stateData_ = stateData_;
+ result.bitField0_ = to_bitField0_;
+ onBuilt();
+ return result;
+ }
+
+ public Builder mergeFrom(com.google.protobuf.Message other) {
+ if (other instanceof org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure) {
+ return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure)other);
+ } else {
+ super.mergeFrom(other);
+ return this;
+ }
+ }
+
+ public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure other) {
+ if (other == org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure.getDefaultInstance()) return this;
+ if (other.hasClassName()) {
+ bitField0_ |= 0x00000001;
+ className_ = other.className_;
+ onChanged();
+ }
+ if (other.hasParentId()) {
+ setParentId(other.getParentId());
+ }
+ if (other.hasProcId()) {
+ setProcId(other.getProcId());
+ }
+ if (other.hasStartTime()) {
+ setStartTime(other.getStartTime());
+ }
+ if (other.hasOwner()) {
+ bitField0_ |= 0x00000010;
+ owner_ = other.owner_;
+ onChanged();
+ }
+ if (other.hasState()) {
+ setState(other.getState());
+ }
+ if (!other.stackId_.isEmpty()) {
+ if (stackId_.isEmpty()) {
+ stackId_ = other.stackId_;
+ bitField0_ = (bitField0_ & ~0x00000040);
+ } else {
+ ensureStackIdIsMutable();
+ stackId_.addAll(other.stackId_);
+ }
+ onChanged();
+ }
+ if (other.hasLastUpdate()) {
+ setLastUpdate(other.getLastUpdate());
+ }
+ if (other.hasTimeout()) {
+ setTimeout(other.getTimeout());
+ }
+ if (other.hasException()) {
+ mergeException(other.getException());
+ }
+ if (other.hasResult()) {
+ setResult(other.getResult());
+ }
+ if (other.hasStateData()) {
+ setStateData(other.getStateData());
+ }
+ this.mergeUnknownFields(other.getUnknownFields());
+ return this;
+ }
+
+ public final boolean isInitialized() {
+ if (!hasClassName()) {
+
+ return false;
+ }
+ if (!hasProcId()) {
+
+ return false;
+ }
+ if (!hasStartTime()) {
+
+ return false;
+ }
+ if (!hasState()) {
+
+ return false;
+ }
+ if (!hasLastUpdate()) {
+
+ return false;
+ }
+ return true;
+ }
+
+ public Builder mergeFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure parsedMessage = null;
+ try {
+ parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ parsedMessage = (org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.Procedure) e.getUnfinishedMessage();
+ throw e;
+ } finally {
+ if (parsedMessage != null) {
+ mergeFrom(parsedMessage);
+ }
+ }
+ return this;
+ }
+ private int bitField0_;
+
+ // required string class_name = 1;
+ private java.lang.Object className_ = "";
+ /**
+ * <code>required string class_name = 1;</code>
+ *
+ * <pre>
+ * internal "static" state
+ * </pre>
+ */
+ public boolean hasClassName() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>required string class_name = 1;</code>
+ *
+ * <pre>
+ * internal "static" state
+ * </pre>
+ */
+ public java.lang.String getClassName() {
+ java.lang.Object ref = className_;
+ if (!(ref instanceof java.lang.String)) {
+ java.lang.String s = ((com.google.protobuf.ByteString) ref)
+ .toStringUtf8();
+ className_ = s;
+ return s;
+ } else {
+ return (java.lang.String) ref;
+ }
+ }
+ /**
+ * <code>required string class_name = 1;</code>
+ *
+ * <pre>
+ * internal "static" state
+ * </pre>
+ */
+ public com.google.protobuf.ByteString
+ getClassNameBytes() {
+ java.lang.Object ref = className_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ className_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+ /**
+ * <code>required string class_name = 1;</code>
+ *
+ * <pre>
+ * internal "static" state
+ * </pre>
+ */
+ public Builder setClassName(
+ java.lang.String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000001;
+ className_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required string class_name = 1;</code>
+ *
+ * <pre>
+ * internal "static" state
+ * </pre>
+ */
+ public Builder clearClassName() {
+ bitField0_ = (bitField0_ & ~0x00000001);
+ className_ = getDefaultInstance().getClassName();
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required string class_name = 1;</code>
+ *
+ * <pre>
+ * internal "static" state
+ * </pre>
+ */
+ public Builder setClassNameBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000001;
+ className_ = value;
+ onChanged();
+ return this;
+ }
+
+ // optional uint64 parent_id = 2;
+ private long parentId_ ;
+ /**
+ * <code>optional uint64 parent_id = 2;</code>
+ *
+ * <pre>
+ * parent if not a root-procedure otherwise not set
+ * </pre>
+ */
+ public boolean hasParentId() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>optional uint64 parent_id = 2;</code>
+ *
+ * <pre>
+ * parent if not a root-procedure otherwise not set
+ * </pre>
+ */
+ public long getParentId() {
+ return parentId_;
+ }
+ /**
+ * <code>optional uint64 parent_id = 2;</code>
+ *
+ * <pre>
+ * parent if not a root-procedure otherwise not set
+ * </pre>
+ */
+ public Builder setParentId(long value) {
+ bitField0_ |= 0x00000002;
+ parentId_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional uint64 parent_id = 2;</code>
+ *
+ * <pre>
+ * parent if not a root-procedure otherwise not set
+ * </pre>
+ */
+ public Builder clearParentId() {
+ bitField0_ = (bitField0_ & ~0x00000002);
+ parentId_ = 0L;
+ onChanged();
+ return this;
+ }
+
+ // required uint64 proc_id = 3;
+ private long procId_ ;
+ /**
+ * <code>required uint64 proc_id = 3;</code>
+ */
+ public boolean hasProcId() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ /**
+ * <code>required uint64 proc_id = 3;</code>
+ */
+ public long getProcId() {
+ return procId_;
+ }
+ /**
+ * <code>required uint64 proc_id = 3;</code>
+ */
+ public Builder setProcId(long value) {
+ bitField0_ |= 0x00000004;
+ procId_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required uint64 proc_id = 3;</code>
+ */
+ public Builder clearProcId() {
+ bitField0_ = (bitField0_ & ~0x00000004);
+ procId_ = 0L;
+ onChanged();
+ return this;
+ }
+
+ // required uint64 start_time = 4;
+ private long startTime_ ;
+ /**
+ * <code>required uint64 start_time = 4;</code>
+ */
+ public boolean hasStartTime() {
+ return ((bitField0_ & 0x00000008) == 0x00000008);
+ }
+ /**
+ * <code>required uint64 start_time = 4;</code>
+ */
+ public long getStartTime() {
+ return startTime_;
+ }
+ /**
+ * <code>required uint64 start_time = 4;</code>
+ */
+ public Builder setStartTime(long value) {
+ bitField0_ |= 0x00000008;
+ startTime_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required uint64 start_time = 4;</code>
+ */
+ public Builder clearStartTime() {
+ bitField0_ = (bitField0_ & ~0x00000008);
+ startTime_ = 0L;
+ onChanged();
+ return this;
+ }
+
+ // optional string owner = 5;
+ private java.lang.Object owner_ = "";
+ /**
+ * <code>optional string owner = 5;</code>
+ */
+ public boolean hasOwner() {
+ return ((bitField0_ & 0x00000010) == 0x00000010);
+ }
+ /**
+ * <code>optional string owner = 5;</code>
+ */
+ public java.lang.String getOwner() {
+ java.lang.Object ref = owner_;
+ if (!(ref instanceof java.lang.String)) {
+ java.lang.String s = ((com.google.protobuf.ByteString) ref)
+ .toStringUtf8();
+ owner_ = s;
+ return s;
+ } else {
+ return (java.lang.String) ref;
+ }
+ }
+ /**
+ * <code>optional string owner = 5;</code>
+ */
+ public com.google.protobuf.ByteString
+ getOwnerBytes() {
+ java.lang.Object ref = owner_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ owner_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+ /**
+ * <code>optional string owner = 5;</code>
+ */
+ public Builder setOwner(
+ java.lang.String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000010;
+ owner_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional string owner = 5;</code>
+ */
+ public Builder clearOwner() {
+ bitField0_ = (bitField0_ & ~0x00000010);
+ owner_ = getDefaultInstance().getOwner();
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional string owner = 5;</code>
+ */
+ public Builder setOwnerBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000010;
+ owner_ = value;
+ onChanged();
+ return this;
+ }
+
+ // required .ProcedureState state = 6;
+ private org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState state_ = org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState.INITIALIZING;
+ /**
+ * <code>required .ProcedureState state = 6;</code>
+ *
+ * <pre>
+ * internal "runtime" state
+ * </pre>
+ */
+ public boolean hasState() {
+ return ((bitField0_ & 0x00000020) == 0x00000020);
+ }
+ /**
+ * <code>required .ProcedureState state = 6;</code>
+ *
+ * <pre>
+ * internal "runtime" state
+ * </pre>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState getState() {
+ return state_;
+ }
+ /**
+ * <code>required .ProcedureState state = 6;</code>
+ *
+ * <pre>
+ * internal "runtime" state
+ * </pre>
+ */
+ public Builder setState(org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000020;
+ state_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required .ProcedureState state = 6;</code>
+ *
+ * <pre>
+ * internal "runtime" state
+ * </pre>
+ */
+ public Builder clearState() {
+ bitField0_ = (bitField0_ & ~0x00000020);
+ state_ = org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState.INITIALIZING;
+ onChanged();
+ return this;
+ }
+
+ // repeated uint32 stack_id = 7;
+ private java.util.List<java.lang.Integer> stackId_ = java.util.Collections.emptyList();
+ private void ensureStackIdIsMutable() {
+ if (!((bitField0_ & 0x00000040) == 0x00000040)) {
+ stackId_ = new java.util.ArrayList<java.lang.Integer>(stackId_);
+ bitField0_ |= 0x00000040;
+ }
+ }
+ /**
+ * <code>repeated uint32 stack_id = 7;</code>
+ *
+ * <pre>
+ * stack indices in case the procedure was running
+ * </pre>
+ */
+ public java.util.List<java.lang.Integer>
+ getStackIdList() {
+ return java.util.Collections.unmodifiableList(stackId_);
+ }
+ /**
+ * <code>repeated uint32 stack_id = 7;</code>
+ *
+ * <pre>
+ * stack indices in case the procedure was running
+ * </pre>
+ */
+ public int getStackIdCount() {
+ return stackId_.size();
+ }
+ /**
+ * <code>repeated uint32 stack_id = 7;</code>
+ *
+ * <pre>
+ * stack indices in case the procedure was running
+ * </pre>
+ */
+ public int getStackId(int index) {
+ return stackId_.get(index);
+ }
+ /**
+ * <code>repeated uint32 stack_id = 7;</code>
+ *
+ * <pre>
+ * stack indices in case the procedure was running
+ * </pre>
+ */
+ public Builder setStackId(
+ int index, int value) {
+ ensureStackIdIsMutable();
+ stackId_.set(index, value);
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>repeated uint32 stack_id = 7;</code>
+ *
+ * <pre>
+ * stack indices in case the procedure was running
+ * </pre>
+ */
+ public Builder addStackId(int value) {
+ ensureStackIdIsMutable();
+ stackId_.add(value);
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>repeated uint32 stack_id = 7;</code>
+ *
+ * <pre>
+ * stack indices in case the procedure was running
+ * </pre>
+ */
+ public Builder addAllStackId(
+ java.lang.Iterable<? extends java.lang.Integer> values) {
+ ensureStackIdIsMutable();
+ super.addAll(values, stackId_);
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>repeated uint32 stack_id = 7;</code>
+ *
+ * <pre>
+ * stack indices in case the procedure was running
+ * </pre>
+ */
+ public Builder clearStackId() {
+ stackId_ = java.util.Collections.emptyList();
+ bitField0_ = (bitField0_ & ~0x00000040);
+ onChanged();
+ return this;
+ }
+
+ // required uint64 last_update = 8;
+ private long lastUpdate_ ;
+ /**
+ * <code>required uint64 last_update = 8;</code>
+ */
+ public boolean hasLastUpdate() {
+ return ((bitField0_ & 0x00000080) == 0x00000080);
+ }
+ /**
+ * <code>required uint64 last_update = 8;</code>
+ */
+ public long getLastUpdate() {
+ return lastUpdate_;
+ }
+ /**
+ * <code>required uint64 last_update = 8;</code>
+ */
+ public Builder setLastUpdate(long value) {
+ bitField0_ |= 0x00000080;
+ lastUpdate_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required uint64 last_update = 8;</code>
+ */
+ public Builder clearLastUpdate() {
+ bitField0_ = (bitField0_ & ~0x00000080);
+ lastUpdate_ = 0L;
+ onChanged();
+ return this;
+ }
+
+ // optional uint32 timeout = 9;
+ private int timeout_ ;
+ /**
+ * <code>optional uint32 timeout = 9;</code>
+ */
+ public boolean hasTimeout() {
+ return ((bitField0_ & 0x00000100) == 0x00000100);
+ }
+ /**
+ * <code>optional uint32 timeout = 9;</code>
+ */
+ public int getTimeout() {
+ return timeout_;
+ }
+ /**
+ * <code>optional uint32 timeout = 9;</code>
+ */
+ public Builder setTimeout(int value) {
+ bitField0_ |= 0x00000100;
+ timeout_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional uint32 timeout = 9;</code>
+ */
+ public Builder clearTimeout() {
+ bitField0_ = (bitField0_ & ~0x00000100);
+ timeout_ = 0;
+ onChanged();
+ return this;
+ }
+
+ // optional .ForeignExceptionMessage exception = 10;
+ private org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage exception_ = org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.getDefaultInstance();
+ private com.google.protobuf.SingleFieldBuilder<
+ org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.Builder, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessageOrBuilder> exceptionBuilder_;
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ public boolean hasException() {
+ return ((bitField0_ & 0x00000200) == 0x00000200);
+ }
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage getException() {
+ if (exceptionBuilder_ == null) {
+ return exception_;
+ } else {
+ return exceptionBuilder_.getMessage();
+ }
+ }
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ public Builder setException(org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage value) {
+ if (exceptionBuilder_ == null) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ exception_ = value;
+ onChanged();
+ } else {
+ exceptionBuilder_.setMessage(value);
+ }
+ bitField0_ |= 0x00000200;
+ return this;
+ }
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ public Builder setException(
+ org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.Builder builderForValue) {
+ if (exceptionBuilder_ == null) {
+ exception_ = builderForValue.build();
+ onChanged();
+ } else {
+ exceptionBuilder_.setMessage(builderForValue.build());
+ }
+ bitField0_ |= 0x00000200;
+ return this;
+ }
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ public Builder mergeException(org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage value) {
+ if (exceptionBuilder_ == null) {
+ if (((bitField0_ & 0x00000200) == 0x00000200) &&
+ exception_ != org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.getDefaultInstance()) {
+ exception_ =
+ org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.newBuilder(exception_).mergeFrom(value).buildPartial();
+ } else {
+ exception_ = value;
+ }
+ onChanged();
+ } else {
+ exceptionBuilder_.mergeFrom(value);
+ }
+ bitField0_ |= 0x00000200;
+ return this;
+ }
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ public Builder clearException() {
+ if (exceptionBuilder_ == null) {
+ exception_ = org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.getDefaultInstance();
+ onChanged();
+ } else {
+ exceptionBuilder_.clear();
+ }
+ bitField0_ = (bitField0_ & ~0x00000200);
+ return this;
+ }
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.Builder getExceptionBuilder() {
+ bitField0_ |= 0x00000200;
+ onChanged();
+ return getExceptionFieldBuilder().getBuilder();
+ }
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ public org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessageOrBuilder getExceptionOrBuilder() {
+ if (exceptionBuilder_ != null) {
+ return exceptionBuilder_.getMessageOrBuilder();
+ } else {
+ return exception_;
+ }
+ }
+ /**
+ * <code>optional .ForeignExceptionMessage exception = 10;</code>
+ *
+ * <pre>
+ * user state/results
+ * </pre>
+ */
+ private com.google.protobuf.SingleFieldBuilder<
+ org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.Builder, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessageOrBuilder>
+ getExceptionFieldBuilder() {
+ if (exceptionBuilder_ == null) {
+ exceptionBuilder_ = new com.google.protobuf.SingleFieldBuilder<
+ org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage.Builder, org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessageOrBuilder>(
+ exception_,
+ getParentForChildren(),
+ isClean());
+ exception_ = null;
+ }
+ return exceptionBuilder_;
+ }
+
+ // optional bytes result = 11;
+ private com.google.protobuf.ByteString result_ = com.google.protobuf.ByteString.EMPTY;
+ /**
+ * <code>optional bytes result = 11;</code>
+ *
+ * <pre>
+ * opaque (user) result structure
+ * </pre>
+ */
+ public boolean hasResult() {
+ return ((bitField0_ & 0x00000400) == 0x00000400);
+ }
+ /**
+ * <code>optional bytes result = 11;</code>
+ *
+ * <pre>
+ * opaque (user) result structure
+ * </pre>
+ */
+ public com.google.protobuf.ByteString getResult() {
+ return result_;
+ }
+ /**
+ * <code>optional bytes result = 11;</code>
+ *
+ * <pre>
+ * opaque (user) result structure
+ * </pre>
+ */
+ public Builder setResult(com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000400;
+ result_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional bytes result = 11;</code>
+ *
+ * <pre>
+ * opaque (user) result structure
+ * </pre>
+ */
+ public Builder clearResult() {
+ bitField0_ = (bitField0_ & ~0x00000400);
+ result_ = getDefaultInstance().getResult();
+ onChanged();
+ return this;
+ }
+
+ // optional bytes state_data = 12;
+ private com.google.protobuf.ByteString stateData_ = com.google.protobuf.ByteString.EMPTY;
+ /**
+ * <code>optional bytes state_data = 12;</code>
+ *
+ * <pre>
+ * opaque (user) procedure internal-state
+ * </pre>
+ */
+ public boolean hasStateData() {
+ return ((bitField0_ & 0x00000800) == 0x00000800);
+ }
+ /**
+ * <code>optional bytes state_data = 12;</code>
+ *
+ * <pre>
+ * opaque (user) procedure internal-state
+ * </pre>
+ */
+ public com.google.protobuf.ByteString getStateData() {
+ return stateData_;
+ }
+ /**
+ * <code>optional bytes state_data = 12;</code>
+ *
+ * <pre>
+ * opaque (user) procedure internal-state
+ * </pre>
+ */
+ public Builder setStateData(com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000800;
+ stateData_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional bytes state_data = 12;</code>
+ *
+ * <pre>
+ * opaque (user) procedure internal-state
+ * </pre>
+ */
+ public Builder clearStateData() {
+ bitField0_ = (bitField0_ & ~0x00000800);
+ stateData_ = getDefaultInstance().getStateData();
+ onChanged();
+ return this;
+ }
+
+ // @@protoc_insertion_point(builder_scope:Procedure)
+ }
+
+ static {
+ defaultInstance = new Procedure(true);
+ defaultInstance.initFields();
+ }
+
+ // @@protoc_insertion_point(class_scope:Procedure)
+ }
+
+ public interface SequentialProcedureDataOrBuilder
+ extends com.google.protobuf.MessageOrBuilder {
+
+ // required bool executed = 1;
+ /**
+ * <code>required bool executed = 1;</code>
+ */
+ boolean hasExecuted();
+ /**
+ * <code>required bool executed = 1;</code>
+ */
+ boolean getExecuted();
+ }
+ /**
+ * Protobuf type {@code SequentialProcedureData}
+ *
+ * <pre>
+ **
+ * SequentialProcedure data
+ * </pre>
+ */
+ public static final class SequentialProcedureData extends
+ com.google.protobuf.GeneratedMessage
+ implements SequentialProcedureDataOrBuilder {
+ // Use SequentialProcedureData.newBuilder() to construct.
+ private SequentialProcedureData(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+ super(builder);
+ this.unknownFields = builder.getUnknownFields();
+ }
+ private SequentialProcedureData(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+ private static final SequentialProcedureData defaultInstance;
+ public static SequentialProcedureData getDefaultInstance() {
+ return defaultInstance;
+ }
+
+ public SequentialProcedureData getDefaultInstanceForType() {
+ return defaultInstance;
+ }
+
+ private final com.google.protobuf.UnknownFieldSet unknownFields;
+ @java.lang.Override
+ public final com.google.protobuf.UnknownFieldSet
+ getUnknownFields() {
+ return this.unknownFields;
+ }
+ private SequentialProcedureData(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ initFields();
+ int mutable_bitField0_ = 0;
+ com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+ com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+ boolean done = false;
+ while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+ case 0:
+ done = true;
+ break;
+ default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+ done = true;
+ }
+ break;
+ }
+ case 8: {
+ bitField0_ |= 0x00000001;
+ executed_ = input.readBool();
+ break;
+ }
+ }
+ }
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+ throw new com.google.protobuf.InvalidProtocolBufferException(
+ e.getMessage()).setUnfinishedMessage(this);
+ } finally {
+ this.unknownFields = unknownFields.build();
+ makeExtensionsImmutable();
+ }
+ }
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.internal_static_SequentialProcedureData_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.internal_static_SequentialProcedureData_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData.class, org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData.Builder.class);
+ }
+
+ public static com.google.protobuf.Parser<SequentialProcedureData> PARSER =
+ new com.google.protobuf.AbstractParser<SequentialProcedureData>() {
+ public SequentialProcedureData parsePartialFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return new SequentialProcedureData(input, extensionRegistry);
+ }
+ };
+
+ @java.lang.Override
+ public com.google.protobuf.Parser<SequentialProcedureData> getParserForType() {
+ return PARSER;
+ }
+
+ private int bitField0_;
+ // required bool executed = 1;
+ public static final int EXECUTED_FIELD_NUMBER = 1;
+ private boolean executed_;
+ /**
+ * <code>required bool executed = 1;</code>
+ */
+ public boolean hasExecuted() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>required bool executed = 1;</code>
+ */
+ public boolean getExecuted() {
+ return executed_;
+ }
+
+ private void initFields() {
+ executed_ = false;
+ }
+ private byte memoizedIsInitialized = -1;
+ public final boolean isInitialized() {
+ byte isInitialized = memoizedIsInitialized;
+ if (isInitialized != -1) return isInitialized == 1;
+
+ if (!hasExecuted()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ memoizedIsInitialized = 1;
+ return true;
+ }
+
+ public void writeTo(com.google.protobuf.CodedOutputStream output)
+ throws java.io.IOException {
+ getSerializedSize();
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ output.writeBool(1, executed_);
+ }
+ getUnknownFields().writeTo(output);
+ }
+
+ private int memoizedSerializedSize = -1;
+ public int getSerializedSize() {
+ int size = memoizedSerializedSize;
+ if (size != -1) return size;
+
+ size = 0;
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBoolSize(1, executed_);
+ }
+ size += getUnknownFields().getSerializedSize();
+ memoizedSerializedSize = size;
+ return size;
+ }
+
+ private static final long serialVersionUID = 0L;
+ @java.lang.Override
+ protected java.lang.Object writeReplace()
+ throws java.io.ObjectStreamException {
+ return super.writeReplace();
+ }
+
+ @java.lang.Override
+ public boolean equals(final java.lang.Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData)) {
+ return super.equals(obj);
+ }
+ org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData other = (org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData) obj;
+
+ boolean result = true;
+ result = result && (hasExecuted() == other.hasExecuted());
+ if (hasExecuted()) {
+ result = result && (getExecuted()
+ == other.getExecuted());
+ }
+ result = result &&
+ getUnknownFields().equals(other.getUnknownFields());
+ return result;
+ }
+
+ private int memoizedHashCode = 0;
+ @java.lang.Override
+ public int hashCode() {
+ if (memoizedHashCode != 0) {
+ return memoizedHashCode;
+ }
+ int hash = 41;
+ hash = (19 * hash) + getDescriptorForType().hashCode();
+ if (hasExecuted()) {
+ hash = (37 * hash) + EXECUTED_FIELD_NUMBER;
+ hash = (53 * hash) + hashBoolean(getExecuted());
+ }
+ hash = (29 * hash) + getUnknownFields().hashCode();
+ memoizedHashCode = hash;
+ return hash;
+ }
+
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData parseFrom(
+ com.google.protobuf.ByteString data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData parseFrom(
+ com.google.protobuf.ByteString data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData parseFrom(byte[] data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData parseFrom(
+ byte[] data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData parseFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData parseFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData parseDelimitedFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData parseDelimitedFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input, extensionRegistry);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData parseFrom(
+ com.google.protobuf.CodedInputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData parseFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+
+ public static Builder newBuilder() { return Builder.create(); }
+ public Builder newBuilderForType() { return newBuilder(); }
+ public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData prototype) {
+ return newBuilder().mergeFrom(prototype);
+ }
+ public Builder toBuilder() { return newBuilder(this); }
+
+ @java.lang.Override
+ protected Builder newBuilderForType(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ Builder builder = new Builder(parent);
+ return builder;
+ }
+ /**
+ * Protobuf type {@code SequentialProcedureData}
+ *
+ * <pre>
+ **
+ * SequentialProcedure data
+ * </pre>
+ */
+ public static final class Builder extends
+ com.google.protobuf.GeneratedMessage.Builder<Builder>
+ implements org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureDataOrBuilder {
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.internal_static_SequentialProcedureData_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.internal_static_SequentialProcedureData_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData.class, org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData.Builder.class);
+ }
+
+ // Construct using org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData.newBuilder()
+ private Builder() {
+ maybeForceBuilderInitialization();
+ }
+
+ private Builder(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ super(parent);
+ maybeForceBuilderInitialization();
+ }
+ private void maybeForceBuilderInitialization() {
+ if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+ }
+ }
+ private static Builder create() {
+ return new Builder();
+ }
+
+ public Builder clear() {
+ super.clear();
+ executed_ = false;
+ bitField0_ = (bitField0_ & ~0x00000001);
+ return this;
+ }
+
+ public Builder clone() {
+ return create().mergeFrom(buildPartial());
+ }
+
+ public com.google.protobuf.Descriptors.Descriptor
+ getDescriptorForType() {
+ return org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.internal_static_SequentialProcedureData_descriptor;
+ }
+
+ public org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData getDefaultInstanceForType() {
+ return org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData.getDefaultInstance();
+ }
+
+ public org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData build() {
+ org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData result = buildPartial();
+ if (!result.isInitialized()) {
+ throw newUninitializedMessageException(result);
+ }
+ return result;
+ }
+
+ public org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData buildPartial() {
+ org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData result = new org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData(this);
+ int from_bitField0_ = bitField0_;
+ int to_bitField0_ = 0;
+ if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+ to_bitField0_ |= 0x00000001;
+ }
+ result.executed_ = executed_;
+ result.bitField0_ = to_bitField0_;
+ onBuilt();
+ return result;
+ }
+
+ public Builder mergeFrom(com.google.protobuf.Message other) {
+ if (other instanceof org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData) {
+ return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData)other);
+ } else {
+ super.mergeFrom(other);
+ return this;
+ }
+ }
+
+ public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData other) {
+ if (other == org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData.getDefaultInstance()) return this;
+ if (other.hasExecuted()) {
+ setExecuted(other.getExecuted());
+ }
+ this.mergeUnknownFields(other.getUnknownFields());
+ return this;
+ }
+
+ public final boolean isInitialized() {
+ if (!hasExecuted()) {
+
+ return false;
+ }
+ return tr
<TRUNCATED>
[05/15] hbase git commit: HBASE-13265 Make thrift2 usable from c++
Posted by mb...@apache.org.
HBASE-13265 Make thrift2 usable from c++
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/97bfc2de
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/97bfc2de
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/97bfc2de
Branch: refs/heads/hbase-12439
Commit: 97bfc2ded7b7881d41a7ef35055bbfa5cbc65afe
Parents: e26c4e4
Author: Elliott Clark <ec...@apache.org>
Authored: Tue Mar 17 10:07:27 2015 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Wed Mar 25 17:17:40 2015 -0700
----------------------------------------------------------------------
.../apache/hadoop/hbase/thrift2/hbase.thrift | 32 ++++++++++----------
1 file changed, 16 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/97bfc2de/hbase-thrift/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift
----------------------------------------------------------------------
diff --git a/hbase-thrift/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift b/hbase-thrift/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift
index 7c6d6a2..bc790e7 100644
--- a/hbase-thrift/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift
+++ b/hbase-thrift/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift
@@ -230,8 +230,8 @@ struct TScan {
* Atomic mutation for the specified row. It can be either Put or Delete.
*/
union TMutation {
- 1: optional TPut put,
- 2: optional TDelete deleteSingle,
+ 1: TPut put,
+ 2: TDelete deleteSingle,
}
/**
@@ -275,7 +275,7 @@ service THBaseService {
1: required binary table,
/** the TGet to check for */
- 2: required TGet get
+ 2: required TGet tget
) throws (1:TIOError io)
/**
@@ -291,7 +291,7 @@ service THBaseService {
1: required binary table,
/** the TGet to fetch */
- 2: required TGet get
+ 2: required TGet tget
) throws (1: TIOError io)
/**
@@ -310,7 +310,7 @@ service THBaseService {
/** a list of TGets to fetch, the Result list
will have the Results at corresponding positions
or null if there was an error */
- 2: required list<TGet> gets
+ 2: required list<TGet> tgets
) throws (1: TIOError io)
/**
@@ -321,7 +321,7 @@ service THBaseService {
1: required binary table,
/** the TPut to put */
- 2: required TPut put
+ 2: required TPut tput
) throws (1: TIOError io)
/**
@@ -349,7 +349,7 @@ service THBaseService {
5: binary value,
/** the TPut to put if the check succeeds */
- 6: required TPut put
+ 6: required TPut tput
) throws (1: TIOError io)
/**
@@ -360,7 +360,7 @@ service THBaseService {
1: required binary table,
/** a list of TPuts to commit */
- 2: required list<TPut> puts
+ 2: required list<TPut> tputs
) throws (1: TIOError io)
/**
@@ -374,7 +374,7 @@ service THBaseService {
1: required binary table,
/** the TDelete to delete */
- 2: required TDelete deleteSingle
+ 2: required TDelete tdelete
) throws (1: TIOError io)
/**
@@ -389,7 +389,7 @@ service THBaseService {
1: required binary table,
/** list of TDeletes to delete */
- 2: required list<TDelete> deletes
+ 2: required list<TDelete> tdeletes
) throws (1: TIOError io)
/**
@@ -417,7 +417,7 @@ service THBaseService {
5: binary value,
/** the TDelete to execute if the check succeeds */
- 6: required TDelete deleteSingle
+ 6: required TDelete tdelete
) throws (1: TIOError io)
TResult increment(
@@ -425,7 +425,7 @@ service THBaseService {
1: required binary table,
/** the TIncrement to increment */
- 2: required TIncrement increment
+ 2: required TIncrement tincrement
) throws (1: TIOError io)
TResult append(
@@ -433,7 +433,7 @@ service THBaseService {
1: required binary table,
/** the TAppend to append */
- 2: required TAppend append
+ 2: required TAppend tappend
) throws (1: TIOError io)
/**
@@ -446,7 +446,7 @@ service THBaseService {
1: required binary table,
/** the scan object to get a Scanner for */
- 2: required TScan scan,
+ 2: required TScan tscan,
) throws (1: TIOError io)
/**
@@ -490,7 +490,7 @@ service THBaseService {
1: required binary table,
/** mutations to apply */
- 2: required TRowMutations rowMutations
+ 2: required TRowMutations trowMutations
) throws (1: TIOError io)
/**
@@ -504,7 +504,7 @@ service THBaseService {
1: required binary table,
/** the scan object to get a Scanner for */
- 2: required TScan scan,
+ 2: required TScan tscan,
/** number of rows to return */
3: i32 numRows = 1
[15/15] hbase git commit: HBASE-13202 Procedure v2 - core framework
Posted by mb...@apache.org.
HBASE-13202 Procedure v2 - core framework
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ddc1bcae
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ddc1bcae
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ddc1bcae
Branch: refs/heads/hbase-12439
Commit: ddc1bcae9d8b81b8890db43947772351d85a05e8
Parents: 69f6a2e
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Wed Mar 25 18:26:20 2015 +0000
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Thu Mar 26 09:52:06 2015 +0000
----------------------------------------------------------------------
.../hadoop/hbase/io/util/StreamUtils.java | 12 +-
.../hadoop/hbase/util/ForeignExceptionUtil.java | 105 +
hbase-procedure/pom.xml | 200 +
.../hbase/procedure2/OnePhaseProcedure.java | 28 +
.../hadoop/hbase/procedure2/Procedure.java | 659 ++
.../procedure2/ProcedureAbortedException.java | 42 +
.../hbase/procedure2/ProcedureException.java | 45 +
.../hbase/procedure2/ProcedureExecutor.java | 1006 +++
.../procedure2/ProcedureFairRunQueues.java | 146 +
.../hbase/procedure2/ProcedureResult.java | 95 +
.../hbase/procedure2/ProcedureRunnableSet.java | 71 +
.../procedure2/ProcedureSimpleRunQueue.java | 116 +
.../procedure2/ProcedureYieldException.java | 40 +
.../procedure2/RemoteProcedureException.java | 116 +
.../hbase/procedure2/RootProcedureState.java | 177 +
.../hbase/procedure2/SequentialProcedure.java | 80 +
.../hbase/procedure2/StateMachineProcedure.java | 140 +
.../hbase/procedure2/TwoPhaseProcedure.java | 28 +
.../hbase/procedure2/store/ProcedureStore.java | 110 +
.../procedure2/store/ProcedureStoreTracker.java | 540 ++
.../CorruptedWALProcedureStoreException.java | 43 +
.../procedure2/store/wal/ProcedureWALFile.java | 152 +
.../store/wal/ProcedureWALFormat.java | 212 +
.../store/wal/ProcedureWALFormatReader.java | 161 +
.../procedure2/store/wal/WALProcedureStore.java | 683 ++
.../hadoop/hbase/procedure2/util/ByteSlot.java | 111 +
.../hbase/procedure2/util/StringUtils.java | 80 +
.../procedure2/util/TimeoutBlockingQueue.java | 216 +
.../procedure2/ProcedureTestingUtility.java | 130 +
.../procedure2/TestProcedureExecution.java | 303 +
.../hbase/procedure2/TestProcedureRecovery.java | 488 ++
.../procedure2/TestProcedureRunQueues.java | 72 +
.../store/TestProcedureStoreTracker.java | 168 +
.../store/wal/TestWALProcedureStore.java | 267 +
.../util/TestTimeoutBlockingQueue.java | 137 +
hbase-protocol/pom.xml | 1 +
.../protobuf/generated/ProcedureProtos.java | 7219 ++++++++++++++++++
.../src/main/protobuf/Procedure.proto | 114 +
pom.xml | 21 +-
39 files changed, 14329 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/StreamUtils.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/StreamUtils.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/StreamUtils.java
index 314ed2b..0b442a5 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/StreamUtils.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/StreamUtils.java
@@ -120,7 +120,7 @@ public class StreamUtils {
/**
* Reads a varInt value stored in an array.
- *
+ *
* @param input
* Input array where the varInt is available
* @param offset
@@ -198,4 +198,14 @@ public class StreamUtils {
out.write((byte) (0xff & (v >> 8)));
out.write((byte) (0xff & v));
}
+
+ public static long readLong(InputStream in) throws IOException {
+ long result = 0;
+ for (int shift = 56; shift >= 0; shift -= 8) {
+ long x = in.read();
+ if (x < 0) throw new IOException("EOF");
+ result |= (x << shift);
+ }
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ForeignExceptionUtil.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ForeignExceptionUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ForeignExceptionUtil.java
new file mode 100644
index 0000000..94a8c8c
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ForeignExceptionUtil.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage;
+import org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.GenericExceptionMessage;
+import org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.StackTraceElementMessage;
+
+/**
+ * Helper to convert Exceptions and StackTraces from/to protobuf.
+ * (see ErrorHandling.proto for the internal of the proto messages)
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public final class ForeignExceptionUtil {
+ private ForeignExceptionUtil() { }
+
+ public static IOException toIOException(final ForeignExceptionMessage eem) {
+ GenericExceptionMessage gem = eem.getGenericException();
+ StackTraceElement[] trace = toStackTrace(gem.getTraceList());
+ RemoteException re = new RemoteException(gem.getClassName(), gem.getMessage());
+ re.setStackTrace(trace);
+ return re.unwrapRemoteException();
+ }
+
+ public static ForeignExceptionMessage toProtoForeignException(String source, Throwable t) {
+ GenericExceptionMessage.Builder gemBuilder = GenericExceptionMessage.newBuilder();
+ gemBuilder.setClassName(t.getClass().getName());
+ if (t.getMessage() != null) {
+ gemBuilder.setMessage(t.getMessage());
+ }
+ // set the stack trace, if there is one
+ List<StackTraceElementMessage> stack = toProtoStackTraceElement(t.getStackTrace());
+ if (stack != null) {
+ gemBuilder.addAllTrace(stack);
+ }
+ GenericExceptionMessage payload = gemBuilder.build();
+ ForeignExceptionMessage.Builder exception = ForeignExceptionMessage.newBuilder();
+ exception.setGenericException(payload).setSource(source);
+ return exception.build();
+ }
+
+ /**
+ * Convert a stack trace to list of {@link StackTraceElement}.
+ * @param trace the stack trace to convert to protobuf message
+ * @return <tt>null</tt> if the passed stack is <tt>null</tt>.
+ */
+ public static List<StackTraceElementMessage> toProtoStackTraceElement(StackTraceElement[] trace) {
+ // if there is no stack trace, ignore it and just return the message
+ if (trace == null) return null;
+ // build the stack trace for the message
+ List<StackTraceElementMessage> pbTrace = new ArrayList<StackTraceElementMessage>(trace.length);
+ for (StackTraceElement elem : trace) {
+ StackTraceElementMessage.Builder stackBuilder = StackTraceElementMessage.newBuilder();
+ stackBuilder.setDeclaringClass(elem.getClassName());
+ stackBuilder.setFileName(elem.getFileName());
+ stackBuilder.setLineNumber(elem.getLineNumber());
+ stackBuilder.setMethodName(elem.getMethodName());
+ pbTrace.add(stackBuilder.build());
+ }
+ return pbTrace;
+ }
+
+ /**
+ * Unwind a serialized array of {@link StackTraceElementMessage}s to a
+ * {@link StackTraceElement}s.
+ * @param traceList list that was serialized
+ * @return the deserialized list or <tt>null</tt> if it couldn't be unwound (e.g. wasn't set on
+ * the sender).
+ */
+ public static StackTraceElement[] toStackTrace(List<StackTraceElementMessage> traceList) {
+ if (traceList == null || traceList.size() == 0) {
+ return new StackTraceElement[0]; // empty array
+ }
+ StackTraceElement[] trace = new StackTraceElement[traceList.size()];
+ for (int i = 0; i < traceList.size(); i++) {
+ StackTraceElementMessage elem = traceList.get(i);
+ trace[i] = new StackTraceElement(
+ elem.getDeclaringClass(), elem.getMethodName(), elem.getFileName(), elem.getLineNumber());
+ }
+ return trace;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-procedure/pom.xml b/hbase-procedure/pom.xml
new file mode 100644
index 0000000..fce3102
--- /dev/null
+++ b/hbase-procedure/pom.xml
@@ -0,0 +1,200 @@
+<?xml version="1.0"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>hbase</artifactId>
+ <groupId>org.apache.hbase</groupId>
+ <version>2.0.0-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>hbase-procedure</artifactId>
+ <name>HBase - Procedure</name>
+ <description>Procedure Framework</description>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>default-compile</id>
+ <configuration>
+ <compilerId>javac-with-errorprone</compilerId>
+ <forceJavacCompilerUse>true</forceJavacCompilerUse>
+ </configuration>
+ </execution>
+ <execution>
+ <id>default-testCompile</id>
+ <configuration>
+ <compilerId>javac-with-errorprone</compilerId>
+ <forceJavacCompilerUse>true</forceJavacCompilerUse>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ <!-- Make a jar and put the sources in the jar -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-source-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <!--Make it so assembly:single does nothing in here-->
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>${maven.assembly.version}</version>
+ <configuration>
+ <skipAssembly>true</skipAssembly>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <!-- Always skip the second part executions, since we only run
+ simple unit tests in this module. -->
+ <executions>
+ <execution>
+ <id>secondPartTestsExecution</id>
+ <phase>test</phase>
+ <goals>
+ <goal>test</goal>
+ </goals>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <version>${project.version}</version>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-annotations</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-protocol</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </dependency>
+ </dependencies>
+
+ <profiles>
+ <!-- Profiles for building against different hadoop versions -->
+ <profile>
+ <id>hadoop-1.1</id>
+ <activation>
+ <property>
+ <!--Below formatting for dev-support/generate-hadoopX-poms.sh-->
+ <!--h1--><name>hadoop.profile</name><value>1.1</value>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+ <profile>
+ <id>hadoop-1.0</id>
+ <activation>
+ <property>
+ <name>hadoop.profile</name>
+ <value>1.0</value>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+ <!--
+ profile for building against Hadoop 2.0.0-alpha. Activate using:
+ mvn -Dhadoop.profile=2.0
+ -->
+ <profile>
+ <id>hadoop-2.0</id>
+ <activation>
+ <property>
+ <!--Below formatting for dev-support/generate-hadoopX-poms.sh-->
+ <!--h2--><name>!hadoop.profile</name>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+ <!--
+ profile for building against Hadoop 3.0.x. Activate using:
+ mvn -Dhadoop.profile=3.0
+ -->
+ <profile>
+ <id>hadoop-3.0</id>
+ <activation>
+ <property>
+ <name>hadoop.profile</name>
+ <value>3.0</value>
+ </property>
+ </activation>
+ <properties>
+ <hadoop.version>3.0-SNAPSHOT</hadoop.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+</project>
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/OnePhaseProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/OnePhaseProcedure.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/OnePhaseProcedure.java
new file mode 100644
index 0000000..1c3be2d
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/OnePhaseProcedure.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public abstract class OnePhaseProcedure<TEnvironment> extends Procedure<TEnvironment> {
+ // TODO (e.g. used by online snapshots)
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
new file mode 100644
index 0000000..899fe8d
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
@@ -0,0 +1,659 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Modifier;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.procedure2.util.StringUtils;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState;
+import org.apache.hadoop.hbase.util.ByteStringer;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.protobuf.ByteString;
+
+/**
+ * Base Procedure class responsible to handle the Procedure Metadata
+ * e.g. state, startTime, lastUpdate, stack-indexes, ...
+ *
+ * execute() is called each time the procedure is executed.
+ * it may be called multiple times in case of failure and restart, so the
+ * code must be idempotent.
+ * the return is a set of sub-procedures or null in case the procedure doesn't
+ * have sub-procedures. Once the sub-procedures are successfully completed
+ * the execute() method is called again, you should think at it as a stack:
+ * -> step 1
+ * ---> step 2
+ * -> step 1
+ *
+ * rollback() is called when the procedure or one of the sub-procedures is failed.
+ * the rollback step is supposed to cleanup the resources created during the
+ * execute() step. in case of failure and restart rollback() may be called
+ * multiple times, so the code must be idempotent.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public abstract class Procedure<TEnvironment> implements Comparable<Procedure> {
+ // unchanged after initialization
+ private String owner = null;
+ private Long parentProcId = null;
+ private Long procId = null;
+ private long startTime;
+
+ // runtime state, updated every operation
+ private ProcedureState state = ProcedureState.INITIALIZING;
+ private Integer timeout = null;
+ private int[] stackIndexes = null;
+ private int childrenLatch = 0;
+ private long lastUpdate;
+
+ private RemoteProcedureException exception = null;
+ private byte[] result = null;
+
+ /**
+ * The main code of the procedure. It must be idempotent since execute()
+ * may be called multiple time in case of machine failure in the middle
+ * of the execution.
+ * @return a set of sub-procedures or null if there is nothing else to execute.
+ */
+ protected abstract Procedure[] execute(TEnvironment env)
+ throws ProcedureYieldException;
+
+ /**
+ * The code to undo what done by the execute() code.
+ * It is called when the procedure or one of the sub-procedure failed or an
+ * abort was requested. It should cleanup all the resources created by
+ * the execute() call. The implementation must be idempotent since rollback()
+ * may be called multiple time in case of machine failure in the middle
+ * of the execution.
+ */
+ protected abstract void rollback(TEnvironment env);
+
+ /**
+ * The abort() call is asynchronous and each procedure must decide how to deal
+ * with that, if they want to be abortable. The simplest implementation
+ * is to have an AtomicBoolean set in the abort() method and then the execute()
+ * will check if the abort flag is set or not.
+ * abort() may be called multiple times from the client, so the implementation
+ * must be idempotent.
+ *
+ * NOTE: abort() is not like Thread.interrupt() it is just a notification
+ * that allows the procedure implementor where to abort to avoid leak and
+ * have a better control on what was executed and what not.
+ */
+ protected abstract boolean abort(TEnvironment env);
+
+ /**
+ * The user-level code of the procedure may have some state to
+ * persist (e.g. input arguments) to be able to resume on failure.
+ * @param stream the stream that will contain the user serialized data
+ */
+ protected abstract void serializeStateData(final OutputStream stream)
+ throws IOException;
+
+ /**
+ * Called on store load to allow the user to decode the previously serialized
+ * state.
+ * @param stream the stream that contains the user serialized data
+ */
+ protected abstract void deserializeStateData(final InputStream stream)
+ throws IOException;
+
+ /**
+ * The user should override this method, and try to take a lock if necessary.
+ * A lock can be anything, and it is up to the implementor.
+ * Example: in our Master we can execute request in parallel for different tables
+ * create t1 and create t2 can be executed at the same time.
+ * anything else on t1/t2 is queued waiting that specific table create to happen.
+ *
+ * @return true if the lock was acquired and false otherwise
+ */
+ protected boolean acquireLock(final TEnvironment env) {
+ return true;
+ }
+
+ /**
+ * The user should override this method, and release lock if necessary.
+ */
+ protected void releaseLock(final TEnvironment env) {
+ // no-op
+ }
+
+ /**
+ * Called when the procedure is marked as completed (success or rollback).
+ * The procedure implementor may use this method to cleanup in-memory states.
+ * This operation will not be retried on failure.
+ */
+ protected void completionCleanup(final TEnvironment env) {
+ // no-op
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(getClass().getName());
+
+ if (procId != null) {
+ sb.append(" id=");
+ sb.append(getProcId());
+ }
+
+ if (hasParent()) {
+ sb.append(" parent=");
+ sb.append(getParentProcId());
+ }
+
+ if (hasOwner()) {
+ sb.append(" owner=");
+ sb.append(getOwner());
+ }
+
+ sb.append(" state=");
+ sb.append(getState());
+ return sb.toString();
+ }
+
+ /**
+ * @return the serialized result if any, otherwise null
+ */
+ public byte[] getResult() {
+ return result;
+ }
+
+ /**
+ * The procedure may leave a "result" on completion.
+ * @param result the serialized result that will be passed to the client
+ */
+ protected void setResult(final byte[] result) {
+ this.result = result;
+ }
+
+ public long getProcId() {
+ return procId;
+ }
+
+ public boolean hasParent() {
+ return parentProcId != null;
+ }
+
+ public boolean hasException() {
+ return exception != null;
+ }
+
+ public boolean hasTimeout() {
+ return timeout != null;
+ }
+
+ public long getParentProcId() {
+ return parentProcId;
+ }
+
+ /**
+ * @return true if the procedure has failed.
+ * true may mean failed but not yet rolledback or failed and rolledback.
+ */
+ public synchronized boolean isFailed() {
+ return exception != null || state == ProcedureState.ROLLEDBACK;
+ }
+
+ /**
+ * @return true if the procedure is finished successfully.
+ */
+ public synchronized boolean isSuccess() {
+ return state == ProcedureState.FINISHED && exception == null;
+ }
+
+ /**
+ * @return true if the procedure is finished. The Procedure may be completed
+ * successfuly or failed and rolledback.
+ */
+ public synchronized boolean isFinished() {
+ switch (state) {
+ case ROLLEDBACK:
+ return true;
+ case FINISHED:
+ return exception == null;
+ default:
+ break;
+ }
+ return false;
+ }
+
+ /**
+ * @return true if the procedure is waiting for a child to finish or for an external event.
+ */
+ public synchronized boolean isWaiting() {
+ switch (state) {
+ case WAITING:
+ case WAITING_TIMEOUT:
+ return true;
+ default:
+ break;
+ }
+ return false;
+ }
+
+ public synchronized RemoteProcedureException getException() {
+ return exception;
+ }
+
+ public long getStartTime() {
+ return startTime;
+ }
+
+ public synchronized long getLastUpdate() {
+ return lastUpdate;
+ }
+
+ public synchronized long elapsedTime() {
+ return lastUpdate - startTime;
+ }
+
+ /**
+ * @param timeout timeout in msec
+ */
+ protected void setTimeout(final int timeout) {
+ this.timeout = timeout;
+ }
+
+ /**
+ * @return the timeout in msec
+ */
+ public int getTimeout() {
+ return timeout;
+ }
+
+ /**
+ * @return the remaining time before the timeout
+ */
+ public long getTimeRemaining() {
+ return Math.max(0, timeout - (EnvironmentEdgeManager.currentTime() - startTime));
+ }
+
+ protected void setOwner(final String owner) {
+ this.owner = StringUtils.isEmpty(owner) ? null : owner;
+ }
+
+ public String getOwner() {
+ return owner;
+ }
+
+ public boolean hasOwner() {
+ return owner != null;
+ }
+
+ @VisibleForTesting
+ @InterfaceAudience.Private
+ protected synchronized void setState(final ProcedureState state) {
+ this.state = state;
+ updateTimestamp();
+ }
+
+ @InterfaceAudience.Private
+ protected synchronized ProcedureState getState() {
+ return state;
+ }
+
+ protected void setFailure(final String source, final Throwable cause) {
+ setFailure(new RemoteProcedureException(source, cause));
+ }
+
+ protected synchronized void setFailure(final RemoteProcedureException exception) {
+ this.exception = exception;
+ if (!isFinished()) {
+ setState(ProcedureState.FINISHED);
+ }
+ }
+
+ protected void setAbortFailure(final String source, final String msg) {
+ setFailure(source, new ProcedureAbortedException(msg));
+ }
+
+ @InterfaceAudience.Private
+ protected synchronized boolean setTimeoutFailure() {
+ if (state == ProcedureState.WAITING_TIMEOUT) {
+ long timeDiff = EnvironmentEdgeManager.currentTime() - lastUpdate;
+ setFailure("ProcedureExecutor", new TimeoutException(
+ "Operation timed out after " + StringUtils.humanTimeDiff(timeDiff)));
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Called by the ProcedureExecutor to assign the ID to the newly created procedure.
+ */
+ @VisibleForTesting
+ @InterfaceAudience.Private
+ protected void setProcId(final long procId) {
+ this.procId = procId;
+ this.startTime = EnvironmentEdgeManager.currentTime();
+ setState(ProcedureState.RUNNABLE);
+ }
+
+ /**
+ * Called by the ProcedureExecutor to assign the parent to the newly created procedure.
+ */
+ @InterfaceAudience.Private
+ protected void setParentProcId(final long parentProcId) {
+ this.parentProcId = parentProcId;
+ }
+
+ /**
+ * Internal method called by the ProcedureExecutor that starts the
+ * user-level code execute().
+ */
+ @InterfaceAudience.Private
+ protected Procedure[] doExecute(final TEnvironment env)
+ throws ProcedureYieldException {
+ try {
+ updateTimestamp();
+ return execute(env);
+ } finally {
+ updateTimestamp();
+ }
+ }
+
+ /**
+ * Internal method called by the ProcedureExecutor that starts the
+ * user-level code rollback().
+ */
+ @InterfaceAudience.Private
+ protected void doRollback(final TEnvironment env) {
+ try {
+ updateTimestamp();
+ rollback(env);
+ } finally {
+ updateTimestamp();
+ }
+ }
+
+ /**
+ * Called on store load to initialize the Procedure internals after
+ * the creation/deserialization.
+ */
+ @InterfaceAudience.Private
+ protected void setStartTime(final long startTime) {
+ this.startTime = startTime;
+ }
+
+ /**
+ * Called on store load to initialize the Procedure internals after
+ * the creation/deserialization.
+ */
+ private synchronized void setLastUpdate(final long lastUpdate) {
+ this.lastUpdate = lastUpdate;
+ }
+
+ protected synchronized void updateTimestamp() {
+ this.lastUpdate = EnvironmentEdgeManager.currentTime();
+ }
+
+ /**
+ * Called by the ProcedureExecutor on procedure-load to restore the latch state
+ */
+ @InterfaceAudience.Private
+ protected synchronized void setChildrenLatch(final int numChildren) {
+ this.childrenLatch = numChildren;
+ }
+
+ /**
+ * Called by the ProcedureExecutor on procedure-load to restore the latch state
+ */
+ @InterfaceAudience.Private
+ protected synchronized void incChildrenLatch() {
+ // TODO: can this be inferred from the stack? I think so...
+ this.childrenLatch++;
+ }
+
+ /**
+ * Called by the ProcedureExecutor to notify that one of the sub-procedures
+ * has completed.
+ */
+ @InterfaceAudience.Private
+ protected synchronized boolean childrenCountDown() {
+ assert childrenLatch > 0;
+ return --childrenLatch == 0;
+ }
+
+ /**
+ * Called by the RootProcedureState on procedure execution.
+ * Each procedure store its stack-index positions.
+ */
+ @InterfaceAudience.Private
+ protected void addStackIndex(final int index) {
+ if (stackIndexes == null) {
+ stackIndexes = new int[] { index };
+ } else {
+ int count = stackIndexes.length;
+ stackIndexes = Arrays.copyOf(stackIndexes, count + 1);
+ stackIndexes[count] = index;
+ }
+ }
+
+ @InterfaceAudience.Private
+ protected boolean removeStackIndex() {
+ if (stackIndexes.length > 1) {
+ stackIndexes = Arrays.copyOf(stackIndexes, stackIndexes.length - 1);
+ return false;
+ } else {
+ stackIndexes = null;
+ return true;
+ }
+ }
+
+ /**
+ * Called on store load to initialize the Procedure internals after
+ * the creation/deserialization.
+ */
+ @InterfaceAudience.Private
+ protected void setStackIndexes(final List<Integer> stackIndexes) {
+ this.stackIndexes = new int[stackIndexes.size()];
+ for (int i = 0; i < this.stackIndexes.length; ++i) {
+ this.stackIndexes[i] = stackIndexes.get(i);
+ }
+ }
+
+ @InterfaceAudience.Private
+ protected boolean wasExecuted() {
+ return stackIndexes != null;
+ }
+
+ @InterfaceAudience.Private
+ protected int[] getStackIndexes() {
+ return stackIndexes;
+ }
+
+ @Override
+ public int compareTo(final Procedure other) {
+ long diff = getProcId() - other.getProcId();
+ return (diff < 0) ? -1 : (diff > 0) ? 1 : 0;
+ }
+
+ /*
+ * Helper to lookup the root Procedure ID given a specified procedure.
+ */
+ @InterfaceAudience.Private
+ protected static Long getRootProcedureId(final Map<Long, Procedure> procedures, Procedure proc) {
+ while (proc.hasParent()) {
+ proc = procedures.get(proc.getParentProcId());
+ if (proc == null) return null;
+ }
+ return proc.getProcId();
+ }
+
+ protected static Procedure newInstance(final String className) throws IOException {
+ try {
+ Class<?> clazz = Class.forName(className);
+ if (!Modifier.isPublic(clazz.getModifiers())) {
+ throw new Exception("the " + clazz + " class is not public");
+ }
+
+ Constructor<?> ctor = clazz.getConstructor();
+ assert ctor != null : "no constructor found";
+ if (!Modifier.isPublic(ctor.getModifiers())) {
+ throw new Exception("the " + clazz + " constructor is not public");
+ }
+ return (Procedure)ctor.newInstance();
+ } catch (Exception e) {
+ throw new IOException("The procedure class " + className +
+ " must be accessible and have an empty constructor", e);
+ }
+ }
+
+ protected static void validateClass(final Procedure proc) throws IOException {
+ try {
+ Class<?> clazz = proc.getClass();
+ if (!Modifier.isPublic(clazz.getModifiers())) {
+ throw new Exception("the " + clazz + " class is not public");
+ }
+
+ Constructor<?> ctor = clazz.getConstructor();
+ assert ctor != null;
+ if (!Modifier.isPublic(ctor.getModifiers())) {
+ throw new Exception("the " + clazz + " constructor is not public");
+ }
+ } catch (Exception e) {
+ throw new IOException("The procedure class " + proc.getClass().getName() +
+ " must be accessible and have an empty constructor", e);
+ }
+ }
+
+ /**
+ * Helper to convert the procedure to protobuf.
+ * Used by ProcedureStore implementations.
+ */
+ @InterfaceAudience.Private
+ public static ProcedureProtos.Procedure convert(final Procedure proc)
+ throws IOException {
+ Preconditions.checkArgument(proc != null);
+ validateClass(proc);
+
+ ProcedureProtos.Procedure.Builder builder = ProcedureProtos.Procedure.newBuilder()
+ .setClassName(proc.getClass().getName())
+ .setProcId(proc.getProcId())
+ .setState(proc.getState())
+ .setStartTime(proc.getStartTime())
+ .setLastUpdate(proc.getLastUpdate());
+
+ if (proc.hasParent()) {
+ builder.setParentId(proc.getParentProcId());
+ }
+
+ if (proc.hasTimeout()) {
+ builder.setTimeout(proc.getTimeout());
+ }
+
+ if (proc.hasOwner()) {
+ builder.setOwner(proc.getOwner());
+ }
+
+ int[] stackIds = proc.getStackIndexes();
+ if (stackIds != null) {
+ for (int i = 0; i < stackIds.length; ++i) {
+ builder.addStackId(stackIds[i]);
+ }
+ }
+
+ if (proc.hasException()) {
+ RemoteProcedureException exception = proc.getException();
+ builder.setException(
+ RemoteProcedureException.toProto(exception.getSource(), exception.getCause()));
+ }
+
+ byte[] result = proc.getResult();
+ if (result != null) {
+ builder.setResult(ByteStringer.wrap(result));
+ }
+
+ ByteString.Output stateStream = ByteString.newOutput();
+ proc.serializeStateData(stateStream);
+ if (stateStream.size() > 0) {
+ builder.setStateData(stateStream.toByteString());
+ }
+
+ return builder.build();
+ }
+
+ /**
+ * Helper to convert the protobuf procedure.
+ * Used by ProcedureStore implementations.
+ *
+ * TODO: OPTIMIZATION: some of the field never change during the execution
+ * (e.g. className, procId, parentId, ...).
+ * We can split in 'data' and 'state', and the store
+ * may take advantage of it by storing the data only on insert().
+ */
+ @InterfaceAudience.Private
+ public static Procedure convert(final ProcedureProtos.Procedure proto)
+ throws IOException {
+ // Procedure from class name
+ Procedure proc = Procedure.newInstance(proto.getClassName());
+
+ // set fields
+ proc.setProcId(proto.getProcId());
+ proc.setState(proto.getState());
+ proc.setStartTime(proto.getStartTime());
+ proc.setLastUpdate(proto.getLastUpdate());
+
+ if (proto.hasParentId()) {
+ proc.setParentProcId(proto.getParentId());
+ }
+
+ if (proto.hasOwner()) {
+ proc.setOwner(proto.getOwner());
+ }
+
+ if (proto.hasTimeout()) {
+ proc.setTimeout(proto.getTimeout());
+ }
+
+ if (proto.getStackIdCount() > 0) {
+ proc.setStackIndexes(proto.getStackIdList());
+ }
+
+ if (proto.hasException()) {
+ assert proc.getState() == ProcedureState.FINISHED ||
+ proc.getState() == ProcedureState.ROLLEDBACK :
+ "The procedure must be failed (waiting to rollback) or rolledback";
+ proc.setFailure(RemoteProcedureException.fromProto(proto.getException()));
+ }
+
+ if (proto.hasResult()) {
+ proc.setResult(proto.getResult().toByteArray());
+ }
+
+ // we want to call deserialize even when the stream is empty, mainly for testing.
+ proc.deserializeStateData(proto.getStateData().newInput());
+
+ return proc;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureAbortedException.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureAbortedException.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureAbortedException.java
new file mode 100644
index 0000000..2e409cf
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureAbortedException.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * Thrown when a procedure is aborted
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class ProcedureAbortedException extends ProcedureException {
+ /** default constructor */
+ public ProcedureAbortedException() {
+ super();
+ }
+
+ /**
+ * Constructor
+ * @param s message
+ */
+ public ProcedureAbortedException(String s) {
+ super(s);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureException.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureException.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureException.java
new file mode 100644
index 0000000..9f922b1
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureException.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class ProcedureException extends IOException {
+ /** default constructor */
+ public ProcedureException() {
+ super();
+ }
+
+ /**
+ * Constructor
+ * @param s message
+ */
+ public ProcedureException(String s) {
+ super(s);
+ }
+
+ public ProcedureException(Throwable t) {
+ super(t);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
new file mode 100644
index 0000000..0379579
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
@@ -0,0 +1,1006 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.HashSet;
+import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureStore;
+import org.apache.hadoop.hbase.procedure2.util.TimeoutBlockingQueue;
+import org.apache.hadoop.hbase.procedure2.util.TimeoutBlockingQueue.TimeoutRetriever;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.procedure2.util.StringUtils;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Thread Pool that executes the submitted procedures.
+ * The executor has a ProcedureStore associated.
+ * Each operation is logged and on restart the pending procedures are resumed.
+ *
+ * Unless the Procedure code throws an error (e.g. invalid user input)
+ * the procedure will complete (at some point in time), On restart the pending
+ * procedures are resumed and the once failed will be rolledback.
+ *
+ * The user can add procedures to the executor via submitProcedure(proc)
+ * check for the finished state via isFinished(procId)
+ * and get the result via getResult(procId)
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class ProcedureExecutor<TEnvironment> {
+ private static final Log LOG = LogFactory.getLog(ProcedureExecutor.class);
+
+ Testing testing = null;
+ public static class Testing {
+ protected boolean killBeforeStoreUpdate = false;
+ protected boolean toggleKillBeforeStoreUpdate = false;
+
+ protected boolean shouldKillBeforeStoreUpdate() {
+ final boolean kill = this.killBeforeStoreUpdate;
+ if (this.toggleKillBeforeStoreUpdate) {
+ this.killBeforeStoreUpdate = !kill;
+ LOG.warn("Toggle Kill before store update to: " + this.killBeforeStoreUpdate);
+ }
+ return kill;
+ }
+ }
+
+ public interface ProcedureExecutorListener {
+ void procedureLoaded(long procId);
+ void procedureAdded(long procId);
+ void procedureFinished(long procId);
+ }
+
+ /**
+ * Used by the TimeoutBlockingQueue to get the timeout interval of the procedure
+ */
+ private static class ProcedureTimeoutRetriever implements TimeoutRetriever<Procedure> {
+ @Override
+ public long getTimeout(Procedure proc) {
+ long delta = (EnvironmentEdgeManager.currentTime() - proc.getLastUpdate());
+ return Math.max(0, proc.getTimeout() - delta);
+ }
+
+ @Override
+ public TimeUnit getTimeUnit(Procedure proc) {
+ return TimeUnit.MILLISECONDS;
+ }
+ }
+
+ /**
+ * Internal cleaner that removes the completed procedure results after a TTL.
+ * NOTE: This is a special case handled in timeoutLoop().
+ *
+ * Since the client code looks more or less like:
+ * procId = master.doOperation()
+ * while (master.getProcResult(procId) == ProcInProgress);
+ * The master should not throw away the proc result as soon as the procedure is done
+ * but should wait a result request from the client (see executor.removeResult(procId))
+ * The client will call something like master.isProcDone() or master.getProcResult()
+ * which will return the result/state to the client, and it will mark the completed
+ * proc as ready to delete. note that the client may not receive the response from
+ * the master (e.g. master failover) so, if we delay a bit the real deletion of
+ * the proc result the client will be able to get the result the next try.
+ */
+ private static class CompletedProcedureCleaner<TEnvironment> extends Procedure<TEnvironment> {
+ private static final Log LOG = LogFactory.getLog(CompletedProcedureCleaner.class);
+
+ private static final String CLEANER_INTERVAL_CONF_KEY = "hbase.procedure.cleaner.interval";
+ private static final int DEFAULT_CLEANER_INTERVAL = 30 * 1000; // 30sec
+
+ private static final String EVICT_TTL_CONF_KEY = "hbase.procedure.cleaner.evict.ttl";
+ private static final int DEFAULT_EVICT_TTL = 15 * 60000; // 15min
+
+ private static final String EVICT_ACKED_TTL_CONF_KEY ="hbase.procedure.cleaner.acked.evict.ttl";
+ private static final int DEFAULT_ACKED_EVICT_TTL = 5 * 60000; // 5min
+
+ private final Map<Long, ProcedureResult> completed;
+ private final ProcedureStore store;
+ private final Configuration conf;
+
+ public CompletedProcedureCleaner(final Configuration conf, final ProcedureStore store,
+ final Map<Long, ProcedureResult> completedMap) {
+ // set the timeout interval that triggers the periodic-procedure
+ setTimeout(conf.getInt(CLEANER_INTERVAL_CONF_KEY, DEFAULT_CLEANER_INTERVAL));
+ this.completed = completedMap;
+ this.store = store;
+ this.conf = conf;
+ }
+
+ public void periodicExecute(final TEnvironment env) {
+ if (completed.isEmpty()) {
+ LOG.debug("no completed procedures to cleanup");
+ return;
+ }
+
+ final long evictTtl = conf.getInt(EVICT_TTL_CONF_KEY, DEFAULT_EVICT_TTL);
+ final long evictAckTtl = conf.getInt(EVICT_ACKED_TTL_CONF_KEY, DEFAULT_ACKED_EVICT_TTL);
+
+ LOG.debug("processing " + completed.size() + " completed procedures");
+ long now = EnvironmentEdgeManager.currentTime();
+ Iterator<Map.Entry<Long, ProcedureResult>> it = completed.entrySet().iterator();
+ while (it.hasNext()) {
+ Map.Entry<Long, ProcedureResult> entry = it.next();
+ ProcedureResult result = entry.getValue();
+
+ // TODO: Select TTL based on Procedure type
+ if ((result.hasClientAckTime() && (now - result.getClientAckTime()) >= evictAckTtl) ||
+ (now - result.getLastUpdate()) >= evictTtl) {
+ LOG.debug("Evict completed procedure " + entry.getKey());
+ store.delete(entry.getKey());
+ it.remove();
+ }
+ }
+ }
+
+ @Override
+ protected Procedure[] execute(final TEnvironment env) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected void rollback(final TEnvironment env) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected boolean abort(final TEnvironment env) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void serializeStateData(final OutputStream stream) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void deserializeStateData(final InputStream stream) {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ /**
+ * Map the the procId returned by submitProcedure(), the Root-ProcID, to the ProcedureResult.
+ * Once a Root-Procedure completes (success or failure), the result will be added to this map.
+ * The user of ProcedureExecutor should call getResult(procId) to get the result.
+ */
+ private final ConcurrentHashMap<Long, ProcedureResult> completed =
+ new ConcurrentHashMap<Long, ProcedureResult>();
+
+ /**
+ * Map the the procId returned by submitProcedure(), the Root-ProcID, to the RootProcedureState.
+ * The RootProcedureState contains the execution stack of the Root-Procedure,
+ * It is added to the map by submitProcedure() and removed on procedure completion.
+ */
+ private final ConcurrentHashMap<Long, RootProcedureState> rollbackStack =
+ new ConcurrentHashMap<Long, RootProcedureState>();
+
+ /**
+ * Helper map to lookup the live procedures by ID.
+ * This map contains every procedure. root-procedures and subprocedures.
+ */
+ private final ConcurrentHashMap<Long, Procedure> procedures =
+ new ConcurrentHashMap<Long, Procedure>();
+
+ /**
+ * Timeout Queue that contains Procedures in a WAITING_TIMEOUT state
+ * or periodic procedures.
+ */
+ private final TimeoutBlockingQueue<Procedure> waitingTimeout =
+ new TimeoutBlockingQueue<Procedure>(new ProcedureTimeoutRetriever());
+
+ /**
+ * Queue that contains runnable procedures.
+ */
+ private final ProcedureRunnableSet runnables;
+
+ private final CopyOnWriteArrayList<ProcedureExecutorListener> listeners =
+ new CopyOnWriteArrayList<ProcedureExecutorListener>();
+
+ private final AtomicInteger activeExecutorCount = new AtomicInteger(0);
+ private final AtomicBoolean running = new AtomicBoolean(false);
+ private final AtomicLong lastProcId = new AtomicLong(-1);
+ private final TEnvironment environment;
+ private final ProcedureStore store;
+ private final Configuration conf;
+
+ private Thread[] threads;
+
+ public ProcedureExecutor(final Configuration conf, final TEnvironment environment,
+ final ProcedureStore store) {
+ this(conf, environment, store, new ProcedureSimpleRunQueue());
+ }
+
+ public ProcedureExecutor(final Configuration conf, final TEnvironment environment,
+ final ProcedureStore store, final ProcedureRunnableSet runqueue) {
+ this.environment = environment;
+ this.runnables = runqueue;
+ this.store = store;
+ this.conf = conf;
+ }
+
+ private List<Map.Entry<Long, RootProcedureState>> load() throws IOException {
+ Preconditions.checkArgument(completed.isEmpty());
+ Preconditions.checkArgument(rollbackStack.isEmpty());
+ Preconditions.checkArgument(procedures.isEmpty());
+ Preconditions.checkArgument(waitingTimeout.isEmpty());
+ Preconditions.checkArgument(runnables.size() == 0);
+
+ // 1. Load the procedures
+ Iterator<Procedure> loader = store.load();
+ if (loader == null) {
+ lastProcId.set(0);
+ return null;
+ }
+
+ long logMaxProcId = 0;
+ int runnablesCount = 0;
+ while (loader.hasNext()) {
+ Procedure proc = loader.next();
+ procedures.put(proc.getProcId(), proc);
+ logMaxProcId = Math.max(logMaxProcId, proc.getProcId());
+ LOG.debug("Loading procedure " + proc +
+ " state=" + proc.getState() + " isFailed=" + proc.hasException());
+ if (!proc.hasParent() && !proc.isFinished()) {
+ rollbackStack.put(proc.getProcId(), new RootProcedureState());
+ }
+ if (proc.getState() == ProcedureState.RUNNABLE) {
+ runnablesCount++;
+ }
+ }
+ assert lastProcId.get() < 0;
+ lastProcId.set(logMaxProcId);
+
+ // 2. Initialize the stacks
+ TreeSet<Procedure> runnableSet = null;
+ HashSet<Procedure> waitingSet = null;
+ for (final Procedure proc: procedures.values()) {
+ Long rootProcId = getRootProcedureId(proc);
+ if (rootProcId == null) {
+ // The 'proc' was ready to run but the root procedure was rolledback?
+ runnables.addBack(proc);
+ continue;
+ }
+
+ if (!proc.hasParent() && proc.isFinished()) {
+ LOG.debug("The procedure is completed " + proc +
+ " state=" + proc.getState() + " isFailed=" + proc.hasException());
+ assert !rollbackStack.containsKey(proc.getProcId());
+ completed.put(proc.getProcId(), newResultFromProcedure(proc));
+ continue;
+ }
+
+ if (proc.hasParent() && !proc.isFinished()) {
+ Procedure parent = procedures.get(proc.getParentProcId());
+ // corrupted procedures are handled later at step 3
+ if (parent != null) {
+ parent.incChildrenLatch();
+ }
+ }
+
+ RootProcedureState procStack = rollbackStack.get(rootProcId);
+ procStack.loadStack(proc);
+
+ switch (proc.getState()) {
+ case RUNNABLE:
+ if (runnableSet == null) {
+ runnableSet = new TreeSet<Procedure>();
+ }
+ runnableSet.add(proc);
+ break;
+ case WAITING_TIMEOUT:
+ if (waitingSet == null) {
+ waitingSet = new HashSet<Procedure>();
+ }
+ waitingSet.add(proc);
+ break;
+ case FINISHED:
+ if (proc.hasException()) {
+ // add the proc to the runnables to perform the rollback
+ runnables.addBack(proc);
+ break;
+ }
+ case ROLLEDBACK:
+ case INITIALIZING:
+ String msg = "Unexpected " + proc.getState() + " state for " + proc;
+ LOG.error(msg);
+ throw new UnsupportedOperationException(msg);
+ default:
+ break;
+ }
+ }
+
+ // 3. Validate the stacks
+ List<Map.Entry<Long, RootProcedureState>> corrupted = null;
+ Iterator<Map.Entry<Long, RootProcedureState>> itStack = rollbackStack.entrySet().iterator();
+ while (itStack.hasNext()) {
+ Map.Entry<Long, RootProcedureState> entry = itStack.next();
+ RootProcedureState procStack = entry.getValue();
+ if (procStack.isValid()) continue;
+
+ for (Procedure proc: procStack.getSubprocedures()) {
+ procedures.remove(proc.getProcId());
+ if (runnableSet != null) runnableSet.remove(proc);
+ if (waitingSet != null) waitingSet.remove(proc);
+ }
+ itStack.remove();
+ if (corrupted == null) {
+ corrupted = new ArrayList<Map.Entry<Long, RootProcedureState>>();
+ }
+ corrupted.add(entry);
+ }
+
+ // 4. Push the runnables
+ if (runnableSet != null) {
+ // TODO: See ProcedureWALFormatReader.readInitEntry() some procedure
+ // may be started way before this stuff.
+ for (Procedure proc: runnableSet) {
+ if (!proc.hasParent()) {
+ sendProcedureLoadedNotification(proc.getProcId());
+ }
+ runnables.addBack(proc);
+ }
+ }
+ return corrupted;
+ }
+
+ public void start(int numThreads) throws IOException {
+ if (running.getAndSet(true)) {
+ LOG.warn("Already running");
+ return;
+ }
+
+ // We have numThreads executor + one timer thread used for timing out
+ // procedures and triggering periodic procedures.
+ threads = new Thread[numThreads + 1];
+ LOG.info("Starting procedure executor threads=" + threads.length);
+
+ // Initialize procedures executor
+ for (int i = 0; i < numThreads; ++i) {
+ threads[i] = new Thread() {
+ @Override
+ public void run() {
+ execLoop();
+ }
+ };
+ }
+
+ // Initialize procedures timeout handler (this is the +1 thread)
+ threads[numThreads] = new Thread() {
+ @Override
+ public void run() {
+ timeoutLoop();
+ }
+ };
+
+ // Acquire the store lease.
+ store.recoverLease();
+
+ // TODO: Split in two steps.
+ // TODO: Handle corrupted procedure returned (probably just a WARN)
+ // The first one will make sure that we have the latest id,
+ // so we can start the threads and accept new procedures.
+ // The second step will do the actual load of old procedures.
+ load();
+
+ // Start the executors. Here we must have the lastProcId set.
+ for (int i = 0; i < threads.length; ++i) {
+ threads[i].start();
+ }
+
+ // Add completed cleaner
+ waitingTimeout.add(new CompletedProcedureCleaner(conf, store, completed));
+ }
+
+ public void stop() {
+ if (!running.getAndSet(false)) {
+ return;
+ }
+
+ LOG.info("Stopping the procedure executor");
+ runnables.signalAll();
+ waitingTimeout.signalAll();
+ }
+
+ public void join() {
+ boolean interrupted = false;
+
+ for (int i = 0; i < threads.length; ++i) {
+ try {
+ threads[i].join();
+ } catch (InterruptedException ex) {
+ interrupted = true;
+ }
+ }
+
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+
+ completed.clear();
+ rollbackStack.clear();
+ procedures.clear();
+ waitingTimeout.clear();
+ runnables.clear();
+ lastProcId.set(-1);
+ }
+
+ public boolean isRunning() {
+ return running.get();
+ }
+
+ /**
+ * @return the number of execution threads.
+ */
+ public int getNumThreads() {
+ return threads == null ? 0 : (threads.length - 1);
+ }
+
+ public int getActiveExecutorCount() {
+ return activeExecutorCount.get();
+ }
+
+ public TEnvironment getEnvironment() {
+ return this.environment;
+ }
+
+ public ProcedureStore getStore() {
+ return this.store;
+ }
+
+ public void registerListener(ProcedureExecutorListener listener) {
+ this.listeners.add(listener);
+ }
+
+ public boolean unregisterListener(ProcedureExecutorListener listener) {
+ return this.listeners.remove(listener);
+ }
+
+ /**
+ * Add a new root-procedure to the executor.
+ * @param proc the new procedure to execute.
+ * @return the procedure id, that can be used to monitor the operation
+ */
+ public long submitProcedure(final Procedure proc) {
+ Preconditions.checkArgument(proc.getState() == ProcedureState.INITIALIZING);
+ Preconditions.checkArgument(isRunning());
+ Preconditions.checkArgument(lastProcId.get() >= 0);
+ Preconditions.checkArgument(!proc.hasParent());
+
+ // Initialize the Procedure ID
+ proc.setProcId(nextProcId());
+
+ // Commit the transaction
+ store.insert(proc, null);
+ LOG.debug("procedure " + proc + " added to the store");
+
+ // Create the rollback stack for the procedure
+ RootProcedureState stack = new RootProcedureState();
+ rollbackStack.put(proc.getProcId(), stack);
+
+ // Submit the new subprocedures
+ assert !procedures.containsKey(proc.getProcId());
+ procedures.put(proc.getProcId(), proc);
+ sendProcedureAddedNotification(proc.getProcId());
+ runnables.addBack(proc);
+ return proc.getProcId();
+ }
+
+ public ProcedureResult getResult(final long procId) {
+ return completed.get(procId);
+ }
+
+ /**
+ * Return true if the procedure is finished.
+ * The state may be "completed successfully" or "failed and rolledback".
+ * Use getResult() to check the state or get the result data.
+ * @param procId the ID of the procedure to check
+ * @return true if the procedure execution is finished, otherwise false.
+ */
+ public boolean isFinished(final long procId) {
+ return completed.containsKey(procId);
+ }
+
+ /**
+ * Mark the specified completed procedure, as ready to remove.
+ * @param procId the ID of the procedure to remove
+ */
+ public void removeResult(final long procId) {
+ ProcedureResult result = completed.get(procId);
+ if (result == null) {
+ assert !procedures.containsKey(procId) : "procId=" + procId + " is still running";
+ LOG.debug("Procedure procId=" + procId + " already removed by the cleaner");
+ return;
+ }
+
+ // The CompletedProcedureCleaner will take care of deletion, once the TTL is expired.
+ result.setClientAckTime(EnvironmentEdgeManager.currentTime());
+ }
+
+ /**
+ * Send an abort notification the specified procedure.
+ * Depending on the procedure implementation the abort can be considered or ignored.
+ * @param procId the procedure to abort
+ * @return true if the procedure exist and has received the abort, otherwise false.
+ */
+ public boolean abort(final long procId) {
+ Procedure proc = procedures.get(procId);
+ if (proc != null) {
+ return proc.abort(getEnvironment());
+ }
+ return false;
+ }
+
+ public Map<Long, ProcedureResult> getResults() {
+ return Collections.unmodifiableMap(completed);
+ }
+
+ public Procedure getProcedure(final long procId) {
+ return procedures.get(procId);
+ }
+
+ /**
+ * Execution loop (N threads)
+ * while the executor is in a running state,
+ * fetch a procedure from the runnables queue and start the execution.
+ */
+ private void execLoop() {
+ while (running.get()) {
+ Long procId = runnables.poll();
+ Procedure proc = procId != null ? procedures.get(procId) : null;
+ if (proc == null) continue;
+
+ try {
+ activeExecutorCount.incrementAndGet();
+ execLoop(proc);
+ } finally {
+ activeExecutorCount.decrementAndGet();
+ }
+ }
+ }
+
+ private void execLoop(Procedure proc) {
+ LOG.debug("starting the execution of " + proc);
+
+ Long rootProcId = getRootProcedureId(proc);
+ if (rootProcId == null) {
+ // The 'proc' was ready to run but the root procedure was rolledback
+ executeRollback(proc);
+ return;
+ }
+
+ RootProcedureState procStack = rollbackStack.get(rootProcId);
+ if (procStack == null) return;
+
+ do {
+ // Try to acquire the execution
+ if (!procStack.acquire(proc)) {
+ if (procStack.setRollback()) {
+ // we have the 'rollback-lock' we can start rollingback
+ executeRollback(rootProcId, procStack);
+ } else {
+ // if we can't rollback means that some child is still running.
+ // the rollback will be executed after all the children are done.
+ // If the procedure was never executed, remove and mark it as rolledback.
+ if (!proc.wasExecuted()) {
+ executeRollback(proc);
+ }
+ }
+ break;
+ }
+
+ // Execute the procedure
+ assert proc.getState() == ProcedureState.RUNNABLE;
+ if (proc.acquireLock(getEnvironment())) {
+ execProcedure(procStack, proc);
+ proc.releaseLock(getEnvironment());
+ } else {
+ runnables.yield(proc);
+ }
+ procStack.release(proc);
+
+ // allows to kill the executor before something is stored to the wal.
+ // useful to test the procedure recovery.
+ if (testing != null && !running.get()) {
+ break;
+ }
+
+ if (proc.getProcId() == rootProcId && proc.isSuccess()) {
+ // Finalize the procedure state
+ LOG.info("Procedure completed in " +
+ StringUtils.humanTimeDiff(proc.elapsedTime()) + ": " + proc);
+ procedureFinished(proc);
+ break;
+ }
+ } while (procStack.isFailed());
+ }
+
+ private void timeoutLoop() {
+ while (running.get()) {
+ Procedure proc = waitingTimeout.poll();
+ if (proc == null) continue;
+
+ if (proc.getTimeRemaining() > 100) {
+ // got an early wake, maybe a stop?
+ // re-enqueue the task in case was not a stop or just a signal
+ waitingTimeout.add(proc);
+ continue;
+ }
+
+ // ----------------------------------------------------------------------------
+ // TODO-MAYBE: Should we provide a notification to the store with the
+ // full set of procedures pending and completed to write a compacted
+ // version of the log (in case is a log)?
+ // In theory no, procedures are have a short life, so at some point the store
+ // will have the tracker saying everything is in the last log.
+ // ----------------------------------------------------------------------------
+
+ // The CompletedProcedureCleaner is a special case, and it acts as a chore.
+ // instead of bringing the Chore class in, we reuse this timeout thread for
+ // this special case.
+ if (proc instanceof CompletedProcedureCleaner) {
+ try {
+ ((CompletedProcedureCleaner)proc).periodicExecute(getEnvironment());
+ } catch (Throwable e) {
+ LOG.error("ignoring CompletedProcedureCleaner exception: " + e.getMessage(), e);
+ }
+ proc.setStartTime(EnvironmentEdgeManager.currentTime());
+ waitingTimeout.add(proc);
+ continue;
+ }
+
+ // The procedure received an "abort-timeout", call abort() and
+ // add the procedure back in the queue for rollback.
+ if (proc.setTimeoutFailure()) {
+ long rootProcId = Procedure.getRootProcedureId(procedures, proc);
+ RootProcedureState procStack = rollbackStack.get(rootProcId);
+ procStack.abort();
+ store.update(proc);
+ runnables.addFront(proc);
+ continue;
+ }
+ }
+ }
+
+ /**
+ * Execute the rollback of the full procedure stack.
+ * Once the procedure is rolledback, the root-procedure will be visible as
+ * finished to user, and the result will be the fatal exception.
+ */
+ private boolean executeRollback(final long rootProcId, final RootProcedureState procStack) {
+ Procedure rootProc = procedures.get(rootProcId);
+ RemoteProcedureException exception = rootProc.getException();
+ if (exception == null) {
+ exception = procStack.getException();
+ rootProc.setFailure(exception);
+ store.update(rootProc);
+ }
+
+ List<Procedure> subprocStack = procStack.getSubprocedures();
+ assert subprocStack != null : "called rollback with no steps executed rootProc=" + rootProc;
+
+ int stackTail = subprocStack.size();
+ boolean reuseLock = false;
+ while (stackTail --> 0) {
+ final Procedure proc = subprocStack.remove(stackTail);
+
+ if (!reuseLock && !proc.acquireLock(getEnvironment())) {
+ // can't take a lock on the procedure, add the root-proc back on the
+ // queue waiting for the lock availability
+ runnables.yield(rootProc);
+ return false;
+ }
+
+ executeRollback(proc);
+ boolean abortRollback = !running.get();
+
+ // If the next procedure is the same to this one
+ // (e.g. StateMachineProcedure reuse the same instance)
+ // we can avoid to lock/unlock each step
+ reuseLock = stackTail > 0 && (subprocStack.get(stackTail - 1) == proc) && !abortRollback;
+ if (!reuseLock) {
+ proc.releaseLock(getEnvironment());
+ }
+
+ // allows to kill the executor before something is stored to the wal.
+ // useful to test the procedure recovery.
+ if (abortRollback) {
+ return false;
+ }
+ }
+
+ // Finalize the procedure state
+ LOG.info("Rolledback procedure " + rootProc +
+ " exec-time=" + StringUtils.humanTimeDiff(rootProc.elapsedTime()) +
+ " exception=" + exception.getMessage());
+ procedureFinished(rootProc);
+ return true;
+ }
+
+ /**
+ * Execute the rollback of the procedure step.
+ * It updates the store with the new state (stack index)
+ * or will remove completly the procedure in case it is a child.
+ */
+ private void executeRollback(final Procedure proc) {
+ try {
+ proc.doRollback(getEnvironment());
+ } catch (Throwable e) {
+ // Catch NullPointerExceptions or similar errors...
+ LOG.fatal("CODE-BUG: uncatched runtime exception for procedure: " + proc, e);
+ }
+
+ // allows to kill the executor before something is stored to the wal.
+ // useful to test the procedure recovery.
+ if (testing != null && testing.shouldKillBeforeStoreUpdate()) {
+ stop();
+ return;
+ }
+
+ if (proc.removeStackIndex()) {
+ proc.setState(ProcedureState.ROLLEDBACK);
+ if (proc.hasParent()) {
+ store.delete(proc.getProcId());
+ procedures.remove(proc.getProcId());
+ } else {
+ store.update(proc);
+ }
+ } else {
+ store.update(proc);
+ }
+ }
+
+ /**
+ * Executes the specified procedure
+ * - calls the doExecute() of the procedure
+ * - if the procedure execution didn't fail (e.g. invalid user input)
+ * - ...and returned subprocedures
+ * - the subprocedures are initialized.
+ * - the subprocedures are added to the store
+ * - the subprocedures are added to the runnable queue
+ * - the procedure is now in a WAITING state, waiting for the subprocedures to complete
+ * - ...if there are no subprocedure
+ * - the procedure completed successfully
+ * - if there is a parent (WAITING)
+ * - the parent state will be set to RUNNABLE
+ * - in case of failure
+ * - the store is updated with the new state
+ * - the executor (caller of this method) will start the rollback of the procedure
+ */
+ private void execProcedure(final RootProcedureState procStack, final Procedure procedure) {
+ Preconditions.checkArgument(procedure.getState() == ProcedureState.RUNNABLE);
+
+ // Execute the procedure
+ boolean reExecute = false;
+ Procedure[] subprocs = null;
+ do {
+ reExecute = false;
+ try {
+ subprocs = procedure.doExecute(getEnvironment());
+ if (subprocs != null && subprocs.length == 0) {
+ subprocs = null;
+ }
+ } catch (ProcedureYieldException e) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("yield procedure: " + procedure);
+ }
+ runnables.yield(procedure);
+ return;
+ } catch (Throwable e) {
+ // Catch NullPointerExceptions or similar errors...
+ String msg = "CODE-BUG: uncatched runtime exception for procedure: " + procedure;
+ LOG.error(msg, e);
+ procedure.setFailure(new RemoteProcedureException(msg, e));
+ }
+
+ if (!procedure.isFailed()) {
+ if (subprocs != null) {
+ if (subprocs.length == 1 && subprocs[0] == procedure) {
+ // quick-shortcut for a state machine like procedure
+ subprocs = null;
+ reExecute = true;
+ } else {
+ // yield the current procedure, and make the subprocedure runnable
+ for (int i = 0; i < subprocs.length; ++i) {
+ Procedure subproc = subprocs[i];
+ if (subproc == null) {
+ String msg = "subproc[" + i + "] is null, aborting the procedure";
+ procedure.setFailure(new RemoteProcedureException(msg,
+ new IllegalArgumentException(msg)));
+ subprocs = null;
+ break;
+ }
+
+ assert subproc.getState() == ProcedureState.INITIALIZING;
+ subproc.setParentProcId(procedure.getProcId());
+ subproc.setProcId(nextProcId());
+ }
+
+ if (!procedure.isFailed()) {
+ procedure.setChildrenLatch(subprocs.length);
+ switch (procedure.getState()) {
+ case RUNNABLE:
+ procedure.setState(ProcedureState.WAITING);
+ break;
+ case WAITING_TIMEOUT:
+ waitingTimeout.add(procedure);
+ break;
+ default:
+ break;
+ }
+ }
+ }
+ } else if (procedure.getState() == ProcedureState.WAITING_TIMEOUT) {
+ waitingTimeout.add(procedure);
+ } else {
+ // No subtask, so we are done
+ procedure.setState(ProcedureState.FINISHED);
+ }
+ }
+
+ // Add the procedure to the stack
+ procStack.addRollbackStep(procedure);
+
+ // allows to kill the executor before something is stored to the wal.
+ // useful to test the procedure recovery.
+ if (testing != null && testing.shouldKillBeforeStoreUpdate()) {
+ stop();
+ return;
+ }
+
+ // Commit the transaction
+ if (subprocs != null && !procedure.isFailed()) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("store add " + procedure + " children " + Arrays.toString(subprocs));
+ }
+ store.insert(procedure, subprocs);
+ } else {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("store update " + procedure);
+ }
+ store.update(procedure);
+ }
+
+ assert (reExecute && subprocs == null) || !reExecute;
+ } while (reExecute);
+
+ // Submit the new subprocedures
+ if (subprocs != null && !procedure.isFailed()) {
+ for (int i = 0; i < subprocs.length; ++i) {
+ Procedure subproc = subprocs[i];
+ assert !procedures.containsKey(subproc.getProcId());
+ procedures.put(subproc.getProcId(), subproc);
+ runnables.addFront(subproc);
+ }
+ }
+
+ if (procedure.isFinished() && procedure.hasParent()) {
+ Procedure parent = procedures.get(procedure.getParentProcId());
+ if (parent == null) {
+ assert procStack.isRollingback();
+ return;
+ }
+
+ // If this procedure is the last child awake the parent procedure
+ if (LOG.isTraceEnabled()) {
+ LOG.trace(parent + " child is done: " + procedure);
+ }
+ if (parent.childrenCountDown() && parent.getState() == ProcedureState.WAITING) {
+ parent.setState(ProcedureState.RUNNABLE);
+ store.update(parent);
+ runnables.addFront(parent);
+ if (LOG.isTraceEnabled()) {
+ LOG.trace(parent + " all the children finished their work, resume.");
+ }
+ return;
+ }
+ }
+ }
+
+ private void sendProcedureLoadedNotification(final long procId) {
+ if (!this.listeners.isEmpty()) {
+ for (ProcedureExecutorListener listener: this.listeners) {
+ try {
+ listener.procedureLoaded(procId);
+ } catch (Throwable e) {
+ LOG.error("the listener " + listener + " had an error: " + e.getMessage(), e);
+ }
+ }
+ }
+ }
+
+ private void sendProcedureAddedNotification(final long procId) {
+ if (!this.listeners.isEmpty()) {
+ for (ProcedureExecutorListener listener: this.listeners) {
+ try {
+ listener.procedureAdded(procId);
+ } catch (Throwable e) {
+ LOG.error("the listener " + listener + " had an error: " + e.getMessage(), e);
+ }
+ }
+ }
+ }
+
+ private void sendProcedureFinishedNotification(final long procId) {
+ if (!this.listeners.isEmpty()) {
+ for (ProcedureExecutorListener listener: this.listeners) {
+ try {
+ listener.procedureFinished(procId);
+ } catch (Throwable e) {
+ LOG.error("the listener " + listener + " had an error: " + e.getMessage(), e);
+ }
+ }
+ }
+ }
+
+ private long nextProcId() {
+ long procId = lastProcId.incrementAndGet();
+ if (procId < 0) {
+ while (!lastProcId.compareAndSet(procId, 0)) {
+ procId = lastProcId.get();
+ if (procId >= 0)
+ break;
+ }
+ while (procedures.containsKey(procId)) {
+ procId = lastProcId.incrementAndGet();
+ }
+ }
+ return procId;
+ }
+
+ private Long getRootProcedureId(Procedure proc) {
+ return Procedure.getRootProcedureId(procedures, proc);
+ }
+
+ private void procedureFinished(final Procedure proc) {
+ proc.completionCleanup(getEnvironment());
+ completed.put(proc.getProcId(), newResultFromProcedure(proc));
+ rollbackStack.remove(proc.getProcId());
+ sendProcedureFinishedNotification(proc.getProcId());
+ }
+
+ private static ProcedureResult newResultFromProcedure(final Procedure proc) {
+ if (proc.isFailed()) {
+ return new ProcedureResult(proc.getStartTime(), proc.getLastUpdate(), proc.getException());
+ }
+ return new ProcedureResult(proc.getStartTime(), proc.getLastUpdate(), proc.getResult());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureFairRunQueues.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureFairRunQueues.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureFairRunQueues.java
new file mode 100644
index 0000000..6a6eda2
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureFairRunQueues.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import java.util.Map;
+
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.ConcurrentSkipListMap;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * This class is a container of queues that allows to select a queue
+ * in a round robin fashion, considering priority of the queue.
+ *
+ * the quantum is just how many poll() will return the same object.
+ * e.g. if quantum is 1 and you have A and B as object you'll get: A B A B
+ * e.g. if quantum is 2 and you have A and B as object you'll get: A A B B A A B B
+ * then the object priority is just a priority * quantum
+ *
+ * Example:
+ * - three queues (A, B, C) with priorities (1, 1, 2)
+ * - The first poll() will return A
+ * - The second poll() will return B
+ * - The third and forth poll() will return C
+ * - and so on again and again.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class ProcedureFairRunQueues<TKey, TQueue extends ProcedureFairRunQueues.FairObject> {
+ private ConcurrentSkipListMap<TKey, TQueue> objMap =
+ new ConcurrentSkipListMap<TKey, TQueue>();
+
+ private final ReentrantLock lock = new ReentrantLock();
+ private final Condition waitCond = lock.newCondition();
+ private final int quantum;
+
+ private Map.Entry<TKey, TQueue> current;
+ private int currentQuantum = 0;
+
+ public interface FairObject {
+ boolean isAvailable();
+ int getPriority();
+ }
+
+ /**
+ * @param quantum how many poll() will return the same object.
+ */
+ public ProcedureFairRunQueues(final int quantum) {
+ this.quantum = quantum;
+ }
+
+ public TQueue get(final TKey key) {
+ return objMap.get(key);
+ }
+
+ public TQueue add(final TKey key, final TQueue queue) {
+ TQueue oldq = objMap.putIfAbsent(key, queue);
+ return oldq != null ? oldq : queue;
+ }
+
+ public TQueue remove(final TKey key) {
+ return objMap.remove(key);
+ }
+
+ public void clear() {
+ lock.lock();
+ try {
+ current = null;
+ objMap.clear();
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ public TQueue poll() {
+ lock.lock();
+ try {
+ TQueue queue;
+ if (currentQuantum == 0) {
+ if (nextObject() == null) {
+ // nothing here
+ return null;
+ }
+
+ queue = current.getValue();
+ currentQuantum = calculateQuantum(queue) - 1;
+ } else {
+ currentQuantum--;
+ queue = current.getValue();
+ }
+
+ if (!queue.isAvailable()) {
+ Map.Entry<TKey, TQueue> last = current;
+ // Try the next one
+ do {
+ if (nextObject() == null)
+ return null;
+ } while (current.getValue() != last.getValue() && !current.getValue().isAvailable());
+
+ queue = current.getValue();
+ currentQuantum = calculateQuantum(queue) - 1;
+ }
+
+ return queue;
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ private Map.Entry<TKey, TQueue> nextObject() {
+ Map.Entry<TKey, TQueue> next = null;
+
+ // If we have already a key, try the next one
+ if (current != null) {
+ next = objMap.higherEntry(current.getKey());
+ }
+
+ // if there is no higher key, go back to the first
+ current = (next != null) ? next : objMap.firstEntry();
+ return current;
+ }
+
+ private int calculateQuantum(final TQueue fairObject) {
+ // TODO
+ return Math.max(1, fairObject.getPriority() * quantum);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureResult.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureResult.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureResult.java
new file mode 100644
index 0000000..0aebd5a
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureResult.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * Once a Procedure completes the ProcedureExecutor takes all the useful
+ * information of the procedure (e.g. exception/result) and creates a ProcedureResult.
+ * The user of the Procedure framework will get the procedure result with
+ * procedureExecutor.getResult(procId)
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class ProcedureResult {
+ private final RemoteProcedureException exception;
+ private final long lastUpdate;
+ private final long startTime;
+ private final byte[] result;
+
+ private long clientAckTime = -1;
+
+ public ProcedureResult(final long startTime, final long lastUpdate,
+ final RemoteProcedureException exception) {
+ this.lastUpdate = lastUpdate;
+ this.startTime = startTime;
+ this.exception = exception;
+ this.result = null;
+ }
+
+ public ProcedureResult(final long startTime, final long lastUpdate, final byte[] result) {
+ this.lastUpdate = lastUpdate;
+ this.startTime = startTime;
+ this.exception = null;
+ this.result = result;
+ }
+
+ public boolean isFailed() {
+ return exception != null;
+ }
+
+ public RemoteProcedureException getException() {
+ return exception;
+ }
+
+ public boolean hasResultData() {
+ return result != null;
+ }
+
+ public byte[] getResult() {
+ return result;
+ }
+
+ public long getStartTime() {
+ return startTime;
+ }
+
+ public long getLastUpdate() {
+ return lastUpdate;
+ }
+
+ public long executionTime() {
+ return lastUpdate - startTime;
+ }
+
+ public boolean hasClientAckTime() {
+ return clientAckTime > 0;
+ }
+
+ public long getClientAckTime() {
+ return clientAckTime;
+ }
+
+ @InterfaceAudience.Private
+ protected void setClientAckTime(final long timestamp) {
+ this.clientAckTime = timestamp;
+ }
+}
\ No newline at end of file
[04/15] hbase git commit: HBASE-5238 Add a log4j category for all
edits to META/ROOT
Posted by mb...@apache.org.
HBASE-5238 Add a log4j category for all edits to META/ROOT
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e26c4e4e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e26c4e4e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e26c4e4e
Branch: refs/heads/hbase-12439
Commit: e26c4e4e6733ea715fb33e4bd9df9ea2d32d5408
Parents: 134b955
Author: Andrey Stepachev <an...@wandisco.com>
Authored: Wed Mar 25 23:03:18 2015 +0000
Committer: Andrey Stepachev <an...@wandisco.com>
Committed: Wed Mar 25 23:03:18 2015 +0000
----------------------------------------------------------------------
conf/log4j.properties | 1 +
.../apache/hadoop/hbase/MetaTableAccessor.java | 45 +++++++++++++++++++-
2 files changed, 45 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e26c4e4e/conf/log4j.properties
----------------------------------------------------------------------
diff --git a/conf/log4j.properties b/conf/log4j.properties
index 7b0acc0..8d6badb 100644
--- a/conf/log4j.properties
+++ b/conf/log4j.properties
@@ -76,6 +76,7 @@ log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%
log4j.logger.org.apache.zookeeper=INFO
#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
log4j.logger.org.apache.hadoop.hbase=INFO
+log4j.logger.org.apache.hadoop.hbase.META=INFO
# Make these two classes INFO-level. Make them DEBUG to see more zk debug.
log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO
log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO
http://git-wip-us.apache.org/repos/asf/hbase/blob/e26c4e4e/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
index 4df58a2..88cc25e 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
@@ -122,6 +122,7 @@ public class MetaTableAccessor {
*/
private static final Log LOG = LogFactory.getLog(MetaTableAccessor.class);
+ private static final Log METALOG = LogFactory.getLog("org.apache.hadoop.hbase.META");
static final byte [] META_REGION_PREFIX;
static {
@@ -1354,6 +1355,9 @@ public class MetaTableAccessor {
*/
private static void put(final Table t, final Put p) throws IOException {
try {
+ if (METALOG.isDebugEnabled()) {
+ METALOG.debug(mutationToString(p));
+ }
t.put(p);
} finally {
t.close();
@@ -1370,6 +1374,9 @@ public class MetaTableAccessor {
throws IOException {
Table t = getMetaHTable(connection);
try {
+ if (METALOG.isDebugEnabled()) {
+ METALOG.debug(mutationsToString(ps));
+ }
t.put(ps);
} finally {
t.close();
@@ -1399,6 +1406,9 @@ public class MetaTableAccessor {
throws IOException {
Table t = getMetaHTable(connection);
try {
+ if (METALOG.isDebugEnabled()) {
+ METALOG.debug(mutationsToString(deletes));
+ }
t.delete(deletes);
} finally {
t.close();
@@ -1443,7 +1453,10 @@ public class MetaTableAccessor {
throws IOException {
Table t = getMetaHTable(connection);
try {
- t.batch(mutations);
+ if (METALOG.isDebugEnabled()) {
+ METALOG.debug(mutationsToString(mutations));
+ }
+ t.batch(mutations, new Object[mutations.size()]);
} catch (InterruptedException e) {
InterruptedIOException ie = new InterruptedIOException(e.getMessage());
ie.initCause(e);
@@ -1494,6 +1507,9 @@ public class MetaTableAccessor {
Put put = makePutFromRegionInfo(regionInfo);
addDaughtersToPut(put, splitA, splitB);
meta.put(put);
+ if (METALOG.isDebugEnabled()) {
+ METALOG.debug(mutationToString(put));
+ }
if (LOG.isDebugEnabled()) {
LOG.debug("Added " + regionInfo.getRegionNameAsString());
}
@@ -1705,6 +1721,9 @@ public class MetaTableAccessor {
CoprocessorRpcChannel channel = table.coprocessorService(row);
MultiRowMutationProtos.MutateRowsRequest.Builder mmrBuilder
= MultiRowMutationProtos.MutateRowsRequest.newBuilder();
+ if (METALOG.isDebugEnabled()) {
+ METALOG.debug(mutationsToString(mutations));
+ }
for (Mutation mutation : mutations) {
if (mutation instanceof Put) {
mmrBuilder.addMutationRequest(ProtobufUtil.toMutation(
@@ -1900,4 +1919,28 @@ public class MetaTableAccessor {
p.addImmutable(getCatalogFamily(), getSeqNumColumn(replicaId), now, null);
return p;
}
+
+ private static String mutationsToString(Mutation ... mutations) throws IOException {
+ StringBuilder sb = new StringBuilder();
+ String prefix = "";
+ for (Mutation mutation : mutations) {
+ sb.append(prefix).append(mutationToString(mutation));
+ prefix = ", ";
+ }
+ return sb.toString();
+ }
+
+ private static String mutationsToString(List<? extends Mutation> mutations) throws IOException {
+ StringBuilder sb = new StringBuilder();
+ String prefix = "";
+ for (Mutation mutation : mutations) {
+ sb.append(prefix).append(mutationToString(mutation));
+ prefix = ", ";
+ }
+ return sb.toString();
+ }
+
+ private static String mutationToString(Mutation p) throws IOException {
+ return p.getClass().getSimpleName() + p.toJSON();
+ }
}
[12/15] hbase git commit: HBASE-13202 Procedure v2 - core framework
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/util/TestTimeoutBlockingQueue.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/util/TestTimeoutBlockingQueue.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/util/TestTimeoutBlockingQueue.java
new file mode 100644
index 0000000..aff536a
--- /dev/null
+++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/util/TestTimeoutBlockingQueue.java
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2.util;
+
+
+import java.util.Arrays;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.procedure2.util.TimeoutBlockingQueue.TimeoutRetriever;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+@Category({MasterTests.class, SmallTests.class})
+public class TestTimeoutBlockingQueue {
+ private static final Log LOG = LogFactory.getLog(TestTimeoutBlockingQueue.class);
+
+ static class TestObject {
+ private long timeout;
+ private int seqId;
+
+ public TestObject(int seqId, long timeout) {
+ this.timeout = timeout;
+ this.seqId = seqId;
+ }
+
+ public long getTimeout() {
+ return timeout;
+ }
+
+ public String toString() {
+ return String.format("(%03d, %03d)", seqId, timeout);
+ }
+ }
+
+ static class TestObjectTimeoutRetriever implements TimeoutRetriever<TestObject> {
+ @Override
+ public long getTimeout(TestObject obj) {
+ return obj.getTimeout();
+ }
+
+ @Override
+ public TimeUnit getTimeUnit(TestObject obj) {
+ return TimeUnit.MILLISECONDS;
+ }
+ }
+
+ @Test
+ public void testOrder() {
+ TimeoutBlockingQueue<TestObject> queue =
+ new TimeoutBlockingQueue<TestObject>(8, new TestObjectTimeoutRetriever());
+
+ long[] timeouts = new long[] {500, 200, 700, 300, 600, 600, 200, 800, 500};
+
+ for (int i = 0; i < timeouts.length; ++i) {
+ for (int j = 0; j <= i; ++j) {
+ queue.add(new TestObject(j, timeouts[j]));
+ queue.dump();
+ }
+
+ long prev = 0;
+ for (int j = 0; j <= i; ++j) {
+ TestObject obj = queue.poll();
+ assertTrue(obj.getTimeout() >= prev);
+ prev = obj.getTimeout();
+ queue.dump();
+ }
+ }
+ }
+
+ @Test
+ public void testTimeoutBlockingQueue() {
+ TimeoutBlockingQueue<TestObject> queue;
+
+ int[][] testArray = new int[][] {
+ {200, 400, 600}, // append
+ {200, 400, 100}, // prepend
+ {200, 400, 300}, // insert
+ };
+
+ for (int i = 0; i < testArray.length; ++i) {
+ int[] sortedArray = Arrays.copyOf(testArray[i], testArray[i].length);
+ Arrays.sort(sortedArray);
+
+ // test with head == 0
+ queue = new TimeoutBlockingQueue<TestObject>(2, new TestObjectTimeoutRetriever());
+ for (int j = 0; j < testArray[i].length; ++j) {
+ queue.add(new TestObject(j, testArray[i][j]));
+ queue.dump();
+ }
+
+ for (int j = 0; !queue.isEmpty(); ++j) {
+ assertEquals(sortedArray[j], queue.poll().getTimeout());
+ }
+
+ queue = new TimeoutBlockingQueue<TestObject>(2, new TestObjectTimeoutRetriever());
+ queue.add(new TestObject(0, 50));
+ assertEquals(50, queue.poll().getTimeout());
+
+ // test with head > 0
+ for (int j = 0; j < testArray[i].length; ++j) {
+ queue.add(new TestObject(j, testArray[i][j]));
+ queue.dump();
+ }
+
+ for (int j = 0; !queue.isEmpty(); ++j) {
+ assertEquals(sortedArray[j], queue.poll().getTimeout());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-protocol/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-protocol/pom.xml b/hbase-protocol/pom.xml
index 7787c52..0d33332 100644
--- a/hbase-protocol/pom.xml
+++ b/hbase-protocol/pom.xml
@@ -176,6 +176,7 @@
<include>MapReduce.proto</include>
<include>Master.proto</include>
<include>MultiRowMutation.proto</include>
+ <include>Procedure.proto</include>
<include>Quota.proto</include>
<include>RegionServerStatus.proto</include>
<include>RowProcessor.proto</include>
[09/15] hbase git commit: HBASE-13334 FindBugs should create precise
report for new bugs introduced (addendum)
Posted by mb...@apache.org.
HBASE-13334 FindBugs should create precise report for new bugs introduced (addendum)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/69f6a2e7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/69f6a2e7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/69f6a2e7
Branch: refs/heads/hbase-12439
Commit: 69f6a2e7b44d33e01866928672700cde1366f429
Parents: a85c1d9
Author: Andrey Stepachev <oc...@gmail.com>
Authored: Thu Mar 26 08:18:15 2015 +0000
Committer: Andrey Stepachev <oc...@gmail.com>
Committed: Thu Mar 26 08:18:15 2015 +0000
----------------------------------------------------------------------
dev-support/test-patch.sh | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/69f6a2e7/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index 64cf8a7..20d5f25 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -324,7 +324,7 @@ setup () {
echo ""
echo ""
echo "$MVN clean package checkstyle:checkstyle-aggregate findbugs:findbugs -DskipTests \
- -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavacWarnings.txt 2>&1"
+ -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavacWarnings.txt 2>&1"
export MAVEN_OPTS="${MAVEN_OPTS}"
# build core and tests
$MVN clean package checkstyle:checkstyle-aggregate findbugs:findbugs -DskipTests \
@@ -757,13 +757,13 @@ $JIRA_COMMENT_FOOTER"
JIRA_COMMENT="$JIRA_COMMENT
{color:red}-1 findbugs{color}. The patch appears to introduce $findbugsWarnings \
- new Findbugs (version ${findbugs_version}) warnings."
+ new Findbugs (version ${findbugs_version}) warnings."
return 1
fi
JIRA_COMMENT="$JIRA_COMMENT
{color:green}+1 findbugs{color}. The patch does not introduce any \
- new Findbugs (version ${findbugs_version}) warnings."
+ new Findbugs (version ${findbugs_version}) warnings."
return 0
}
[03/15] hbase git commit: HBASE-13325 Protocol Buffers 2.5 no longer
available for download on code.google.com
Posted by mb...@apache.org.
HBASE-13325 Protocol Buffers 2.5 no longer available for download on code.google.com
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/134b9557
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/134b9557
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/134b9557
Branch: refs/heads/hbase-12439
Commit: 134b95579c0d02d54f477e437f03ca1a825256f4
Parents: 2e31e83
Author: Elliott Clark <ec...@apache.org>
Authored: Wed Mar 25 09:27:12 2015 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Wed Mar 25 14:05:34 2015 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hbase/protobuf/package.html | 2 +-
hbase-protocol/README.txt | 2 +-
src/main/asciidoc/_chapters/rpc.adoc | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/134b9557/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/package.html
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/package.html b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/package.html
index 292cac5..d324653 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/package.html
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/package.html
@@ -20,7 +20,7 @@
<head />
<body bgcolor="white">
-Holds classes generated from <a href="http://code.google.com/apis/protocolbuffers/">protobuf</a>
+Holds classes generated from <a href="https://developers.google.com/protocol-buffers/">protobuf</a>
<code>src/main/protobuf</code> definition files.
<p>See under <code>src/main/protobuf</code> for instruction on how to generate the content under
http://git-wip-us.apache.org/repos/asf/hbase/blob/134b9557/hbase-protocol/README.txt
----------------------------------------------------------------------
diff --git a/hbase-protocol/README.txt b/hbase-protocol/README.txt
index b67f0c3..e651411 100644
--- a/hbase-protocol/README.txt
+++ b/hbase-protocol/README.txt
@@ -5,7 +5,7 @@ and then checked in. The reasoning is that they change infrequently.
To regenerate the classes after making definition file changes, ensure first that
the protobuf protoc tool is in your $PATH (You may need to download it and build
it first; its part of the protobuf package obtainable from here:
-http://code.google.com/p/protobuf/downloads/list).
+https://github.com/google/protobuf/releases/tag/v2.5.0).
HBase uses hadoop-maven-plugins:protoc goal to invoke the protoc command. You can
compile the protoc definitions by invoking maven with profile compile-protobuf or
http://git-wip-us.apache.org/repos/asf/hbase/blob/134b9557/src/main/asciidoc/_chapters/rpc.adoc
----------------------------------------------------------------------
diff --git a/src/main/asciidoc/_chapters/rpc.adoc b/src/main/asciidoc/_chapters/rpc.adoc
index 5d8b230..43e7156 100644
--- a/src/main/asciidoc/_chapters/rpc.adoc
+++ b/src/main/asciidoc/_chapters/rpc.adoc
@@ -28,7 +28,7 @@
:icons: font
:experimental:
-In 0.95, all client/server communication is done with link:https://code.google.com/p/protobuf/[protobuf'ed] Messages rather than with link:http://hadoop.apache.org/docs/current/api/org/apache/hadoop/io/Writable.html[Hadoop
+In 0.95, all client/server communication is done with link:https://developers.google.com/protocol-buffers/[protobuf'ed] Messages rather than with link:http://hadoop.apache.org/docs/current/api/org/apache/hadoop/io/Writable.html[Hadoop
Writables].
Our RPC wire format therefore changes.
This document describes the client/server request/response protocol and our new RPC wire-format.
[14/15] hbase git commit: HBASE-13202 Procedure v2 - core framework
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureRunnableSet.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureRunnableSet.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureRunnableSet.java
new file mode 100644
index 0000000..126ec19
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureRunnableSet.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * Keep track of the runnable procedures
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface ProcedureRunnableSet {
+ /**
+ * Inserts the specified element at the front of this queue.
+ * @param proc the Procedure to add
+ */
+ void addFront(Procedure proc);
+
+ /**
+ * Inserts the specified element at the end of this queue.
+ * @param proc the Procedure to add
+ */
+ void addBack(Procedure proc);
+
+ /**
+ * The procedure can't run at the moment.
+ * add it back to the queue, giving priority to someone else.
+ * @param proc the Procedure to add back to the list
+ */
+ void yield(Procedure proc);
+
+ /**
+ * Fetch one Procedure from the queue
+ * @return the Procedure ID to execute, or null if nothing present.
+ */
+ Long poll();
+
+ /**
+ * In case the class is blocking on poll() waiting for items to be added,
+ * this method should awake poll() and poll() should return.
+ */
+ void signalAll();
+
+ /**
+ * Returns the number of elements in this collection.
+ * @return the number of elements in this collection.
+ */
+ int size();
+
+ /**
+ * Removes all of the elements from this collection.
+ */
+ void clear();
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureSimpleRunQueue.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureSimpleRunQueue.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureSimpleRunQueue.java
new file mode 100644
index 0000000..0a00ac5
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureSimpleRunQueue.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * Simple runqueue for the procedures
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class ProcedureSimpleRunQueue implements ProcedureRunnableSet {
+ private final Deque<Long> runnables = new ArrayDeque<Long>();
+ private final ReentrantLock lock = new ReentrantLock();
+ private final Condition waitCond = lock.newCondition();
+
+ @Override
+ public void addFront(final Procedure proc) {
+ lock.lock();
+ try {
+ runnables.addFirst(proc.getProcId());
+ waitCond.signal();
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ @Override
+ public void addBack(final Procedure proc) {
+ lock.lock();
+ try {
+ runnables.addLast(proc.getProcId());
+ waitCond.signal();
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ @Override
+ public void yield(final Procedure proc) {
+ addBack(proc);
+ }
+
+ @Override
+ @edu.umd.cs.findbugs.annotations.SuppressWarnings("WA_AWAIT_NOT_IN_LOOP")
+ public Long poll() {
+ lock.lock();
+ try {
+ if (runnables.isEmpty()) {
+ waitCond.await();
+ if (!runnables.isEmpty()) {
+ return runnables.pop();
+ }
+ } else {
+ return runnables.pop();
+ }
+ } catch (InterruptedException e) {
+ return null;
+ } finally {
+ lock.unlock();
+ }
+ return null;
+ }
+
+ @Override
+ public void signalAll() {
+ lock.lock();
+ try {
+ waitCond.signalAll();
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ @Override
+ public void clear() {
+ lock.lock();
+ try {
+ runnables.clear();
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ @Override
+ public int size() {
+ lock.lock();
+ try {
+ return runnables.size();
+ } finally {
+ lock.unlock();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureYieldException.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureYieldException.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureYieldException.java
new file mode 100644
index 0000000..177ff5b
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureYieldException.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+// TODO: Not used yet
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class ProcedureYieldException extends ProcedureException {
+ /** default constructor */
+ public ProcedureYieldException() {
+ super();
+ }
+
+ /**
+ * Constructor
+ * @param s message
+ */
+ public ProcedureYieldException(String s) {
+ super(s);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureException.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureException.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureException.java
new file mode 100644
index 0000000..6be512d
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RemoteProcedureException.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure2;
+
+import java.io.IOException;
+
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage;
+import org.apache.hadoop.hbase.util.ForeignExceptionUtil;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+
+/**
+ * A RemoteProcedureException is an exception from another thread or process.
+ * <p>
+ * RemoteProcedureExceptions are sent to 'remote' peers to signal an abort in the face of failures.
+ * When serialized for transmission we encode using Protobufs to ensure version compatibility.
+ * <p>
+ * RemoteProcedureException exceptions contain a Throwable as its cause.
+ * This can be a "regular" exception generated locally or a ProxyThrowable that is a representation
+ * of the original exception created on original 'remote' source. These ProxyThrowables have their
+ * their stacks traces and messages overridden to reflect the original 'remote' exception.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+@SuppressWarnings("serial")
+public class RemoteProcedureException extends ProcedureException {
+
+ /**
+ * Name of the throwable's source such as a host or thread name. Must be non-null.
+ */
+ private final String source;
+
+ /**
+ * Create a new RemoteProcedureException that can be serialized.
+ * It is assumed that this came form a local source.
+ * @param source
+ * @param cause
+ */
+ public RemoteProcedureException(String source, Throwable cause) {
+ super(cause);
+ assert source != null;
+ assert cause != null;
+ this.source = source;
+ }
+
+ public String getSource() {
+ return source;
+ }
+
+ public IOException unwrapRemoteException() {
+ if (getCause() instanceof RemoteException) {
+ return ((RemoteException)getCause()).unwrapRemoteException();
+ }
+ if (getCause() instanceof IOException) {
+ return (IOException)getCause();
+ }
+ return new IOException(getCause());
+ }
+
+ @Override
+ public String toString() {
+ String className = getCause().getClass().getName();
+ return className + " via " + getSource() + ":" + getLocalizedMessage();
+ }
+
+ /**
+ * Converts a RemoteProcedureException to an array of bytes.
+ * @param source the name of the external exception source
+ * @param t the "local" external exception (local)
+ * @return protobuf serialized version of RemoteProcedureException
+ */
+ public static byte[] serialize(String source, Throwable t) {
+ return toProto(source, t).toByteArray();
+ }
+
+ /**
+ * Takes a series of bytes and tries to generate an RemoteProcedureException instance for it.
+ * @param bytes
+ * @return the ForeignExcpetion instance
+ * @throws InvalidProtocolBufferException if there was deserialization problem this is thrown.
+ */
+ public static RemoteProcedureException deserialize(byte[] bytes)
+ throws InvalidProtocolBufferException {
+ return fromProto(ForeignExceptionMessage.parseFrom(bytes));
+ }
+
+ public ForeignExceptionMessage convert() {
+ return ForeignExceptionUtil.toProtoForeignException(getSource(), getCause());
+ }
+
+ public static ForeignExceptionMessage toProto(String source, Throwable t) {
+ return ForeignExceptionUtil.toProtoForeignException(source, t);
+ }
+
+ public static RemoteProcedureException fromProto(final ForeignExceptionMessage eem) {
+ return new RemoteProcedureException(eem.getSource(), ForeignExceptionUtil.toIOException(eem));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RootProcedureState.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RootProcedureState.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RootProcedureState.java
new file mode 100644
index 0000000..2f9c007
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/RootProcedureState.java
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState;
+
+/**
+ * Internal state of the ProcedureExecutor that describes the state of a "Root Procedure".
+ * A "Root Procedure" is a Procedure without parent, each subprocedure will be
+ * added to the "Root Procedure" stack (or rollback-stack).
+ *
+ * RootProcedureState is used and managed only by the ProcedureExecutor.
+ * Long rootProcId = getRootProcedureId(proc);
+ * rollbackStack.get(rootProcId).acquire(proc)
+ * rollbackStack.get(rootProcId).release(proc)
+ * ...
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+class RootProcedureState {
+ private static final Log LOG = LogFactory.getLog(RootProcedureState.class);
+
+ private enum State {
+ RUNNING, // The Procedure is running or ready to run
+ FAILED, // The Procedure failed, waiting for the rollback executing
+ ROLLINGBACK, // The Procedure failed and the execution was rolledback
+ }
+
+ private ArrayList<Procedure> subprocedures = null;
+ private State state = State.RUNNING;
+ private int running = 0;
+
+ public synchronized boolean isFailed() {
+ switch (state) {
+ case ROLLINGBACK:
+ case FAILED:
+ return true;
+ default:
+ break;
+ }
+ return false;
+ }
+
+ public synchronized boolean isRollingback() {
+ return state == State.ROLLINGBACK;
+ }
+
+ /**
+ * Called by the ProcedureExecutor to mark rollback execution
+ */
+ protected synchronized boolean setRollback() {
+ if (running == 0 && state == State.FAILED) {
+ state = State.ROLLINGBACK;
+ return true;
+ }
+ return false;
+ }
+
+ protected synchronized List<Procedure> getSubprocedures() {
+ return subprocedures;
+ }
+
+ protected synchronized RemoteProcedureException getException() {
+ if (subprocedures != null) {
+ for (Procedure proc: subprocedures) {
+ if (proc.hasException()) {
+ return proc.getException();
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Called by the ProcedureExecutor to mark the procedure step as running.
+ */
+ protected synchronized boolean acquire(final Procedure proc) {
+ if (state != State.RUNNING) return false;
+
+ running++;
+ return true;
+ }
+
+ /**
+ * Called by the ProcedureExecutor to mark the procedure step as finished.
+ */
+ protected synchronized void release(final Procedure proc) {
+ running--;
+ }
+
+ protected synchronized void abort() {
+ if (state == State.RUNNING) {
+ state = State.FAILED;
+ }
+ }
+
+ /**
+ * Called by the ProcedureExecutor after the procedure step is completed,
+ * to add the step to the rollback list (or procedure stack)
+ */
+ protected synchronized void addRollbackStep(final Procedure proc) {
+ if (proc.isFailed()) {
+ state = State.FAILED;
+ }
+ if (subprocedures == null) {
+ subprocedures = new ArrayList<Procedure>();
+ }
+ proc.addStackIndex(subprocedures.size());
+ subprocedures.add(proc);
+ }
+
+ /**
+ * Called on store load by the ProcedureExecutor to load part of the stack.
+ *
+ * Each procedure has its own stack-positions. Which means we have to write
+ * to the store only the Procedure we executed, and nothing else.
+ * on load we recreate the full stack by aggregating each procedure stack-positions.
+ */
+ protected synchronized void loadStack(final Procedure proc) {
+ int[] stackIndexes = proc.getStackIndexes();
+ if (stackIndexes != null) {
+ if (subprocedures == null) {
+ subprocedures = new ArrayList<Procedure>();
+ }
+ int diff = (1 + stackIndexes[stackIndexes.length - 1]) - subprocedures.size();
+ if (diff > 0) {
+ subprocedures.ensureCapacity(1 + stackIndexes[stackIndexes.length - 1]);
+ while (diff-- > 0) subprocedures.add(null);
+ }
+ for (int i = 0; i < stackIndexes.length; ++i) {
+ subprocedures.set(stackIndexes[i], proc);
+ }
+ }
+ if (proc.getState() == ProcedureState.ROLLEDBACK) {
+ state = State.ROLLINGBACK;
+ } else if (proc.isFailed()) {
+ state = State.FAILED;
+ }
+ }
+
+ /**
+ * Called on store load by the ProcedureExecutor to validate the procedure stack.
+ */
+ protected synchronized boolean isValid() {
+ if (subprocedures != null) {
+ for (Procedure proc: subprocedures) {
+ if (proc == null) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/SequentialProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/SequentialProcedure.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/SequentialProcedure.java
new file mode 100644
index 0000000..7bf959a
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/SequentialProcedure.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.SequentialProcedureData;
+
+/**
+ * A SequentialProcedure describes one step in a procedure chain.
+ * -> Step 1 -> Step 2 -> Step 3
+ *
+ * The main difference from a base Procedure is that the execute() of a
+ * SequentialProcedure will be called only once, there will be no second
+ * execute() call once the child are finished. which means once the child
+ * of a SequentialProcedure are completed the SequentialProcedure is completed too.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public abstract class SequentialProcedure<TEnvironment> extends Procedure<TEnvironment> {
+ private boolean executed = false;
+
+ @Override
+ protected Procedure[] doExecute(final TEnvironment env)
+ throws ProcedureYieldException {
+ updateTimestamp();
+ try {
+ executed = !executed;
+ return executed ? execute(env) : null;
+ } finally {
+ updateTimestamp();
+ }
+ }
+
+ @Override
+ protected void doRollback(final TEnvironment env) {
+ updateTimestamp();
+ executed = !executed;
+ if (!executed) {
+ try {
+ rollback(env);
+ } finally {
+ updateTimestamp();
+ }
+ }
+ }
+
+ @Override
+ protected void serializeStateData(final OutputStream stream) throws IOException {
+ SequentialProcedureData.Builder data = SequentialProcedureData.newBuilder();
+ data.setExecuted(executed);
+ data.build().writeDelimitedTo(stream);
+ }
+
+ @Override
+ protected void deserializeStateData(final InputStream stream) throws IOException {
+ SequentialProcedureData data = SequentialProcedureData.parseDelimitedFrom(stream);
+ executed = data.getExecuted();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
new file mode 100644
index 0000000..f0a3765
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
@@ -0,0 +1,140 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.StateMachineProcedureData;
+
+/**
+ * Procedure described by a series of steps.
+ *
+ * The procedure implementor must have an enum of 'states', describing
+ * the various step of the procedure.
+ * Once the procedure is running, the procedure-framework will call executeFromState()
+ * using the 'state' provided by the user. The first call to executeFromState()
+ * will be performed with 'state = null'. The implementor can jump between
+ * states using setNextState(MyStateEnum.ordinal()).
+ * The rollback will call rollbackState() for each state that was executed, in reverse order.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public abstract class StateMachineProcedure<TEnvironment, TState>
+ extends Procedure<TEnvironment> {
+ private int stateCount = 0;
+ private int[] states = null;
+
+ protected enum Flow {
+ HAS_MORE_STATE,
+ NO_MORE_STATE,
+ }
+
+ /**
+ * called to perform a single step of the specified 'state' of the procedure
+ * @param state state to execute
+ * @return Flow.NO_MORE_STATE if the procedure is completed,
+ * Flow.HAS_MORE_STATE if there is another step.
+ */
+ protected abstract Flow executeFromState(TEnvironment env, TState state)
+ throws ProcedureYieldException;
+
+ /**
+ * called to perform the rollback of the specified state
+ * @param state state to rollback
+ */
+ protected abstract void rollbackState(TEnvironment env, TState state);
+
+ /**
+ * Convert an ordinal (or state id) to an Enum (or more descriptive) state object.
+ * @param stateId the ordinal() of the state enum (or state id)
+ * @return the state enum object
+ */
+ protected abstract TState getState(int stateId);
+
+ @Override
+ protected Procedure[] execute(final TEnvironment env)
+ throws ProcedureYieldException {
+ updateTimestamp();
+ try {
+ TState state = stateCount > 0 ? getState(states[stateCount-1]) : null;
+ if (executeFromState(env, state) == Flow.NO_MORE_STATE) {
+ // completed
+ return null;
+ }
+ return (isWaiting() || isFailed()) ? null : new Procedure[] {this};
+ } finally {
+ updateTimestamp();
+ }
+ }
+
+ @Override
+ protected void rollback(final TEnvironment env) {
+ try {
+ assert stateCount > 0 : "called rollback with no steps executed proc=" + this;
+ updateTimestamp();
+ rollbackState(env, getState(states[--stateCount]));
+ } finally {
+ updateTimestamp();
+ }
+ }
+
+ /**
+ * Set the next state for the procedure.
+ * @param stateId the ordinal() of the state enum (or state id)
+ */
+ protected void setNextState(final int stateId) {
+ if (states == null || states.length == stateCount) {
+ int newCapacity = stateCount + 8;
+ if (states != null) {
+ states = Arrays.copyOf(states, newCapacity);
+ } else {
+ states = new int[newCapacity];
+ }
+ }
+ states[stateCount++] = stateId;
+ }
+
+ @Override
+ protected void serializeStateData(final OutputStream stream) throws IOException {
+ StateMachineProcedureData.Builder data = StateMachineProcedureData.newBuilder();
+ for (int i = 0; i < stateCount; ++i) {
+ data.addState(states[i]);
+ }
+ data.build().writeDelimitedTo(stream);
+ }
+
+ @Override
+ protected void deserializeStateData(final InputStream stream) throws IOException {
+ StateMachineProcedureData data = StateMachineProcedureData.parseDelimitedFrom(stream);
+ stateCount = data.getStateCount();
+ if (stateCount > 0) {
+ states = new int[stateCount];
+ for (int i = 0; i < stateCount; ++i) {
+ states[i] = data.getState(i);
+ }
+ } else {
+ states = null;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/TwoPhaseProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/TwoPhaseProcedure.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/TwoPhaseProcedure.java
new file mode 100644
index 0000000..cd6b0a7
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/TwoPhaseProcedure.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public abstract class TwoPhaseProcedure<TEnvironment> extends Procedure<TEnvironment> {
+ // TODO (e.g. used by ACLs/VisibilityTags updates)
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/ProcedureStore.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/ProcedureStore.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/ProcedureStore.java
new file mode 100644
index 0000000..b440580
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/ProcedureStore.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2.store;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.procedure2.Procedure;
+
+/**
+ * The ProcedureStore is used by the executor to persist the state of each procedure execution.
+ * This allows to resume the execution of pending/in-progress procedures in case
+ * of machine failure or service shutdown.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public interface ProcedureStore {
+ /**
+ * Store listener interface.
+ * The main process should register a listener and respond to the store events.
+ */
+ public interface ProcedureStoreListener {
+ /**
+ * triggered when the store is not able to write out data.
+ * the main process should abort.
+ */
+ void abortProcess();
+ }
+
+ /**
+ * Add the listener to the notification list.
+ * @param listener The AssignmentListener to register
+ */
+ void registerListener(ProcedureStoreListener listener);
+
+ /**
+ * Remove the listener from the notification list.
+ * @param listener The AssignmentListener to unregister
+ * @return true if the listner was in the list and it was removed, otherwise false.
+ */
+ boolean unregisterListener(ProcedureStoreListener listener);
+
+ /**
+ * Start/Open the procedure store
+ * @param numThreads
+ */
+ void start(int numThreads) throws IOException;
+
+ /**
+ * Stop/Close the procedure store
+ */
+ void stop();
+
+ /**
+ * Acquire the lease for the procedure store.
+ */
+ void recoverLease() throws IOException;
+
+ /**
+ * Load the Procedures in the store.
+ * @return the set of procedures present in the store
+ */
+ Iterator<Procedure> load() throws IOException;
+
+ /**
+ * When a procedure is submitted to the executor insert(proc, null) will be called.
+ * 'proc' has a 'RUNNABLE' state and the initial information required to start up.
+ *
+ * When a procedure is executed and it returns children insert(proc, subprocs) will be called.
+ * 'proc' has a 'WAITING' state and an update state.
+ * 'subprocs' are the children in 'RUNNABLE' state with the initial information.
+ *
+ * @param proc the procedure to serialize and write to the store.
+ * @param subprocs the newly created child of the proc.
+ */
+ void insert(Procedure proc, Procedure[] subprocs);
+
+ /**
+ * The specified procedure was executed,
+ * and the new state should be written to the store.
+ * @param proc the procedure to serialize and write to the store.
+ */
+ void update(Procedure proc);
+
+ /**
+ * The specified procId was removed from the executor,
+ * due to completion, abort or failure.
+ * The store implementor should remove all the information about the specified procId.
+ * @param procId the ID of the procedure to remove.
+ */
+ void delete(long procId);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.java
new file mode 100644
index 0000000..4e4653a
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.java
@@ -0,0 +1,540 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2.store;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos;
+
+/**
+ * Keeps track of live procedures.
+ *
+ * It can be used by the ProcedureStore to identify which procedures are already
+ * deleted/completed to avoid the deserialization step on restart.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class ProcedureStoreTracker {
+ private final TreeMap<Long, BitSetNode> map = new TreeMap<Long, BitSetNode>();
+
+ private boolean keepDeletes = false;
+ private boolean partial = false;
+
+ public enum DeleteState { YES, NO, MAYBE }
+
+ public static class BitSetNode {
+ private final static long WORD_MASK = 0xffffffffffffffffL;
+ private final static int ADDRESS_BITS_PER_WORD = 6;
+ private final static int BITS_PER_WORD = 1 << ADDRESS_BITS_PER_WORD;
+ private final static int MAX_NODE_SIZE = 4 << ADDRESS_BITS_PER_WORD;
+
+ private long[] updated;
+ private long[] deleted;
+ private long start;
+
+ public void dump() {
+ System.out.printf("%06d:%06d min=%d max=%d%n", getStart(), getEnd(),
+ getMinProcId(), getMaxProcId());
+ System.out.println("Update:");
+ for (int i = 0; i < updated.length; ++i) {
+ for (int j = 0; j < BITS_PER_WORD; ++j) {
+ System.out.print((updated[i] & (1L << j)) != 0 ? "1" : "0");
+ }
+ System.out.println(" " + i);
+ }
+ System.out.println();
+ System.out.println("Delete:");
+ for (int i = 0; i < deleted.length; ++i) {
+ for (int j = 0; j < BITS_PER_WORD; ++j) {
+ System.out.print((deleted[i] & (1L << j)) != 0 ? "1" : "0");
+ }
+ System.out.println(" " + i);
+ }
+ System.out.println();
+ }
+
+ public BitSetNode(final long procId, final boolean partial) {
+ start = alignDown(procId);
+
+ int count = 2;
+ updated = new long[count];
+ deleted = new long[count];
+ for (int i = 0; i < count; ++i) {
+ updated[i] = 0;
+ deleted[i] = partial ? 0 : WORD_MASK;
+ }
+
+ updateState(procId, false);
+ }
+
+ protected BitSetNode(final long start, final long[] updated, final long[] deleted) {
+ this.start = start;
+ this.updated = updated;
+ this.deleted = deleted;
+ }
+
+ public void update(final long procId) {
+ updateState(procId, false);
+ }
+
+ public void delete(final long procId) {
+ updateState(procId, true);
+ }
+
+ public Long getStart() {
+ return start;
+ }
+
+ public Long getEnd() {
+ return start + (updated.length << ADDRESS_BITS_PER_WORD) - 1;
+ }
+
+ public boolean contains(final long procId) {
+ return start <= procId && procId <= getEnd();
+ }
+
+ public DeleteState isDeleted(final long procId) {
+ int bitmapIndex = getBitmapIndex(procId);
+ int wordIndex = bitmapIndex >> ADDRESS_BITS_PER_WORD;
+ if (wordIndex >= deleted.length) {
+ return DeleteState.MAYBE;
+ }
+ return (deleted[wordIndex] & (1L << bitmapIndex)) != 0 ? DeleteState.YES : DeleteState.NO;
+ }
+
+ private boolean isUpdated(final long procId) {
+ int bitmapIndex = getBitmapIndex(procId);
+ int wordIndex = bitmapIndex >> ADDRESS_BITS_PER_WORD;
+ if (wordIndex >= updated.length) {
+ return false;
+ }
+ return (updated[wordIndex] & (1L << bitmapIndex)) != 0;
+ }
+
+ public boolean isUpdated() {
+ // TODO: cache the value
+ for (int i = 0; i < updated.length; ++i) {
+ long deleteMask = ~deleted[i];
+ if ((updated[i] & deleteMask) != (WORD_MASK & deleteMask)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public boolean isEmpty() {
+ // TODO: cache the value
+ for (int i = 0; i < deleted.length; ++i) {
+ if (deleted[i] != WORD_MASK) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public void resetUpdates() {
+ for (int i = 0; i < updated.length; ++i) {
+ updated[i] = 0;
+ }
+ }
+
+ public void undeleteAll() {
+ for (int i = 0; i < updated.length; ++i) {
+ deleted[i] = 0;
+ }
+ }
+
+ public ProcedureProtos.ProcedureStoreTracker.TrackerNode convert() {
+ ProcedureProtos.ProcedureStoreTracker.TrackerNode.Builder builder =
+ ProcedureProtos.ProcedureStoreTracker.TrackerNode.newBuilder();
+ builder.setStartId(start);
+ for (int i = 0; i < updated.length; ++i) {
+ builder.addUpdated(updated[i]);
+ builder.addDeleted(deleted[i]);
+ }
+ return builder.build();
+ }
+
+ public static BitSetNode convert(ProcedureProtos.ProcedureStoreTracker.TrackerNode data) {
+ long start = data.getStartId();
+ int size = data.getUpdatedCount();
+ long[] updated = new long[size];
+ long[] deleted = new long[size];
+ for (int i = 0; i < size; ++i) {
+ updated[i] = data.getUpdated(i);
+ deleted[i] = data.getDeleted(i);
+ }
+ return new BitSetNode(start, updated, deleted);
+ }
+
+ // ========================================================================
+ // Grow/Merge Helpers
+ // ========================================================================
+ public boolean canGrow(final long procId) {
+ return (procId - start) < MAX_NODE_SIZE;
+ }
+
+ public boolean canMerge(final BitSetNode rightNode) {
+ return (start + rightNode.getEnd()) < MAX_NODE_SIZE;
+ }
+
+ public void grow(final long procId) {
+ int delta, offset;
+
+ if (procId < start) {
+ // add to head
+ long newStart = alignDown(procId);
+ delta = (int)(start - newStart) >> ADDRESS_BITS_PER_WORD;
+ offset = delta;
+ } else {
+ // Add to tail
+ long newEnd = alignUp(procId + 1);
+ delta = (int)(newEnd - getEnd()) >> ADDRESS_BITS_PER_WORD;
+ offset = 0;
+ }
+
+ long[] newBitmap;
+ int oldSize = updated.length;
+
+ newBitmap = new long[oldSize + delta];
+ System.arraycopy(updated, 0, newBitmap, offset, oldSize);
+ updated = newBitmap;
+
+ newBitmap = new long[deleted.length + delta];
+ System.arraycopy(deleted, 0, newBitmap, offset, oldSize);
+ deleted = newBitmap;
+
+ for (int i = 0; i < delta; ++i) {
+ updated[oldSize + i] = 0;
+ deleted[oldSize + i] = WORD_MASK;
+ }
+ }
+
+ public void merge(final BitSetNode rightNode) {
+ int delta = (int)(rightNode.getEnd() - getEnd()) >> ADDRESS_BITS_PER_WORD;
+
+ long[] newBitmap;
+ int oldSize = updated.length;
+ int newSize = (delta - rightNode.updated.length);
+ int offset = oldSize + newSize;
+
+ newBitmap = new long[oldSize + delta];
+ System.arraycopy(updated, 0, newBitmap, 0, oldSize);
+ System.arraycopy(rightNode.updated, 0, newBitmap, offset, rightNode.updated.length);
+ updated = newBitmap;
+
+ newBitmap = new long[oldSize + delta];
+ System.arraycopy(deleted, 0, newBitmap, 0, oldSize);
+ System.arraycopy(rightNode.deleted, 0, newBitmap, offset, rightNode.deleted.length);
+ deleted = newBitmap;
+
+ for (int i = 0; i < newSize; ++i) {
+ updated[offset + i] = 0;
+ deleted[offset + i] = WORD_MASK;
+ }
+ }
+
+ // ========================================================================
+ // Min/Max Helpers
+ // ========================================================================
+ public long getMinProcId() {
+ long minProcId = start;
+ for (int i = 0; i < deleted.length; ++i) {
+ if (deleted[i] == 0) {
+ return(minProcId);
+ }
+
+ if (deleted[i] != WORD_MASK) {
+ for (int j = 0; j < BITS_PER_WORD; ++j) {
+ if ((deleted[i] & (1L << j)) != 0) {
+ return minProcId + j;
+ }
+ }
+ }
+
+ minProcId += BITS_PER_WORD;
+ }
+ return minProcId;
+ }
+
+ public long getMaxProcId() {
+ long maxProcId = getEnd();
+ for (int i = deleted.length - 1; i >= 0; --i) {
+ if (deleted[i] == 0) {
+ return maxProcId;
+ }
+
+ if (deleted[i] != WORD_MASK) {
+ for (int j = BITS_PER_WORD - 1; j >= 0; --j) {
+ if ((deleted[i] & (1L << j)) == 0) {
+ return maxProcId - (BITS_PER_WORD - 1 - j);
+ }
+ }
+ }
+ maxProcId -= BITS_PER_WORD;
+ }
+ return maxProcId;
+ }
+
+ // ========================================================================
+ // Bitmap Helpers
+ // ========================================================================
+ private int getBitmapIndex(final long procId) {
+ return (int)(procId - start);
+ }
+
+ private void updateState(final long procId, final boolean isDeleted) {
+ int bitmapIndex = getBitmapIndex(procId);
+ int wordIndex = bitmapIndex >> ADDRESS_BITS_PER_WORD;
+ long value = (1L << bitmapIndex);
+
+ if (isDeleted) {
+ updated[wordIndex] |= value;
+ deleted[wordIndex] |= value;
+ } else {
+ updated[wordIndex] |= value;
+ deleted[wordIndex] &= ~value;
+ }
+ }
+
+ // ========================================================================
+ // Helpers
+ // ========================================================================
+ private static long alignUp(final long x) {
+ return (x + (BITS_PER_WORD - 1)) & -BITS_PER_WORD;
+ }
+
+ private static long alignDown(final long x) {
+ return x & -BITS_PER_WORD;
+ }
+ }
+
+ public void insert(final Procedure proc, final Procedure[] subprocs) {
+ insert(proc.getProcId());
+ if (subprocs != null) {
+ for (int i = 0; i < subprocs.length; ++i) {
+ insert(subprocs[i].getProcId());
+ }
+ }
+ }
+
+ public void update(final Procedure proc) {
+ update(proc.getProcId());
+ }
+
+ public void insert(long procId) {
+ BitSetNode node = getOrCreateNode(procId);
+ node.update(procId);
+ }
+
+ public void update(long procId) {
+ Map.Entry<Long, BitSetNode> entry = map.floorEntry(procId);
+ assert entry != null : "expected node to update procId=" + procId;
+
+ BitSetNode node = entry.getValue();
+ assert node.contains(procId);
+ node.update(procId);
+ }
+
+ public void delete(long procId) {
+ Map.Entry<Long, BitSetNode> entry = map.floorEntry(procId);
+ assert entry != null : "expected node to delete procId=" + procId;
+
+ BitSetNode node = entry.getValue();
+ assert node.contains(procId) : "expected procId in the node";
+ node.delete(procId);
+
+ if (!keepDeletes && node.isEmpty()) {
+ // TODO: RESET if (map.size() == 1)
+ map.remove(entry.getKey());
+ }
+ }
+
+ @InterfaceAudience.Private
+ public void setDeleted(final long procId, final boolean isDeleted) {
+ BitSetNode node = getOrCreateNode(procId);
+ node.updateState(procId, isDeleted);
+ }
+
+ public void clear() {
+ this.map.clear();
+ }
+
+ public DeleteState isDeleted(long procId) {
+ Map.Entry<Long, BitSetNode> entry = map.floorEntry(procId);
+ if (entry != null) {
+ BitSetNode node = entry.getValue();
+ DeleteState state = node.isDeleted(procId);
+ return partial && !node.isUpdated(procId) ? DeleteState.MAYBE : state;
+ }
+ return partial ? DeleteState.MAYBE : DeleteState.YES;
+ }
+
+ public long getMinProcId() {
+ // TODO: Cache?
+ Map.Entry<Long, BitSetNode> entry = map.firstEntry();
+ return entry == null ? 0 : entry.getValue().getMinProcId();
+ }
+
+ public void setKeepDeletes(boolean keepDeletes) {
+ this.keepDeletes = keepDeletes;
+ if (!keepDeletes) {
+ Iterator<Map.Entry<Long, BitSetNode>> it = map.entrySet().iterator();
+ while (it.hasNext()) {
+ Map.Entry<Long, BitSetNode> entry = it.next();
+ if (entry.getValue().isEmpty()) {
+ it.remove();
+ }
+ }
+ }
+ }
+
+ public void setPartialFlag(boolean isPartial) {
+ this.partial = isPartial;
+ }
+
+ public boolean isEmpty() {
+ for (Map.Entry<Long, BitSetNode> entry : map.entrySet()) {
+ if (entry.getValue().isEmpty() == false) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public boolean isUpdated() {
+ for (Map.Entry<Long, BitSetNode> entry : map.entrySet()) {
+ if (entry.getValue().isUpdated() == false) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public void resetUpdates() {
+ for (Map.Entry<Long, BitSetNode> entry : map.entrySet()) {
+ entry.getValue().resetUpdates();
+ }
+ }
+
+ public void undeleteAll() {
+ for (Map.Entry<Long, BitSetNode> entry : map.entrySet()) {
+ entry.getValue().undeleteAll();
+ }
+ }
+
+ private BitSetNode getOrCreateNode(final long procId) {
+ // can procId fit in the left node?
+ BitSetNode leftNode = null;
+ boolean leftCanGrow = false;
+ Map.Entry<Long, BitSetNode> leftEntry = map.floorEntry(procId);
+ if (leftEntry != null) {
+ leftNode = leftEntry.getValue();
+ if (leftNode.contains(procId)) {
+ return leftNode;
+ }
+ leftCanGrow = leftNode.canGrow(procId);
+ }
+
+ BitSetNode rightNode = null;
+ boolean rightCanGrow = false;
+ Map.Entry<Long, BitSetNode> rightEntry = map.ceilingEntry(procId);
+ if (rightEntry != null) {
+ rightNode = rightEntry.getValue();
+ rightCanGrow = rightNode.canGrow(procId);
+ if (leftNode != null) {
+ if (leftNode.canMerge(rightNode)) {
+ // merge left and right node
+ return mergeNodes(leftNode, rightNode);
+ }
+
+ if (leftCanGrow && rightCanGrow) {
+ if ((procId - leftNode.getEnd()) <= (rightNode.getStart() - procId)) {
+ // grow the left node
+ return growNode(leftNode, procId);
+ }
+ // grow the right node
+ return growNode(rightNode, procId);
+ }
+ }
+ }
+
+ // grow the left node
+ if (leftCanGrow) {
+ return growNode(leftNode, procId);
+ }
+
+ // grow the right node
+ if (rightCanGrow) {
+ return growNode(rightNode, procId);
+ }
+
+ // add new node
+ BitSetNode node = new BitSetNode(procId, partial);
+ map.put(node.getStart(), node);
+ return node;
+ }
+
+ private BitSetNode growNode(BitSetNode node, long procId) {
+ map.remove(node.getStart());
+ node.grow(procId);
+ map.put(node.getStart(), node);
+ return node;
+ }
+
+ private BitSetNode mergeNodes(BitSetNode leftNode, BitSetNode rightNode) {
+ leftNode.merge(rightNode);
+ map.remove(rightNode.getStart());
+ return leftNode;
+ }
+
+ public void dump() {
+ System.out.println("map " + map.size());
+ for (Map.Entry<Long, BitSetNode> entry : map.entrySet()) {
+ entry.getValue().dump();
+ }
+ }
+
+ public void writeTo(final OutputStream stream) throws IOException {
+ ProcedureProtos.ProcedureStoreTracker.Builder builder =
+ ProcedureProtos.ProcedureStoreTracker.newBuilder();
+ for (Map.Entry<Long, BitSetNode> entry : map.entrySet()) {
+ builder.addNode(entry.getValue().convert());
+ }
+ builder.build().writeDelimitedTo(stream);
+ }
+
+ public void readFrom(final InputStream stream) throws IOException {
+ ProcedureProtos.ProcedureStoreTracker data =
+ ProcedureProtos.ProcedureStoreTracker.parseDelimitedFrom(stream);
+ map.clear();
+ for (ProcedureProtos.ProcedureStoreTracker.TrackerNode protoNode: data.getNodeList()) {
+ BitSetNode node = BitSetNode.convert(protoNode);
+ map.put(node.getStart(), node);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/CorruptedWALProcedureStoreException.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/CorruptedWALProcedureStoreException.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/CorruptedWALProcedureStoreException.java
new file mode 100644
index 0000000..29db3bf
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/CorruptedWALProcedureStoreException.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2.store.wal;
+
+import org.apache.hadoop.hbase.HBaseIOException;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * Thrown when a procedure WAL is corrupted
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class CorruptedWALProcedureStoreException extends HBaseIOException {
+ /** default constructor */
+ public CorruptedWALProcedureStoreException() {
+ super();
+ }
+
+ /**
+ * Constructor
+ * @param s message
+ */
+ public CorruptedWALProcedureStoreException(String s) {
+ super(s);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFile.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFile.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFile.java
new file mode 100644
index 0000000..859b3cb
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFile.java
@@ -0,0 +1,152 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2.store.wal;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureWALHeader;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureWALTrailer;
+
+/**
+ * Describes a WAL File
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class ProcedureWALFile implements Comparable<ProcedureWALFile> {
+ private static final Log LOG = LogFactory.getLog(ProcedureWALFile.class);
+
+ private ProcedureWALHeader header;
+ private FSDataInputStream stream;
+ private FileStatus logStatus;
+ private FileSystem fs;
+ private Path logFile;
+ private long startPos;
+
+ public ProcedureWALFile(final FileSystem fs, final FileStatus logStatus) {
+ this.fs = fs;
+ this.logStatus = logStatus;
+ this.logFile = logStatus.getPath();
+ }
+
+ public ProcedureWALFile(FileSystem fs, Path logFile, ProcedureWALHeader header, long startPos) {
+ this.fs = fs;
+ this.logFile = logFile;
+ this.header = header;
+ this.startPos = startPos;
+ }
+
+ public void open() throws IOException {
+ if (stream == null) {
+ stream = fs.open(logFile);
+ }
+
+ if (header == null) {
+ header = ProcedureWALFormat.readHeader(stream);
+ startPos = stream.getPos();
+ } else {
+ stream.seek(startPos);
+ }
+ }
+
+ public ProcedureWALTrailer readTrailer() throws IOException {
+ try {
+ return ProcedureWALFormat.readTrailer(stream, startPos, logStatus.getLen());
+ } finally {
+ stream.seek(startPos);
+ }
+ }
+
+ public void readTracker(ProcedureStoreTracker tracker) throws IOException {
+ ProcedureWALTrailer trailer = readTrailer();
+ try {
+ stream.seek(trailer.getTrackerPos());
+ tracker.readFrom(stream);
+ } finally {
+ stream.seek(startPos);
+ }
+ }
+
+ public void close() {
+ if (stream == null) return;
+ try {
+ stream.close();
+ } catch (IOException e) {
+ LOG.warn("unable to close the wal file: " + logFile, e);
+ } finally {
+ stream = null;
+ }
+ }
+
+ public FSDataInputStream getStream() {
+ return stream;
+ }
+
+ public ProcedureWALHeader getHeader() {
+ return header;
+ }
+
+ public boolean isCompacted() {
+ return header.getType() == ProcedureWALFormat.LOG_TYPE_COMPACTED;
+ }
+
+ public long getLogId() {
+ return header.getLogId();
+ }
+
+ public long getSize() {
+ return logStatus.getLen();
+ }
+
+ public void removeFile() throws IOException {
+ close();
+ fs.delete(logFile, false);
+ }
+
+ @Override
+ public int compareTo(final ProcedureWALFile other) {
+ long diff = header.getLogId() - other.header.getLogId();
+ return (diff < 0) ? -1 : (diff > 0) ? 1 : 0;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof ProcedureWALFile)) return false;
+ return compareTo((ProcedureWALFile)o) == 0;
+ }
+
+ @Override
+ public int hashCode() {
+ return logFile.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return logFile.toString();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.java
new file mode 100644
index 0000000..b6b4622
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.java
@@ -0,0 +1,212 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2.store.wal;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Iterator;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.hbase.io.util.StreamUtils;
+import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker;
+import org.apache.hadoop.hbase.procedure2.util.ByteSlot;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureWALEntry;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureWALHeader;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureWALTrailer;
+
+/**
+ * Helper class that contains the WAL serialization utils.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public final class ProcedureWALFormat {
+ static final byte LOG_TYPE_STREAM = 0;
+ static final byte LOG_TYPE_COMPACTED = 1;
+ static final byte LOG_TYPE_MAX_VALID = 1;
+
+ static final byte HEADER_VERSION = 1;
+ static final byte TRAILER_VERSION = 1;
+ static final long HEADER_MAGIC = 0x31764c4157637250L;
+ static final long TRAILER_MAGIC = 0x50726357414c7631L;
+
+ interface Loader {
+ void removeLog(ProcedureWALFile log);
+ void markCorruptedWAL(ProcedureWALFile log, IOException e);
+ }
+
+ private ProcedureWALFormat() {}
+
+ public static Iterator<Procedure> load(final Iterator<ProcedureWALFile> logs,
+ final ProcedureStoreTracker tracker, final Loader loader) throws IOException {
+ ProcedureWALFormatReader reader = new ProcedureWALFormatReader(tracker);
+ tracker.setKeepDeletes(true);
+ try {
+ while (logs.hasNext()) {
+ ProcedureWALFile log = logs.next();
+ log.open();
+ try {
+ reader.read(log, loader);
+ } finally {
+ log.close();
+ }
+ }
+ // The tracker is now updated with all the procedures read from the logs
+ tracker.setPartialFlag(false);
+ tracker.resetUpdates();
+ } finally {
+ tracker.setKeepDeletes(false);
+ }
+ // TODO: Write compacted version?
+ return reader.getProcedures();
+ }
+
+ public static void writeHeader(OutputStream stream, ProcedureWALHeader header)
+ throws IOException {
+ header.writeDelimitedTo(stream);
+ }
+
+ /*
+ * +-----------------+
+ * | END OF WAL DATA | <---+
+ * +-----------------+ |
+ * | | |
+ * | Tracker | |
+ * | | |
+ * +-----------------+ |
+ * | version | |
+ * +-----------------+ |
+ * | TRAILER_MAGIC | |
+ * +-----------------+ |
+ * | offset |-----+
+ * +-----------------+
+ */
+ public static void writeTrailer(FSDataOutputStream stream, ProcedureStoreTracker tracker)
+ throws IOException {
+ long offset = stream.getPos();
+
+ // Write EOF Entry
+ ProcedureWALEntry.newBuilder()
+ .setType(ProcedureWALEntry.Type.EOF)
+ .build().writeDelimitedTo(stream);
+
+ // Write Tracker
+ tracker.writeTo(stream);
+
+ stream.write(TRAILER_VERSION);
+ StreamUtils.writeLong(stream, TRAILER_MAGIC);
+ StreamUtils.writeLong(stream, offset);
+ }
+
+ public static ProcedureWALHeader readHeader(InputStream stream)
+ throws IOException {
+ ProcedureWALHeader header = ProcedureWALHeader.parseDelimitedFrom(stream);
+
+ if (header.getVersion() < 0 || header.getVersion() != HEADER_VERSION) {
+ throw new IOException("Invalid Header version. got " + header.getVersion() +
+ " expected " + HEADER_VERSION);
+ }
+
+ if (header.getType() < 0 || header.getType() > LOG_TYPE_MAX_VALID) {
+ throw new IOException("Invalid header type. got " + header.getType());
+ }
+
+ return header;
+ }
+
+ public static ProcedureWALTrailer readTrailer(FSDataInputStream stream, long startPos, long size)
+ throws IOException {
+ long trailerPos = size - 17; // Beginning of the Trailer Jump
+
+ if (trailerPos < startPos) {
+ throw new IOException("Missing trailer: size=" + size + " startPos=" + startPos);
+ }
+
+ stream.seek(trailerPos);
+ int version = stream.read();
+ if (version != TRAILER_VERSION) {
+ throw new IOException("Invalid Trailer version. got " + version +
+ " expected " + TRAILER_VERSION);
+ }
+
+ long magic = StreamUtils.readLong(stream);
+ if (magic != TRAILER_MAGIC) {
+ throw new IOException("Invalid Trailer magic. got " + magic +
+ " expected " + TRAILER_MAGIC);
+ }
+
+ long trailerOffset = StreamUtils.readLong(stream);
+ stream.seek(trailerOffset);
+
+ ProcedureWALEntry entry = readEntry(stream);
+ if (entry.getType() != ProcedureWALEntry.Type.EOF) {
+ throw new IOException("Invalid Trailer begin");
+ }
+
+ ProcedureWALTrailer trailer = ProcedureWALTrailer.newBuilder()
+ .setVersion(version)
+ .setTrackerPos(stream.getPos())
+ .build();
+ return trailer;
+ }
+
+ public static ProcedureWALEntry readEntry(InputStream stream) throws IOException {
+ return ProcedureWALEntry.parseDelimitedFrom(stream);
+ }
+
+ public static void writeEntry(ByteSlot slot, ProcedureWALEntry.Type type,
+ Procedure proc, Procedure[] subprocs) throws IOException {
+ ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();
+ builder.setType(type);
+ builder.addProcedure(Procedure.convert(proc));
+ if (subprocs != null) {
+ for (int i = 0; i < subprocs.length; ++i) {
+ builder.addProcedure(Procedure.convert(subprocs[i]));
+ }
+ }
+ builder.build().writeDelimitedTo(slot);
+ }
+
+ public static void writeInsert(ByteSlot slot, Procedure proc)
+ throws IOException {
+ writeEntry(slot, ProcedureWALEntry.Type.INIT, proc, null);
+ }
+
+ public static void writeInsert(ByteSlot slot, Procedure proc, Procedure[] subprocs)
+ throws IOException {
+ writeEntry(slot, ProcedureWALEntry.Type.INSERT, proc, subprocs);
+ }
+
+ public static void writeUpdate(ByteSlot slot, Procedure proc)
+ throws IOException {
+ writeEntry(slot, ProcedureWALEntry.Type.UPDATE, proc, null);
+ }
+
+ public static void writeDelete(ByteSlot slot, long procId)
+ throws IOException {
+ ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();
+ builder.setType(ProcedureWALEntry.Type.DELETE);
+ builder.setProcId(procId);
+ builder.build().writeDelimitedTo(slot);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.java
new file mode 100644
index 0000000..66c374e
--- /dev/null
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.java
@@ -0,0 +1,161 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2.store.wal;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.HashMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureWALEntry;
+
+/**
+ * Helper class that loads the procedures stored in a WAL
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class ProcedureWALFormatReader {
+ private static final Log LOG = LogFactory.getLog(ProcedureWALFormatReader.class);
+
+ private final ProcedureStoreTracker tracker;
+ //private final long compactionLogId;
+
+ private final Map<Long, Procedure> procedures = new HashMap<Long, Procedure>();
+ private final Map<Long, ProcedureProtos.Procedure> localProcedures =
+ new HashMap<Long, ProcedureProtos.Procedure>();
+
+ private long maxProcId = 0;
+
+ public ProcedureWALFormatReader(final ProcedureStoreTracker tracker) {
+ this.tracker = tracker;
+ }
+
+ public void read(ProcedureWALFile log, ProcedureWALFormat.Loader loader) throws IOException {
+ FSDataInputStream stream = log.getStream();
+ try {
+ boolean hasMore = true;
+ while (hasMore) {
+ ProcedureWALEntry entry = ProcedureWALFormat.readEntry(stream);
+ switch (entry.getType()) {
+ case INIT:
+ readInitEntry(entry);
+ break;
+ case INSERT:
+ readInsertEntry(entry);
+ break;
+ case UPDATE:
+ case COMPACT:
+ readUpdateEntry(entry);
+ break;
+ case DELETE:
+ readDeleteEntry(entry);
+ break;
+ case EOF:
+ hasMore = false;
+ break;
+ default:
+ throw new CorruptedWALProcedureStoreException("Invalid entry: " + entry);
+ }
+ }
+ } catch (IOException e) {
+ LOG.error("got an exception while reading the procedure WAL: " + log, e);
+ loader.markCorruptedWAL(log, e);
+ }
+
+ if (localProcedures.isEmpty()) {
+ LOG.info("No active entry found in state log " + log + ". removing it");
+ loader.removeLog(log);
+ } else {
+ Iterator<Map.Entry<Long, ProcedureProtos.Procedure>> itd =
+ localProcedures.entrySet().iterator();
+ while (itd.hasNext()) {
+ Map.Entry<Long, ProcedureProtos.Procedure> entry = itd.next();
+ itd.remove();
+
+ // Deserialize the procedure
+ Procedure proc = Procedure.convert(entry.getValue());
+ procedures.put(entry.getKey(), proc);
+ }
+
+ // TODO: Some procedure may be already runnables (see readInitEntry())
+ // (we can also check the "update map" in the log trackers)
+ }
+ }
+
+ public Iterator<Procedure> getProcedures() {
+ return procedures.values().iterator();
+ }
+
+ private void loadEntries(final ProcedureWALEntry entry) {
+ for (ProcedureProtos.Procedure proc: entry.getProcedureList()) {
+ maxProcId = Math.max(maxProcId, proc.getProcId());
+ if (isRequired(proc.getProcId())) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("read " + entry.getType() + " entry " + proc.getProcId());
+ }
+ localProcedures.put(proc.getProcId(), proc);
+ tracker.setDeleted(proc.getProcId(), false);
+ }
+ }
+ }
+
+ private void readInitEntry(final ProcedureWALEntry entry)
+ throws IOException {
+ assert entry.getProcedureCount() == 1 : "Expected only one procedure";
+ // TODO: Make it runnable, before reading other files
+ loadEntries(entry);
+ }
+
+ private void readInsertEntry(final ProcedureWALEntry entry) throws IOException {
+ assert entry.getProcedureCount() >= 1 : "Expected one or more procedures";
+ loadEntries(entry);
+ }
+
+ private void readUpdateEntry(final ProcedureWALEntry entry) throws IOException {
+ assert entry.getProcedureCount() == 1 : "Expected only one procedure";
+ loadEntries(entry);
+ }
+
+ private void readDeleteEntry(final ProcedureWALEntry entry) throws IOException {
+ assert entry.getProcedureCount() == 0 : "Expected no procedures";
+ assert entry.hasProcId() : "expected ProcID";
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("read delete entry " + entry.getProcId());
+ }
+ maxProcId = Math.max(maxProcId, entry.getProcId());
+ localProcedures.remove(entry.getProcId());
+ tracker.setDeleted(entry.getProcId(), true);
+ }
+
+ private boolean isDeleted(final long procId) {
+ return tracker.isDeleted(procId) == ProcedureStoreTracker.DeleteState.YES;
+ }
+
+ private boolean isRequired(final long procId) {
+ return !isDeleted(procId) && !procedures.containsKey(procId);
+ }
+}
\ No newline at end of file
[10/15] hbase git commit: HBASE-13202 Procedure v2 - core framework
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/hbase-protocol/src/main/protobuf/Procedure.proto
----------------------------------------------------------------------
diff --git a/hbase-protocol/src/main/protobuf/Procedure.proto b/hbase-protocol/src/main/protobuf/Procedure.proto
new file mode 100644
index 0000000..232c290
--- /dev/null
+++ b/hbase-protocol/src/main/protobuf/Procedure.proto
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "ProcedureProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "ErrorHandling.proto";
+
+enum ProcedureState {
+ INITIALIZING = 1; // Procedure in construction, not yet added to the executor
+ RUNNABLE = 2; // Procedure added to the executor, and ready to be executed
+ WAITING = 3; // The procedure is waiting on children to be completed
+ WAITING_TIMEOUT = 4; // The procedure is waiting a timout or an external event
+ ROLLEDBACK = 5; // The procedure failed and was rolledback
+ FINISHED = 6; // The procedure execution is completed. may need a rollback if failed.
+}
+
+/**
+ * Procedure metadata, serialized by the ProcedureStore to be able to recover the old state.
+ */
+message Procedure {
+ // internal "static" state
+ required string class_name = 1; // full classname to be able to instantiate the procedure
+ optional uint64 parent_id = 2; // parent if not a root-procedure otherwise not set
+ required uint64 proc_id = 3;
+ required uint64 start_time = 4;
+ optional string owner = 5;
+
+ // internal "runtime" state
+ required ProcedureState state = 6;
+ repeated uint32 stack_id = 7; // stack indices in case the procedure was running
+ required uint64 last_update = 8;
+ optional uint32 timeout = 9;
+
+ // user state/results
+ optional ForeignExceptionMessage exception = 10;
+ optional bytes result = 11; // opaque (user) result structure
+ optional bytes state_data = 12; // opaque (user) procedure internal-state
+}
+
+/**
+ * SequentialProcedure data
+ */
+message SequentialProcedureData {
+ required bool executed = 1;
+}
+
+/**
+ * StateMachineProcedure data
+ */
+message StateMachineProcedureData {
+ repeated uint32 state = 1;
+}
+
+/**
+ * Procedure WAL header
+ */
+message ProcedureWALHeader {
+ required uint32 version = 1;
+ required uint32 type = 2;
+ required uint64 log_id = 3;
+ required uint64 min_proc_id = 4;
+}
+
+/**
+ * Procedure WAL trailer
+ */
+message ProcedureWALTrailer {
+ required uint32 version = 1;
+ required uint64 tracker_pos = 2;
+}
+
+message ProcedureStoreTracker {
+ message TrackerNode {
+ required uint64 start_id = 1;
+ repeated uint64 updated = 2;
+ repeated uint64 deleted = 3;
+ }
+
+ repeated TrackerNode node = 1;
+}
+
+message ProcedureWALEntry {
+ enum Type {
+ EOF = 1;
+ INIT = 2;
+ INSERT = 3;
+ UPDATE = 4;
+ DELETE = 5;
+ COMPACT = 6;
+ }
+
+ required Type type = 1;
+ repeated Procedure procedure = 2;
+ optional uint64 proc_id = 3;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ddc1bcae/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index d5b974c..9f71d98 100644
--- a/pom.xml
+++ b/pom.xml
@@ -56,6 +56,7 @@
<module>hbase-client</module>
<module>hbase-hadoop-compat</module>
<module>hbase-common</module>
+ <module>hbase-procedure</module>
<module>hbase-it</module>
<module>hbase-examples</module>
<module>hbase-prefix-tree</module>
@@ -872,7 +873,7 @@
</fileMapper>
</fileMappers>
<outputDir>${basedir}/target/asciidoc</outputDir>
- </transformationSet>
+ </transformationSet>
</transformationSets>
</configuration>
</plugin>
@@ -1015,7 +1016,7 @@
<plugin>
<groupId>org.asciidoctor</groupId>
<artifactId>asciidoctor-maven-plugin</artifactId>
- <version>1.5.2</version>
+ <version>1.5.2</version>
<inherited>false</inherited>
<dependencies>
<dependency>
@@ -1035,10 +1036,10 @@
</configuration>
<executions>
<execution>
- <id>output-html</id>
+ <id>output-html</id>
<phase>site</phase>
<goals>
- <goal>process-asciidoc</goal>
+ <goal>process-asciidoc</goal>
</goals>
<configuration>
<attributes>
@@ -1192,6 +1193,7 @@
Modules are pretty heavy-weight things, so doing this work isn't too bad. -->
<server.test.jar>hbase-server-${project.version}-tests.jar</server.test.jar>
<common.test.jar>hbase-common-${project.version}-tests.jar</common.test.jar>
+ <procedure.test.jar>hbase-procedure-${project.version}-tests.jar</procedure.test.jar>
<it.test.jar>hbase-it-${project.version}-tests.jar</it.test.jar>
<annotations.test.jar>hbase-annotations-${project.version}-tests.jar</annotations.test.jar>
<surefire.version>2.18</surefire.version>
@@ -1263,6 +1265,17 @@
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-procedure</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-procedure</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
<artifactId>hbase-hadoop-compat</artifactId>
<version>${project.version}</version>
</dependency>
[02/15] hbase git commit: Fix company name for garyh
Posted by mb...@apache.org.
Fix company name for garyh
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2e31e834
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2e31e834
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2e31e834
Branch: refs/heads/hbase-12439
Commit: 2e31e834a67be8a88afaaca7691eadaf55f84b49
Parents: 50daed2
Author: Gary Helmling <ga...@cask.co>
Authored: Wed Mar 25 13:56:30 2015 -0700
Committer: Gary Helmling <ga...@cask.co>
Committed: Wed Mar 25 13:56:54 2015 -0700
----------------------------------------------------------------------
pom.xml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e31e834/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 3f94abc..d5b974c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -198,8 +198,8 @@
<name>Gary Helmling</name>
<email>garyh@apache.org</email>
<timezone>-8</timezone>
- <organization>Continuuity</organization>
- <organizationUrl>http://www.continuuity.com</organizationUrl>
+ <organization>Cask</organization>
+ <organizationUrl>http://www.cask.co</organizationUrl>
</developer>
<developer>
<id>gchanan</id>
[06/15] hbase git commit: HBASE-13334 FindBugs should create precise
report for new bugs introduced
Posted by mb...@apache.org.
HBASE-13334 FindBugs should create precise report for new bugs introduced
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/948a6a63
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/948a6a63
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/948a6a63
Branch: refs/heads/hbase-12439
Commit: 948a6a63e8c1a347ff18903cba8fe6ffb6190d37
Parents: 97bfc2d
Author: Andrey Stepachev <oc...@gmail.com>
Authored: Thu Mar 26 00:27:14 2015 +0000
Committer: Andrey Stepachev <oc...@gmail.com>
Committed: Thu Mar 26 00:27:14 2015 +0000
----------------------------------------------------------------------
dev-support/test-patch.properties | 1 -
dev-support/test-patch.sh | 88 +++++++++++++++++++++-------------
2 files changed, 56 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/948a6a63/dev-support/test-patch.properties
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.properties b/dev-support/test-patch.properties
index 9bf5842..5f5cc32 100644
--- a/dev-support/test-patch.properties
+++ b/dev-support/test-patch.properties
@@ -19,7 +19,6 @@ MAVEN_OPTS="${MAVEN_OPTS:-"-Xmx3100M"}"
# Please update the per-module test-patch.properties if you update this file.
OK_RELEASEAUDIT_WARNINGS=0
-OK_FINDBUGS_WARNINGS=95
# Allow two warnings. Javadoc complains about sun.misc.Unsafe use. See HBASE-7457
OK_JAVADOC_WARNINGS=2
http://git-wip-us.apache.org/repos/asf/hbase/blob/948a6a63/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index 86f7685..64cf8a7 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -240,6 +240,33 @@ checkoutBranch() {
}
###############################################################################
+### Collect findbugs reports
+collectFindbugsReports() {
+ name=$1
+ basedir=$2
+ patch_dir=$3
+ for file in $(find $basedir -name findbugsXml.xml)
+ do
+ relative_file=${file#$basedir/} # strip leading $basedir prefix
+ if [ ! $relative_file == "target/findbugsXml.xml" ]; then
+ module_suffix=${relative_file%/target/findbugsXml.xml} # strip trailing path
+ module_suffix=`basename ${module_suffix}`
+ fi
+
+ cp $file $patch_dir/${name}FindbugsWarnings${module_suffix}.xml
+ $FINDBUGS_HOME/bin/setBugDatabaseInfo -name $name \
+ $patch_dir/${name}FindbugsWarnings${module_suffix}.xml \
+ $patch_dir/${name}FindbugsWarnings${module_suffix}.xml
+ done
+ xml_file=$patch_dir/${name}FindbugsWarnings.xml
+ html_file=$patch_dir/${name}FindbugsWarnings.html
+ $FINDBUGS_HOME/bin/unionBugs -withMessages \
+ -output $xml_file $patch_dir/${name}FindbugsWarnings*.xml
+ $FINDBUGS_HOME/bin/convertXmlToText -html $xml_file $html_file
+ file $xml_file $html_file
+}
+
+###############################################################################
setup () {
### Download latest patch file (ignoring .htm and .html) when run from patch process
if [[ $JENKINS == "true" ]] ; then
@@ -281,9 +308,8 @@ setup () {
fi
fi
### exit if warnings are NOT defined in the properties file
- if [ -z "$OK_FINDBUGS_WARNINGS" ] || [[ -z "$OK_JAVADOC_WARNINGS" ]] || [[ -z $OK_RELEASEAUDIT_WARNINGS ]] ; then
+ if [[ -z "$OK_JAVADOC_WARNINGS" ]] || [[ -z $OK_RELEASEAUDIT_WARNINGS ]] ; then
echo "Please define the following properties in test-patch.properties file"
- echo "OK_FINDBUGS_WARNINGS"
echo "OK_RELEASEAUDIT_WARNINGS"
echo "OK_JAVADOC_WARNINGS"
cleanupAndExit 1
@@ -297,10 +323,12 @@ setup () {
echo "======================================================================"
echo ""
echo ""
- echo "$MVN clean package checkstyle:checkstyle-aggregate -DskipTests -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavacWarnings.txt 2>&1"
+ echo "$MVN clean package checkstyle:checkstyle-aggregate findbugs:findbugs -DskipTests \
+ -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavacWarnings.txt 2>&1"
export MAVEN_OPTS="${MAVEN_OPTS}"
# build core and tests
- $MVN clean package checkstyle:checkstyle-aggregate -DskipTests -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavacWarnings.txt 2>&1
+ $MVN clean package checkstyle:checkstyle-aggregate findbugs:findbugs -DskipTests \
+ -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavacWarnings.txt 2>&1
if [[ $? != 0 ]] ; then
ERR=`$GREP -A 5 'Compilation failure' $PATCH_DIR/trunkJavacWarnings.txt`
echo "Trunk compilation is broken?
@@ -308,6 +336,7 @@ setup () {
cleanupAndExit 1
fi
mv target/checkstyle-result.xml $PATCH_DIR/trunkCheckstyle.xml
+ collectFindbugsReports trunk $BASEDIR $PATCH_DIR
}
###############################################################################
@@ -705,41 +734,36 @@ checkFindbugsWarnings () {
{color:red}-1 findbugs{color}. The patch appears to cause Findbugs (version ${findbugs_version}) to fail."
return 1
fi
-
- findbugsWarnings=0
- for file in $(find $BASEDIR -name findbugsXml.xml)
- do
- relative_file=${file#$BASEDIR/} # strip leading $BASEDIR prefix
- if [ ! $relative_file == "target/findbugsXml.xml" ]; then
- module_suffix=${relative_file%/target/findbugsXml.xml} # strip trailing path
- module_suffix=`basename ${module_suffix}`
- fi
-
- cp $file $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml
- $FINDBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/2000" \
- $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \
- $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml
- newFindbugsWarnings=`$FINDBUGS_HOME/bin/filterBugs -first "01/01/2000" $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \
- $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml | $AWK '{print $1}'`
- echo "Found $newFindbugsWarnings Findbugs warnings ($file)"
- findbugsWarnings=$((findbugsWarnings+newFindbugsWarnings))
- echo "$FINDBUGS_HOME/bin/convertXmlToText -html $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.html"
- $FINDBUGS_HOME/bin/convertXmlToText -html $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.html
- file $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.html
- JIRA_COMMENT_FOOTER="Findbugs warnings: $BUILD_URL/artifact/patchprocess/newPatchFindbugsWarnings${module_suffix}.html
-$JIRA_COMMENT_FOOTER"
- done
- ### if current warnings greater than OK_FINDBUGS_WARNINGS
- if [[ $findbugsWarnings -gt $OK_FINDBUGS_WARNINGS ]] ; then
+ collectFindbugsReports patch $BASEDIR $PATCH_DIR
+ #this files are generated by collectFindbugsReports() named with its first argument
+ patch_xml=$PATCH_DIR/patchFindbugsWarnings.xml
+ trunk_xml=$PATCH_DIR/trunkFindbugsWarnings.xml
+ # combine them to one database
+ combined_xml=$PATCH_DIR/combinedFindbugsWarnings.xml
+ new_xml=$PATCH_DIR/newFindbugsWarnings.xml
+ new_html=$PATCH_DIR/newFindbugsWarnings.html
+ $FINDBUGS_HOME/bin/computeBugHistory -useAnalysisTimes -withMessages \
+ -output $combined_xml $trunk_xml $patch_xml
+ findbugsWarnings=$($FINDBUGS_HOME/bin/filterBugs -first patch $combined_xml $new_xml)
+ findbugsFixedWarnings=$($FINDBUGS_HOME/bin/filterBugs -fixed patch $combined_xml $new_xml)
+ $FINDBUGS_HOME/bin/convertXmlToText -html $new_xml $new_html
+ file $new_xml $new_html
+ JIRA_COMMENT_FOOTER="Release Findbugs (version ${findbugs_version}) \
+ warnings: $BUILD_URL/artifact/patchprocess/newFindbugsWarnings.html
+$JIRA_COMMENT_FOOTER"
+ ### if current warnings greater than 0, fail
+ if [[ $findbugsWarnings -gt 0 ]] ; then
JIRA_COMMENT="$JIRA_COMMENT
- {color:red}-1 findbugs{color}. The patch appears to introduce `expr $(($findbugsWarnings-$OK_FINDBUGS_WARNINGS))` new Findbugs (version ${findbugs_version}) warnings."
+ {color:red}-1 findbugs{color}. The patch appears to introduce $findbugsWarnings \
+ new Findbugs (version ${findbugs_version}) warnings."
return 1
fi
JIRA_COMMENT="$JIRA_COMMENT
- {color:green}+1 findbugs{color}. The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings."
+ {color:green}+1 findbugs{color}. The patch does not introduce any \
+ new Findbugs (version ${findbugs_version}) warnings."
return 0
}
[08/15] hbase git commit: Revert "HBASE-13326 Disabled table can't be
enabled after HBase is restarted (addendum)"
Posted by mb...@apache.org.
Revert "HBASE-13326 Disabled table can't be enabled after HBase is restarted (addendum)"
This reverts commit bfeb067e3a50513cfc20bb8ee57c49f1d5c1f19f.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a85c1d9d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a85c1d9d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a85c1d9d
Branch: refs/heads/hbase-12439
Commit: a85c1d9d30db473c048f778b211a2730261020df
Parents: bfeb067
Author: Andrey Stepachev <oc...@gmail.com>
Authored: Thu Mar 26 08:17:33 2015 +0000
Committer: Andrey Stepachev <oc...@gmail.com>
Committed: Thu Mar 26 08:17:33 2015 +0000
----------------------------------------------------------------------
dev-support/test-patch.sh | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/a85c1d9d/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index 20d5f25..64cf8a7 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -324,7 +324,7 @@ setup () {
echo ""
echo ""
echo "$MVN clean package checkstyle:checkstyle-aggregate findbugs:findbugs -DskipTests \
- -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavacWarnings.txt 2>&1"
+ -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavacWarnings.txt 2>&1"
export MAVEN_OPTS="${MAVEN_OPTS}"
# build core and tests
$MVN clean package checkstyle:checkstyle-aggregate findbugs:findbugs -DskipTests \
@@ -757,13 +757,13 @@ $JIRA_COMMENT_FOOTER"
JIRA_COMMENT="$JIRA_COMMENT
{color:red}-1 findbugs{color}. The patch appears to introduce $findbugsWarnings \
- new Findbugs (version ${findbugs_version}) warnings."
+ new Findbugs (version ${findbugs_version}) warnings."
return 1
fi
JIRA_COMMENT="$JIRA_COMMENT
{color:green}+1 findbugs{color}. The patch does not introduce any \
- new Findbugs (version ${findbugs_version}) warnings."
+ new Findbugs (version ${findbugs_version}) warnings."
return 0
}