You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2012/10/04 03:27:16 UTC
svn commit: r1393868 - in /accumulo/branches/1.4:
src/server/src/main/java/org/apache/accumulo/server/fate/ZooStore.java
src/server/src/main/java/org/apache/accumulo/server/test/functional/FateStarvationTest.java
test/system/auto/simple/fateStartvation.py
Author: kturner
Date: Thu Oct 4 01:27:15 2012
New Revision: 1393868
URL: http://svn.apache.org/viewvc?rev=1393868&view=rev
Log:
ACCUMULO-779 fixed Fate starvation bug
Added:
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/functional/FateStarvationTest.java
accumulo/branches/1.4/test/system/auto/simple/fateStartvation.py (with props)
Modified:
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/fate/ZooStore.java
Modified: accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/fate/ZooStore.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/fate/ZooStore.java?rev=1393868&r1=1393867&r2=1393868&view=diff
==============================================================================
--- accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/fate/ZooStore.java (original)
+++ accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/fate/ZooStore.java Thu Oct 4 01:27:15 2012
@@ -47,6 +47,7 @@ public class ZooStore<T> implements TSto
private String path;
private IZooReaderWriter zk;
+ private String lastReserved = "";
private Set<Long> reserved;
private Map<Long,Long> defered;
private SecureRandom idgenerator;
@@ -123,20 +124,33 @@ public class ZooStore<T> implements TSto
events = statusChangeEvents;
}
- List<String> txdirs = zk.getChildren(path);
+ List<String> txdirs = new ArrayList<String>(zk.getChildren(path));
+ Collections.sort(txdirs);
+
+ synchronized (this) {
+ if (txdirs.size() > 0 && txdirs.get(txdirs.size() - 1).compareTo(lastReserved) <= 0)
+ lastReserved = "";
+ }
for (String txdir : txdirs) {
long tid = parseTid(txdir);
synchronized (this) {
+ // this check makes reserve pick up where it left off, so that it cycles through all as it is repeatedly called.... failing to do so can lead to
+ // starvation where fate ops that sort higher and hold a lock are never reserved.
+ if (txdir.compareTo(lastReserved) <= 0)
+ continue;
+
if (defered.containsKey(tid)) {
if (defered.get(tid) < System.currentTimeMillis())
defered.remove(tid);
else
continue;
}
- if (!reserved.contains(tid))
+ if (!reserved.contains(tid)) {
reserved.add(tid);
+ lastReserved = txdir;
+ }
else
continue;
}
Added: accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/functional/FateStarvationTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/functional/FateStarvationTest.java?rev=1393868&view=auto
==============================================================================
--- accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/functional/FateStarvationTest.java (added)
+++ accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/test/functional/FateStarvationTest.java Thu Oct 4 01:27:15 2012
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.test.functional;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.accumulo.server.test.TestIngest;
+import org.apache.hadoop.io.Text;
+
+/**
+ * See ACCUMULO-779
+ */
+public class FateStarvationTest extends FunctionalTest {
+
+ @Override
+ public void cleanup() throws Exception {}
+
+ @Override
+ public Map<String,String> getInitialConfig() {
+ return Collections.emptyMap();
+ }
+
+ @Override
+ public List<TableSetup> getTablesToCreate() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public void run() throws Exception {
+ getConnector().tableOperations().create("test_ingest");
+
+ getConnector().tableOperations().addSplits("test_ingest", TestIngest.CreateTable.getSplitPoints(0, 100000, 50));
+
+ TestIngest.main(new String[] {"-random", "89", "-timestamp", "7", "-size", "" + 50, "100000", "0", "1"});
+
+ getConnector().tableOperations().flush("test_ingest", null, null, true);
+
+ List<Text> splits = new ArrayList<Text>(TestIngest.CreateTable.getSplitPoints(0, 100000, 67));
+ Random rand = new Random();
+
+ for (int i = 0; i < 100; i++) {
+ int idx1 = rand.nextInt(splits.size() - 1);
+ int idx2 = rand.nextInt(splits.size() - (idx1 + 1)) + idx1 + 1;
+
+ getConnector().tableOperations().compact("test_ingest", splits.get(idx1), splits.get(idx2), false, false);
+ }
+
+ getConnector().tableOperations().offline("test_ingest");
+ }
+
+ public static void main(String[] args) throws Exception {
+ ArrayList<String> argsList = new ArrayList<String>();
+ argsList.addAll(Arrays.asList(args));
+ argsList.addAll(Arrays.asList(FateStarvationTest.class.getName(), "run"));
+ FunctionalTest.main(argsList.toArray(new String[0]));
+ }
+
+}
Added: accumulo/branches/1.4/test/system/auto/simple/fateStartvation.py
URL: http://svn.apache.org/viewvc/accumulo/branches/1.4/test/system/auto/simple/fateStartvation.py?rev=1393868&view=auto
==============================================================================
--- accumulo/branches/1.4/test/system/auto/simple/fateStartvation.py (added)
+++ accumulo/branches/1.4/test/system/auto/simple/fateStartvation.py Thu Oct 4 01:27:15 2012
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from JavaTest import JavaTest
+
+import unittest
+
+class FateStarvationTest(JavaTest):
+ "Try to trigger a bug that was found in FATE"
+
+ order = 21
+ testClass="org.apache.accumulo.server.test.functional.FateStarvationTest"
+
+
+def suite():
+ result = unittest.TestSuite()
+ result.addTest(FateStarvationTest())
+ return result
Propchange: accumulo/branches/1.4/test/system/auto/simple/fateStartvation.py
------------------------------------------------------------------------------
svn:executable = *