You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2013/04/02 16:06:18 UTC
svn commit: r1463552 - in /accumulo/branches/1.5/server/src:
main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java
Author: ecn
Date: Tue Apr 2 14:06:18 2013
New Revision: 1463552
URL: http://svn.apache.org/r1463552
Log:
ACCUMULO-1044 add a metadata table constraint to require the bulk load transaction to be alive when writing the bulk-loaded flags
Modified:
accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
accumulo/branches/1.5/server/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java
Modified: accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java?rev=1463552&r1=1463551&r2=1463552&view=diff
==============================================================================
--- accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java (original)
+++ accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java Tue Apr 2 14:06:18 2013
@@ -34,6 +34,7 @@ import org.apache.accumulo.core.zookeepe
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.zookeeper.ZooCache;
import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.TransactionWatcher.ZooArbitrator;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
@@ -72,6 +73,22 @@ public class MetadataConstraints impleme
return false;
}
+ static private ArrayList<Short> addViolation(ArrayList<Short> lst, int violation) {
+ if (lst == null)
+ lst = new ArrayList<Short>();
+ lst.add((short)violation);
+ return lst;
+ }
+
+ static private ArrayList<Short> addIfNotPresent(ArrayList<Short> lst, int intViolation) {
+ if (lst == null)
+ return addViolation(lst, intViolation);
+ short violation = (short)intViolation;
+ if (!lst.contains(violation))
+ return addViolation(lst, intViolation);
+ return lst;
+ }
+
public List<Short> check(Environment env, Mutation mutation) {
ArrayList<Short> violations = null;
@@ -96,44 +113,30 @@ public class MetadataConstraints impleme
break;
if (!validTableNameChars[0xff & b]) {
- if (violations == null)
- violations = new ArrayList<Short>();
- if (!violations.contains((short) 4))
- violations.add((short) 4);
+ violations = addIfNotPresent(violations, 4);
}
}
if (!containsSemiC) {
// see if last row char is <
if (row.length == 0 || row[row.length - 1] != '<') {
- if (violations == null)
- violations = new ArrayList<Short>();
- if (!violations.contains((short) 4))
- violations.add((short) 4);
+ violations = addIfNotPresent(violations, 4);
}
} else {
if (row.length == 0) {
- if (violations == null)
- violations = new ArrayList<Short>();
- if (!violations.contains((short) 4))
- violations.add((short) 4);
+ violations = addIfNotPresent(violations, 4);
}
}
if (row.length > 0 && row[0] == '!') {
if (row.length < 3 || row[1] != '0' || (row[2] != '<' && row[2] != ';')) {
- if (violations == null)
- violations = new ArrayList<Short>();
- if (!violations.contains((short) 4))
- violations.add((short) 4);
+ violations = addIfNotPresent(violations, 4);
}
}
// ensure row is not less than Constants.METADATA_TABLE_ID
if (new Text(row).compareTo(new Text(Constants.METADATA_TABLE_ID)) < 0) {
- if (violations == null)
- violations = new ArrayList<Short>();
- violations.add((short) 5);
+ violations = addViolation(violations, 5);
}
for (ColumnUpdate columnUpdate : colUpdates) {
@@ -141,17 +144,13 @@ public class MetadataConstraints impleme
if (columnUpdate.isDeleted()) {
if (!isValidColumn(columnUpdate)) {
- if (violations == null)
- violations = new ArrayList<Short>();
- violations.add((short) 2);
+ violations = addViolation(violations, 2);
}
continue;
}
if (columnUpdate.getValue().length == 0 && !columnFamily.equals(Constants.METADATA_SCANFILE_COLUMN_FAMILY)) {
- if (violations == null)
- violations = new ArrayList<Short>();
- violations.add((short) 6);
+ violations = addViolation(violations, 6);
}
if (columnFamily.equals(Constants.METADATA_DATAFILE_COLUMN_FAMILY)) {
@@ -159,26 +158,49 @@ public class MetadataConstraints impleme
DataFileValue dfv = new DataFileValue(columnUpdate.getValue());
if (dfv.getSize() < 0 || dfv.getNumEntries() < 0) {
- if (violations == null)
- violations = new ArrayList<Short>();
- violations.add((short) 1);
+ violations = addViolation(violations, 1);
}
} catch (NumberFormatException nfe) {
- if (violations == null)
- violations = new ArrayList<Short>();
- violations.add((short) 1);
+ violations = addViolation(violations, 1);
} catch (ArrayIndexOutOfBoundsException aiooe) {
- if (violations == null)
- violations = new ArrayList<Short>();
- violations.add((short) 1);
+ violations = addViolation(violations, 1);
}
} else if (columnFamily.equals(Constants.METADATA_SCANFILE_COLUMN_FAMILY)) {
+ } else if (columnFamily.equals(Constants.METADATA_BULKFILE_COLUMN_FAMILY)) {
+ if (!columnUpdate.isDeleted()) {
+ // splits, which also write the time reference, are allowed to write this reference even when
+ // the transaction is not running because the other half of the tablet is holding a reference
+ // to the file.
+ boolean isSplitMutation = false;
+ // When a tablet is assigned, it re-writes the metadata. It should probably only update the location information,
+ // but it writes everything. We allow it to re-write the bulk information if it is setting the location.
+ // See ACCUMULO-1230.
+ boolean isLocationMutation = false;
+ for (ColumnUpdate update : mutation.getUpdates()) {
+ if (new ColumnFQ(update).equals(Constants.METADATA_TIME_COLUMN)) {
+ isSplitMutation = true;
+ }
+ if (update.getColumnFamily().equals(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY)) {
+ isLocationMutation = true;
+ }
+ }
+
+ if (!isSplitMutation && !isLocationMutation) {
+ String tidString = new String(columnUpdate.getValue());
+ long tid = Long.parseLong(tidString);
+ try {
+ if (!new ZooArbitrator().transactionAlive(Constants.BULK_ARBITRATOR_TYPE, tid)) {
+ violations = addViolation(violations, 8);
+ }
+ } catch (Exception ex) {
+ violations = addViolation(violations, 8);
+ }
+ }
+ }
} else {
if (!isValidColumn(columnUpdate)) {
- if (violations == null)
- violations = new ArrayList<Short>();
- violations.add((short) 2);
+ violations = addViolation(violations, 2);
} else if (new ColumnFQ(columnUpdate).equals(Constants.METADATA_PREV_ROW_COLUMN) && columnUpdate.getValue().length > 0
&& (violations == null || !violations.contains((short) 4))) {
KeyExtent ke = new KeyExtent(new Text(mutation.getRow()), (Text) null);
@@ -188,9 +210,7 @@ public class MetadataConstraints impleme
boolean prevEndRowLessThanEndRow = per == null || ke.getEndRow() == null || per.compareTo(ke.getEndRow()) < 0;
if (!prevEndRowLessThanEndRow) {
- if (violations == null)
- violations = new ArrayList<Short>();
- violations.add((short) 3);
+ violations = addViolation(violations, 3);
}
} else if (new ColumnFQ(columnUpdate).equals(Constants.METADATA_LOCK_COLUMN)) {
if (zooCache == null) {
@@ -211,9 +231,7 @@ public class MetadataConstraints impleme
}
if (!lockHeld) {
- if (violations == null)
- violations = new ArrayList<Short>();
- violations.add((short) 7);
+ violations = addViolation(violations, 7);
}
}
@@ -221,7 +239,10 @@ public class MetadataConstraints impleme
}
if (violations != null) {
- log.debug(" violating metadata mutation : " + mutation);
+ log.debug("violating metadata mutation : " + new String(mutation.getRow()));
+ for (ColumnUpdate update : mutation.getUpdates()) {
+ log.debug(" update: " + new String(update.getColumnFamily()) + ":" + new String(update.getColumnQualifier()) + " value " + (update.isDeleted() ? "[delete]" : new String(update.getValue())));
+ }
}
return violations;
@@ -243,6 +264,8 @@ public class MetadataConstraints impleme
return "Empty values are not allowed for any " + Constants.METADATA_TABLE_NAME + " column";
case 7:
return "Lock not held in zookeeper by writer";
+ case 8:
+ return "Bulk load transaction no longer running";
}
return null;
}
Modified: accumulo/branches/1.5/server/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/server/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java?rev=1463552&r1=1463551&r2=1463552&view=diff
==============================================================================
--- accumulo/branches/1.5/server/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java (original)
+++ accumulo/branches/1.5/server/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java Tue Apr 2 14:06:18 2013
@@ -105,7 +105,14 @@ public class MetadataConstraintsTest {
assertNotNull(violations);
assertEquals(1, violations.size());
assertEquals(Short.valueOf((short) 4), violations.get(0));
+
+ m = new Mutation(new Text("0;foo"));
+ m.put(Constants.METADATA_BULKFILE_COLUMN_FAMILY, new Text("/someFile"), new Value("12345".getBytes()));
+ violations = mc.check(null, m);
+ assertNotNull(violations);
+ assertEquals(1, violations.size());
+ assertEquals(Short.valueOf((short)8), violations.get(0));
}
}