You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2009/06/23 07:05:14 UTC
svn commit: r787544 - in /hadoop/hdfs/trunk: ./ ivy/ src/test/ src/test/aop/
src/test/aop/org/ src/test/aop/org/apache/ src/test/aop/org/apache/hadoop/
src/test/aop/org/apache/hadoop/fi/ src/test/aop/org/apache/hadoop/hdfs/
src/test/aop/org/apache/hado...
Author: szetszwo
Date: Tue Jun 23 05:05:14 2009
New Revision: 787544
URL: http://svn.apache.org/viewvc?rev=787544&view=rev
Log:
HDFS-436. Introduce AspectJ framework for HDFS code and tests. Contributed by Konstantin Boudnik
Added:
hadoop/hdfs/trunk/src/test/aop/
hadoop/hdfs/trunk/src/test/aop/org/
hadoop/hdfs/trunk/src/test/aop/org/apache/
hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/
hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/fi/
hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/fi/FiConfig.java
hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/fi/ProbabilityModel.java
hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/
hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/server/
hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/server/datanode/
hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/server/datanode/BlockReceiverAspects.aj
hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/server/datanode/FSDatasetAspects.aj
hadoop/hdfs/trunk/src/test/fi-site.xml
Modified:
hadoop/hdfs/trunk/CHANGES.txt
hadoop/hdfs/trunk/build.xml
hadoop/hdfs/trunk/ivy.xml
hadoop/hdfs/trunk/ivy/libraries.properties
Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=787544&r1=787543&r2=787544&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Tue Jun 23 05:05:14 2009
@@ -2,6 +2,11 @@
Trunk (unreleased changes)
+ NEW FEATURES
+
+ HDFS-436. Introduce AspectJ framework for HDFS code and tests.
+ (Konstantin Boudnik via szetszwo)
+
IMPROVEMENTS
HDFS-381. Remove blocks from DataNode maps when corresponding file
Modified: hadoop/hdfs/trunk/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/build.xml?rev=787544&r1=787543&r2=787544&view=diff
==============================================================================
--- hadoop/hdfs/trunk/build.xml (original)
+++ hadoop/hdfs/trunk/build.xml Tue Jun 23 05:05:14 2009
@@ -315,6 +315,31 @@
</copy>
</target>
+ <!-- Weaving aspects in place
+ Later on one can run 'ant jar' to create Hadoop jar file with instrumented classes
+ -->
+ <target name="injectfaults" depends="compile" description="Weaves aspects into precomplied HDFS classes">
+ <!-- AspectJ task definition -->
+ <taskdef resource="org/aspectj/tools/ant/taskdefs/aspectjTaskdefs.properties">
+ <classpath>
+ <pathelement location="${common.ivy.lib.dir}/aspectjtools-1.6.4.jar"/>
+ </classpath>
+ </taskdef>
+ <echo message="Start weaving aspects in place"/>
+ <iajc
+ encoding="${build.encoding}"
+ srcdir="${hdfs.src.dir};${build.src};${test.src.dir}/aop"
+ includes="org/apache/hadoop/**/*.java, org/apache/hadoop/**/*.aj"
+ destDir="${build.classes}"
+ debug="${javac.debug}"
+ target="${javac.version}"
+ source="${javac.version}"
+ deprecation="${javac.deprecation}">
+ <classpath refid="classpath" />
+ </iajc>
+ <echo message="Weaving of aspects is finished"/>
+ </target>
+
<target name="compile-core" depends="clover, compile-hdfs-classes" description="Compile"/>
<target name="compile-contrib" depends="compile-core">
@@ -449,6 +474,8 @@
<mkdir dir="${test.log.dir}"/>
<copy file="${test.src.dir}/hadoop-policy.xml"
todir="${test.build.extraconf}" />
+ <copy file="${test.src.dir}/fi-site.xml"
+ todir="${test.build.extraconf}" />
<junit showoutput="${test.output}"
printsummary="${test.junit.printsummary}"
haltonfailure="${test.junit.haltonfailure}"
@@ -465,6 +492,10 @@
<sysproperty key="test.build.extraconf" value="${test.build.extraconf}" />
<sysproperty key="hadoop.policy.file" value="hadoop-policy.xml"/>
<classpath refid="test.classpath"/>
+ <!-- Pass probability specifications to the spawn JVM -->
+ <syspropertyset id="FaultProbabilityProperties">
+ <propertyref regex="fi.*"/>
+ </syspropertyset>
<formatter type="${test.junit.output.format}" />
<batchtest todir="${test.build.dir}" unless="testcase">
<fileset dir="${test.src.dir}/hdfs"
@@ -486,6 +517,8 @@
<mkdir dir="${test.log.dir}"/>
<copy file="${test.src.dir}/hadoop-policy.xml"
todir="${test.build.extraconf}" />
+ <copy file="${test.src.dir}/fi-site.xml"
+ todir="${test.build.extraconf}" />
<junit showoutput="${test.output}"
printsummary="${test.junit.printsummary}"
haltonfailure="${test.junit.haltonfailure}"
Modified: hadoop/hdfs/trunk/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/ivy.xml?rev=787544&r1=787543&r2=787544&view=diff
==============================================================================
--- hadoop/hdfs/trunk/ivy.xml (original)
+++ hadoop/hdfs/trunk/ivy.xml Tue Jun 23 05:05:14 2009
@@ -265,6 +265,16 @@
rev="${slf4j-log4j12.version}"
conf="common->master">
</dependency>
+ <dependency org="org.aspectj"
+ name="aspectjrt"
+ rev="${aspectj.version}"
+ conf="common->default">
+ </dependency>
+ <dependency org="org.aspectj"
+ name="aspectjtools"
+ rev="${aspectj.version}"
+ conf="common->default">
+ </dependency>
</dependencies>
</ivy-module>
Modified: hadoop/hdfs/trunk/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/ivy/libraries.properties?rev=787544&r1=787543&r2=787544&view=diff
==============================================================================
--- hadoop/hdfs/trunk/ivy/libraries.properties (original)
+++ hadoop/hdfs/trunk/ivy/libraries.properties Tue Jun 23 05:05:14 2009
@@ -70,3 +70,5 @@
xmlenc.version=0.52
xerces.version=1.4.4
+
+aspectj.version=1.6.4
Added: hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/fi/FiConfig.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/fi/FiConfig.java?rev=787544&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/fi/FiConfig.java (added)
+++ hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/fi/FiConfig.java Tue Jun 23 05:05:14 2009
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fi;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * This class wraps the logic around fault injection configuration file
+ * Default file is expected to be found in src/test/fi-site.xml
+ * This default file should be copied by JUnit Ant's tasks to
+ * build/test/extraconf folder before tests are ran
+ * An alternative location can be set through
+ * -Dfi.config=<file_name>
+ */
+public class FiConfig {
+ private static final String CONFIG_PARAMETER = ProbabilityModel.FPROB_NAME + "config";
+ private static final String DEFAULT_CONFIG = "fi-site.xml";
+ private static Configuration conf;
+ static {
+ init();
+ }
+
+ protected static void init () {
+ if (conf == null) {
+ conf = new Configuration(false);
+ System.out.println(System.getProperties());
+ String configName = System.getProperty(CONFIG_PARAMETER, DEFAULT_CONFIG);
+ conf.addResource(configName);
+ }
+ }
+
+ /**
+ * Method provides access to local Configuration
+ *
+ * @return Configuration initialized with fault injection's parameters
+ */
+ public static Configuration getConfig() {
+ return conf;
+ }
+}
Added: hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/fi/ProbabilityModel.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/fi/ProbabilityModel.java?rev=787544&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/fi/ProbabilityModel.java (added)
+++ hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/fi/ProbabilityModel.java Tue Jun 23 05:05:14 2009
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fi;
+
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * This class is responsible for the decision of when a fault
+ * has to be triggered within a class of Hadoop
+ *
+ * Default probability of injection is set to 0%. To change it
+ * one can set the sys. prop. -Dfi.*=<new probability level>
+ * Another way to do so is to set this level through FI config file,
+ * located under src/test/fi-site.conf
+ *
+ * To change the level one has to specify the following sys,prop.:
+ * -Dfi.<name of fault location>=<probability level> in the runtime
+ * Probability level is specified by a float between 0.0 and 1.0
+ *
+ * <name of fault location> might be represented by a short classname
+ * or otherwise. This decision is left up to the discretion of aspects
+ * developer, but has to be consistent through the code
+ */
+public class ProbabilityModel {
+ private static Random generator = new Random();
+ private static final Log LOG = LogFactory.getLog(ProbabilityModel.class);
+
+ static final String FPROB_NAME = "fi.";
+ private static final String ALL_PROBABILITIES = FPROB_NAME + "*";
+ private static final float DEFAULT_PROB = 0.00f; // Default probability rate is 0%
+
+ private static Configuration conf = FiConfig.getConfig();
+
+ static {
+ // Set new default probability if specified through a system.property
+ // If neither is specified set default probability to DEFAULT_PROB
+ conf.set(ALL_PROBABILITIES,
+ System.getProperty(ALL_PROBABILITIES,
+ conf.get(ALL_PROBABILITIES, Float.toString(DEFAULT_PROB))));
+
+ LOG.info(ALL_PROBABILITIES + "=" + conf.get(ALL_PROBABILITIES));
+ }
+
+ // Simplistic method to check if we have reached the point of injection
+ public static boolean injectCriteria(String klassName) {
+ boolean trigger = false;
+ // TODO fix this: make it more sophisticated!!!
+ if (generator.nextFloat() < getProbability(klassName)) {
+ trigger = true;
+ }
+ return trigger;
+ }
+
+ // This primitive checks for arbitrary set of desired probability and
+ // uses default setting if it wasn't
+ // The probability expected to be set as an float between 0 and 100
+ protected static float getProbability(final String klass) {
+ String newProbName = FPROB_NAME + klass;
+
+ conf.setIfUnset(newProbName, System.getProperty(newProbName, conf.get(ALL_PROBABILITIES)));
+ float ret = conf.getFloat(newProbName, conf.getFloat(ALL_PROBABILITIES, DEFAULT_PROB));
+ LOG.debug("Request for " + newProbName + " returns=" + ret);
+ // Make sure that probability level is valid.
+ if (ret < 0.00 || ret > 1.00)
+ ret = conf.getFloat(ALL_PROBABILITIES, DEFAULT_PROB);
+
+ return ret;
+ }
+}
Added: hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/server/datanode/BlockReceiverAspects.aj
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/server/datanode/BlockReceiverAspects.aj?rev=787544&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/server/datanode/BlockReceiverAspects.aj (added)
+++ hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/server/datanode/BlockReceiverAspects.aj Tue Jun 23 05:05:14 2009
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.datanode;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fi.ProbabilityModel;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.util.DiskChecker.*;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.DataOutputStream;
+
+/**
+ * This aspect takes care about faults injected into datanode.BlockReceiver
+ * class
+ */
+public aspect BlockReceiverAspects {
+ public static final Log LOG = LogFactory.getLog(BlockReceiverAspects.class);
+
+ pointcut callReceivePacket() :
+ call (* OutputStream.write(..))
+ && withincode (* BlockReceiver.receivePacket(..))
+// to further limit the application of this aspect a very narrow 'target' can be used as follows
+// && target(DataOutputStream)
+ && !within(BlockReceiverAspects +);
+
+ before () throws IOException : callReceivePacket () {
+ if (ProbabilityModel.injectCriteria(BlockReceiver.class.getSimpleName())) {
+ LOG.info("Before the injection point");
+ Thread.dumpStack();
+ throw new DiskOutOfSpaceException ("FI: injected fault point at " +
+ thisJoinPoint.getStaticPart( ).getSourceLocation());
+ }
+ }
+}
Added: hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/server/datanode/FSDatasetAspects.aj
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/server/datanode/FSDatasetAspects.aj?rev=787544&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/server/datanode/FSDatasetAspects.aj (added)
+++ hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/server/datanode/FSDatasetAspects.aj Tue Jun 23 05:05:14 2009
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.datanode;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fi.ProbabilityModel;
+import org.apache.hadoop.hdfs.server.datanode.FSDataset;
+import org.apache.hadoop.hdfs.server.datanode.FSDatasetInterface.BlockWriteStreams;
+import org.apache.hadoop.util.DiskChecker.*;
+
+
+/**
+ * This aspect takes care about faults injected into datanode.FSDatase class
+ */
+public aspect FSDatasetAspects {
+ public static final Log LOG = LogFactory.getLog(BlockReceiverAspects.class);
+
+ pointcut execGetBlockFile() :
+ // the following will inject faults inside of the method in question
+ execution (* FSDataset.getBlockFile(..)) && !within(FSDatasetAspects +);
+
+ // the following will inject faults before the actual call of the method
+ // call (* FSDataset.getBlockFile(..)) && !within(FSDatasetAspects +);
+
+ pointcut callCreateBlockWriteStream() :
+ call (BlockWriteStreams FSDataset.createBlockWriteStreams(..))
+ && !within(FSDatasetAspects +);
+
+ // This aspect specifies the logic of our fault point.
+ // In this case it simply throws DiskErrorException at the very beginning of
+ // invocation of the method, specified by callGetBlockFile() pointcut
+ before() throws DiskErrorException : execGetBlockFile() {
+ if (ProbabilityModel.injectCriteria(FSDataset.class.getSimpleName())) {
+ LOG.info("Before the injection point");
+ Thread.dumpStack();
+ throw new DiskErrorException("FI: injected fault point at "
+ + thisJoinPoint.getStaticPart().getSourceLocation());
+ }
+ }
+
+ before() throws DiskOutOfSpaceException : callCreateBlockWriteStream() {
+ if (ProbabilityModel.injectCriteria(FSDataset.class.getSimpleName())) {
+ LOG.info("Before the injection point");
+ Thread.dumpStack();
+ throw new DiskOutOfSpaceException("FI: injected fault point at "
+ + thisJoinPoint.getStaticPart().getSourceLocation());
+ }
+ }
+}
Added: hadoop/hdfs/trunk/src/test/fi-site.xml
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/fi-site.xml?rev=787544&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/test/fi-site.xml (added)
+++ hadoop/hdfs/trunk/src/test/fi-site.xml Tue Jun 23 05:05:14 2009
@@ -0,0 +1,15 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put fault injection specific property overrides in this file. -->
+
+<configuration>
+ <property>
+ <name>fi.*</name>
+ <value>0.00</value>
+ <description>
+ Default probability level for all injected faults specified
+ as a floating number between 0 and 1.00
+ </description>
+ </property>
+</configuration>