You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maven.apache.org by st...@apache.org on 2017/02/01 22:30:42 UTC
maven-integration-testing git commit: Provide a mechanism whereby
tests can indicate versions of Maven expected to fail
Repository: maven-integration-testing
Updated Branches:
refs/heads/master 63bb5189a -> 92a11a968
Provide a mechanism whereby tests can indicate versions of Maven expected to fail
- This will let us fix broken tests while confirming that the versions of Maven that were released using the broken test contain the bug that escaped the broken test
Project: http://git-wip-us.apache.org/repos/asf/maven-integration-testing/repo
Commit: http://git-wip-us.apache.org/repos/asf/maven-integration-testing/commit/92a11a96
Tree: http://git-wip-us.apache.org/repos/asf/maven-integration-testing/tree/92a11a96
Diff: http://git-wip-us.apache.org/repos/asf/maven-integration-testing/diff/92a11a96
Branch: refs/heads/master
Commit: 92a11a96877c275d90d3502ff1df70c637850f6c
Parents: 63bb518
Author: Stephen Connolly <st...@gmail.com>
Authored: Wed Feb 1 22:30:25 2017 +0000
Committer: Stephen Connolly <st...@gmail.com>
Committed: Wed Feb 1 22:30:25 2017 +0000
----------------------------------------------------------------------
.../it/AbstractMavenIntegrationTestCase.java | 78 +++++++++++++++++++-
1 file changed, 76 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/maven-integration-testing/blob/92a11a96/core-it-support/maven-it-helper/src/main/java/org/apache/maven/it/AbstractMavenIntegrationTestCase.java
----------------------------------------------------------------------
diff --git a/core-it-support/maven-it-helper/src/main/java/org/apache/maven/it/AbstractMavenIntegrationTestCase.java b/core-it-support/maven-it-helper/src/main/java/org/apache/maven/it/AbstractMavenIntegrationTestCase.java
index 8966dce..78c6e63 100644
--- a/core-it-support/maven-it-helper/src/main/java/org/apache/maven/it/AbstractMavenIntegrationTestCase.java
+++ b/core-it-support/maven-it-helper/src/main/java/org/apache/maven/it/AbstractMavenIntegrationTestCase.java
@@ -60,6 +60,8 @@ public abstract class AbstractMavenIntegrationTestCase
private boolean skip;
+ private BrokenMavenVersionException invert;
+
private static ArtifactVersion javaVersion;
private ArtifactVersion mavenVersion;
@@ -213,11 +215,16 @@ public abstract class AbstractMavenIntegrationTestCase
setupLocalRepo();
}
+ invert = null;
long milliseconds = System.currentTimeMillis();
try
{
super.runTest();
milliseconds = System.currentTimeMillis() - milliseconds;
+ if ( invert != null )
+ {
+ throw invert;
+ }
out.println( "OK " + formatTime( milliseconds ) );
}
catch ( UnsupportedJavaVersionException e )
@@ -230,11 +237,25 @@ public abstract class AbstractMavenIntegrationTestCase
out.println( "SKIPPED - Maven version " + e.mavenVersion + " not in range " + e.supportedRange );
return;
}
+ catch ( BrokenMavenVersionException e )
+ {
+ out.println( "UNEXPECTED OK - Maven version " + e.mavenVersion + " expected to fail "
+ + formatTime( milliseconds ) );
+ fail( "Expected failure when with Maven version " + e.mavenVersion );
+ }
catch ( Throwable t )
{
milliseconds = System.currentTimeMillis() - milliseconds;
- out.println( "FAILURE " + formatTime( milliseconds ) );
- throw t;
+ if ( invert != null )
+ {
+ out.println( "EXPECTED FAIL - Maven version " + invert.mavenVersion + " expected to fail "
+ + formatTime( milliseconds ) );
+ }
+ else
+ {
+ out.println( "FAILURE " + formatTime( milliseconds ) );
+ throw t;
+ }
}
}
@@ -300,6 +321,42 @@ public abstract class AbstractMavenIntegrationTestCase
}
}
+ /**
+ * Inverts the execution of a test case for cases where we discovered a bug in the test case, have corrected the
+ * test case and shipped versions of Maven with a bug because of the faulty test case. This method allows the
+ * tests to continue passing against the historical releases as they historically would, as well as verifying that
+ * the test is no longer providing a false positive.
+ *
+ * @param versionRange
+ */
+ protected void failingMavenVersions( String versionRange )
+ {
+ assertNull( "Only call failingMavenVersions at most once per test", invert );
+ VersionRange range;
+ try
+ {
+ range = VersionRange.createFromVersionSpec( versionRange );
+ }
+ catch ( InvalidVersionSpecificationException e )
+ {
+ throw (RuntimeException) new IllegalArgumentException( "Invalid version range: " + versionRange, e );
+ }
+
+ ArtifactVersion version = getMavenVersion();
+ if ( version != null )
+ {
+ if ( range.containsVersion( removePattern( version ) ) )
+ {
+ invert = new BrokenMavenVersionException( version, range );
+ }
+ }
+ else
+ {
+ out.println( "WARNING: " + getITName() + ": version range '" + versionRange
+ + "' supplied but no Maven version found - not marking test as expected to fail." );
+ }
+ }
+
private class UnsupportedJavaVersionException
extends RuntimeException
{
@@ -334,6 +391,23 @@ public abstract class AbstractMavenIntegrationTestCase
}
+ private class BrokenMavenVersionException
+ extends RuntimeException
+ {
+ @SuppressWarnings( "checkstyle:visibilitymodifier" )
+ public ArtifactVersion mavenVersion;
+
+ @SuppressWarnings( "checkstyle:visibilitymodifier" )
+ public VersionRange supportedRange;
+
+ public BrokenMavenVersionException( ArtifactVersion mavenVersion, VersionRange supportedRange )
+ {
+ this.mavenVersion = mavenVersion;
+ this.supportedRange = supportedRange;
+ }
+
+ }
+
private String getITName()
{
String simpleName = getClass().getName();
Re: maven-integration-testing git commit: Provide a mechanism whereby
tests can indicate versions of Maven expected to fail
Posted by Christian Schulte <cs...@schulte.it>.
Am 02/02/17 um 01:35 schrieb Stephen Connolly:
> Well we want that test the other way, we want it to pass everywhere except
> the buggy releases 3.3.0 and 3.3.9
Ok. Means I do not need to change anything about that test and can leave
it the way it was. I just need a way to indicate the expected failures.
That's what your recent changes would provide. Got it.
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@maven.apache.org
For additional commands, e-mail: dev-help@maven.apache.org
Re: maven-integration-testing git commit: Provide a mechanism whereby
tests can indicate versions of Maven expected to fail
Posted by Stephen Connolly <st...@gmail.com>.
Well we want that test the other way, we want it to pass everywhere except
the buggy releases 3.3.0 and 3.3.9
Having a test to cover broken behaviour is exactly not what we want imho
On Wed 1 Feb 2017 at 23:15, Christian Schulte <cs...@schulte.it> wrote:
> Am 02/02/17 um 00:10 schrieb Stephen Connolly:
> > Sorry we want the inverse,
> >
> > "(,3.3.0),(3.3.0,3.3.9),(3.3.9,)"
> >
> > If the test is expected to fail on everything except 3.3.0 and 3.3.9 but
> > such a test is not the usecase for this tool
>
> That's what I was looking for. I have that use case for that MNG-2199 IT
> I asked about. Class
> 'MavenITmng2199DeprecatedParentVersionRangeTest.java' from the following
> commit to branch MNG-2199.
>
> <
> https://git-wip-us.apache.org/repos/asf?p=maven-integration-testing.git;a=commit;h=2348fbc31805560fdd6e1a68dbf7b52e00e0e992
> >
>
> Prior to this commit the test from that class was in class
> 'MavenITmng2199ParentVersionRangeTest.java' with range [3.2.2,) although
> that test is expected to fail with all Maven versions but 3.3.0 and 3.3.9.
>
> Regards,
> --
> Christian
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@maven.apache.org
> For additional commands, e-mail: dev-help@maven.apache.org
>
> --
Sent from my phone
Re: maven-integration-testing git commit: Provide a mechanism whereby
tests can indicate versions of Maven expected to fail
Posted by Christian Schulte <cs...@schulte.it>.
Am 02/02/17 um 00:10 schrieb Stephen Connolly:
> Sorry we want the inverse,
>
> "(,3.3.0),(3.3.0,3.3.9),(3.3.9,)"
>
> If the test is expected to fail on everything except 3.3.0 and 3.3.9 but
> such a test is not the usecase for this tool
That's what I was looking for. I have that use case for that MNG-2199 IT
I asked about. Class
'MavenITmng2199DeprecatedParentVersionRangeTest.java' from the following
commit to branch MNG-2199.
<https://git-wip-us.apache.org/repos/asf?p=maven-integration-testing.git;a=commit;h=2348fbc31805560fdd6e1a68dbf7b52e00e0e992>
Prior to this commit the test from that class was in class
'MavenITmng2199ParentVersionRangeTest.java' with range [3.2.2,) although
that test is expected to fail with all Maven versions but 3.3.0 and 3.3.9.
Regards,
--
Christian
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@maven.apache.org
For additional commands, e-mail: dev-help@maven.apache.org
Re: maven-integration-testing git commit: Provide a mechanism whereby
tests can indicate versions of Maven expected to fail
Posted by Stephen Connolly <st...@gmail.com>.
Sorry we want the inverse,
"(,3.3.0),(3.3.0,3.3.9),(3.3.9,)"
If the test is expected to fail on everything except 3.3.0 and 3.3.9 but
such a test is not the usecase for this tool
On Wed 1 Feb 2017 at 23:09, Stephen Connolly <
stephen.alan.connolly@gmail.com> wrote:
> With the range "[3.3.0],[3.3.9]"
>
> On Wed 1 Feb 2017 at 23:04, Christian Schulte <cs...@schulte.it> wrote:
>
> Out of curiosity: How would I specify the test to fail on all Maven
> versions but 3.3.0 and 3.3.9?
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@maven.apache.org
> For additional commands, e-mail: dev-help@maven.apache.org
>
> --
> Sent from my phone
>
--
Sent from my phone
Re: maven-integration-testing git commit: Provide a mechanism whereby
tests can indicate versions of Maven expected to fail
Posted by Stephen Connolly <st...@gmail.com>.
With the range "[3.3.0],[3.3.9]"
On Wed 1 Feb 2017 at 23:04, Christian Schulte <cs...@schulte.it> wrote:
> Out of curiosity: How would I specify the test to fail on all Maven
> versions but 3.3.0 and 3.3.9?
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@maven.apache.org
> For additional commands, e-mail: dev-help@maven.apache.org
>
> --
Sent from my phone
Re: maven-integration-testing git commit: Provide a mechanism whereby
tests can indicate versions of Maven expected to fail
Posted by Christian Schulte <cs...@schulte.it>.
Out of curiosity: How would I specify the test to fail on all Maven
versions but 3.3.0 and 3.3.9?
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@maven.apache.org
For additional commands, e-mail: dev-help@maven.apache.org
Re: maven-integration-testing git commit: Provide a mechanism whereby
tests can indicate versions of Maven expected to fail
Posted by Stephen Connolly <st...@gmail.com>.
An example of use would be:
diff --git
> a/core-it-suite/src/test/java/org/apache/maven/it/MavenITmng5958LifecyclePhaseBinaryCompat.java
> b/core-it-suite/src/test/java/org/apache/maven/it/MavenITmng5958LifecyclePhaseBinaryCompat.java
> index 3ee3fe10..370bdf37 100644
> ---
> a/core-it-suite/src/test/java/org/apache/maven/it/MavenITmng5958LifecyclePhaseBinaryCompat.java
> +++
> b/core-it-suite/src/test/java/org/apache/maven/it/MavenITmng5958LifecyclePhaseBinaryCompat.java
> @@ -10,12 +10,13 @@ public class MavenITmng5958LifecyclePhaseBinaryCompat
>
> public MavenITmng5958LifecyclePhaseBinaryCompat()
> {
> - super( "(3.3.9,)" );
> + super( "[3.3.9,)" );
> }
>
> public void testGood()
> throws Exception
> {
> + failingMavenVersions("[3.3.9]");
> File testDir = ResourceExtractor.simpleExtractResources(
> getClass(), "/mng-5958-lifecycle-phases/good" );
>
> Verifier verifier = newVerifier( testDir.getAbsolutePath() );
Which will result in the test output like so:
> -------------------------------------------------------
> T E S T S
> -------------------------------------------------------
> Running org.apache.maven.it.MavenITmng5958LifecyclePhaseBinaryCompat
> mng5958LifecyclePhaseBinaryCompat(Bad)......................OK (1.2 s)
> mng5958LifecyclePhaseBinaryCompat(Good).....................EXPECTED FAIL
> - Maven version 3.3.9 expected to fail (0.1 s)
> Tests run: 2, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 1.37 sec -
> in org.apache.maven.it.MavenITmng5958LifecyclePhaseBinaryCompat
> Results :
> Tests run: 2, Failures: 0, Errors: 0, Skipped: 0
If we also put the failingMavenVersions("[3.3.9]"); into testBad we get
> -------------------------------------------------------
> T E S T S
> -------------------------------------------------------
> Running org.apache.maven.it.MavenITmng5958LifecyclePhaseBinaryCompat
> mng5958LifecyclePhaseBinaryCompat(Bad)......................UNEXPECTED OK
> - Maven version 3.3.9 expected to fail (1.3 s)
> mng5958LifecyclePhaseBinaryCompat(Good).....................EXPECTED FAIL
> - Maven version 3.3.9 expected to fail (0.1 s)
> Tests run: 2, Failures: 1, Errors: 0, Skipped: 0, Time elapsed: 1.496 sec
> <<< FAILURE! - in
> org.apache.maven.it.MavenITmng5958LifecyclePhaseBinaryCompat
> testBad(org.apache.maven.it.MavenITmng5958LifecyclePhaseBinaryCompat)
> Time elapsed: 1.326 sec <<< FAILURE!
> junit.framework.AssertionFailedError: Expected failure when with Maven
> version 3.3.9
> at junit.framework.Assert.fail(Assert.java:47)
> at
> org.apache.maven.it.AbstractMavenIntegrationTestCase.runTest(AbstractMavenIntegrationTestCase.java:244)
> at junit.framework.TestCase.runBare(TestCase.java:134)
> at junit.framework.TestResult$1.protect(TestResult.java:110)
> at junit.framework.TestResult.runProtected(TestResult.java:128)
> at junit.framework.TestResult.run(TestResult.java:113)
> at junit.framework.TestCase.run(TestCase.java:124)
> at junit.framework.TestSuite.runTest(TestSuite.java:243)
> at junit.framework.TestSuite.run(TestSuite.java:238)
> at
> org.junit.internal.runners.JUnit38ClassRunner.run(JUnit38ClassRunner.java:83)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:264)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:153)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:124)
> at
> org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:200)
> at
> org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:153)
> at
> org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:103)
>
> Results :
> Failed tests:
>
> MavenITmng5958LifecyclePhaseBinaryCompat>AbstractMavenIntegrationTestCase.runTest:244
> Expected failure when with Maven version 3.3.9
> Tests run: 2, Failures: 1, Errors: 0, Skipped: 0
Which to my mind is exactly what we want
This should let us have the best of all worlds for fixing false-positive
tests
On 1 February 2017 at 22:30, <st...@apache.org> wrote:
> Repository: maven-integration-testing
> Updated Branches:
> refs/heads/master 63bb5189a -> 92a11a968
>
>
> Provide a mechanism whereby tests can indicate versions of Maven expected
> to fail
>
> - This will let us fix broken tests while confirming that the versions of
> Maven that were released using the broken test contain the bug that escaped
> the broken test
>
>
> Project: http://git-wip-us.apache.org/repos/asf/maven-integration-
> testing/repo
> Commit: http://git-wip-us.apache.org/repos/asf/maven-integration-
> testing/commit/92a11a96
> Tree: http://git-wip-us.apache.org/repos/asf/maven-integration-
> testing/tree/92a11a96
> Diff: http://git-wip-us.apache.org/repos/asf/maven-integration-
> testing/diff/92a11a96
>
> Branch: refs/heads/master
> Commit: 92a11a96877c275d90d3502ff1df70c637850f6c
> Parents: 63bb518
> Author: Stephen Connolly <st...@gmail.com>
> Authored: Wed Feb 1 22:30:25 2017 +0000
> Committer: Stephen Connolly <st...@gmail.com>
> Committed: Wed Feb 1 22:30:25 2017 +0000
>
> ----------------------------------------------------------------------
> .../it/AbstractMavenIntegrationTestCase.java | 78 +++++++++++++++++++-
> 1 file changed, 76 insertions(+), 2 deletions(-)
> ----------------------------------------------------------------------
>
>
> http://git-wip-us.apache.org/repos/asf/maven-integration-
> testing/blob/92a11a96/core-it-support/maven-it-helper/src/
> main/java/org/apache/maven/it/AbstractMavenIntegrationTestCase.java
> ----------------------------------------------------------------------
> diff --git a/core-it-support/maven-it-helper/src/main/java/org/
> apache/maven/it/AbstractMavenIntegrationTestCase.java
> b/core-it-support/maven-it-helper/src/main/java/org/apache/maven/it/
> AbstractMavenIntegrationTestCase.java
> index 8966dce..78c6e63 100644
> --- a/core-it-support/maven-it-helper/src/main/java/org/apache/maven/it/
> AbstractMavenIntegrationTestCase.java
> +++ b/core-it-support/maven-it-helper/src/main/java/org/apache/maven/it/
> AbstractMavenIntegrationTestCase.java
> @@ -60,6 +60,8 @@ public abstract class AbstractMavenIntegrationTestCase
>
> private boolean skip;
>
> + private BrokenMavenVersionException invert;
> +
> private static ArtifactVersion javaVersion;
>
> private ArtifactVersion mavenVersion;
> @@ -213,11 +215,16 @@ public abstract class AbstractMavenIntegrationTestCa
> se
> setupLocalRepo();
> }
>
> + invert = null;
> long milliseconds = System.currentTimeMillis();
> try
> {
> super.runTest();
> milliseconds = System.currentTimeMillis() - milliseconds;
> + if ( invert != null )
> + {
> + throw invert;
> + }
> out.println( "OK " + formatTime( milliseconds ) );
> }
> catch ( UnsupportedJavaVersionException e )
> @@ -230,11 +237,25 @@ public abstract class AbstractMavenIntegrationTestCa
> se
> out.println( "SKIPPED - Maven version " + e.mavenVersion + "
> not in range " + e.supportedRange );
> return;
> }
> + catch ( BrokenMavenVersionException e )
> + {
> + out.println( "UNEXPECTED OK - Maven version " +
> e.mavenVersion + " expected to fail "
> + + formatTime( milliseconds ) );
> + fail( "Expected failure when with Maven version " +
> e.mavenVersion );
> + }
> catch ( Throwable t )
> {
> milliseconds = System.currentTimeMillis() - milliseconds;
> - out.println( "FAILURE " + formatTime( milliseconds ) );
> - throw t;
> + if ( invert != null )
> + {
> + out.println( "EXPECTED FAIL - Maven version " +
> invert.mavenVersion + " expected to fail "
> + + formatTime( milliseconds ) );
> + }
> + else
> + {
> + out.println( "FAILURE " + formatTime( milliseconds ) );
> + throw t;
> + }
> }
> }
>
> @@ -300,6 +321,42 @@ public abstract class AbstractMavenIntegrationTestCa
> se
> }
> }
>
> + /**
> + * Inverts the execution of a test case for cases where we discovered
> a bug in the test case, have corrected the
> + * test case and shipped versions of Maven with a bug because of the
> faulty test case. This method allows the
> + * tests to continue passing against the historical releases as they
> historically would, as well as verifying that
> + * the test is no longer providing a false positive.
> + *
> + * @param versionRange
> + */
> + protected void failingMavenVersions( String versionRange )
> + {
> + assertNull( "Only call failingMavenVersions at most once per
> test", invert );
> + VersionRange range;
> + try
> + {
> + range = VersionRange.createFromVersionSpec( versionRange );
> + }
> + catch ( InvalidVersionSpecificationException e )
> + {
> + throw (RuntimeException) new IllegalArgumentException(
> "Invalid version range: " + versionRange, e );
> + }
> +
> + ArtifactVersion version = getMavenVersion();
> + if ( version != null )
> + {
> + if ( range.containsVersion( removePattern( version ) ) )
> + {
> + invert = new BrokenMavenVersionException( version, range
> );
> + }
> + }
> + else
> + {
> + out.println( "WARNING: " + getITName() + ": version range '"
> + versionRange
> + + "' supplied but no Maven version found -
> not marking test as expected to fail." );
> + }
> + }
> +
> private class UnsupportedJavaVersionException
> extends RuntimeException
> {
> @@ -334,6 +391,23 @@ public abstract class AbstractMavenIntegrationTestCa
> se
>
> }
>
> + private class BrokenMavenVersionException
> + extends RuntimeException
> + {
> + @SuppressWarnings( "checkstyle:visibilitymodifier" )
> + public ArtifactVersion mavenVersion;
> +
> + @SuppressWarnings( "checkstyle:visibilitymodifier" )
> + public VersionRange supportedRange;
> +
> + public BrokenMavenVersionException( ArtifactVersion mavenVersion,
> VersionRange supportedRange )
> + {
> + this.mavenVersion = mavenVersion;
> + this.supportedRange = supportedRange;
> + }
> +
> + }
> +
> private String getITName()
> {
> String simpleName = getClass().getName();
>
>