You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by aj...@apache.org on 2015/08/14 04:40:30 UTC
[1/2] falcon git commit: FALCON-1398 CrossEntityValidations contains
incorrect validations. Contributed by Pragya Mittal.
Repository: falcon
Updated Branches:
refs/heads/master 016999b6d -> f8ee60986
FALCON-1398 CrossEntityValidations contains incorrect validations. Contributed by Pragya Mittal.
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/e3dd84ed
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/e3dd84ed
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/e3dd84ed
Branch: refs/heads/master
Commit: e3dd84edf0651a7f51087e14ac3c6116e86dbdd1
Parents: 016999b
Author: Ajay Yadava <aj...@gmail.com>
Authored: Fri Aug 14 07:36:25 2015 +0530
Committer: Ajay Yadava <aj...@gmail.com>
Committed: Fri Aug 14 07:36:25 2015 +0530
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../entity/parser/CrossEntityValidations.java | 8 +--
.../entity/parser/ProcessEntityParserTest.java | 62 ++++++++++++++++----
3 files changed, 55 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/e3dd84ed/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index fa154ac..a790384 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -87,6 +87,8 @@ Trunk (Unreleased)
(Suhas Vasu)
BUG FIXES
+ FALCON-1398 CrossEntityValidations contains incorrect validations(Pragya Mittal via Ajay Yadava)
+
FALCON-1396 Disable the faulty test(Ajay Yadava via Sowmya Ramesh)
FALCON-1251 FeedEvictor UT fails intermittently(Sandeep Samudrala via Ajay Yadava)
http://git-wip-us.apache.org/repos/asf/falcon/blob/e3dd84ed/common/src/main/java/org/apache/falcon/entity/parser/CrossEntityValidations.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/parser/CrossEntityValidations.java b/common/src/main/java/org/apache/falcon/entity/parser/CrossEntityValidations.java
index 7a01d1e..2696552 100644
--- a/common/src/main/java/org/apache/falcon/entity/parser/CrossEntityValidations.java
+++ b/common/src/main/java/org/apache/falcon/entity/parser/CrossEntityValidations.java
@@ -55,19 +55,13 @@ public final class CrossEntityValidations {
Validity processValidity = ProcessHelper.getCluster(process, clusterName).getValidity();
ExpressionHelper.setReferenceDate(processValidity.getStart());
Date instStart = evaluator.evaluate(instStartEL, Date.class);
+ Date instEnd = evaluator.evaluate(instEndEL, Date.class);
if (instStart.before(feedStart)) {
throw new ValidationException("Start instance " + instStartEL + " of feed " + feed.getName()
+ " is before the start of feed " + feedValidity.getStart() + " for cluster "
+ clusterName);
}
- Date instEnd = evaluator.evaluate(instEndEL, Date.class);
- if (instEnd.after(feedEnd)) {
- throw new ValidationException("End instance " + instEndEL + " of feed " + feed.getName()
- + " is before the start of feed " + feedValidity.getStart() + " for cluster "
- + clusterName);
- }
-
if (instEnd.before(instStart)) {
throw new ValidationException("End instance " + instEndEL + " for feed " + feed.getName()
+ " is before the start instance " + instStartEL + " for cluster " + clusterName);
http://git-wip-us.apache.org/repos/asf/falcon/blob/e3dd84ed/common/src/test/java/org/apache/falcon/entity/parser/ProcessEntityParserTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/parser/ProcessEntityParserTest.java b/common/src/test/java/org/apache/falcon/entity/parser/ProcessEntityParserTest.java
index 8783081..675fd46 100644
--- a/common/src/test/java/org/apache/falcon/entity/parser/ProcessEntityParserTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/parser/ProcessEntityParserTest.java
@@ -231,16 +231,6 @@ public class ProcessEntityParserTest extends AbstractTestBase {
//RESUME CHECKSTYLE CHECK HiddenFieldCheck
@Test(expectedExceptions = ValidationException.class)
- public void testInvalidProcessValidity() throws Exception {
- Process process = parser
- .parseAndValidate((ProcessEntityParserTest.class
- .getResourceAsStream(PROCESS_XML)));
- process.getClusters().getClusters().get(0).getValidity().setStart(
- SchemaHelper.parseDateUTC("2011-12-31T00:00Z"));
- parser.validate(process);
- }
-
- @Test(expectedExceptions = ValidationException.class)
public void testInvalidDependentFeedsRetentionLimit() throws Exception {
Process process = parser
.parseAndValidate((ProcessEntityParserTest.class
@@ -508,4 +498,56 @@ public class ProcessEntityParserTest extends AbstractTestBase {
Assert.assertEquals(javax.xml.bind.UnmarshalException.class, e.getCause().getClass());
}
}
+
+ @Test(expectedExceptions = ValidationException.class)
+ public void testEndTimeProcessBeforeStartTime() throws Exception {
+ Process process = parser
+ .parseAndValidate((ProcessEntityParserTest.class
+ .getResourceAsStream(PROCESS_XML)));
+ process.getClusters().getClusters().get(0).getValidity().setEnd(
+ SchemaHelper.parseDateUTC("2010-12-31T00:00Z"));
+ parser.validate(process);
+ }
+
+ @Test(expectedExceptions = ValidationException.class)
+ public void testInstanceStartTimeBeforeFeedStartTimeForInput() throws Exception {
+ Process process = parser
+ .parseAndValidate((ProcessEntityParserTest.class
+ .getResourceAsStream(PROCESS_XML)));
+ process.getClusters().getClusters().get(0).getValidity().setStart(
+ SchemaHelper.parseDateUTC("2011-10-31T00:00Z"));
+ parser.validate(process);
+ }
+
+ @Test(expectedExceptions = ValidationException.class)
+ public void testInstanceEndTimeAfterFeedEndTimeForInput() throws Exception {
+ Process process = parser
+ .parseAndValidate((ProcessEntityParserTest.class
+ .getResourceAsStream(PROCESS_XML)));
+ process.getClusters().getClusters().get(0).getValidity().setStart(
+ SchemaHelper.parseDateUTC("2011-12-31T00:00Z"));
+ parser.validate(process);
+ }
+
+ @Test(expectedExceptions = ValidationException.class)
+ public void testInstanceTimeBeforeFeedStartTimeForOutput() throws Exception {
+ Process process = parser
+ .parseAndValidate((ProcessEntityParserTest.class
+ .getResourceAsStream(PROCESS_XML)));
+ process.getClusters().getClusters().get(0).getValidity().setStart(
+ SchemaHelper.parseDateUTC("2011-11-02T00:00Z"));
+ process.getOutputs().getOutputs().get(0).setInstance("yesterday(-60,0)");
+ parser.validate(process);
+ }
+
+ @Test(expectedExceptions = ValidationException.class)
+ public void testInstanceTimeAfterFeedEndTimeForOutput() throws Exception {
+ Process process = parser
+ .parseAndValidate((ProcessEntityParserTest.class
+ .getResourceAsStream(PROCESS_XML)));
+ process.getClusters().getClusters().get(0).getValidity().setStart(
+ SchemaHelper.parseDateUTC("2011-12-30T00:00Z"));
+ process.getOutputs().getOutputs().get(0).setInstance("today(120,0)");
+ parser.validate(process);
+ }
}
[2/2] falcon git commit: FALCON-1374 Remove the cap on numResults.
Contributed by Pragya Mittal.
Posted by aj...@apache.org.
FALCON-1374 Remove the cap on numResults. Contributed by Pragya Mittal.
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/f8ee6098
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/f8ee6098
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/f8ee6098
Branch: refs/heads/master
Commit: f8ee609867e65ee9b62895fa8cf03ab074fe1c25
Parents: e3dd84e
Author: Ajay Yadava <aj...@gmail.com>
Authored: Fri Aug 14 07:43:24 2015 +0530
Committer: Ajay Yadava <aj...@gmail.com>
Committed: Fri Aug 14 07:43:24 2015 +0530
----------------------------------------------------------------------
CHANGES.txt | 2 ++
common/src/main/resources/runtime.properties | 1 -
.../apache/falcon/resource/AbstractEntityManager.java | 14 --------------
.../org/apache/falcon/resource/EntityManagerTest.java | 8 ++++----
src/conf/runtime.properties | 4 ----
5 files changed, 6 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/f8ee6098/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index a790384..1b71717 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -13,6 +13,8 @@ Trunk (Unreleased)
FALCON-796 Enable users to triage data processing issues through falcon (Ajay Yadava)
IMPROVEMENTS
+ FALCON-1374 Remove the cap on numResults(Pragya Mittal via Ajay Yadava)
+
FALCON-1379 Doc describes retention incorrectly(Ajay Yadava)
FALCON-1359 Improve output format for Feed Instance Listing(Ajay Yadava)
http://git-wip-us.apache.org/repos/asf/falcon/blob/f8ee6098/common/src/main/resources/runtime.properties
----------------------------------------------------------------------
diff --git a/common/src/main/resources/runtime.properties b/common/src/main/resources/runtime.properties
index 2159e1a..f0253e6 100644
--- a/common/src/main/resources/runtime.properties
+++ b/common/src/main/resources/runtime.properties
@@ -23,7 +23,6 @@
*.falcon.replication.workflow.maxmaps=5
*.falcon.replication.workflow.mapbandwidth=100
-*.webservices.default.max.results.per.page=3000
*.webservices.default.results.per.page=3000
# Default configs to handle replication for late arriving feeds.
http://git-wip-us.apache.org/repos/asf/falcon/blob/f8ee6098/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java b/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java
index 43461c8..ff5dc04 100644
--- a/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java
+++ b/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java
@@ -78,7 +78,6 @@ public abstract class AbstractEntityManager {
protected static final int XML_DEBUG_LEN = 10 * 1024;
protected static final Integer DEFAULT_NUM_RESULTS = getDefaultResultsPerPage();
- protected static final int MAX_RESULTS = getMaxResultsPerPage();
private AbstractWorkflowEngine workflowEngine;
protected ConfigurationStore configStore = ConfigurationStore.get();
@@ -91,18 +90,6 @@ public abstract class AbstractEntityManager {
}
}
- private static int getMaxResultsPerPage() {
- Integer result = 3000;
- final String key = "webservices.default.max.results.per.page";
- String value = RuntimeProperties.get().getProperty(key, result.toString());
- try {
- result = Integer.valueOf(value);
- } catch (NumberFormatException e) {
- LOG.warn("Invalid value:{} for key:{} in runtime.properties", value, key);
- }
- return result;
- }
-
private static int getDefaultResultsPerPage() {
Integer result = 3000;
final String key = "webservices.default.results.per.page";
@@ -969,7 +956,6 @@ public abstract class AbstractEntityManager {
return 0;
}
- numresults = numresults <= MAX_RESULTS ? numresults : MAX_RESULTS;
int retLen = arraySize - offset;
if (retLen > numresults) {
retLen = numresults;
http://git-wip-us.apache.org/repos/asf/falcon/blob/f8ee6098/prism/src/test/java/org/apache/falcon/resource/EntityManagerTest.java
----------------------------------------------------------------------
diff --git a/prism/src/test/java/org/apache/falcon/resource/EntityManagerTest.java b/prism/src/test/java/org/apache/falcon/resource/EntityManagerTest.java
index c37da40..be1fe1f 100644
--- a/prism/src/test/java/org/apache/falcon/resource/EntityManagerTest.java
+++ b/prism/src/test/java/org/apache/falcon/resource/EntityManagerTest.java
@@ -168,11 +168,11 @@ public class EntityManagerTest extends AbstractEntityManager {
CurrentUser.authenticate(System.getProperty("user.name"));
}
-
@Test
- public void testCapOnNumberOfResults() {
- Assert.assertNotEquals(getRequiredNumberOfResults(10000, 0, 10000), 10000);
- Assert.assertEquals(getRequiredNumberOfResults(10000, 0, 10000), MAX_RESULTS);
+ public void testNumberOfResults() {
+ Assert.assertEquals(getRequiredNumberOfResults(10000, 0, 10000), 10000);
+ Assert.assertEquals(getRequiredNumberOfResults(10000, 0, 4000), 4000);
+ Assert.assertNotEquals(getRequiredNumberOfResults(10000, 0, 10000), 3000);
}
@Test
http://git-wip-us.apache.org/repos/asf/falcon/blob/f8ee6098/src/conf/runtime.properties
----------------------------------------------------------------------
diff --git a/src/conf/runtime.properties b/src/conf/runtime.properties
index 4d3088b..1eb3f1c 100644
--- a/src/conf/runtime.properties
+++ b/src/conf/runtime.properties
@@ -30,12 +30,8 @@ prism.falcon.local.endpoint=https://localhost:15443
# falcon server should have the following properties
falcon.current.colo=local
-
#default number of results per call, if numResults option is not specified then this value is used.
*.webservices.default.results.per.page=3000
-# maximum number of results allowed to be returned from server, acts as an upper limit for numResults option
-*.webservices.default.max.results.per.page=3000
-
# retry count - to fetch the status from the workflow engine
*.workflow.status.retry.count=30