You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@oozie.apache.org by Purshotam Shah <pu...@yahoo-inc.com> on 2015/09/18 02:20:23 UTC
Review Request 38474: OOZIE-1976- Specifying coordinator input
datasets in more logical ways
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/38474/
-----------------------------------------------------------
Review request for oozie.
Bugs: OOZIE-1976
https://issues.apache.org/jira/browse/OOZIE-1976
Repository: oozie-git
Description
-------
There are three components in this patch
1. User interface
A new tag is added to coordinator.xml
ex.
<input-check>
<or name="test">
<and>
<data-in dataset="A"/>"
<data-in dataset="B"/>
</and>
<and>
<data-in dataset="C"/>
<data-in dataset="D"/>
</and>
</or>;
<input-check>
input-check will have nested and/or/combine operation. It can have min and wait at operator or at date-in.
If input-check tag is missing then it consider to be old approach where all data dependency are needed.
2. Processing
input-check is converted into logical expression
(a&&B)||(c&&d)
We use jexl to parse the logical expression.
There are three phase in parsing.
phase 1 : only resolved dataset are parsed ( only current).
phase 2 : once all current are resolved, then future/latest are parsed.
phase 3 : Doesn't do any filecheck, just return what is being parsed by phase1 and phase2. Is used for EL functions
3. Storage.
if inputcheck is enable, push_missing_dependencies and missing_dependencies are serialized and stored in DB.
If then not then it's old approach, where they are stored in plan text. This is backward compatible.
Diffs
-----
client/src/main/resources/oozie-coordinator-0.5.xsd e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/pom.xml ca40e2e22293a3df2841764ce725420857425139
core/src/main/java/org/apache/oozie/CoordinatorActionBean.java 188b70e2e76858228b4d42e5798952383719a93d
core/src/main/java/org/apache/oozie/action/ActionExecutor.java ff836fbbbe95ca03aace1136abea9548306b2cf2
core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java a975f6edd40ef674638d1c32c36d5234b78beb0f
core/src/main/java/org/apache/oozie/command/coord/CoordActionUpdatePushMissingDependency.java 4e1c5b3392358cb6b1e98e16e469310338f27fed
core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java ca8175e8b8019a42e2780fba4f8fbd313577494c
core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java 548946f05beacbba032b1f411fdae17ca7dd1f44
core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java cc346274e2f28fd3eb062e8d3550281486c23954
core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java 8a279c06a62bd1e6502104f0760d7a1a9792b4bd
core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java 7f59186bb508536e5662bc3ed25c7ec68ba1e2a7
core/src/main/java/org/apache/oozie/coord/SyncCoordAction.java 44258eb5be40bf6769e32c8780117e8533d80d7e
core/src/main/java/org/apache/oozie/coord/dependency/CoordDependenciesInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/dependency/CoordDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/dependency/CoordInputCheckerPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/dependency/CoordInputCheckerPhaseOnePush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/dependency/CoordInputCheckerPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/dependency/CoordInputCheckerPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/dependency/CoordInputCheckerUtility.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/dependency/CoordInputDependencies.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/dependency/CoordInputInstance.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/dependency/CoordPullDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/dependency/CoordPushDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/dependency/CoordUnResolvedDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/dependency/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/dependency/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/dependency/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/dependency/OozieJexlNode.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/dependency/DependencyChecker.java a5507575a3403103133f85a78c873c9976419857
core/src/main/java/org/apache/oozie/service/SchemaService.java 32105857f51eea9b2e4fd4d9d8cb74900fcdbac8
core/src/main/java/org/apache/oozie/util/Pair.java 1bf45b41edf19ad0a3115a7bafc010daef11b5c1
core/src/main/resources/oozie-default.xml 19cae9d4e07ea1c3cd3b287d23cacb9b342b406e
core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java f79c9a06c8238aa17b7d331afcb3f137f0bb83f9
core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java 59b8b48b564b04ddd6eb263f8766bf3a8919a429
core/src/test/java/org/apache/oozie/coord/dependency/TestCoordInputCheckPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/dependency/TestCoordinatorInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/dependency/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputcheck-combine.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputcheck-hcat.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputcheck-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputcheck-range-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputcheck-range.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputcheck.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
Diff: https://reviews.apache.org/r/38474/diff/
Testing
-------
Thanks,
Purshotam Shah
Re: Review Request 38474: OOZIE-1976- Specifying coordinator input
datasets in more logical ways
Posted by Purshotam Shah <pu...@yahoo-inc.com>.
> On Dec. 19, 2015, 1:43 a.m., Robert Kanter wrote:
> > client/src/main/resources/oozie-coordinator-0.5.xsd, line 19
> > <https://reviews.apache.org/r/38474/diff/3/?file=1165028#file1165028line19>
> >
> > Please add some documentation on the new features. It should explain all of the options, and include some examples.
> >
> > It would also be good to include an example coordinator or two in the examples tarball, though this can be a followup JIRA if you want.
https://issues.apache.org/jira/browse/OOZIE-2445
- Purshotam
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/38474/#review111340
-----------------------------------------------------------
On Jan. 25, 2016, 11:32 p.m., Purshotam Shah wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/38474/
> -----------------------------------------------------------
>
> (Updated Jan. 25, 2016, 11:32 p.m.)
>
>
> Review request for oozie.
>
>
> Bugs: OOZIE-1976
> https://issues.apache.org/jira/browse/OOZIE-1976
>
>
> Repository: oozie-git
>
>
> Description
> -------
>
> There are three components in this patch
>
> 1. User interface
> A new tag is added to coordinator.xml
> ex.
> <input-check>
> <or name="test">
> <and>
> <data-in dataset="A"/>"
> <data-in dataset="B"/>
> </and>
> <and>
> <data-in dataset="C"/>
> <data-in dataset="D"/>
> </and>
>
> </or>;
> <input-check>
>
>
> input-check will have nested and/or/combine operation. It can have min and wait at operator or at date-in.
> If input-check tag is missing then it consider to be old approach where all data dependency are needed.
>
> 2. Processing
> input-check is converted into logical expression
> (a&&B)||(c&&d)
> We use jexl to parse the logical expression.
>
> There are three phase in parsing.
> phase 1 : only resolved dataset are parsed ( only current).
> phase 2 : once all current are resolved, then future/latest are parsed.
> phase 3 : Doesn't do any filecheck, just return what is being parsed by phase1 and phase2. Is used for EL functions
>
>
> 3. Storage.
> if inputcheck is enable, push_missing_dependencies and missing_dependencies are serialized and stored in DB.
> If then not then it's old approach, where they are stored in plan text. This is backward compatible.
>
>
> Diffs
> -----
>
> client/src/main/resources/oozie-coordinator-0.5.xsd e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/pom.xml b063dab79415447a86c1a33f5c3f5304e0dffca0
> core/src/main/java/org/apache/oozie/CoordinatorActionBean.java 91bff4dca2ef2dece68ca7260724ba3e43b1a08e
> core/src/main/java/org/apache/oozie/ErrorCode.java 6c1e3997c9a1cb0bb0de39d687a083af0a7b5f04
> core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java 742d00dd47aab55392abc8fbc207f8100728b832
> core/src/main/java/org/apache/oozie/command/coord/CoordActionUpdatePushMissingDependency.java 4e1c5b3392358cb6b1e98e16e469310338f27fed
> core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java 58ef483272264e0d391d4a1e6533dc1cab9940da
> core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java 39e6ac15ce3a3ea7f2ed9178688537f7b1d7842d
> core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java b05344d89e8df0e11fe69c1aa725d19a18eb0a2b
> core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java d4d1c08141088e64e0551dccbd23a6185cfdbb7b
> core/src/main/java/org/apache/oozie/coord/CoordELConstants.java f010a817fc900821c0e429fc16e1d3902a98d8bb
> core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java 8b2f4560ae66bbcd707a446382e647663ea67be1
> core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java 5d238663aa94f0dd55a9190b60bfe621439c7b53
> core/src/main/java/org/apache/oozie/coord/CoordUtils.java 94c69740618110ea180b188ab0c5a02db76f8b4d
> core/src/main/java/org/apache/oozie/coord/SyncCoordAction.java 44258eb5be40bf6769e32c8780117e8533d80d7e
> core/src/main/java/org/apache/oozie/coord/input/dependency/AbstractCoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesChecker.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyFactory.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputInstance.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPullInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPushInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluator.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorResult.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/dependency/ActionDependency.java c280d1dc2387c9b3df96d4e83ad5563e10b1e1ef
> core/src/main/java/org/apache/oozie/dependency/DependencyChecker.java a5507575a3403103133f85a78c873c9976419857
> core/src/main/java/org/apache/oozie/dependency/FSURIHandler.java 7c1aadf273aeca70c110f29ccf527c073250b2e7
> core/src/main/java/org/apache/oozie/dependency/HCatURIHandler.java 1bbf37d8d15c603666ddb081f83d515730740bbc
> core/src/main/java/org/apache/oozie/dependency/URIHandler.java bc947168558f1970cc2fd1c40b87a1c43ce860ee
> core/src/main/java/org/apache/oozie/util/WritableUtils.java 76a68953541125cab05bd05c94aa73b50e5363ff
> core/src/main/resources/oozie-default.xml faf37405757719554a5b6e3671d0c87723eb9959
> core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java 1fe1b3adb5ceaa83985ef31278e49a3224cde0fb
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordInputCheckPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordinatorInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordInputLogicPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordinatorInputLogic.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/logic/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-action-sla.xml f3f1bc09c51272a35d65edd7271599e7e8ba1ba4
> core/src/test/resources/coord-inputlogic-combine.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-hcat.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-range-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-range.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> pom.xml a74ffab081defd15a1c5dbbf15355aa4e0455029
>
> Diff: https://reviews.apache.org/r/38474/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Purshotam Shah
>
>
Re: Review Request 38474: OOZIE-1976- Specifying coordinator input
datasets in more logical ways
Posted by Robert Kanter <rk...@cloudera.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/38474/#review111340
-----------------------------------------------------------
client/src/main/resources/oozie-coordinator-0.5.xsd (line 19)
<https://reviews.apache.org/r/38474/#comment171473>
Please add some documentation on the new features. It should explain all of the options, and include some examples.
It would also be good to include an example coordinator or two in the examples tarball, though this can be a followup JIRA if you want.
- Robert Kanter
On Dec. 15, 2015, 7:33 p.m., Purshotam Shah wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/38474/
> -----------------------------------------------------------
>
> (Updated Dec. 15, 2015, 7:33 p.m.)
>
>
> Review request for oozie.
>
>
> Bugs: OOZIE-1976
> https://issues.apache.org/jira/browse/OOZIE-1976
>
>
> Repository: oozie-git
>
>
> Description
> -------
>
> There are three components in this patch
>
> 1. User interface
> A new tag is added to coordinator.xml
> ex.
> <input-check>
> <or name="test">
> <and>
> <data-in dataset="A"/>"
> <data-in dataset="B"/>
> </and>
> <and>
> <data-in dataset="C"/>
> <data-in dataset="D"/>
> </and>
>
> </or>;
> <input-check>
>
>
> input-check will have nested and/or/combine operation. It can have min and wait at operator or at date-in.
> If input-check tag is missing then it consider to be old approach where all data dependency are needed.
>
> 2. Processing
> input-check is converted into logical expression
> (a&&B)||(c&&d)
> We use jexl to parse the logical expression.
>
> There are three phase in parsing.
> phase 1 : only resolved dataset are parsed ( only current).
> phase 2 : once all current are resolved, then future/latest are parsed.
> phase 3 : Doesn't do any filecheck, just return what is being parsed by phase1 and phase2. Is used for EL functions
>
>
> 3. Storage.
> if inputcheck is enable, push_missing_dependencies and missing_dependencies are serialized and stored in DB.
> If then not then it's old approach, where they are stored in plan text. This is backward compatible.
>
>
> Diffs
> -----
>
> client/src/main/resources/oozie-coordinator-0.5.xsd e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/pom.xml b063dab79415447a86c1a33f5c3f5304e0dffca0
> core/src/main/java/org/apache/oozie/CoordinatorActionBean.java 91bff4dca2ef2dece68ca7260724ba3e43b1a08e
> core/src/main/java/org/apache/oozie/ErrorCode.java 6c1e3997c9a1cb0bb0de39d687a083af0a7b5f04
> core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java 742d00dd47aab55392abc8fbc207f8100728b832
> core/src/main/java/org/apache/oozie/command/coord/CoordActionUpdatePushMissingDependency.java 4e1c5b3392358cb6b1e98e16e469310338f27fed
> core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java 58ef483272264e0d391d4a1e6533dc1cab9940da
> core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java 39e6ac15ce3a3ea7f2ed9178688537f7b1d7842d
> core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java b05344d89e8df0e11fe69c1aa725d19a18eb0a2b
> core/src/main/java/org/apache/oozie/coord/CoordELConstants.java f010a817fc900821c0e429fc16e1d3902a98d8bb
> core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java 8b2f4560ae66bbcd707a446382e647663ea67be1
> core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java 5d238663aa94f0dd55a9190b60bfe621439c7b53
> core/src/main/java/org/apache/oozie/coord/CoordUtils.java 94c69740618110ea180b188ab0c5a02db76f8b4d
> core/src/main/java/org/apache/oozie/coord/SyncCoordAction.java 44258eb5be40bf6769e32c8780117e8533d80d7e
> core/src/main/java/org/apache/oozie/coord/input/dependency/AbstractCoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesChecker.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyFactory.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputInstance.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPullInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPushInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/dependency/ActionDependency.java c280d1dc2387c9b3df96d4e83ad5563e10b1e1ef
> core/src/main/java/org/apache/oozie/dependency/DependencyChecker.java a5507575a3403103133f85a78c873c9976419857
> core/src/main/java/org/apache/oozie/util/WritableUtils.java 76a68953541125cab05bd05c94aa73b50e5363ff
> core/src/main/resources/oozie-default.xml faf37405757719554a5b6e3671d0c87723eb9959
> core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java 1fe1b3adb5ceaa83985ef31278e49a3224cde0fb
> core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java 7062e697e5162ac25688d9cb62352bd139a7013a
> core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java 29e7ca145612fadee661fe975aa0ea2cab1cbfa3
> core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java 52eb9dd1fb903ac86294a55d6e2a6918390bf4e9
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordInputCheckPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordinatorInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-action-sla.xml f3f1bc09c51272a35d65edd7271599e7e8ba1ba4
> core/src/test/resources/coord-dataset-offset.xml 5dfbb1b40ee86dfc3336031eea3c5ff3144b9f5a
> core/src/test/resources/coord-inputcheck-combine.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-hcat.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-range-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-range.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> pom.xml a74ffab081defd15a1c5dbbf15355aa4e0455029
>
> Diff: https://reviews.apache.org/r/38474/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Purshotam Shah
>
>
Re: Review Request 38474: OOZIE-1976- Specifying coordinator input
datasets in more logical ways
Posted by Rohini Palaniswamy <ro...@gmail.com>.
> On Dec. 29, 2015, 10:05 p.m., Rohini Palaniswamy wrote:
> > client/src/main/resources/oozie-coordinator-0.5.xsd, lines 116-117
> > <https://reviews.apache.org/r/38474/diff/3/?file=1165028#file1165028line116>
> >
> > min and wait seems to be supported only at the data-in level.
Dropped the issue. It is supported on and and or as well.
- Rohini
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/38474/#review112102
-----------------------------------------------------------
On Dec. 15, 2015, 7:33 p.m., Purshotam Shah wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/38474/
> -----------------------------------------------------------
>
> (Updated Dec. 15, 2015, 7:33 p.m.)
>
>
> Review request for oozie.
>
>
> Bugs: OOZIE-1976
> https://issues.apache.org/jira/browse/OOZIE-1976
>
>
> Repository: oozie-git
>
>
> Description
> -------
>
> There are three components in this patch
>
> 1. User interface
> A new tag is added to coordinator.xml
> ex.
> <input-check>
> <or name="test">
> <and>
> <data-in dataset="A"/>"
> <data-in dataset="B"/>
> </and>
> <and>
> <data-in dataset="C"/>
> <data-in dataset="D"/>
> </and>
>
> </or>;
> <input-check>
>
>
> input-check will have nested and/or/combine operation. It can have min and wait at operator or at date-in.
> If input-check tag is missing then it consider to be old approach where all data dependency are needed.
>
> 2. Processing
> input-check is converted into logical expression
> (a&&B)||(c&&d)
> We use jexl to parse the logical expression.
>
> There are three phase in parsing.
> phase 1 : only resolved dataset are parsed ( only current).
> phase 2 : once all current are resolved, then future/latest are parsed.
> phase 3 : Doesn't do any filecheck, just return what is being parsed by phase1 and phase2. Is used for EL functions
>
>
> 3. Storage.
> if inputcheck is enable, push_missing_dependencies and missing_dependencies are serialized and stored in DB.
> If then not then it's old approach, where they are stored in plan text. This is backward compatible.
>
>
> Diffs
> -----
>
> client/src/main/resources/oozie-coordinator-0.5.xsd e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/pom.xml b063dab79415447a86c1a33f5c3f5304e0dffca0
> core/src/main/java/org/apache/oozie/CoordinatorActionBean.java 91bff4dca2ef2dece68ca7260724ba3e43b1a08e
> core/src/main/java/org/apache/oozie/ErrorCode.java 6c1e3997c9a1cb0bb0de39d687a083af0a7b5f04
> core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java 742d00dd47aab55392abc8fbc207f8100728b832
> core/src/main/java/org/apache/oozie/command/coord/CoordActionUpdatePushMissingDependency.java 4e1c5b3392358cb6b1e98e16e469310338f27fed
> core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java 58ef483272264e0d391d4a1e6533dc1cab9940da
> core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java 39e6ac15ce3a3ea7f2ed9178688537f7b1d7842d
> core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java b05344d89e8df0e11fe69c1aa725d19a18eb0a2b
> core/src/main/java/org/apache/oozie/coord/CoordELConstants.java f010a817fc900821c0e429fc16e1d3902a98d8bb
> core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java 8b2f4560ae66bbcd707a446382e647663ea67be1
> core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java 5d238663aa94f0dd55a9190b60bfe621439c7b53
> core/src/main/java/org/apache/oozie/coord/CoordUtils.java 94c69740618110ea180b188ab0c5a02db76f8b4d
> core/src/main/java/org/apache/oozie/coord/SyncCoordAction.java 44258eb5be40bf6769e32c8780117e8533d80d7e
> core/src/main/java/org/apache/oozie/coord/input/dependency/AbstractCoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesChecker.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyFactory.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputInstance.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPullInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPushInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/dependency/ActionDependency.java c280d1dc2387c9b3df96d4e83ad5563e10b1e1ef
> core/src/main/java/org/apache/oozie/dependency/DependencyChecker.java a5507575a3403103133f85a78c873c9976419857
> core/src/main/java/org/apache/oozie/util/WritableUtils.java 76a68953541125cab05bd05c94aa73b50e5363ff
> core/src/main/resources/oozie-default.xml faf37405757719554a5b6e3671d0c87723eb9959
> core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java 1fe1b3adb5ceaa83985ef31278e49a3224cde0fb
> core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java 7062e697e5162ac25688d9cb62352bd139a7013a
> core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java 29e7ca145612fadee661fe975aa0ea2cab1cbfa3
> core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java 52eb9dd1fb903ac86294a55d6e2a6918390bf4e9
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordInputCheckPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordinatorInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-action-sla.xml f3f1bc09c51272a35d65edd7271599e7e8ba1ba4
> core/src/test/resources/coord-dataset-offset.xml 5dfbb1b40ee86dfc3336031eea3c5ff3144b9f5a
> core/src/test/resources/coord-inputcheck-combine.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-hcat.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-range-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-range.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> pom.xml a74ffab081defd15a1c5dbbf15355aa4e0455029
>
> Diff: https://reviews.apache.org/r/38474/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Purshotam Shah
>
>
Re: Review Request 38474: OOZIE-1976- Specifying coordinator input
datasets in more logical ways
Posted by Rohini Palaniswamy <ro...@gmail.com>.
> On Dec. 29, 2015, 10:05 p.m., Rohini Palaniswamy wrote:
> > client/src/main/resources/oozie-coordinator-0.5.xsd, line 115
> > <https://reviews.apache.org/r/38474/diff/3/?file=1165028#file1165028line115>
> >
> > type="coordinator:IDENTIFIER" . Same for other names.
> >
> > Shouldn't it be use="required" ?
>
> Purshotam Shah wrote:
> and/or can be nested. If needed user can only provide at root, not at every level.
use=optional is fine. But type="coordinator:IDENTIFIER" needs to be there for all names.
- Rohini
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/38474/#review112102
-----------------------------------------------------------
On Jan. 8, 2016, 2:32 a.m., Purshotam Shah wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/38474/
> -----------------------------------------------------------
>
> (Updated Jan. 8, 2016, 2:32 a.m.)
>
>
> Review request for oozie.
>
>
> Bugs: OOZIE-1976
> https://issues.apache.org/jira/browse/OOZIE-1976
>
>
> Repository: oozie-git
>
>
> Description
> -------
>
> There are three components in this patch
>
> 1. User interface
> A new tag is added to coordinator.xml
> ex.
> <input-check>
> <or name="test">
> <and>
> <data-in dataset="A"/>"
> <data-in dataset="B"/>
> </and>
> <and>
> <data-in dataset="C"/>
> <data-in dataset="D"/>
> </and>
>
> </or>;
> <input-check>
>
>
> input-check will have nested and/or/combine operation. It can have min and wait at operator or at date-in.
> If input-check tag is missing then it consider to be old approach where all data dependency are needed.
>
> 2. Processing
> input-check is converted into logical expression
> (a&&B)||(c&&d)
> We use jexl to parse the logical expression.
>
> There are three phase in parsing.
> phase 1 : only resolved dataset are parsed ( only current).
> phase 2 : once all current are resolved, then future/latest are parsed.
> phase 3 : Doesn't do any filecheck, just return what is being parsed by phase1 and phase2. Is used for EL functions
>
>
> 3. Storage.
> if inputcheck is enable, push_missing_dependencies and missing_dependencies are serialized and stored in DB.
> If then not then it's old approach, where they are stored in plan text. This is backward compatible.
>
>
> Diffs
> -----
>
> client/src/main/resources/oozie-coordinator-0.5.xsd e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/pom.xml b063dab79415447a86c1a33f5c3f5304e0dffca0
> core/src/main/java/org/apache/oozie/CoordinatorActionBean.java 91bff4dca2ef2dece68ca7260724ba3e43b1a08e
> core/src/main/java/org/apache/oozie/ErrorCode.java 6c1e3997c9a1cb0bb0de39d687a083af0a7b5f04
> core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java 742d00dd47aab55392abc8fbc207f8100728b832
> core/src/main/java/org/apache/oozie/command/coord/CoordActionUpdatePushMissingDependency.java 4e1c5b3392358cb6b1e98e16e469310338f27fed
> core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java 58ef483272264e0d391d4a1e6533dc1cab9940da
> core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java 39e6ac15ce3a3ea7f2ed9178688537f7b1d7842d
> core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java b05344d89e8df0e11fe69c1aa725d19a18eb0a2b
> core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java d4d1c08141088e64e0551dccbd23a6185cfdbb7b
> core/src/main/java/org/apache/oozie/coord/CoordELConstants.java f010a817fc900821c0e429fc16e1d3902a98d8bb
> core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java 8b2f4560ae66bbcd707a446382e647663ea67be1
> core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java 5d238663aa94f0dd55a9190b60bfe621439c7b53
> core/src/main/java/org/apache/oozie/coord/CoordUtils.java 94c69740618110ea180b188ab0c5a02db76f8b4d
> core/src/main/java/org/apache/oozie/coord/SyncCoordAction.java 44258eb5be40bf6769e32c8780117e8533d80d7e
> core/src/main/java/org/apache/oozie/coord/input/dependency/AbstractCoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesChecker.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyFactory.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputInstance.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPullInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPushInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluator.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorResult.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/dependency/ActionDependency.java c280d1dc2387c9b3df96d4e83ad5563e10b1e1ef
> core/src/main/java/org/apache/oozie/dependency/DependencyChecker.java a5507575a3403103133f85a78c873c9976419857
> core/src/main/java/org/apache/oozie/dependency/FSURIHandler.java 7c1aadf273aeca70c110f29ccf527c073250b2e7
> core/src/main/java/org/apache/oozie/dependency/HCatURIHandler.java 1bbf37d8d15c603666ddb081f83d515730740bbc
> core/src/main/java/org/apache/oozie/dependency/URIHandler.java bc947168558f1970cc2fd1c40b87a1c43ce860ee
> core/src/main/java/org/apache/oozie/util/WritableUtils.java 76a68953541125cab05bd05c94aa73b50e5363ff
> core/src/main/resources/oozie-default.xml faf37405757719554a5b6e3671d0c87723eb9959
> core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java 1fe1b3adb5ceaa83985ef31278e49a3224cde0fb
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordInputCheckPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordinatorInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordInputLogicPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordinatorInputLogic.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/logic/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-action-sla.xml f3f1bc09c51272a35d65edd7271599e7e8ba1ba4
> core/src/test/resources/coord-inputlogic-combine.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-hcat.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-range-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-range.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> pom.xml a74ffab081defd15a1c5dbbf15355aa4e0455029
>
> Diff: https://reviews.apache.org/r/38474/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Purshotam Shah
>
>
Re: Review Request 38474: OOZIE-1976- Specifying coordinator input
datasets in more logical ways
Posted by Purshotam Shah <pu...@yahoo-inc.com>.
> On Dec. 29, 2015, 10:05 p.m., Rohini Palaniswamy wrote:
> > client/src/main/resources/oozie-coordinator-0.5.xsd, line 115
> > <https://reviews.apache.org/r/38474/diff/3/?file=1165028#file1165028line115>
> >
> > type="coordinator:IDENTIFIER" . Same for other names.
> >
> > Shouldn't it be use="required" ?
>
> Purshotam Shah wrote:
> and/or can be nested. If needed user can only provide at root, not at every level.
>
> Rohini Palaniswamy wrote:
> use=optional is fine. But type="coordinator:IDENTIFIER" needs to be there for all names.
coordinator:IDENTIFIER deosn't support parameterization.
- Purshotam
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/38474/#review112102
-----------------------------------------------------------
On Jan. 25, 2016, 11:32 p.m., Purshotam Shah wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/38474/
> -----------------------------------------------------------
>
> (Updated Jan. 25, 2016, 11:32 p.m.)
>
>
> Review request for oozie.
>
>
> Bugs: OOZIE-1976
> https://issues.apache.org/jira/browse/OOZIE-1976
>
>
> Repository: oozie-git
>
>
> Description
> -------
>
> There are three components in this patch
>
> 1. User interface
> A new tag is added to coordinator.xml
> ex.
> <input-check>
> <or name="test">
> <and>
> <data-in dataset="A"/>"
> <data-in dataset="B"/>
> </and>
> <and>
> <data-in dataset="C"/>
> <data-in dataset="D"/>
> </and>
>
> </or>;
> <input-check>
>
>
> input-check will have nested and/or/combine operation. It can have min and wait at operator or at date-in.
> If input-check tag is missing then it consider to be old approach where all data dependency are needed.
>
> 2. Processing
> input-check is converted into logical expression
> (a&&B)||(c&&d)
> We use jexl to parse the logical expression.
>
> There are three phase in parsing.
> phase 1 : only resolved dataset are parsed ( only current).
> phase 2 : once all current are resolved, then future/latest are parsed.
> phase 3 : Doesn't do any filecheck, just return what is being parsed by phase1 and phase2. Is used for EL functions
>
>
> 3. Storage.
> if inputcheck is enable, push_missing_dependencies and missing_dependencies are serialized and stored in DB.
> If then not then it's old approach, where they are stored in plan text. This is backward compatible.
>
>
> Diffs
> -----
>
> client/src/main/resources/oozie-coordinator-0.5.xsd e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/pom.xml b063dab79415447a86c1a33f5c3f5304e0dffca0
> core/src/main/java/org/apache/oozie/CoordinatorActionBean.java 91bff4dca2ef2dece68ca7260724ba3e43b1a08e
> core/src/main/java/org/apache/oozie/ErrorCode.java 6c1e3997c9a1cb0bb0de39d687a083af0a7b5f04
> core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java 742d00dd47aab55392abc8fbc207f8100728b832
> core/src/main/java/org/apache/oozie/command/coord/CoordActionUpdatePushMissingDependency.java 4e1c5b3392358cb6b1e98e16e469310338f27fed
> core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java 58ef483272264e0d391d4a1e6533dc1cab9940da
> core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java 39e6ac15ce3a3ea7f2ed9178688537f7b1d7842d
> core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java b05344d89e8df0e11fe69c1aa725d19a18eb0a2b
> core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java d4d1c08141088e64e0551dccbd23a6185cfdbb7b
> core/src/main/java/org/apache/oozie/coord/CoordELConstants.java f010a817fc900821c0e429fc16e1d3902a98d8bb
> core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java 8b2f4560ae66bbcd707a446382e647663ea67be1
> core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java 5d238663aa94f0dd55a9190b60bfe621439c7b53
> core/src/main/java/org/apache/oozie/coord/CoordUtils.java 94c69740618110ea180b188ab0c5a02db76f8b4d
> core/src/main/java/org/apache/oozie/coord/SyncCoordAction.java 44258eb5be40bf6769e32c8780117e8533d80d7e
> core/src/main/java/org/apache/oozie/coord/input/dependency/AbstractCoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesChecker.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyFactory.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputInstance.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPullInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPushInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluator.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorResult.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/dependency/ActionDependency.java c280d1dc2387c9b3df96d4e83ad5563e10b1e1ef
> core/src/main/java/org/apache/oozie/dependency/DependencyChecker.java a5507575a3403103133f85a78c873c9976419857
> core/src/main/java/org/apache/oozie/dependency/FSURIHandler.java 7c1aadf273aeca70c110f29ccf527c073250b2e7
> core/src/main/java/org/apache/oozie/dependency/HCatURIHandler.java 1bbf37d8d15c603666ddb081f83d515730740bbc
> core/src/main/java/org/apache/oozie/dependency/URIHandler.java bc947168558f1970cc2fd1c40b87a1c43ce860ee
> core/src/main/java/org/apache/oozie/util/WritableUtils.java 76a68953541125cab05bd05c94aa73b50e5363ff
> core/src/main/resources/oozie-default.xml faf37405757719554a5b6e3671d0c87723eb9959
> core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java 1fe1b3adb5ceaa83985ef31278e49a3224cde0fb
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordInputCheckPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordinatorInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordInputLogicPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordinatorInputLogic.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/logic/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-action-sla.xml f3f1bc09c51272a35d65edd7271599e7e8ba1ba4
> core/src/test/resources/coord-inputlogic-combine.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-hcat.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-range-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-range.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> pom.xml a74ffab081defd15a1c5dbbf15355aa4e0455029
>
> Diff: https://reviews.apache.org/r/38474/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Purshotam Shah
>
>
Re: Review Request 38474: OOZIE-1976- Specifying coordinator input
datasets in more logical ways
Posted by Purshotam Shah <pu...@yahoo-inc.com>.
> On Dec. 29, 2015, 10:05 p.m., Rohini Palaniswamy wrote:
> > client/src/main/resources/oozie-coordinator-0.5.xsd, line 115
> > <https://reviews.apache.org/r/38474/diff/3/?file=1165028#file1165028line115>
> >
> > type="coordinator:IDENTIFIER" . Same for other names.
> >
> > Shouldn't it be use="required" ?
and/or can be nested. If needed user can only provide at root, not at every level.
> On Dec. 29, 2015, 10:05 p.m., Rohini Palaniswamy wrote:
> > client/src/main/resources/oozie-coordinator-0.5.xsd, line 139
> > <https://reviews.apache.org/r/38474/diff/3/?file=1165028#file1165028line139>
> >
> > Why does this need a name?
<data-in name="date-in-input-logic" dataset="A" min="3">
Users can use a different name to differentiate data return by input-logic and actual data.
> On Dec. 29, 2015, 10:05 p.m., Rohini Palaniswamy wrote:
> > core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java, line 177
> > <https://reviews.apache.org/r/38474/diff/3/?file=1165036#file1165036line177>
> >
> > coordPushInputDependency.isDependencyMet() is redundant. Pushdependency cannot be met without changeindependency
It can. If you use inputlogic without any datain from Pushdependency. There won't be any change is dependency, but it will met the
dependency.
- Purshotam
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/38474/#review112102
-----------------------------------------------------------
On Dec. 15, 2015, 7:33 p.m., Purshotam Shah wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/38474/
> -----------------------------------------------------------
>
> (Updated Dec. 15, 2015, 7:33 p.m.)
>
>
> Review request for oozie.
>
>
> Bugs: OOZIE-1976
> https://issues.apache.org/jira/browse/OOZIE-1976
>
>
> Repository: oozie-git
>
>
> Description
> -------
>
> There are three components in this patch
>
> 1. User interface
> A new tag is added to coordinator.xml
> ex.
> <input-check>
> <or name="test">
> <and>
> <data-in dataset="A"/>"
> <data-in dataset="B"/>
> </and>
> <and>
> <data-in dataset="C"/>
> <data-in dataset="D"/>
> </and>
>
> </or>;
> <input-check>
>
>
> input-check will have nested and/or/combine operation. It can have min and wait at operator or at date-in.
> If input-check tag is missing then it consider to be old approach where all data dependency are needed.
>
> 2. Processing
> input-check is converted into logical expression
> (a&&B)||(c&&d)
> We use jexl to parse the logical expression.
>
> There are three phase in parsing.
> phase 1 : only resolved dataset are parsed ( only current).
> phase 2 : once all current are resolved, then future/latest are parsed.
> phase 3 : Doesn't do any filecheck, just return what is being parsed by phase1 and phase2. Is used for EL functions
>
>
> 3. Storage.
> if inputcheck is enable, push_missing_dependencies and missing_dependencies are serialized and stored in DB.
> If then not then it's old approach, where they are stored in plan text. This is backward compatible.
>
>
> Diffs
> -----
>
> client/src/main/resources/oozie-coordinator-0.5.xsd e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/pom.xml b063dab79415447a86c1a33f5c3f5304e0dffca0
> core/src/main/java/org/apache/oozie/CoordinatorActionBean.java 91bff4dca2ef2dece68ca7260724ba3e43b1a08e
> core/src/main/java/org/apache/oozie/ErrorCode.java 6c1e3997c9a1cb0bb0de39d687a083af0a7b5f04
> core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java 742d00dd47aab55392abc8fbc207f8100728b832
> core/src/main/java/org/apache/oozie/command/coord/CoordActionUpdatePushMissingDependency.java 4e1c5b3392358cb6b1e98e16e469310338f27fed
> core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java 58ef483272264e0d391d4a1e6533dc1cab9940da
> core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java 39e6ac15ce3a3ea7f2ed9178688537f7b1d7842d
> core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java b05344d89e8df0e11fe69c1aa725d19a18eb0a2b
> core/src/main/java/org/apache/oozie/coord/CoordELConstants.java f010a817fc900821c0e429fc16e1d3902a98d8bb
> core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java 8b2f4560ae66bbcd707a446382e647663ea67be1
> core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java 5d238663aa94f0dd55a9190b60bfe621439c7b53
> core/src/main/java/org/apache/oozie/coord/CoordUtils.java 94c69740618110ea180b188ab0c5a02db76f8b4d
> core/src/main/java/org/apache/oozie/coord/SyncCoordAction.java 44258eb5be40bf6769e32c8780117e8533d80d7e
> core/src/main/java/org/apache/oozie/coord/input/dependency/AbstractCoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesChecker.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyFactory.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputInstance.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPullInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPushInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/dependency/ActionDependency.java c280d1dc2387c9b3df96d4e83ad5563e10b1e1ef
> core/src/main/java/org/apache/oozie/dependency/DependencyChecker.java a5507575a3403103133f85a78c873c9976419857
> core/src/main/java/org/apache/oozie/util/WritableUtils.java 76a68953541125cab05bd05c94aa73b50e5363ff
> core/src/main/resources/oozie-default.xml faf37405757719554a5b6e3671d0c87723eb9959
> core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java 1fe1b3adb5ceaa83985ef31278e49a3224cde0fb
> core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java 7062e697e5162ac25688d9cb62352bd139a7013a
> core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java 29e7ca145612fadee661fe975aa0ea2cab1cbfa3
> core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java 52eb9dd1fb903ac86294a55d6e2a6918390bf4e9
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordInputCheckPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordinatorInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-action-sla.xml f3f1bc09c51272a35d65edd7271599e7e8ba1ba4
> core/src/test/resources/coord-dataset-offset.xml 5dfbb1b40ee86dfc3336031eea3c5ff3144b9f5a
> core/src/test/resources/coord-inputcheck-combine.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-hcat.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-range-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-range.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> pom.xml a74ffab081defd15a1c5dbbf15355aa4e0455029
>
> Diff: https://reviews.apache.org/r/38474/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Purshotam Shah
>
>
Re: Review Request 38474: OOZIE-1976- Specifying coordinator input
datasets in more logical ways
Posted by Rohini Palaniswamy <ro...@gmail.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/38474/#review112102
-----------------------------------------------------------
These are comments for Page 1 and 2. Still I have not fully gone through and understood few of the classes. Will comment on those and the testcases/Page 3 in next review.
At high level,
1) Noticed naming and EL function changed from design doc you had put up earlier. Liked those better.
- Possible to rename <input-check> to <input-logic> as per the design doc
- Possible to reuse the existing EL also as per design doc? i.e coord:dataIn(CorD) instead of coord:inputCheckDataIn(CorD) Keeps it simple for user.
2) User documentation missing as mentioned by Robert.
3) Couple of comments are my usual nitpicks on typos and variable naming.
client/src/main/resources/oozie-coordinator-0.5.xsd (line 115)
<https://reviews.apache.org/r/38474/#comment172502>
type="coordinator:IDENTIFIER" . Same for other names.
Shouldn't it be use="required" ?
client/src/main/resources/oozie-coordinator-0.5.xsd (lines 116 - 117)
<https://reviews.apache.org/r/38474/#comment172504>
min and wait seems to be supported only at the data-in level.
client/src/main/resources/oozie-coordinator-0.5.xsd (line 139)
<https://reviews.apache.org/r/38474/#comment172503>
Why does this need a name?
core/src/main/java/org/apache/oozie/CoordinatorActionBean.java (lines 822 - 826)
<https://reviews.apache.org/r/38474/#comment172561>
Declare variables at the top along with other variables
core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java (line 169)
<https://reviews.apache.org/r/38474/#comment172506>
Nitpick. isPushDependenciesMeet -> isPushDependenciesMet .
There are couple of other places as well. Please grep for meet and change to met.
core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java
<https://reviews.apache.org/r/38474/#comment172552>
Can we keep this log statement? Might not be exactly correct anymore if input-logic is defined, but still can be helpful to some extent in determining what to look for
core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java (lines 172 - 174)
<https://reviews.apache.org/r/38474/#comment172553>
Seems redundant.
core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java (line 275)
<https://reviews.apache.org/r/38474/#comment172562>
This is redundant. UPDATE_COORD_ACTION_FOR_INPUTCHECK only serializes pull missing dependencies
core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java (line 436)
<https://reviews.apache.org/r/38474/#comment172565>
Why public? Also method seems to be unused. To be removed?
core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java (line 494)
<https://reviews.apache.org/r/38474/#comment172566>
Why public? Also method seems to be unused as it is copied to CoordOldInputDependency. To be removed?
core/src/main/java/org/apache/oozie/command/coord/CoordActionUpdatePushMissingDependency.java (line 43)
<https://reviews.apache.org/r/38474/#comment172567>
Seems wrong. Getter is that of push dependencies instead of pull.
core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java (line 533)
<https://reviews.apache.org/r/38474/#comment172507>
isInputLogicSpecified
core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java (line 534)
<https://reviews.apache.org/r/38474/#comment172508>
validateInputLogic
core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java (line 123)
<https://reviews.apache.org/r/38474/#comment172576>
Keep the info log statement. May not be fully correct when OR is involved, but still would be useful.
List<String> availableList - This is not required. It will try to unregister already available stuff. Can just do unregisterAvailableDependencies(actionDep.getAvailableDependencies()); as before
core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java (line 171)
<https://reviews.apache.org/r/38474/#comment172581>
coordPushInputDependency.isDependencyMet() is redundant. Pushdependency cannot be met without changeindependency
core/src/main/java/org/apache/oozie/coord/CoordUtils.java (line 434)
<https://reviews.apache.org/r/38474/#comment172511>
getInputLogic
core/src/main/java/org/apache/oozie/coord/CoordUtils.java (line 439)
<https://reviews.apache.org/r/38474/#comment172512>
Nitpick. getDataInputFromString
core/src/main/java/org/apache/oozie/coord/input/dependency/AbstractCoordInputDependency.java (line 64)
<https://reviews.apache.org/r/38474/#comment172530>
Nitpick. Does nothing and can be removed
core/src/main/java/org/apache/oozie/coord/input/dependency/AbstractCoordInputDependency.java (lines 299 - 300)
<https://reviews.apache.org/r/38474/#comment172548>
Can write the boolean first and then map. Can stop deserializing if dependency is met for display purposes.
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependency.java (line 33)
<https://reviews.apache.org/r/38474/#comment172519>
V=1
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependency.java (lines 96 - 98)
<https://reviews.apache.org/r/38474/#comment172547>
join -> missingDependencies
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyFactory.java (line 43)
<https://reviews.apache.org/r/38474/#comment172554>
isInputCheck -> hasInputLogic. Other places too
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyFactory.java (line 58)
<https://reviews.apache.org/r/38474/#comment172555>
Nitpick. Pushinput -> PushInput
core/src/main/java/org/apache/oozie/util/WritableUtils.java (line 130)
<https://reviews.apache.org/r/38474/#comment172517>
readStringList
core/src/main/java/org/apache/oozie/util/WritableUtils.java (line 154)
<https://reviews.apache.org/r/38474/#comment172518>
writeStringList
- Rohini Palaniswamy
On Dec. 15, 2015, 7:33 p.m., Purshotam Shah wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/38474/
> -----------------------------------------------------------
>
> (Updated Dec. 15, 2015, 7:33 p.m.)
>
>
> Review request for oozie.
>
>
> Bugs: OOZIE-1976
> https://issues.apache.org/jira/browse/OOZIE-1976
>
>
> Repository: oozie-git
>
>
> Description
> -------
>
> There are three components in this patch
>
> 1. User interface
> A new tag is added to coordinator.xml
> ex.
> <input-check>
> <or name="test">
> <and>
> <data-in dataset="A"/>"
> <data-in dataset="B"/>
> </and>
> <and>
> <data-in dataset="C"/>
> <data-in dataset="D"/>
> </and>
>
> </or>;
> <input-check>
>
>
> input-check will have nested and/or/combine operation. It can have min and wait at operator or at date-in.
> If input-check tag is missing then it consider to be old approach where all data dependency are needed.
>
> 2. Processing
> input-check is converted into logical expression
> (a&&B)||(c&&d)
> We use jexl to parse the logical expression.
>
> There are three phase in parsing.
> phase 1 : only resolved dataset are parsed ( only current).
> phase 2 : once all current are resolved, then future/latest are parsed.
> phase 3 : Doesn't do any filecheck, just return what is being parsed by phase1 and phase2. Is used for EL functions
>
>
> 3. Storage.
> if inputcheck is enable, push_missing_dependencies and missing_dependencies are serialized and stored in DB.
> If then not then it's old approach, where they are stored in plan text. This is backward compatible.
>
>
> Diffs
> -----
>
> client/src/main/resources/oozie-coordinator-0.5.xsd e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/pom.xml b063dab79415447a86c1a33f5c3f5304e0dffca0
> core/src/main/java/org/apache/oozie/CoordinatorActionBean.java 91bff4dca2ef2dece68ca7260724ba3e43b1a08e
> core/src/main/java/org/apache/oozie/ErrorCode.java 6c1e3997c9a1cb0bb0de39d687a083af0a7b5f04
> core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java 742d00dd47aab55392abc8fbc207f8100728b832
> core/src/main/java/org/apache/oozie/command/coord/CoordActionUpdatePushMissingDependency.java 4e1c5b3392358cb6b1e98e16e469310338f27fed
> core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java 58ef483272264e0d391d4a1e6533dc1cab9940da
> core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java 39e6ac15ce3a3ea7f2ed9178688537f7b1d7842d
> core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java b05344d89e8df0e11fe69c1aa725d19a18eb0a2b
> core/src/main/java/org/apache/oozie/coord/CoordELConstants.java f010a817fc900821c0e429fc16e1d3902a98d8bb
> core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java 8b2f4560ae66bbcd707a446382e647663ea67be1
> core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java 5d238663aa94f0dd55a9190b60bfe621439c7b53
> core/src/main/java/org/apache/oozie/coord/CoordUtils.java 94c69740618110ea180b188ab0c5a02db76f8b4d
> core/src/main/java/org/apache/oozie/coord/SyncCoordAction.java 44258eb5be40bf6769e32c8780117e8533d80d7e
> core/src/main/java/org/apache/oozie/coord/input/dependency/AbstractCoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesChecker.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyFactory.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputInstance.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPullInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPushInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/dependency/ActionDependency.java c280d1dc2387c9b3df96d4e83ad5563e10b1e1ef
> core/src/main/java/org/apache/oozie/dependency/DependencyChecker.java a5507575a3403103133f85a78c873c9976419857
> core/src/main/java/org/apache/oozie/util/WritableUtils.java 76a68953541125cab05bd05c94aa73b50e5363ff
> core/src/main/resources/oozie-default.xml faf37405757719554a5b6e3671d0c87723eb9959
> core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java 1fe1b3adb5ceaa83985ef31278e49a3224cde0fb
> core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java 7062e697e5162ac25688d9cb62352bd139a7013a
> core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java 29e7ca145612fadee661fe975aa0ea2cab1cbfa3
> core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java 52eb9dd1fb903ac86294a55d6e2a6918390bf4e9
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordInputCheckPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordinatorInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-action-sla.xml f3f1bc09c51272a35d65edd7271599e7e8ba1ba4
> core/src/test/resources/coord-dataset-offset.xml 5dfbb1b40ee86dfc3336031eea3c5ff3144b9f5a
> core/src/test/resources/coord-inputcheck-combine.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-hcat.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-range-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-range.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> pom.xml a74ffab081defd15a1c5dbbf15355aa4e0455029
>
> Diff: https://reviews.apache.org/r/38474/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Purshotam Shah
>
>
Re: Review Request 38474: OOZIE-1976- Specifying coordinator input
datasets in more logical ways
Posted by Rohini Palaniswamy <ro...@gmail.com>.
> On Dec. 30, 2015, 6:01 p.m., Rohini Palaniswamy wrote:
> > core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java, line 140
> > <https://reviews.apache.org/r/38474/diff/3/?file=1165054#file1165054line140>
> >
> > Should be CoordELFunctions.INSTANCE_SEPARATOR
>
> Purshotam Shah wrote:
> this is for unresolved list
Instances in unresolved list should be separated by CoordELFunctions.INSTANCE_SEPARATOR
> On Dec. 30, 2015, 6:01 p.m., Rohini Palaniswamy wrote:
> > core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java, line 71
> > <https://reviews.apache.org/r/38474/diff/3/?file=1165057#file1165057line71>
> >
> > Avoid adding separator here.
>
> Purshotam Shah wrote:
> This is only display part. Making it similar to oldDependency
This is not right. It ends up adding multiple separators as it is called in a loop.
- Rohini
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/38474/#review112235
-----------------------------------------------------------
On Jan. 8, 2016, 2:32 a.m., Purshotam Shah wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/38474/
> -----------------------------------------------------------
>
> (Updated Jan. 8, 2016, 2:32 a.m.)
>
>
> Review request for oozie.
>
>
> Bugs: OOZIE-1976
> https://issues.apache.org/jira/browse/OOZIE-1976
>
>
> Repository: oozie-git
>
>
> Description
> -------
>
> There are three components in this patch
>
> 1. User interface
> A new tag is added to coordinator.xml
> ex.
> <input-check>
> <or name="test">
> <and>
> <data-in dataset="A"/>"
> <data-in dataset="B"/>
> </and>
> <and>
> <data-in dataset="C"/>
> <data-in dataset="D"/>
> </and>
>
> </or>;
> <input-check>
>
>
> input-check will have nested and/or/combine operation. It can have min and wait at operator or at date-in.
> If input-check tag is missing then it consider to be old approach where all data dependency are needed.
>
> 2. Processing
> input-check is converted into logical expression
> (a&&B)||(c&&d)
> We use jexl to parse the logical expression.
>
> There are three phase in parsing.
> phase 1 : only resolved dataset are parsed ( only current).
> phase 2 : once all current are resolved, then future/latest are parsed.
> phase 3 : Doesn't do any filecheck, just return what is being parsed by phase1 and phase2. Is used for EL functions
>
>
> 3. Storage.
> if inputcheck is enable, push_missing_dependencies and missing_dependencies are serialized and stored in DB.
> If then not then it's old approach, where they are stored in plan text. This is backward compatible.
>
>
> Diffs
> -----
>
> client/src/main/resources/oozie-coordinator-0.5.xsd e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/pom.xml b063dab79415447a86c1a33f5c3f5304e0dffca0
> core/src/main/java/org/apache/oozie/CoordinatorActionBean.java 91bff4dca2ef2dece68ca7260724ba3e43b1a08e
> core/src/main/java/org/apache/oozie/ErrorCode.java 6c1e3997c9a1cb0bb0de39d687a083af0a7b5f04
> core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java 742d00dd47aab55392abc8fbc207f8100728b832
> core/src/main/java/org/apache/oozie/command/coord/CoordActionUpdatePushMissingDependency.java 4e1c5b3392358cb6b1e98e16e469310338f27fed
> core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java 58ef483272264e0d391d4a1e6533dc1cab9940da
> core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java 39e6ac15ce3a3ea7f2ed9178688537f7b1d7842d
> core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java b05344d89e8df0e11fe69c1aa725d19a18eb0a2b
> core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java d4d1c08141088e64e0551dccbd23a6185cfdbb7b
> core/src/main/java/org/apache/oozie/coord/CoordELConstants.java f010a817fc900821c0e429fc16e1d3902a98d8bb
> core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java 8b2f4560ae66bbcd707a446382e647663ea67be1
> core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java 5d238663aa94f0dd55a9190b60bfe621439c7b53
> core/src/main/java/org/apache/oozie/coord/CoordUtils.java 94c69740618110ea180b188ab0c5a02db76f8b4d
> core/src/main/java/org/apache/oozie/coord/SyncCoordAction.java 44258eb5be40bf6769e32c8780117e8533d80d7e
> core/src/main/java/org/apache/oozie/coord/input/dependency/AbstractCoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesChecker.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyFactory.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputInstance.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPullInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPushInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluator.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorResult.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/dependency/ActionDependency.java c280d1dc2387c9b3df96d4e83ad5563e10b1e1ef
> core/src/main/java/org/apache/oozie/dependency/DependencyChecker.java a5507575a3403103133f85a78c873c9976419857
> core/src/main/java/org/apache/oozie/dependency/FSURIHandler.java 7c1aadf273aeca70c110f29ccf527c073250b2e7
> core/src/main/java/org/apache/oozie/dependency/HCatURIHandler.java 1bbf37d8d15c603666ddb081f83d515730740bbc
> core/src/main/java/org/apache/oozie/dependency/URIHandler.java bc947168558f1970cc2fd1c40b87a1c43ce860ee
> core/src/main/java/org/apache/oozie/util/WritableUtils.java 76a68953541125cab05bd05c94aa73b50e5363ff
> core/src/main/resources/oozie-default.xml faf37405757719554a5b6e3671d0c87723eb9959
> core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java 1fe1b3adb5ceaa83985ef31278e49a3224cde0fb
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordInputCheckPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordinatorInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordInputLogicPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordinatorInputLogic.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/logic/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-action-sla.xml f3f1bc09c51272a35d65edd7271599e7e8ba1ba4
> core/src/test/resources/coord-inputlogic-combine.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-hcat.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-range-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-range.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> pom.xml a74ffab081defd15a1c5dbbf15355aa4e0455029
>
> Diff: https://reviews.apache.org/r/38474/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Purshotam Shah
>
>
Re: Review Request 38474: OOZIE-1976- Specifying coordinator input
datasets in more logical ways
Posted by Purshotam Shah <pu...@yahoo-inc.com>.
> On Dec. 30, 2015, 6:01 p.m., Rohini Palaniswamy wrote:
> > core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java, line 47
> > <https://reviews.apache.org/r/38474/diff/3/?file=1165048#file1165048line47>
> >
> > private
not needed. removed.
> On Dec. 30, 2015, 6:01 p.m., Rohini Palaniswamy wrote:
> > core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java, lines 247-251
> > <https://reviews.apache.org/r/38474/diff/3/?file=1165048#file1165048line247>
> >
> > Method seems redundant as it just does call another method. Can be just inlined.
It's overridden in phasethree where we don't have to check hdfs. It always return false from phasethree.
> On Dec. 30, 2015, 6:01 p.m., Rohini Palaniswamy wrote:
> > core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java, line 71
> > <https://reviews.apache.org/r/38474/diff/3/?file=1165057#file1165057line71>
> >
> > Avoid adding separator here.
This is only display part. Making it similar to oldDependency
> On Dec. 30, 2015, 6:01 p.m., Rohini Palaniswamy wrote:
> > core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java, line 140
> > <https://reviews.apache.org/r/38474/diff/3/?file=1165054#file1165054line140>
> >
> > Should be CoordELFunctions.INSTANCE_SEPARATOR
this is for unresolved list
- Purshotam
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/38474/#review112235
-----------------------------------------------------------
On Dec. 15, 2015, 7:33 p.m., Purshotam Shah wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/38474/
> -----------------------------------------------------------
>
> (Updated Dec. 15, 2015, 7:33 p.m.)
>
>
> Review request for oozie.
>
>
> Bugs: OOZIE-1976
> https://issues.apache.org/jira/browse/OOZIE-1976
>
>
> Repository: oozie-git
>
>
> Description
> -------
>
> There are three components in this patch
>
> 1. User interface
> A new tag is added to coordinator.xml
> ex.
> <input-check>
> <or name="test">
> <and>
> <data-in dataset="A"/>"
> <data-in dataset="B"/>
> </and>
> <and>
> <data-in dataset="C"/>
> <data-in dataset="D"/>
> </and>
>
> </or>;
> <input-check>
>
>
> input-check will have nested and/or/combine operation. It can have min and wait at operator or at date-in.
> If input-check tag is missing then it consider to be old approach where all data dependency are needed.
>
> 2. Processing
> input-check is converted into logical expression
> (a&&B)||(c&&d)
> We use jexl to parse the logical expression.
>
> There are three phase in parsing.
> phase 1 : only resolved dataset are parsed ( only current).
> phase 2 : once all current are resolved, then future/latest are parsed.
> phase 3 : Doesn't do any filecheck, just return what is being parsed by phase1 and phase2. Is used for EL functions
>
>
> 3. Storage.
> if inputcheck is enable, push_missing_dependencies and missing_dependencies are serialized and stored in DB.
> If then not then it's old approach, where they are stored in plan text. This is backward compatible.
>
>
> Diffs
> -----
>
> client/src/main/resources/oozie-coordinator-0.5.xsd e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/pom.xml b063dab79415447a86c1a33f5c3f5304e0dffca0
> core/src/main/java/org/apache/oozie/CoordinatorActionBean.java 91bff4dca2ef2dece68ca7260724ba3e43b1a08e
> core/src/main/java/org/apache/oozie/ErrorCode.java 6c1e3997c9a1cb0bb0de39d687a083af0a7b5f04
> core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java 742d00dd47aab55392abc8fbc207f8100728b832
> core/src/main/java/org/apache/oozie/command/coord/CoordActionUpdatePushMissingDependency.java 4e1c5b3392358cb6b1e98e16e469310338f27fed
> core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java 58ef483272264e0d391d4a1e6533dc1cab9940da
> core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java 39e6ac15ce3a3ea7f2ed9178688537f7b1d7842d
> core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java b05344d89e8df0e11fe69c1aa725d19a18eb0a2b
> core/src/main/java/org/apache/oozie/coord/CoordELConstants.java f010a817fc900821c0e429fc16e1d3902a98d8bb
> core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java 8b2f4560ae66bbcd707a446382e647663ea67be1
> core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java 5d238663aa94f0dd55a9190b60bfe621439c7b53
> core/src/main/java/org/apache/oozie/coord/CoordUtils.java 94c69740618110ea180b188ab0c5a02db76f8b4d
> core/src/main/java/org/apache/oozie/coord/SyncCoordAction.java 44258eb5be40bf6769e32c8780117e8533d80d7e
> core/src/main/java/org/apache/oozie/coord/input/dependency/AbstractCoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesChecker.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyFactory.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputInstance.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPullInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPushInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/dependency/ActionDependency.java c280d1dc2387c9b3df96d4e83ad5563e10b1e1ef
> core/src/main/java/org/apache/oozie/dependency/DependencyChecker.java a5507575a3403103133f85a78c873c9976419857
> core/src/main/java/org/apache/oozie/util/WritableUtils.java 76a68953541125cab05bd05c94aa73b50e5363ff
> core/src/main/resources/oozie-default.xml faf37405757719554a5b6e3671d0c87723eb9959
> core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java 1fe1b3adb5ceaa83985ef31278e49a3224cde0fb
> core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java 7062e697e5162ac25688d9cb62352bd139a7013a
> core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java 29e7ca145612fadee661fe975aa0ea2cab1cbfa3
> core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java 52eb9dd1fb903ac86294a55d6e2a6918390bf4e9
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordInputCheckPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordinatorInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-action-sla.xml f3f1bc09c51272a35d65edd7271599e7e8ba1ba4
> core/src/test/resources/coord-dataset-offset.xml 5dfbb1b40ee86dfc3336031eea3c5ff3144b9f5a
> core/src/test/resources/coord-inputcheck-combine.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-hcat.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-range-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-range.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> pom.xml a74ffab081defd15a1c5dbbf15355aa4e0455029
>
> Diff: https://reviews.apache.org/r/38474/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Purshotam Shah
>
>
Re: Review Request 38474: OOZIE-1976- Specifying coordinator input
datasets in more logical ways
Posted by Rohini Palaniswamy <ro...@gmail.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/38474/#review112235
-----------------------------------------------------------
core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java (line 78)
<https://reviews.apache.org/r/38474/#comment172613>
RESOLVED_UNRESOLVED_SEPARATOR. Same as before as it is used to separate resolved and unresolved(latest/future) instances. Looking at places in code where it is used to separate unresolved instances there could be some issue or backward incompatibility with it. Earlier unresolved_separator were separated by CoordELFunctions.INSTANCE_SEPARATOR similar to resolved instances.
core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java (line 575)
<https://reviews.apache.org/r/38474/#comment172604>
actionBean.getPushInputDependencies().getMissingDependencies() can be a local variable instead of calling twice. It does string operations.
core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java (line 576)
<https://reviews.apache.org/r/38474/#comment172603>
Redundant. Can assign directly
core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java (lines 703 - 705)
<https://reviews.apache.org/r/38474/#comment172610>
for loop can be outside of if-else block
core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java (line 375)
<https://reviews.apache.org/r/38474/#comment172779>
This should be either !isResolved or eval.getVariable(CoordELConstants.RESOLVED_PATH) == null if you want to store partial paths when it is not resolved fully for min processing
core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java (line 513)
<https://reviews.apache.org/r/38474/#comment172594>
Remove line
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesBuilder.java (line 45)
<https://reviews.apache.org/r/38474/#comment172758>
input -> dataset for both method and param
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesBuilder.java (line 56)
<https://reviews.apache.org/r/38474/#comment172760>
combine -> combineDatasets for both method and param
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesBuilder.java (lines 74 - 75)
<https://reviews.apache.org/r/38474/#comment172761>
input -> dataset
combine -> combineDatasets
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesBuilder.java (line 159)
<https://reviews.apache.org/r/38474/#comment172755>
Get rid of code for m and h. Only support numeric value to be consistent with other parameters in coordinator xml like timeout, sla start/end/duration. In future can add support for EL and multiplication. I understand that we added support for m and h in APIs for querying, but configuration in general (hadoop and other hadoop projects) has always been just numeric and lets keep it standard.
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesUtil.java (line 141)
<https://reviews.apache.org/r/38474/#comment172762>
Use [{0}], [{1}] instead of MessageFormat to be consistent with rest of Oozie code. In other places in the code as well. XLog does do MessageFormat internally.
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheck.java (line 31)
<https://reviews.apache.org/r/38474/#comment172630>
Please explain in javadoc what the return string is for both input and combine functions.
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheck.java (line 33)
<https://reviews.apache.org/r/38474/#comment172647>
evalInput
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheck.java (line 43)
<https://reviews.apache.org/r/38474/#comment172648>
evalCombineInput
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java (line 47)
<https://reviews.apache.org/r/38474/#comment172639>
private
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java (line 98)
<https://reviews.apache.org/r/38474/#comment172765>
Looks like you are not stopping when one path is not found. This means you will hit the namenode lot more and is bad. Need to stop if one path does not exist as the old code did.
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java (line 152)
<https://reviews.apache.org/r/38474/#comment172768>
In combine, you need to stop if a path does not exist in all datasets to avoid hitting namenode unnecessarily.
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java (lines 155 - 156)
<https://reviews.apache.org/r/38474/#comment172767>
log message should be moved into if (!pathExists(coordInputInstance.getInputDataInstance(), jobConf)) block
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java (lines 247 - 251)
<https://reviews.apache.org/r/38474/#comment172606>
Method seems redundant as it just does call another method. Can be just inlined.
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java (line 286)
<https://reviews.apache.org/r/38474/#comment172770>
Using true and false as special case values in the string makes the code hacky and was also very hard to follow the code and understand the logic.
Can you use a object instead of String and store the path in one variable and this state in another.
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseTwo.java (line 73)
<https://reviews.apache.org/r/38474/#comment172785>
Why do we need RESOLVED_PATH_WITH_DONE_FLAG instead of RESOLVED_PATH?
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseTwo.java (line 116)
<https://reviews.apache.org/r/38474/#comment172774>
Why do we need to do this? Where do old instances come from? Can you elaborate that on the comment.
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java (line 138)
<https://reviews.apache.org/r/38474/#comment172614>
dependency -> unresolvedDependencies
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java (line 140)
<https://reviews.apache.org/r/38474/#comment172621>
Should be CoordELFunctions.INSTANCE_SEPARATOR
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java (line 282)
<https://reviews.apache.org/r/38474/#comment172782>
Use the new constant added for is_resolved
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java (line 290)
<https://reviews.apache.org/r/38474/#comment172783>
Use the new constant added for resolved_path
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java (line 35)
<https://reviews.apache.org/r/38474/#comment172625>
private
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java (lines 53 - 54)
<https://reviews.apache.org/r/38474/#comment172627>
Plural naming for variable and getter. Either dependencies or dependencyList
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java (line 71)
<https://reviews.apache.org/r/38474/#comment172628>
Avoid adding separator here.
core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlInterpreter.java (line 132)
<https://reviews.apache.org/r/38474/#comment172638>
PHASE_TWO_EVALUATION and TIMED_WAITING could be part of a single enum CoordInputDependencyCheck.STATE
Can use static import of the constant to avoid repeating the class name everywhere.
- Rohini Palaniswamy
On Dec. 15, 2015, 7:33 p.m., Purshotam Shah wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/38474/
> -----------------------------------------------------------
>
> (Updated Dec. 15, 2015, 7:33 p.m.)
>
>
> Review request for oozie.
>
>
> Bugs: OOZIE-1976
> https://issues.apache.org/jira/browse/OOZIE-1976
>
>
> Repository: oozie-git
>
>
> Description
> -------
>
> There are three components in this patch
>
> 1. User interface
> A new tag is added to coordinator.xml
> ex.
> <input-check>
> <or name="test">
> <and>
> <data-in dataset="A"/>"
> <data-in dataset="B"/>
> </and>
> <and>
> <data-in dataset="C"/>
> <data-in dataset="D"/>
> </and>
>
> </or>;
> <input-check>
>
>
> input-check will have nested and/or/combine operation. It can have min and wait at operator or at date-in.
> If input-check tag is missing then it consider to be old approach where all data dependency are needed.
>
> 2. Processing
> input-check is converted into logical expression
> (a&&B)||(c&&d)
> We use jexl to parse the logical expression.
>
> There are three phase in parsing.
> phase 1 : only resolved dataset are parsed ( only current).
> phase 2 : once all current are resolved, then future/latest are parsed.
> phase 3 : Doesn't do any filecheck, just return what is being parsed by phase1 and phase2. Is used for EL functions
>
>
> 3. Storage.
> if inputcheck is enable, push_missing_dependencies and missing_dependencies are serialized and stored in DB.
> If then not then it's old approach, where they are stored in plan text. This is backward compatible.
>
>
> Diffs
> -----
>
> client/src/main/resources/oozie-coordinator-0.5.xsd e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/pom.xml b063dab79415447a86c1a33f5c3f5304e0dffca0
> core/src/main/java/org/apache/oozie/CoordinatorActionBean.java 91bff4dca2ef2dece68ca7260724ba3e43b1a08e
> core/src/main/java/org/apache/oozie/ErrorCode.java 6c1e3997c9a1cb0bb0de39d687a083af0a7b5f04
> core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java 742d00dd47aab55392abc8fbc207f8100728b832
> core/src/main/java/org/apache/oozie/command/coord/CoordActionUpdatePushMissingDependency.java 4e1c5b3392358cb6b1e98e16e469310338f27fed
> core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java 58ef483272264e0d391d4a1e6533dc1cab9940da
> core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java 39e6ac15ce3a3ea7f2ed9178688537f7b1d7842d
> core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java b05344d89e8df0e11fe69c1aa725d19a18eb0a2b
> core/src/main/java/org/apache/oozie/coord/CoordELConstants.java f010a817fc900821c0e429fc16e1d3902a98d8bb
> core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java 8b2f4560ae66bbcd707a446382e647663ea67be1
> core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java 5d238663aa94f0dd55a9190b60bfe621439c7b53
> core/src/main/java/org/apache/oozie/coord/CoordUtils.java 94c69740618110ea180b188ab0c5a02db76f8b4d
> core/src/main/java/org/apache/oozie/coord/SyncCoordAction.java 44258eb5be40bf6769e32c8780117e8533d80d7e
> core/src/main/java/org/apache/oozie/coord/input/dependency/AbstractCoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesChecker.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyFactory.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputInstance.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPullInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPushInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/dependency/ActionDependency.java c280d1dc2387c9b3df96d4e83ad5563e10b1e1ef
> core/src/main/java/org/apache/oozie/dependency/DependencyChecker.java a5507575a3403103133f85a78c873c9976419857
> core/src/main/java/org/apache/oozie/util/WritableUtils.java 76a68953541125cab05bd05c94aa73b50e5363ff
> core/src/main/resources/oozie-default.xml faf37405757719554a5b6e3671d0c87723eb9959
> core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java 1fe1b3adb5ceaa83985ef31278e49a3224cde0fb
> core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java 7062e697e5162ac25688d9cb62352bd139a7013a
> core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java 29e7ca145612fadee661fe975aa0ea2cab1cbfa3
> core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java 52eb9dd1fb903ac86294a55d6e2a6918390bf4e9
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordInputCheckPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordinatorInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-action-sla.xml f3f1bc09c51272a35d65edd7271599e7e8ba1ba4
> core/src/test/resources/coord-dataset-offset.xml 5dfbb1b40ee86dfc3336031eea3c5ff3144b9f5a
> core/src/test/resources/coord-inputcheck-combine.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-hcat.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-range-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-range.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> pom.xml a74ffab081defd15a1c5dbbf15355aa4e0455029
>
> Diff: https://reviews.apache.org/r/38474/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Purshotam Shah
>
>
Re: Review Request 38474: OOZIE-1976- Specifying coordinator input
datasets in more logical ways
Posted by Rohini Palaniswamy <ro...@gmail.com>.
> On Jan. 25, 2016, 10:24 p.m., Rohini Palaniswamy wrote:
> >
Please create a separate jira for documentation
- Rohini
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/38474/#review116181
-----------------------------------------------------------
On Jan. 8, 2016, 2:32 a.m., Purshotam Shah wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/38474/
> -----------------------------------------------------------
>
> (Updated Jan. 8, 2016, 2:32 a.m.)
>
>
> Review request for oozie.
>
>
> Bugs: OOZIE-1976
> https://issues.apache.org/jira/browse/OOZIE-1976
>
>
> Repository: oozie-git
>
>
> Description
> -------
>
> There are three components in this patch
>
> 1. User interface
> A new tag is added to coordinator.xml
> ex.
> <input-check>
> <or name="test">
> <and>
> <data-in dataset="A"/>"
> <data-in dataset="B"/>
> </and>
> <and>
> <data-in dataset="C"/>
> <data-in dataset="D"/>
> </and>
>
> </or>;
> <input-check>
>
>
> input-check will have nested and/or/combine operation. It can have min and wait at operator or at date-in.
> If input-check tag is missing then it consider to be old approach where all data dependency are needed.
>
> 2. Processing
> input-check is converted into logical expression
> (a&&B)||(c&&d)
> We use jexl to parse the logical expression.
>
> There are three phase in parsing.
> phase 1 : only resolved dataset are parsed ( only current).
> phase 2 : once all current are resolved, then future/latest are parsed.
> phase 3 : Doesn't do any filecheck, just return what is being parsed by phase1 and phase2. Is used for EL functions
>
>
> 3. Storage.
> if inputcheck is enable, push_missing_dependencies and missing_dependencies are serialized and stored in DB.
> If then not then it's old approach, where they are stored in plan text. This is backward compatible.
>
>
> Diffs
> -----
>
> client/src/main/resources/oozie-coordinator-0.5.xsd e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/pom.xml b063dab79415447a86c1a33f5c3f5304e0dffca0
> core/src/main/java/org/apache/oozie/CoordinatorActionBean.java 91bff4dca2ef2dece68ca7260724ba3e43b1a08e
> core/src/main/java/org/apache/oozie/ErrorCode.java 6c1e3997c9a1cb0bb0de39d687a083af0a7b5f04
> core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java 742d00dd47aab55392abc8fbc207f8100728b832
> core/src/main/java/org/apache/oozie/command/coord/CoordActionUpdatePushMissingDependency.java 4e1c5b3392358cb6b1e98e16e469310338f27fed
> core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java 58ef483272264e0d391d4a1e6533dc1cab9940da
> core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java 39e6ac15ce3a3ea7f2ed9178688537f7b1d7842d
> core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java b05344d89e8df0e11fe69c1aa725d19a18eb0a2b
> core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java d4d1c08141088e64e0551dccbd23a6185cfdbb7b
> core/src/main/java/org/apache/oozie/coord/CoordELConstants.java f010a817fc900821c0e429fc16e1d3902a98d8bb
> core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java 8b2f4560ae66bbcd707a446382e647663ea67be1
> core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java 5d238663aa94f0dd55a9190b60bfe621439c7b53
> core/src/main/java/org/apache/oozie/coord/CoordUtils.java 94c69740618110ea180b188ab0c5a02db76f8b4d
> core/src/main/java/org/apache/oozie/coord/SyncCoordAction.java 44258eb5be40bf6769e32c8780117e8533d80d7e
> core/src/main/java/org/apache/oozie/coord/input/dependency/AbstractCoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesChecker.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyFactory.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputInstance.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPullInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPushInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluator.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorResult.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/dependency/ActionDependency.java c280d1dc2387c9b3df96d4e83ad5563e10b1e1ef
> core/src/main/java/org/apache/oozie/dependency/DependencyChecker.java a5507575a3403103133f85a78c873c9976419857
> core/src/main/java/org/apache/oozie/dependency/FSURIHandler.java 7c1aadf273aeca70c110f29ccf527c073250b2e7
> core/src/main/java/org/apache/oozie/dependency/HCatURIHandler.java 1bbf37d8d15c603666ddb081f83d515730740bbc
> core/src/main/java/org/apache/oozie/dependency/URIHandler.java bc947168558f1970cc2fd1c40b87a1c43ce860ee
> core/src/main/java/org/apache/oozie/util/WritableUtils.java 76a68953541125cab05bd05c94aa73b50e5363ff
> core/src/main/resources/oozie-default.xml faf37405757719554a5b6e3671d0c87723eb9959
> core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java 1fe1b3adb5ceaa83985ef31278e49a3224cde0fb
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordInputCheckPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordinatorInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordInputLogicPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordinatorInputLogic.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/logic/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-action-sla.xml f3f1bc09c51272a35d65edd7271599e7e8ba1ba4
> core/src/test/resources/coord-inputlogic-combine.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-hcat.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-range-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-range.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> pom.xml a74ffab081defd15a1c5dbbf15355aa4e0455029
>
> Diff: https://reviews.apache.org/r/38474/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Purshotam Shah
>
>
Re: Review Request 38474: OOZIE-1976- Specifying coordinator input
datasets in more logical ways
Posted by Rohini Palaniswamy <ro...@gmail.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/38474/#review116181
-----------------------------------------------------------
core/src/main/java/org/apache/oozie/coord/CoordELConstants.java (line 37)
<https://reviews.apache.org/r/38474/#comment177189>
Not used anymore. To be removed
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseOne.java (line 199)
<https://reviews.apache.org/r/38474/#comment177204>
Not required.
- Rohini Palaniswamy
On Jan. 8, 2016, 2:32 a.m., Purshotam Shah wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/38474/
> -----------------------------------------------------------
>
> (Updated Jan. 8, 2016, 2:32 a.m.)
>
>
> Review request for oozie.
>
>
> Bugs: OOZIE-1976
> https://issues.apache.org/jira/browse/OOZIE-1976
>
>
> Repository: oozie-git
>
>
> Description
> -------
>
> There are three components in this patch
>
> 1. User interface
> A new tag is added to coordinator.xml
> ex.
> <input-check>
> <or name="test">
> <and>
> <data-in dataset="A"/>"
> <data-in dataset="B"/>
> </and>
> <and>
> <data-in dataset="C"/>
> <data-in dataset="D"/>
> </and>
>
> </or>;
> <input-check>
>
>
> input-check will have nested and/or/combine operation. It can have min and wait at operator or at date-in.
> If input-check tag is missing then it consider to be old approach where all data dependency are needed.
>
> 2. Processing
> input-check is converted into logical expression
> (a&&B)||(c&&d)
> We use jexl to parse the logical expression.
>
> There are three phase in parsing.
> phase 1 : only resolved dataset are parsed ( only current).
> phase 2 : once all current are resolved, then future/latest are parsed.
> phase 3 : Doesn't do any filecheck, just return what is being parsed by phase1 and phase2. Is used for EL functions
>
>
> 3. Storage.
> if inputcheck is enable, push_missing_dependencies and missing_dependencies are serialized and stored in DB.
> If then not then it's old approach, where they are stored in plan text. This is backward compatible.
>
>
> Diffs
> -----
>
> client/src/main/resources/oozie-coordinator-0.5.xsd e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/pom.xml b063dab79415447a86c1a33f5c3f5304e0dffca0
> core/src/main/java/org/apache/oozie/CoordinatorActionBean.java 91bff4dca2ef2dece68ca7260724ba3e43b1a08e
> core/src/main/java/org/apache/oozie/ErrorCode.java 6c1e3997c9a1cb0bb0de39d687a083af0a7b5f04
> core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java 742d00dd47aab55392abc8fbc207f8100728b832
> core/src/main/java/org/apache/oozie/command/coord/CoordActionUpdatePushMissingDependency.java 4e1c5b3392358cb6b1e98e16e469310338f27fed
> core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java 58ef483272264e0d391d4a1e6533dc1cab9940da
> core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java 39e6ac15ce3a3ea7f2ed9178688537f7b1d7842d
> core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java b05344d89e8df0e11fe69c1aa725d19a18eb0a2b
> core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java d4d1c08141088e64e0551dccbd23a6185cfdbb7b
> core/src/main/java/org/apache/oozie/coord/CoordELConstants.java f010a817fc900821c0e429fc16e1d3902a98d8bb
> core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java 8b2f4560ae66bbcd707a446382e647663ea67be1
> core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java 5d238663aa94f0dd55a9190b60bfe621439c7b53
> core/src/main/java/org/apache/oozie/coord/CoordUtils.java 94c69740618110ea180b188ab0c5a02db76f8b4d
> core/src/main/java/org/apache/oozie/coord/SyncCoordAction.java 44258eb5be40bf6769e32c8780117e8533d80d7e
> core/src/main/java/org/apache/oozie/coord/input/dependency/AbstractCoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesChecker.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyFactory.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputInstance.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPullInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPushInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluator.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorResult.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/dependency/ActionDependency.java c280d1dc2387c9b3df96d4e83ad5563e10b1e1ef
> core/src/main/java/org/apache/oozie/dependency/DependencyChecker.java a5507575a3403103133f85a78c873c9976419857
> core/src/main/java/org/apache/oozie/dependency/FSURIHandler.java 7c1aadf273aeca70c110f29ccf527c073250b2e7
> core/src/main/java/org/apache/oozie/dependency/HCatURIHandler.java 1bbf37d8d15c603666ddb081f83d515730740bbc
> core/src/main/java/org/apache/oozie/dependency/URIHandler.java bc947168558f1970cc2fd1c40b87a1c43ce860ee
> core/src/main/java/org/apache/oozie/util/WritableUtils.java 76a68953541125cab05bd05c94aa73b50e5363ff
> core/src/main/resources/oozie-default.xml faf37405757719554a5b6e3671d0c87723eb9959
> core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java 1fe1b3adb5ceaa83985ef31278e49a3224cde0fb
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordInputCheckPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordinatorInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordInputLogicPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordinatorInputLogic.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/logic/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-action-sla.xml f3f1bc09c51272a35d65edd7271599e7e8ba1ba4
> core/src/test/resources/coord-inputlogic-combine.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-hcat.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-range-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-range.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> pom.xml a74ffab081defd15a1c5dbbf15355aa4e0455029
>
> Diff: https://reviews.apache.org/r/38474/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Purshotam Shah
>
>
Re: Review Request 38474: OOZIE-1976- Specifying coordinator input
datasets in more logical ways
Posted by Rohini Palaniswamy <ro...@gmail.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/38474/#review116226
-----------------------------------------------------------
Ship it!
Ship It!
- Rohini Palaniswamy
On Jan. 26, 2016, 12:08 a.m., Purshotam Shah wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/38474/
> -----------------------------------------------------------
>
> (Updated Jan. 26, 2016, 12:08 a.m.)
>
>
> Review request for oozie.
>
>
> Bugs: OOZIE-1976
> https://issues.apache.org/jira/browse/OOZIE-1976
>
>
> Repository: oozie-git
>
>
> Description
> -------
>
> There are three components in this patch
>
> 1. User interface
> A new tag is added to coordinator.xml
> ex.
> <input-check>
> <or name="test">
> <and>
> <data-in dataset="A"/>"
> <data-in dataset="B"/>
> </and>
> <and>
> <data-in dataset="C"/>
> <data-in dataset="D"/>
> </and>
>
> </or>;
> <input-check>
>
>
> input-check will have nested and/or/combine operation. It can have min and wait at operator or at date-in.
> If input-check tag is missing then it consider to be old approach where all data dependency are needed.
>
> 2. Processing
> input-check is converted into logical expression
> (a&&B)||(c&&d)
> We use jexl to parse the logical expression.
>
> There are three phase in parsing.
> phase 1 : only resolved dataset are parsed ( only current).
> phase 2 : once all current are resolved, then future/latest are parsed.
> phase 3 : Doesn't do any filecheck, just return what is being parsed by phase1 and phase2. Is used for EL functions
>
>
> 3. Storage.
> if inputcheck is enable, push_missing_dependencies and missing_dependencies are serialized and stored in DB.
> If then not then it's old approach, where they are stored in plan text. This is backward compatible.
>
>
> Diffs
> -----
>
> client/src/main/resources/oozie-coordinator-0.5.xsd e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/pom.xml b063dab79415447a86c1a33f5c3f5304e0dffca0
> core/src/main/java/org/apache/oozie/CoordinatorActionBean.java 91bff4dca2ef2dece68ca7260724ba3e43b1a08e
> core/src/main/java/org/apache/oozie/ErrorCode.java 6c1e3997c9a1cb0bb0de39d687a083af0a7b5f04
> core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java 742d00dd47aab55392abc8fbc207f8100728b832
> core/src/main/java/org/apache/oozie/command/coord/CoordActionUpdatePushMissingDependency.java 4e1c5b3392358cb6b1e98e16e469310338f27fed
> core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java 58ef483272264e0d391d4a1e6533dc1cab9940da
> core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java 39e6ac15ce3a3ea7f2ed9178688537f7b1d7842d
> core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java b05344d89e8df0e11fe69c1aa725d19a18eb0a2b
> core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java d4d1c08141088e64e0551dccbd23a6185cfdbb7b
> core/src/main/java/org/apache/oozie/coord/CoordELConstants.java f010a817fc900821c0e429fc16e1d3902a98d8bb
> core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java 8b2f4560ae66bbcd707a446382e647663ea67be1
> core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java 5d238663aa94f0dd55a9190b60bfe621439c7b53
> core/src/main/java/org/apache/oozie/coord/CoordUtils.java 94c69740618110ea180b188ab0c5a02db76f8b4d
> core/src/main/java/org/apache/oozie/coord/SyncCoordAction.java 44258eb5be40bf6769e32c8780117e8533d80d7e
> core/src/main/java/org/apache/oozie/coord/input/dependency/AbstractCoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesChecker.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyFactory.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputInstance.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPullInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPushInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluator.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorResult.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/dependency/ActionDependency.java c280d1dc2387c9b3df96d4e83ad5563e10b1e1ef
> core/src/main/java/org/apache/oozie/dependency/DependencyChecker.java a5507575a3403103133f85a78c873c9976419857
> core/src/main/java/org/apache/oozie/dependency/FSURIHandler.java 7c1aadf273aeca70c110f29ccf527c073250b2e7
> core/src/main/java/org/apache/oozie/dependency/HCatURIHandler.java 1bbf37d8d15c603666ddb081f83d515730740bbc
> core/src/main/java/org/apache/oozie/dependency/URIHandler.java bc947168558f1970cc2fd1c40b87a1c43ce860ee
> core/src/main/java/org/apache/oozie/util/WritableUtils.java 76a68953541125cab05bd05c94aa73b50e5363ff
> core/src/main/resources/oozie-default.xml faf37405757719554a5b6e3671d0c87723eb9959
> core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java 1fe1b3adb5ceaa83985ef31278e49a3224cde0fb
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordInputCheckPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordinatorInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordInputLogicPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordinatorInputLogic.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/logic/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-action-sla.xml f3f1bc09c51272a35d65edd7271599e7e8ba1ba4
> core/src/test/resources/coord-inputlogic-combine.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-hcat.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-range-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-range.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> pom.xml a74ffab081defd15a1c5dbbf15355aa4e0455029
>
> Diff: https://reviews.apache.org/r/38474/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Purshotam Shah
>
>
Re: Review Request 38474: OOZIE-1976- Specifying coordinator input
datasets in more logical ways
Posted by Rohini Palaniswamy <ro...@gmail.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/38474/#review116227
-----------------------------------------------------------
Ship it!
Ship It!
- Rohini Palaniswamy
On Jan. 26, 2016, 12:15 a.m., Purshotam Shah wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/38474/
> -----------------------------------------------------------
>
> (Updated Jan. 26, 2016, 12:15 a.m.)
>
>
> Review request for oozie.
>
>
> Bugs: OOZIE-1976
> https://issues.apache.org/jira/browse/OOZIE-1976
>
>
> Repository: oozie-git
>
>
> Description
> -------
>
> There are three components in this patch
>
> 1. User interface
> A new tag is added to coordinator.xml
> ex.
> <input-check>
> <or name="test">
> <and>
> <data-in dataset="A"/>"
> <data-in dataset="B"/>
> </and>
> <and>
> <data-in dataset="C"/>
> <data-in dataset="D"/>
> </and>
>
> </or>;
> <input-check>
>
>
> input-check will have nested and/or/combine operation. It can have min and wait at operator or at date-in.
> If input-check tag is missing then it consider to be old approach where all data dependency are needed.
>
> 2. Processing
> input-check is converted into logical expression
> (a&&B)||(c&&d)
> We use jexl to parse the logical expression.
>
> There are three phase in parsing.
> phase 1 : only resolved dataset are parsed ( only current).
> phase 2 : once all current are resolved, then future/latest are parsed.
> phase 3 : Doesn't do any filecheck, just return what is being parsed by phase1 and phase2. Is used for EL functions
>
>
> 3. Storage.
> if inputcheck is enable, push_missing_dependencies and missing_dependencies are serialized and stored in DB.
> If then not then it's old approach, where they are stored in plan text. This is backward compatible.
>
>
> Diffs
> -----
>
> client/src/main/resources/oozie-coordinator-0.5.xsd e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/pom.xml b063dab79415447a86c1a33f5c3f5304e0dffca0
> core/src/main/java/org/apache/oozie/CoordinatorActionBean.java 91bff4dca2ef2dece68ca7260724ba3e43b1a08e
> core/src/main/java/org/apache/oozie/ErrorCode.java 6c1e3997c9a1cb0bb0de39d687a083af0a7b5f04
> core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java 742d00dd47aab55392abc8fbc207f8100728b832
> core/src/main/java/org/apache/oozie/command/coord/CoordActionUpdatePushMissingDependency.java 4e1c5b3392358cb6b1e98e16e469310338f27fed
> core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java 58ef483272264e0d391d4a1e6533dc1cab9940da
> core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java 39e6ac15ce3a3ea7f2ed9178688537f7b1d7842d
> core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java b05344d89e8df0e11fe69c1aa725d19a18eb0a2b
> core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java d4d1c08141088e64e0551dccbd23a6185cfdbb7b
> core/src/main/java/org/apache/oozie/coord/CoordELConstants.java f010a817fc900821c0e429fc16e1d3902a98d8bb
> core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java 8b2f4560ae66bbcd707a446382e647663ea67be1
> core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java 5d238663aa94f0dd55a9190b60bfe621439c7b53
> core/src/main/java/org/apache/oozie/coord/CoordUtils.java 94c69740618110ea180b188ab0c5a02db76f8b4d
> core/src/main/java/org/apache/oozie/coord/SyncCoordAction.java 44258eb5be40bf6769e32c8780117e8533d80d7e
> core/src/main/java/org/apache/oozie/coord/input/dependency/AbstractCoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesChecker.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyFactory.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputInstance.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPullInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPushInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluator.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorResult.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/logic/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/dependency/ActionDependency.java c280d1dc2387c9b3df96d4e83ad5563e10b1e1ef
> core/src/main/java/org/apache/oozie/dependency/DependencyChecker.java a5507575a3403103133f85a78c873c9976419857
> core/src/main/java/org/apache/oozie/dependency/FSURIHandler.java 7c1aadf273aeca70c110f29ccf527c073250b2e7
> core/src/main/java/org/apache/oozie/dependency/HCatURIHandler.java 1bbf37d8d15c603666ddb081f83d515730740bbc
> core/src/main/java/org/apache/oozie/dependency/URIHandler.java bc947168558f1970cc2fd1c40b87a1c43ce860ee
> core/src/main/java/org/apache/oozie/util/WritableUtils.java 76a68953541125cab05bd05c94aa73b50e5363ff
> core/src/main/resources/oozie-default.xml faf37405757719554a5b6e3671d0c87723eb9959
> core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java 1fe1b3adb5ceaa83985ef31278e49a3224cde0fb
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordInputCheckPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordinatorInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordInputLogicPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordinatorInputLogic.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/logic/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-action-sla.xml f3f1bc09c51272a35d65edd7271599e7e8ba1ba4
> core/src/test/resources/coord-inputlogic-combine.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-hcat.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-range-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic-range.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputlogic.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> pom.xml a74ffab081defd15a1c5dbbf15355aa4e0455029
>
> Diff: https://reviews.apache.org/r/38474/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Purshotam Shah
>
>
Re: Review Request 38474: OOZIE-1976- Specifying coordinator input
datasets in more logical ways
Posted by Purshotam Shah <pu...@yahoo-inc.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/38474/
-----------------------------------------------------------
(Updated Jan. 26, 2016, 12:15 a.m.)
Review request for oozie.
Bugs: OOZIE-1976
https://issues.apache.org/jira/browse/OOZIE-1976
Repository: oozie-git
Description
-------
There are three components in this patch
1. User interface
A new tag is added to coordinator.xml
ex.
<input-check>
<or name="test">
<and>
<data-in dataset="A"/>"
<data-in dataset="B"/>
</and>
<and>
<data-in dataset="C"/>
<data-in dataset="D"/>
</and>
</or>;
<input-check>
input-check will have nested and/or/combine operation. It can have min and wait at operator or at date-in.
If input-check tag is missing then it consider to be old approach where all data dependency are needed.
2. Processing
input-check is converted into logical expression
(a&&B)||(c&&d)
We use jexl to parse the logical expression.
There are three phase in parsing.
phase 1 : only resolved dataset are parsed ( only current).
phase 2 : once all current are resolved, then future/latest are parsed.
phase 3 : Doesn't do any filecheck, just return what is being parsed by phase1 and phase2. Is used for EL functions
3. Storage.
if inputcheck is enable, push_missing_dependencies and missing_dependencies are serialized and stored in DB.
If then not then it's old approach, where they are stored in plan text. This is backward compatible.
Diffs (updated)
-----
client/src/main/resources/oozie-coordinator-0.5.xsd e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/pom.xml b063dab79415447a86c1a33f5c3f5304e0dffca0
core/src/main/java/org/apache/oozie/CoordinatorActionBean.java 91bff4dca2ef2dece68ca7260724ba3e43b1a08e
core/src/main/java/org/apache/oozie/ErrorCode.java 6c1e3997c9a1cb0bb0de39d687a083af0a7b5f04
core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java 742d00dd47aab55392abc8fbc207f8100728b832
core/src/main/java/org/apache/oozie/command/coord/CoordActionUpdatePushMissingDependency.java 4e1c5b3392358cb6b1e98e16e469310338f27fed
core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java 58ef483272264e0d391d4a1e6533dc1cab9940da
core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java 39e6ac15ce3a3ea7f2ed9178688537f7b1d7842d
core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java b05344d89e8df0e11fe69c1aa725d19a18eb0a2b
core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java d4d1c08141088e64e0551dccbd23a6185cfdbb7b
core/src/main/java/org/apache/oozie/coord/CoordELConstants.java f010a817fc900821c0e429fc16e1d3902a98d8bb
core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java 8b2f4560ae66bbcd707a446382e647663ea67be1
core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java 5d238663aa94f0dd55a9190b60bfe621439c7b53
core/src/main/java/org/apache/oozie/coord/CoordUtils.java 94c69740618110ea180b188ab0c5a02db76f8b4d
core/src/main/java/org/apache/oozie/coord/SyncCoordAction.java 44258eb5be40bf6769e32c8780117e8533d80d7e
core/src/main/java/org/apache/oozie/coord/input/dependency/AbstractCoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesChecker.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyFactory.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputInstance.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPullInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPushInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluator.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorResult.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/dependency/ActionDependency.java c280d1dc2387c9b3df96d4e83ad5563e10b1e1ef
core/src/main/java/org/apache/oozie/dependency/DependencyChecker.java a5507575a3403103133f85a78c873c9976419857
core/src/main/java/org/apache/oozie/dependency/FSURIHandler.java 7c1aadf273aeca70c110f29ccf527c073250b2e7
core/src/main/java/org/apache/oozie/dependency/HCatURIHandler.java 1bbf37d8d15c603666ddb081f83d515730740bbc
core/src/main/java/org/apache/oozie/dependency/URIHandler.java bc947168558f1970cc2fd1c40b87a1c43ce860ee
core/src/main/java/org/apache/oozie/util/WritableUtils.java 76a68953541125cab05bd05c94aa73b50e5363ff
core/src/main/resources/oozie-default.xml faf37405757719554a5b6e3671d0c87723eb9959
core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java 1fe1b3adb5ceaa83985ef31278e49a3224cde0fb
core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordInputCheckPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordinatorInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/dependency/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordInputLogicPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordinatorInputLogic.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/logic/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-action-sla.xml f3f1bc09c51272a35d65edd7271599e7e8ba1ba4
core/src/test/resources/coord-inputlogic-combine.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputlogic-hcat.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputlogic-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputlogic-range-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputlogic-range.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputlogic.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
pom.xml a74ffab081defd15a1c5dbbf15355aa4e0455029
Diff: https://reviews.apache.org/r/38474/diff/
Testing
-------
Thanks,
Purshotam Shah
Re: Review Request 38474: OOZIE-1976- Specifying coordinator input
datasets in more logical ways
Posted by Purshotam Shah <pu...@yahoo-inc.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/38474/
-----------------------------------------------------------
(Updated Jan. 26, 2016, 12:08 a.m.)
Review request for oozie.
Bugs: OOZIE-1976
https://issues.apache.org/jira/browse/OOZIE-1976
Repository: oozie-git
Description
-------
There are three components in this patch
1. User interface
A new tag is added to coordinator.xml
ex.
<input-check>
<or name="test">
<and>
<data-in dataset="A"/>"
<data-in dataset="B"/>
</and>
<and>
<data-in dataset="C"/>
<data-in dataset="D"/>
</and>
</or>;
<input-check>
input-check will have nested and/or/combine operation. It can have min and wait at operator or at date-in.
If input-check tag is missing then it consider to be old approach where all data dependency are needed.
2. Processing
input-check is converted into logical expression
(a&&B)||(c&&d)
We use jexl to parse the logical expression.
There are three phase in parsing.
phase 1 : only resolved dataset are parsed ( only current).
phase 2 : once all current are resolved, then future/latest are parsed.
phase 3 : Doesn't do any filecheck, just return what is being parsed by phase1 and phase2. Is used for EL functions
3. Storage.
if inputcheck is enable, push_missing_dependencies and missing_dependencies are serialized and stored in DB.
If then not then it's old approach, where they are stored in plan text. This is backward compatible.
Diffs (updated)
-----
client/src/main/resources/oozie-coordinator-0.5.xsd e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/pom.xml b063dab79415447a86c1a33f5c3f5304e0dffca0
core/src/main/java/org/apache/oozie/CoordinatorActionBean.java 91bff4dca2ef2dece68ca7260724ba3e43b1a08e
core/src/main/java/org/apache/oozie/ErrorCode.java 6c1e3997c9a1cb0bb0de39d687a083af0a7b5f04
core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java 742d00dd47aab55392abc8fbc207f8100728b832
core/src/main/java/org/apache/oozie/command/coord/CoordActionUpdatePushMissingDependency.java 4e1c5b3392358cb6b1e98e16e469310338f27fed
core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java 58ef483272264e0d391d4a1e6533dc1cab9940da
core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java 39e6ac15ce3a3ea7f2ed9178688537f7b1d7842d
core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java b05344d89e8df0e11fe69c1aa725d19a18eb0a2b
core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java d4d1c08141088e64e0551dccbd23a6185cfdbb7b
core/src/main/java/org/apache/oozie/coord/CoordELConstants.java f010a817fc900821c0e429fc16e1d3902a98d8bb
core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java 8b2f4560ae66bbcd707a446382e647663ea67be1
core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java 5d238663aa94f0dd55a9190b60bfe621439c7b53
core/src/main/java/org/apache/oozie/coord/CoordUtils.java 94c69740618110ea180b188ab0c5a02db76f8b4d
core/src/main/java/org/apache/oozie/coord/SyncCoordAction.java 44258eb5be40bf6769e32c8780117e8533d80d7e
core/src/main/java/org/apache/oozie/coord/input/dependency/AbstractCoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesChecker.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyFactory.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputInstance.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPullInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPushInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluator.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorResult.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/dependency/ActionDependency.java c280d1dc2387c9b3df96d4e83ad5563e10b1e1ef
core/src/main/java/org/apache/oozie/dependency/DependencyChecker.java a5507575a3403103133f85a78c873c9976419857
core/src/main/java/org/apache/oozie/dependency/FSURIHandler.java 7c1aadf273aeca70c110f29ccf527c073250b2e7
core/src/main/java/org/apache/oozie/dependency/HCatURIHandler.java 1bbf37d8d15c603666ddb081f83d515730740bbc
core/src/main/java/org/apache/oozie/dependency/URIHandler.java bc947168558f1970cc2fd1c40b87a1c43ce860ee
core/src/main/java/org/apache/oozie/util/WritableUtils.java 76a68953541125cab05bd05c94aa73b50e5363ff
core/src/main/resources/oozie-default.xml faf37405757719554a5b6e3671d0c87723eb9959
core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java 1fe1b3adb5ceaa83985ef31278e49a3224cde0fb
core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordInputCheckPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordinatorInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/dependency/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordInputLogicPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordinatorInputLogic.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/logic/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-action-sla.xml f3f1bc09c51272a35d65edd7271599e7e8ba1ba4
core/src/test/resources/coord-inputlogic-combine.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputlogic-hcat.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputlogic-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputlogic-range-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputlogic-range.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputlogic.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
pom.xml a74ffab081defd15a1c5dbbf15355aa4e0455029
Diff: https://reviews.apache.org/r/38474/diff/
Testing
-------
Thanks,
Purshotam Shah
Re: Review Request 38474: OOZIE-1976- Specifying coordinator input
datasets in more logical ways
Posted by Purshotam Shah <pu...@yahoo-inc.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/38474/
-----------------------------------------------------------
(Updated Jan. 25, 2016, 11:32 p.m.)
Review request for oozie.
Bugs: OOZIE-1976
https://issues.apache.org/jira/browse/OOZIE-1976
Repository: oozie-git
Description
-------
There are three components in this patch
1. User interface
A new tag is added to coordinator.xml
ex.
<input-check>
<or name="test">
<and>
<data-in dataset="A"/>"
<data-in dataset="B"/>
</and>
<and>
<data-in dataset="C"/>
<data-in dataset="D"/>
</and>
</or>;
<input-check>
input-check will have nested and/or/combine operation. It can have min and wait at operator or at date-in.
If input-check tag is missing then it consider to be old approach where all data dependency are needed.
2. Processing
input-check is converted into logical expression
(a&&B)||(c&&d)
We use jexl to parse the logical expression.
There are three phase in parsing.
phase 1 : only resolved dataset are parsed ( only current).
phase 2 : once all current are resolved, then future/latest are parsed.
phase 3 : Doesn't do any filecheck, just return what is being parsed by phase1 and phase2. Is used for EL functions
3. Storage.
if inputcheck is enable, push_missing_dependencies and missing_dependencies are serialized and stored in DB.
If then not then it's old approach, where they are stored in plan text. This is backward compatible.
Diffs (updated)
-----
client/src/main/resources/oozie-coordinator-0.5.xsd e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/pom.xml b063dab79415447a86c1a33f5c3f5304e0dffca0
core/src/main/java/org/apache/oozie/CoordinatorActionBean.java 91bff4dca2ef2dece68ca7260724ba3e43b1a08e
core/src/main/java/org/apache/oozie/ErrorCode.java 6c1e3997c9a1cb0bb0de39d687a083af0a7b5f04
core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java 742d00dd47aab55392abc8fbc207f8100728b832
core/src/main/java/org/apache/oozie/command/coord/CoordActionUpdatePushMissingDependency.java 4e1c5b3392358cb6b1e98e16e469310338f27fed
core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java 58ef483272264e0d391d4a1e6533dc1cab9940da
core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java 39e6ac15ce3a3ea7f2ed9178688537f7b1d7842d
core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java b05344d89e8df0e11fe69c1aa725d19a18eb0a2b
core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java d4d1c08141088e64e0551dccbd23a6185cfdbb7b
core/src/main/java/org/apache/oozie/coord/CoordELConstants.java f010a817fc900821c0e429fc16e1d3902a98d8bb
core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java 8b2f4560ae66bbcd707a446382e647663ea67be1
core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java 5d238663aa94f0dd55a9190b60bfe621439c7b53
core/src/main/java/org/apache/oozie/coord/CoordUtils.java 94c69740618110ea180b188ab0c5a02db76f8b4d
core/src/main/java/org/apache/oozie/coord/SyncCoordAction.java 44258eb5be40bf6769e32c8780117e8533d80d7e
core/src/main/java/org/apache/oozie/coord/input/dependency/AbstractCoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesChecker.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyFactory.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputInstance.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPullInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPushInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluator.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorResult.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/dependency/ActionDependency.java c280d1dc2387c9b3df96d4e83ad5563e10b1e1ef
core/src/main/java/org/apache/oozie/dependency/DependencyChecker.java a5507575a3403103133f85a78c873c9976419857
core/src/main/java/org/apache/oozie/dependency/FSURIHandler.java 7c1aadf273aeca70c110f29ccf527c073250b2e7
core/src/main/java/org/apache/oozie/dependency/HCatURIHandler.java 1bbf37d8d15c603666ddb081f83d515730740bbc
core/src/main/java/org/apache/oozie/dependency/URIHandler.java bc947168558f1970cc2fd1c40b87a1c43ce860ee
core/src/main/java/org/apache/oozie/util/WritableUtils.java 76a68953541125cab05bd05c94aa73b50e5363ff
core/src/main/resources/oozie-default.xml faf37405757719554a5b6e3671d0c87723eb9959
core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java 1fe1b3adb5ceaa83985ef31278e49a3224cde0fb
core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordInputCheckPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordinatorInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/dependency/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordInputLogicPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordinatorInputLogic.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/logic/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-action-sla.xml f3f1bc09c51272a35d65edd7271599e7e8ba1ba4
core/src/test/resources/coord-inputlogic-combine.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputlogic-hcat.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputlogic-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputlogic-range-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputlogic-range.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputlogic.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
pom.xml a74ffab081defd15a1c5dbbf15355aa4e0455029
Diff: https://reviews.apache.org/r/38474/diff/
Testing
-------
Thanks,
Purshotam Shah
Re: Review Request 38474: OOZIE-1976- Specifying coordinator input
datasets in more logical ways
Posted by Purshotam Shah <pu...@yahoo-inc.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/38474/
-----------------------------------------------------------
(Updated Jan. 8, 2016, 2:32 a.m.)
Review request for oozie.
Bugs: OOZIE-1976
https://issues.apache.org/jira/browse/OOZIE-1976
Repository: oozie-git
Description
-------
There are three components in this patch
1. User interface
A new tag is added to coordinator.xml
ex.
<input-check>
<or name="test">
<and>
<data-in dataset="A"/>"
<data-in dataset="B"/>
</and>
<and>
<data-in dataset="C"/>
<data-in dataset="D"/>
</and>
</or>;
<input-check>
input-check will have nested and/or/combine operation. It can have min and wait at operator or at date-in.
If input-check tag is missing then it consider to be old approach where all data dependency are needed.
2. Processing
input-check is converted into logical expression
(a&&B)||(c&&d)
We use jexl to parse the logical expression.
There are three phase in parsing.
phase 1 : only resolved dataset are parsed ( only current).
phase 2 : once all current are resolved, then future/latest are parsed.
phase 3 : Doesn't do any filecheck, just return what is being parsed by phase1 and phase2. Is used for EL functions
3. Storage.
if inputcheck is enable, push_missing_dependencies and missing_dependencies are serialized and stored in DB.
If then not then it's old approach, where they are stored in plan text. This is backward compatible.
Diffs (updated)
-----
client/src/main/resources/oozie-coordinator-0.5.xsd e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/pom.xml b063dab79415447a86c1a33f5c3f5304e0dffca0
core/src/main/java/org/apache/oozie/CoordinatorActionBean.java 91bff4dca2ef2dece68ca7260724ba3e43b1a08e
core/src/main/java/org/apache/oozie/ErrorCode.java 6c1e3997c9a1cb0bb0de39d687a083af0a7b5f04
core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java 742d00dd47aab55392abc8fbc207f8100728b832
core/src/main/java/org/apache/oozie/command/coord/CoordActionUpdatePushMissingDependency.java 4e1c5b3392358cb6b1e98e16e469310338f27fed
core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java 58ef483272264e0d391d4a1e6533dc1cab9940da
core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java 39e6ac15ce3a3ea7f2ed9178688537f7b1d7842d
core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java b05344d89e8df0e11fe69c1aa725d19a18eb0a2b
core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java d4d1c08141088e64e0551dccbd23a6185cfdbb7b
core/src/main/java/org/apache/oozie/coord/CoordELConstants.java f010a817fc900821c0e429fc16e1d3902a98d8bb
core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java 8b2f4560ae66bbcd707a446382e647663ea67be1
core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java 5d238663aa94f0dd55a9190b60bfe621439c7b53
core/src/main/java/org/apache/oozie/coord/CoordUtils.java 94c69740618110ea180b188ab0c5a02db76f8b4d
core/src/main/java/org/apache/oozie/coord/SyncCoordAction.java 44258eb5be40bf6769e32c8780117e8533d80d7e
core/src/main/java/org/apache/oozie/coord/input/dependency/AbstractCoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesChecker.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyFactory.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputInstance.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPullInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPushInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluator.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorResult.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/dependency/ActionDependency.java c280d1dc2387c9b3df96d4e83ad5563e10b1e1ef
core/src/main/java/org/apache/oozie/dependency/DependencyChecker.java a5507575a3403103133f85a78c873c9976419857
core/src/main/java/org/apache/oozie/dependency/FSURIHandler.java 7c1aadf273aeca70c110f29ccf527c073250b2e7
core/src/main/java/org/apache/oozie/dependency/HCatURIHandler.java 1bbf37d8d15c603666ddb081f83d515730740bbc
core/src/main/java/org/apache/oozie/dependency/URIHandler.java bc947168558f1970cc2fd1c40b87a1c43ce860ee
core/src/main/java/org/apache/oozie/util/WritableUtils.java 76a68953541125cab05bd05c94aa73b50e5363ff
core/src/main/resources/oozie-default.xml faf37405757719554a5b6e3671d0c87723eb9959
core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java 1fe1b3adb5ceaa83985ef31278e49a3224cde0fb
core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordInputCheckPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordinatorInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/dependency/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordInputLogicPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordinatorInputLogic.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/logic/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-action-sla.xml f3f1bc09c51272a35d65edd7271599e7e8ba1ba4
core/src/test/resources/coord-inputlogic-combine.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputlogic-hcat.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputlogic-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputlogic-range-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputlogic-range.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputlogic.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
pom.xml a74ffab081defd15a1c5dbbf15355aa4e0455029
Diff: https://reviews.apache.org/r/38474/diff/
Testing
-------
Thanks,
Purshotam Shah
Re: Review Request 38474: OOZIE-1976- Specifying coordinator input
datasets in more logical ways
Posted by Purshotam Shah <pu...@yahoo-inc.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/38474/
-----------------------------------------------------------
(Updated Jan. 8, 2016, 2:24 a.m.)
Review request for oozie.
Bugs: OOZIE-1976
https://issues.apache.org/jira/browse/OOZIE-1976
Repository: oozie-git
Description
-------
There are three components in this patch
1. User interface
A new tag is added to coordinator.xml
ex.
<input-check>
<or name="test">
<and>
<data-in dataset="A"/>"
<data-in dataset="B"/>
</and>
<and>
<data-in dataset="C"/>
<data-in dataset="D"/>
</and>
</or>;
<input-check>
input-check will have nested and/or/combine operation. It can have min and wait at operator or at date-in.
If input-check tag is missing then it consider to be old approach where all data dependency are needed.
2. Processing
input-check is converted into logical expression
(a&&B)||(c&&d)
We use jexl to parse the logical expression.
There are three phase in parsing.
phase 1 : only resolved dataset are parsed ( only current).
phase 2 : once all current are resolved, then future/latest are parsed.
phase 3 : Doesn't do any filecheck, just return what is being parsed by phase1 and phase2. Is used for EL functions
3. Storage.
if inputcheck is enable, push_missing_dependencies and missing_dependencies are serialized and stored in DB.
If then not then it's old approach, where they are stored in plan text. This is backward compatible.
Diffs (updated)
-----
client/src/main/resources/oozie-coordinator-0.5.xsd e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/pom.xml b063dab79415447a86c1a33f5c3f5304e0dffca0
core/src/main/java/org/apache/oozie/CoordinatorActionBean.java 91bff4dca2ef2dece68ca7260724ba3e43b1a08e
core/src/main/java/org/apache/oozie/ErrorCode.java 6c1e3997c9a1cb0bb0de39d687a083af0a7b5f04
core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java 742d00dd47aab55392abc8fbc207f8100728b832
core/src/main/java/org/apache/oozie/command/coord/CoordActionUpdatePushMissingDependency.java 4e1c5b3392358cb6b1e98e16e469310338f27fed
core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java 58ef483272264e0d391d4a1e6533dc1cab9940da
core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java 39e6ac15ce3a3ea7f2ed9178688537f7b1d7842d
core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java b05344d89e8df0e11fe69c1aa725d19a18eb0a2b
core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java d4d1c08141088e64e0551dccbd23a6185cfdbb7b
core/src/main/java/org/apache/oozie/coord/CoordELConstants.java f010a817fc900821c0e429fc16e1d3902a98d8bb
core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java 8b2f4560ae66bbcd707a446382e647663ea67be1
core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java 5d238663aa94f0dd55a9190b60bfe621439c7b53
core/src/main/java/org/apache/oozie/coord/CoordUtils.java 94c69740618110ea180b188ab0c5a02db76f8b4d
core/src/main/java/org/apache/oozie/coord/SyncCoordAction.java 44258eb5be40bf6769e32c8780117e8533d80d7e
core/src/main/java/org/apache/oozie/coord/input/dependency/AbstractCoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesChecker.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyFactory.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputInstance.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPullInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPushInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluator.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorResult.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/CoordInputLogicEvaluatorUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/logic/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/dependency/ActionDependency.java c280d1dc2387c9b3df96d4e83ad5563e10b1e1ef
core/src/main/java/org/apache/oozie/dependency/DependencyChecker.java a5507575a3403103133f85a78c873c9976419857
core/src/main/java/org/apache/oozie/dependency/FSURIHandler.java 7c1aadf273aeca70c110f29ccf527c073250b2e7
core/src/main/java/org/apache/oozie/dependency/HCatURIHandler.java 1bbf37d8d15c603666ddb081f83d515730740bbc
core/src/main/java/org/apache/oozie/dependency/URIHandler.java bc947168558f1970cc2fd1c40b87a1c43ce860ee
core/src/main/java/org/apache/oozie/util/WritableUtils.java 76a68953541125cab05bd05c94aa73b50e5363ff
core/src/main/resources/oozie-default.xml faf37405757719554a5b6e3671d0c87723eb9959
core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java 1fe1b3adb5ceaa83985ef31278e49a3224cde0fb
core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordInputCheckPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordinatorInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/dependency/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordInputLogicPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordinatorInputLogic.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/logic/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-action-sla.xml f3f1bc09c51272a35d65edd7271599e7e8ba1ba4
core/src/test/resources/coord-inputlogic-combine.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputlogic-hcat.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputlogic-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputlogic-range-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputlogic-range.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputlogic.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
pom.xml a74ffab081defd15a1c5dbbf15355aa4e0455029
Diff: https://reviews.apache.org/r/38474/diff/
Testing
-------
Thanks,
Purshotam Shah
Re: Review Request 38474: OOZIE-1976- Specifying coordinator input
datasets in more logical ways
Posted by Purshotam Shah <pu...@yahoo-inc.com>.
> On Dec. 30, 2015, 6:58 p.m., Rohini Palaniswamy wrote:
> > core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java, line 1525
> > <https://reviews.apache.org/r/38474/diff/3/?file=1165068#file1165068line1525>
> >
> > Why is this testcase removed?
merge issue
> On Dec. 30, 2015, 6:58 p.m., Rohini Palaniswamy wrote:
> > core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java, lines 329-330
> > <https://reviews.apache.org/r/38474/diff/3/?file=1165066#file1165066line329>
> >
> > Why remove the hourly part of the testcase?
merge issue
> On Dec. 30, 2015, 6:58 p.m., Rohini Palaniswamy wrote:
> > core/src/test/resources/coord-action-sla.xml, line 26
> > <https://reviews.apache.org/r/38474/diff/3/?file=1165072#file1165072line26>
> >
> > Good fixing this. But tow was this working before when # was the separator?
I think path were never used in assert.
- Purshotam
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/38474/#review112330
-----------------------------------------------------------
On Dec. 15, 2015, 7:33 p.m., Purshotam Shah wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/38474/
> -----------------------------------------------------------
>
> (Updated Dec. 15, 2015, 7:33 p.m.)
>
>
> Review request for oozie.
>
>
> Bugs: OOZIE-1976
> https://issues.apache.org/jira/browse/OOZIE-1976
>
>
> Repository: oozie-git
>
>
> Description
> -------
>
> There are three components in this patch
>
> 1. User interface
> A new tag is added to coordinator.xml
> ex.
> <input-check>
> <or name="test">
> <and>
> <data-in dataset="A"/>"
> <data-in dataset="B"/>
> </and>
> <and>
> <data-in dataset="C"/>
> <data-in dataset="D"/>
> </and>
>
> </or>;
> <input-check>
>
>
> input-check will have nested and/or/combine operation. It can have min and wait at operator or at date-in.
> If input-check tag is missing then it consider to be old approach where all data dependency are needed.
>
> 2. Processing
> input-check is converted into logical expression
> (a&&B)||(c&&d)
> We use jexl to parse the logical expression.
>
> There are three phase in parsing.
> phase 1 : only resolved dataset are parsed ( only current).
> phase 2 : once all current are resolved, then future/latest are parsed.
> phase 3 : Doesn't do any filecheck, just return what is being parsed by phase1 and phase2. Is used for EL functions
>
>
> 3. Storage.
> if inputcheck is enable, push_missing_dependencies and missing_dependencies are serialized and stored in DB.
> If then not then it's old approach, where they are stored in plan text. This is backward compatible.
>
>
> Diffs
> -----
>
> client/src/main/resources/oozie-coordinator-0.5.xsd e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/pom.xml b063dab79415447a86c1a33f5c3f5304e0dffca0
> core/src/main/java/org/apache/oozie/CoordinatorActionBean.java 91bff4dca2ef2dece68ca7260724ba3e43b1a08e
> core/src/main/java/org/apache/oozie/ErrorCode.java 6c1e3997c9a1cb0bb0de39d687a083af0a7b5f04
> core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java 742d00dd47aab55392abc8fbc207f8100728b832
> core/src/main/java/org/apache/oozie/command/coord/CoordActionUpdatePushMissingDependency.java 4e1c5b3392358cb6b1e98e16e469310338f27fed
> core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java 58ef483272264e0d391d4a1e6533dc1cab9940da
> core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java 39e6ac15ce3a3ea7f2ed9178688537f7b1d7842d
> core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java b05344d89e8df0e11fe69c1aa725d19a18eb0a2b
> core/src/main/java/org/apache/oozie/coord/CoordELConstants.java f010a817fc900821c0e429fc16e1d3902a98d8bb
> core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java 8b2f4560ae66bbcd707a446382e647663ea67be1
> core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java 5d238663aa94f0dd55a9190b60bfe621439c7b53
> core/src/main/java/org/apache/oozie/coord/CoordUtils.java 94c69740618110ea180b188ab0c5a02db76f8b4d
> core/src/main/java/org/apache/oozie/coord/SyncCoordAction.java 44258eb5be40bf6769e32c8780117e8533d80d7e
> core/src/main/java/org/apache/oozie/coord/input/dependency/AbstractCoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesChecker.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyFactory.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputInstance.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPullInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPushInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/dependency/ActionDependency.java c280d1dc2387c9b3df96d4e83ad5563e10b1e1ef
> core/src/main/java/org/apache/oozie/dependency/DependencyChecker.java a5507575a3403103133f85a78c873c9976419857
> core/src/main/java/org/apache/oozie/util/WritableUtils.java 76a68953541125cab05bd05c94aa73b50e5363ff
> core/src/main/resources/oozie-default.xml faf37405757719554a5b6e3671d0c87723eb9959
> core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java 1fe1b3adb5ceaa83985ef31278e49a3224cde0fb
> core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java 7062e697e5162ac25688d9cb62352bd139a7013a
> core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java 29e7ca145612fadee661fe975aa0ea2cab1cbfa3
> core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java 52eb9dd1fb903ac86294a55d6e2a6918390bf4e9
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordInputCheckPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordinatorInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-action-sla.xml f3f1bc09c51272a35d65edd7271599e7e8ba1ba4
> core/src/test/resources/coord-dataset-offset.xml 5dfbb1b40ee86dfc3336031eea3c5ff3144b9f5a
> core/src/test/resources/coord-inputcheck-combine.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-hcat.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-range-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-range.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> pom.xml a74ffab081defd15a1c5dbbf15355aa4e0455029
>
> Diff: https://reviews.apache.org/r/38474/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Purshotam Shah
>
>
Re: Review Request 38474: OOZIE-1976- Specifying coordinator input
datasets in more logical ways
Posted by Rohini Palaniswamy <ro...@gmail.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/38474/#review112330
-----------------------------------------------------------
core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java
<https://reviews.apache.org/r/38474/#comment172807>
Why remove the hourly part of the testcase?
core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
<https://reviews.apache.org/r/38474/#comment172794>
Why is this testcase removed?
core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordInputCheckPush.java (line 183)
<https://reviews.apache.org/r/38474/#comment172796>
Can we assert for actual value returned so that can also be verified? It will also be a test for the order in which datasets are returned. Same for all the new input check testcases added.
core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordInputCheckPush.java (line 228)
<https://reviews.apache.org/r/38474/#comment172799>
If nested combine is not supported please add a TODO comment here
core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordinatorInputCheck.java (line 831)
<https://reviews.apache.org/r/38474/#comment172805>
Add TODO saying not supported now
core/src/test/resources/coord-action-sla.xml (line 26)
<https://reviews.apache.org/r/38474/#comment172806>
Good fixing this. But tow was this working before when # was the separator?
- Rohini Palaniswamy
On Dec. 15, 2015, 7:33 p.m., Purshotam Shah wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/38474/
> -----------------------------------------------------------
>
> (Updated Dec. 15, 2015, 7:33 p.m.)
>
>
> Review request for oozie.
>
>
> Bugs: OOZIE-1976
> https://issues.apache.org/jira/browse/OOZIE-1976
>
>
> Repository: oozie-git
>
>
> Description
> -------
>
> There are three components in this patch
>
> 1. User interface
> A new tag is added to coordinator.xml
> ex.
> <input-check>
> <or name="test">
> <and>
> <data-in dataset="A"/>"
> <data-in dataset="B"/>
> </and>
> <and>
> <data-in dataset="C"/>
> <data-in dataset="D"/>
> </and>
>
> </or>;
> <input-check>
>
>
> input-check will have nested and/or/combine operation. It can have min and wait at operator or at date-in.
> If input-check tag is missing then it consider to be old approach where all data dependency are needed.
>
> 2. Processing
> input-check is converted into logical expression
> (a&&B)||(c&&d)
> We use jexl to parse the logical expression.
>
> There are three phase in parsing.
> phase 1 : only resolved dataset are parsed ( only current).
> phase 2 : once all current are resolved, then future/latest are parsed.
> phase 3 : Doesn't do any filecheck, just return what is being parsed by phase1 and phase2. Is used for EL functions
>
>
> 3. Storage.
> if inputcheck is enable, push_missing_dependencies and missing_dependencies are serialized and stored in DB.
> If then not then it's old approach, where they are stored in plan text. This is backward compatible.
>
>
> Diffs
> -----
>
> client/src/main/resources/oozie-coordinator-0.5.xsd e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/pom.xml b063dab79415447a86c1a33f5c3f5304e0dffca0
> core/src/main/java/org/apache/oozie/CoordinatorActionBean.java 91bff4dca2ef2dece68ca7260724ba3e43b1a08e
> core/src/main/java/org/apache/oozie/ErrorCode.java 6c1e3997c9a1cb0bb0de39d687a083af0a7b5f04
> core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java 742d00dd47aab55392abc8fbc207f8100728b832
> core/src/main/java/org/apache/oozie/command/coord/CoordActionUpdatePushMissingDependency.java 4e1c5b3392358cb6b1e98e16e469310338f27fed
> core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java 58ef483272264e0d391d4a1e6533dc1cab9940da
> core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java 39e6ac15ce3a3ea7f2ed9178688537f7b1d7842d
> core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java b05344d89e8df0e11fe69c1aa725d19a18eb0a2b
> core/src/main/java/org/apache/oozie/coord/CoordELConstants.java f010a817fc900821c0e429fc16e1d3902a98d8bb
> core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java 8b2f4560ae66bbcd707a446382e647663ea67be1
> core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java 5d238663aa94f0dd55a9190b60bfe621439c7b53
> core/src/main/java/org/apache/oozie/coord/CoordUtils.java 94c69740618110ea180b188ab0c5a02db76f8b4d
> core/src/main/java/org/apache/oozie/coord/SyncCoordAction.java 44258eb5be40bf6769e32c8780117e8533d80d7e
> core/src/main/java/org/apache/oozie/coord/input/dependency/AbstractCoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesChecker.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyFactory.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputInstance.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPullInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPushInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/dependency/ActionDependency.java c280d1dc2387c9b3df96d4e83ad5563e10b1e1ef
> core/src/main/java/org/apache/oozie/dependency/DependencyChecker.java a5507575a3403103133f85a78c873c9976419857
> core/src/main/java/org/apache/oozie/util/WritableUtils.java 76a68953541125cab05bd05c94aa73b50e5363ff
> core/src/main/resources/oozie-default.xml faf37405757719554a5b6e3671d0c87723eb9959
> core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java 1fe1b3adb5ceaa83985ef31278e49a3224cde0fb
> core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java 7062e697e5162ac25688d9cb62352bd139a7013a
> core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java 29e7ca145612fadee661fe975aa0ea2cab1cbfa3
> core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java 52eb9dd1fb903ac86294a55d6e2a6918390bf4e9
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordInputCheckPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordinatorInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-action-sla.xml f3f1bc09c51272a35d65edd7271599e7e8ba1ba4
> core/src/test/resources/coord-dataset-offset.xml 5dfbb1b40ee86dfc3336031eea3c5ff3144b9f5a
> core/src/test/resources/coord-inputcheck-combine.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-hcat.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-range-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-range.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> pom.xml a74ffab081defd15a1c5dbbf15355aa4e0455029
>
> Diff: https://reviews.apache.org/r/38474/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Purshotam Shah
>
>
Re: Review Request 38474: OOZIE-1976- Specifying coordinator input
datasets in more logical ways
Posted by Purshotam Shah <pu...@yahoo-inc.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/38474/
-----------------------------------------------------------
(Updated Dec. 15, 2015, 7:33 p.m.)
Review request for oozie.
Bugs: OOZIE-1976
https://issues.apache.org/jira/browse/OOZIE-1976
Repository: oozie-git
Description
-------
There are three components in this patch
1. User interface
A new tag is added to coordinator.xml
ex.
<input-check>
<or name="test">
<and>
<data-in dataset="A"/>"
<data-in dataset="B"/>
</and>
<and>
<data-in dataset="C"/>
<data-in dataset="D"/>
</and>
</or>;
<input-check>
input-check will have nested and/or/combine operation. It can have min and wait at operator or at date-in.
If input-check tag is missing then it consider to be old approach where all data dependency are needed.
2. Processing
input-check is converted into logical expression
(a&&B)||(c&&d)
We use jexl to parse the logical expression.
There are three phase in parsing.
phase 1 : only resolved dataset are parsed ( only current).
phase 2 : once all current are resolved, then future/latest are parsed.
phase 3 : Doesn't do any filecheck, just return what is being parsed by phase1 and phase2. Is used for EL functions
3. Storage.
if inputcheck is enable, push_missing_dependencies and missing_dependencies are serialized and stored in DB.
If then not then it's old approach, where they are stored in plan text. This is backward compatible.
Diffs (updated)
-----
client/src/main/resources/oozie-coordinator-0.5.xsd e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/pom.xml b063dab79415447a86c1a33f5c3f5304e0dffca0
core/src/main/java/org/apache/oozie/CoordinatorActionBean.java 91bff4dca2ef2dece68ca7260724ba3e43b1a08e
core/src/main/java/org/apache/oozie/ErrorCode.java 6c1e3997c9a1cb0bb0de39d687a083af0a7b5f04
core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java 742d00dd47aab55392abc8fbc207f8100728b832
core/src/main/java/org/apache/oozie/command/coord/CoordActionUpdatePushMissingDependency.java 4e1c5b3392358cb6b1e98e16e469310338f27fed
core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java 58ef483272264e0d391d4a1e6533dc1cab9940da
core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java 39e6ac15ce3a3ea7f2ed9178688537f7b1d7842d
core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java b05344d89e8df0e11fe69c1aa725d19a18eb0a2b
core/src/main/java/org/apache/oozie/coord/CoordELConstants.java f010a817fc900821c0e429fc16e1d3902a98d8bb
core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java 8b2f4560ae66bbcd707a446382e647663ea67be1
core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java 5d238663aa94f0dd55a9190b60bfe621439c7b53
core/src/main/java/org/apache/oozie/coord/CoordUtils.java 94c69740618110ea180b188ab0c5a02db76f8b4d
core/src/main/java/org/apache/oozie/coord/SyncCoordAction.java 44258eb5be40bf6769e32c8780117e8533d80d7e
core/src/main/java/org/apache/oozie/coord/input/dependency/AbstractCoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesChecker.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyFactory.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputInstance.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPullInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPushInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/dependency/ActionDependency.java c280d1dc2387c9b3df96d4e83ad5563e10b1e1ef
core/src/main/java/org/apache/oozie/dependency/DependencyChecker.java a5507575a3403103133f85a78c873c9976419857
core/src/main/java/org/apache/oozie/util/WritableUtils.java 76a68953541125cab05bd05c94aa73b50e5363ff
core/src/main/resources/oozie-default.xml faf37405757719554a5b6e3671d0c87723eb9959
core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java 1fe1b3adb5ceaa83985ef31278e49a3224cde0fb
core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java 7062e697e5162ac25688d9cb62352bd139a7013a
core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java 29e7ca145612fadee661fe975aa0ea2cab1cbfa3
core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java 52eb9dd1fb903ac86294a55d6e2a6918390bf4e9
core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordInputCheckPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordinatorInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/dependency/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-action-sla.xml f3f1bc09c51272a35d65edd7271599e7e8ba1ba4
core/src/test/resources/coord-dataset-offset.xml 5dfbb1b40ee86dfc3336031eea3c5ff3144b9f5a
core/src/test/resources/coord-inputcheck-combine.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputcheck-hcat.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputcheck-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputcheck-range-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputcheck-range.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputcheck.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
pom.xml a74ffab081defd15a1c5dbbf15355aa4e0455029
Diff: https://reviews.apache.org/r/38474/diff/
Testing
-------
Thanks,
Purshotam Shah
Re: Review Request 38474: OOZIE-1976- Specifying coordinator input
datasets in more logical ways
Posted by Purshotam Shah <pu...@yahoo-inc.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/38474/
-----------------------------------------------------------
(Updated Nov. 24, 2015, 1:17 a.m.)
Review request for oozie.
Bugs: OOZIE-1976
https://issues.apache.org/jira/browse/OOZIE-1976
Repository: oozie-git
Description
-------
There are three components in this patch
1. User interface
A new tag is added to coordinator.xml
ex.
<input-check>
<or name="test">
<and>
<data-in dataset="A"/>"
<data-in dataset="B"/>
</and>
<and>
<data-in dataset="C"/>
<data-in dataset="D"/>
</and>
</or>;
<input-check>
input-check will have nested and/or/combine operation. It can have min and wait at operator or at date-in.
If input-check tag is missing then it consider to be old approach where all data dependency are needed.
2. Processing
input-check is converted into logical expression
(a&&B)||(c&&d)
We use jexl to parse the logical expression.
There are three phase in parsing.
phase 1 : only resolved dataset are parsed ( only current).
phase 2 : once all current are resolved, then future/latest are parsed.
phase 3 : Doesn't do any filecheck, just return what is being parsed by phase1 and phase2. Is used for EL functions
3. Storage.
if inputcheck is enable, push_missing_dependencies and missing_dependencies are serialized and stored in DB.
If then not then it's old approach, where they are stored in plan text. This is backward compatible.
Diffs (updated)
-----
core/pom.xml b063dab79415447a86c1a33f5c3f5304e0dffca0
core/src/main/java/org/apache/oozie/CoordinatorActionBean.java c3d4bb4d49eb25bdc4f7fe44c3fc0be678a66838
core/src/main/java/org/apache/oozie/ErrorCode.java 6c1e3997c9a1cb0bb0de39d687a083af0a7b5f04
core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java 179dfbf3616e78b4992118fe7d243af9c6df5c57
core/src/main/java/org/apache/oozie/command/coord/CoordActionUpdatePushMissingDependency.java 4e1c5b3392358cb6b1e98e16e469310338f27fed
core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java 131368a7ef248c2d6bf263fb1a8e01fbe94da97f
core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java 39e6ac15ce3a3ea7f2ed9178688537f7b1d7842d
core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java b05344d89e8df0e11fe69c1aa725d19a18eb0a2b
core/src/main/java/org/apache/oozie/coord/CoordELConstants.java f010a817fc900821c0e429fc16e1d3902a98d8bb
core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java 8b2f4560ae66bbcd707a446382e647663ea67be1
core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java 5d238663aa94f0dd55a9190b60bfe621439c7b53
core/src/main/java/org/apache/oozie/coord/CoordUtils.java 94c69740618110ea180b188ab0c5a02db76f8b4d
core/src/main/java/org/apache/oozie/coord/SyncCoordAction.java 44258eb5be40bf6769e32c8780117e8533d80d7e
core/src/main/java/org/apache/oozie/coord/input/dependency/AbstractCoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencies.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesChecker.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyFactory.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputInstance.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPullInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPushInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/main/java/org/apache/oozie/dependency/ActionDependency.java c280d1dc2387c9b3df96d4e83ad5563e10b1e1ef
core/src/main/java/org/apache/oozie/dependency/DependencyChecker.java a5507575a3403103133f85a78c873c9976419857
core/src/main/java/org/apache/oozie/util/WritableUtils.java 76a68953541125cab05bd05c94aa73b50e5363ff
core/src/main/resources/oozie-default.xml faf37405757719554a5b6e3671d0c87723eb9959
core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java f78dafda17c0113d3b0855b340a95cf27fdd2f19
core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java 652d7922b6597b3272d9d42ef034cbcef21682bd
core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java 77f5518cfd4dced3e9f3da11fc267fe7ec71a341
core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java 52eb9dd1fb903ac86294a55d6e2a6918390bf4e9
core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordInputCheckPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordinatorInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/java/org/apache/oozie/coord/input/dependency/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-action-sla.xml f3f1bc09c51272a35d65edd7271599e7e8ba1ba4
core/src/test/resources/coord-dataset-offset.xml 5dfbb1b40ee86dfc3336031eea3c5ff3144b9f5a
core/src/test/resources/coord-inputcheck-combine.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputcheck-hcat.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputcheck-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputcheck-range-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputcheck-range.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
core/src/test/resources/coord-inputcheck.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
pom.xml a74ffab081defd15a1c5dbbf15355aa4e0455029
Diff: https://reviews.apache.org/r/38474/diff/
Testing
-------
Thanks,
Purshotam Shah
Re: Review Request 38474: OOZIE-1976- Specifying coordinator input
datasets in more logical ways
Posted by Purshotam Shah <pu...@yahoo-inc.com>.
> On Oct. 9, 2015, 12:53 a.m., Robert Kanter wrote:
> > client/src/main/resources/oozie-coordinator-0.5.xsd, line 110
> > <https://reviews.apache.org/r/38474/diff/1/?file=1076234#file1076234line110>
> >
> > Is there any way to enforce a max depth on nested "and"s and "or"s via the schema?
Not sure. We are not enforcing anything. if its needed we can do it in future.
> On Oct. 9, 2015, 12:53 a.m., Robert Kanter wrote:
> > core/src/main/java/org/apache/oozie/coord/dependency/CoordDependenciesInputCheck.java, line 75
> > <https://reviews.apache.org/r/38474/diff/1/?file=1076246#file1076246line75>
> >
> > Is there anyway to reuse this without always creating a new one everywhere? (is it not thread safe?)
Haven't checked that, but OozieJexlEngine is very simple.
public class OozieJexlEngine extends JexlEngine {
OozieJexlInterpreter oozieInterpreter;
public OozieJexlEngine() {
}
- Purshotam
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/38474/#review101982
-----------------------------------------------------------
On Dec. 15, 2015, 7:33 p.m., Purshotam Shah wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/38474/
> -----------------------------------------------------------
>
> (Updated Dec. 15, 2015, 7:33 p.m.)
>
>
> Review request for oozie.
>
>
> Bugs: OOZIE-1976
> https://issues.apache.org/jira/browse/OOZIE-1976
>
>
> Repository: oozie-git
>
>
> Description
> -------
>
> There are three components in this patch
>
> 1. User interface
> A new tag is added to coordinator.xml
> ex.
> <input-check>
> <or name="test">
> <and>
> <data-in dataset="A"/>"
> <data-in dataset="B"/>
> </and>
> <and>
> <data-in dataset="C"/>
> <data-in dataset="D"/>
> </and>
>
> </or>;
> <input-check>
>
>
> input-check will have nested and/or/combine operation. It can have min and wait at operator or at date-in.
> If input-check tag is missing then it consider to be old approach where all data dependency are needed.
>
> 2. Processing
> input-check is converted into logical expression
> (a&&B)||(c&&d)
> We use jexl to parse the logical expression.
>
> There are three phase in parsing.
> phase 1 : only resolved dataset are parsed ( only current).
> phase 2 : once all current are resolved, then future/latest are parsed.
> phase 3 : Doesn't do any filecheck, just return what is being parsed by phase1 and phase2. Is used for EL functions
>
>
> 3. Storage.
> if inputcheck is enable, push_missing_dependencies and missing_dependencies are serialized and stored in DB.
> If then not then it's old approach, where they are stored in plan text. This is backward compatible.
>
>
> Diffs
> -----
>
> client/src/main/resources/oozie-coordinator-0.5.xsd e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/pom.xml b063dab79415447a86c1a33f5c3f5304e0dffca0
> core/src/main/java/org/apache/oozie/CoordinatorActionBean.java 91bff4dca2ef2dece68ca7260724ba3e43b1a08e
> core/src/main/java/org/apache/oozie/ErrorCode.java 6c1e3997c9a1cb0bb0de39d687a083af0a7b5f04
> core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java 742d00dd47aab55392abc8fbc207f8100728b832
> core/src/main/java/org/apache/oozie/command/coord/CoordActionUpdatePushMissingDependency.java 4e1c5b3392358cb6b1e98e16e469310338f27fed
> core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java 58ef483272264e0d391d4a1e6533dc1cab9940da
> core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java 39e6ac15ce3a3ea7f2ed9178688537f7b1d7842d
> core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java b05344d89e8df0e11fe69c1aa725d19a18eb0a2b
> core/src/main/java/org/apache/oozie/coord/CoordELConstants.java f010a817fc900821c0e429fc16e1d3902a98d8bb
> core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java 8b2f4560ae66bbcd707a446382e647663ea67be1
> core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java 5d238663aa94f0dd55a9190b60bfe621439c7b53
> core/src/main/java/org/apache/oozie/coord/CoordUtils.java 94c69740618110ea180b188ab0c5a02db76f8b4d
> core/src/main/java/org/apache/oozie/coord/SyncCoordAction.java 44258eb5be40bf6769e32c8780117e8533d80d7e
> core/src/main/java/org/apache/oozie/coord/input/dependency/AbstractCoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesBuilder.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesChecker.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependenciesUtil.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyCheckPhaseValidate.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputDependencyFactory.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordInputInstance.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPullInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordPushInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/CoordUnResolvedInputDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/input/dependency/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/dependency/ActionDependency.java c280d1dc2387c9b3df96d4e83ad5563e10b1e1ef
> core/src/main/java/org/apache/oozie/dependency/DependencyChecker.java a5507575a3403103133f85a78c873c9976419857
> core/src/main/java/org/apache/oozie/util/WritableUtils.java 76a68953541125cab05bd05c94aa73b50e5363ff
> core/src/main/resources/oozie-default.xml faf37405757719554a5b6e3671d0c87723eb9959
> core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java 1fe1b3adb5ceaa83985ef31278e49a3224cde0fb
> core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java 7062e697e5162ac25688d9cb62352bd139a7013a
> core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java 29e7ca145612fadee661fe975aa0ea2cab1cbfa3
> core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java 52eb9dd1fb903ac86294a55d6e2a6918390bf4e9
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordInputCheckPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordinatorInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/input/dependency/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-action-sla.xml f3f1bc09c51272a35d65edd7271599e7e8ba1ba4
> core/src/test/resources/coord-dataset-offset.xml 5dfbb1b40ee86dfc3336031eea3c5ff3144b9f5a
> core/src/test/resources/coord-inputcheck-combine.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-hcat.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-range-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-range.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> pom.xml a74ffab081defd15a1c5dbbf15355aa4e0455029
>
> Diff: https://reviews.apache.org/r/38474/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Purshotam Shah
>
>
Re: Review Request 38474: OOZIE-1976- Specifying coordinator input
datasets in more logical ways
Posted by Robert Kanter <rk...@cloudera.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/38474/#review101982
-----------------------------------------------------------
As a general point, can you add some additional comments/javadoc? Especially for the large new areas of code and the Jexl stuff.
client/src/main/resources/oozie-coordinator-0.5.xsd (line 106)
<https://reviews.apache.org/r/38474/#comment159491>
Trailing whitespace. Also more of these throughout the patch.
client/src/main/resources/oozie-coordinator-0.5.xsd (line 110)
<https://reviews.apache.org/r/38474/#comment159492>
Is there any way to enforce a max depth on nested "and"s and "or"s via the schema?
client/src/main/resources/oozie-coordinator-0.5.xsd (line 132)
<https://reviews.apache.org/r/38474/#comment159514>
xs:int
client/src/main/resources/oozie-coordinator-0.5.xsd (line 137)
<https://reviews.apache.org/r/38474/#comment159515>
xs:int
core/pom.xml (line 373)
<https://reviews.apache.org/r/38474/#comment159493>
Version should be defined in root pom
core/src/main/java/org/apache/oozie/CoordinatorActionBean.java
<https://reviews.apache.org/r/38474/#comment159494>
I'm not sure what this does exactly, but is it okay to delete?
core/src/main/java/org/apache/oozie/action/ActionExecutor.java
<https://reviews.apache.org/r/38474/#comment159498>
"fake" change.
core/src/main/java/org/apache/oozie/coord/dependency/CoordDependenciesInputCheck.java (line 69)
<https://reviews.apache.org/r/38474/#comment159504>
so we don't lose the stack trace:
````
throw new IOException(e.getMessage, e);
````
core/src/main/java/org/apache/oozie/coord/dependency/CoordDependenciesInputCheck.java (line 75)
<https://reviews.apache.org/r/38474/#comment159505>
Is there anyway to reuse this without always creating a new one everywhere? (is it not thread safe?)
core/src/main/java/org/apache/oozie/coord/dependency/CoordDependenciesInputCheck.java (line 120)
<https://reviews.apache.org/r/38474/#comment159506>
Should this be the OozieJexlEngine?
core/src/main/java/org/apache/oozie/coord/dependency/CoordDependenciesInputCheck.java (line 137)
<https://reviews.apache.org/r/38474/#comment159507>
Should this be the OozieJexlEngine?
core/src/main/java/org/apache/oozie/coord/dependency/CoordDependency.java (lines 48 - 50)
<https://reviews.apache.org/r/38474/#comment159508>
Most of the time, these are re-assigned in the constructors. I think we can simpley declare them here instead of making new objects.
core/src/main/java/org/apache/oozie/coord/dependency/CoordDependency.java (line 290)
<https://reviews.apache.org/r/38474/#comment159509>
isDependencyMet()
core/src/main/java/org/apache/oozie/coord/dependency/CoordInputCheckerPhaseOne.java (line 111)
<https://reviews.apache.org/r/38474/#comment159510>
We should probably do something better here
core/src/main/java/org/apache/oozie/coord/dependency/CoordInputCheckerPhaseOne.java (line 180)
<https://reviews.apache.org/r/38474/#comment159511>
We should probably do something better here
core/src/main/java/org/apache/oozie/coord/dependency/CoordInputCheckerPhaseThree.java (line 78)
<https://reviews.apache.org/r/38474/#comment159512>
We should probably do something better here
core/src/main/java/org/apache/oozie/coord/dependency/InputLogicParser.java (line 114)
<https://reviews.apache.org/r/38474/#comment159513>
wait isn't used?
We also need to check that it's a valid int > 0
core/src/main/java/org/apache/oozie/service/SchemaService.java (line 75)
<https://reviews.apache.org/r/38474/#comment159516>
This is no longer in the code; it's now in oozie-default under oozie.service.SchemaService.coord.schemas. You'll probably hit this when you rebase.
- Robert Kanter
On Sept. 18, 2015, 12:20 a.m., Purshotam Shah wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/38474/
> -----------------------------------------------------------
>
> (Updated Sept. 18, 2015, 12:20 a.m.)
>
>
> Review request for oozie.
>
>
> Bugs: OOZIE-1976
> https://issues.apache.org/jira/browse/OOZIE-1976
>
>
> Repository: oozie-git
>
>
> Description
> -------
>
> There are three components in this patch
>
> 1. User interface
> A new tag is added to coordinator.xml
> ex.
> <input-check>
> <or name="test">
> <and>
> <data-in dataset="A"/>"
> <data-in dataset="B"/>
> </and>
> <and>
> <data-in dataset="C"/>
> <data-in dataset="D"/>
> </and>
>
> </or>;
> <input-check>
>
>
> input-check will have nested and/or/combine operation. It can have min and wait at operator or at date-in.
> If input-check tag is missing then it consider to be old approach where all data dependency are needed.
>
> 2. Processing
> input-check is converted into logical expression
> (a&&B)||(c&&d)
> We use jexl to parse the logical expression.
>
> There are three phase in parsing.
> phase 1 : only resolved dataset are parsed ( only current).
> phase 2 : once all current are resolved, then future/latest are parsed.
> phase 3 : Doesn't do any filecheck, just return what is being parsed by phase1 and phase2. Is used for EL functions
>
>
> 3. Storage.
> if inputcheck is enable, push_missing_dependencies and missing_dependencies are serialized and stored in DB.
> If then not then it's old approach, where they are stored in plan text. This is backward compatible.
>
>
> Diffs
> -----
>
> client/src/main/resources/oozie-coordinator-0.5.xsd e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/pom.xml ca40e2e22293a3df2841764ce725420857425139
> core/src/main/java/org/apache/oozie/CoordinatorActionBean.java 188b70e2e76858228b4d42e5798952383719a93d
> core/src/main/java/org/apache/oozie/action/ActionExecutor.java ff836fbbbe95ca03aace1136abea9548306b2cf2
> core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java a975f6edd40ef674638d1c32c36d5234b78beb0f
> core/src/main/java/org/apache/oozie/command/coord/CoordActionUpdatePushMissingDependency.java 4e1c5b3392358cb6b1e98e16e469310338f27fed
> core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java ca8175e8b8019a42e2780fba4f8fbd313577494c
> core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java 548946f05beacbba032b1f411fdae17ca7dd1f44
> core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java cc346274e2f28fd3eb062e8d3550281486c23954
> core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java 8a279c06a62bd1e6502104f0760d7a1a9792b4bd
> core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java 7f59186bb508536e5662bc3ed25c7ec68ba1e2a7
> core/src/main/java/org/apache/oozie/coord/SyncCoordAction.java 44258eb5be40bf6769e32c8780117e8533d80d7e
> core/src/main/java/org/apache/oozie/coord/dependency/CoordDependenciesInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/CoordDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/CoordInputCheckerPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/CoordInputCheckerPhaseOnePush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/CoordInputCheckerPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/CoordInputCheckerPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/CoordInputCheckerUtility.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/CoordInputDependencies.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/CoordInputInstance.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/CoordPullDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/CoordPushDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/CoordUnResolvedDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/OozieJexlNode.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/dependency/DependencyChecker.java a5507575a3403103133f85a78c873c9976419857
> core/src/main/java/org/apache/oozie/service/SchemaService.java 32105857f51eea9b2e4fd4d9d8cb74900fcdbac8
> core/src/main/java/org/apache/oozie/util/Pair.java 1bf45b41edf19ad0a3115a7bafc010daef11b5c1
> core/src/main/resources/oozie-default.xml 19cae9d4e07ea1c3cd3b287d23cacb9b342b406e
> core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java f79c9a06c8238aa17b7d331afcb3f137f0bb83f9
> core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java 59b8b48b564b04ddd6eb263f8766bf3a8919a429
> core/src/test/java/org/apache/oozie/coord/dependency/TestCoordInputCheckPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/dependency/TestCoordinatorInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/dependency/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-combine.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-hcat.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-range-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-range.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
>
> Diff: https://reviews.apache.org/r/38474/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Purshotam Shah
>
>
Re: Review Request 38474: OOZIE-1976- Specifying coordinator input
datasets in more logical ways
Posted by Mona Chitnis <mo...@yahoo.in>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/38474/#review103325
-----------------------------------------------------------
Having a little trouble zero-ing in on the code that checks that logically the dependencies have been met. Can you point me to that class? Or is that abstracted by the JexlEngine? (I'm not familiar with it)
client/src/main/resources/oozie-coordinator-0.5.xsd (line 110)
<https://reviews.apache.org/r/38474/#comment161313>
there is through 'maxOccurs'. but I'm thinking this patch is supporting any arbitrary depth
client/src/main/resources/oozie-coordinator-0.5.xsd (line 126)
<https://reviews.apache.org/r/38474/#comment161314>
what is 'combine' used for?
core/src/main/java/org/apache/oozie/CoordinatorActionBean.java (line 172)
<https://reviews.apache.org/r/38474/#comment161315>
this is part of a different change
core/src/main/java/org/apache/oozie/CoordinatorActionBean.java (line 859)
<https://reviews.apache.org/r/38474/#comment161316>
Log the exception here
core/src/main/java/org/apache/oozie/CoordinatorActionBean.java (line 882)
<https://reviews.apache.org/r/38474/#comment161318>
same as above, log the exception here
core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java (line 155)
<https://reviews.apache.org/r/38474/#comment161319>
let's make "input-check" a private static final String element and used in the multiple places in the code, so its a single place in case the name changes later
core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java (line 139)
<https://reviews.apache.org/r/38474/#comment161320>
typo availableList
core/src/main/java/org/apache/oozie/coord/dependency/CoordDependenciesInputCheck.java (line 75)
<https://reviews.apache.org/r/38474/#comment161322>
ditto
core/src/main/java/org/apache/oozie/coord/dependency/CoordDependency.java (line 242)
<https://reviews.apache.org/r/38474/#comment161323>
doesn't close() and flush() on the enclosing OutputStream suffice and close/flush the enclosed stream too? Safeguard against closing a stream that's already closed
core/src/main/java/org/apache/oozie/coord/dependency/CoordInputCheckerPhaseOne.java (line 97)
<https://reviews.apache.org/r/38474/#comment161324>
typo getFirst (not getFist)
- Mona Chitnis
On Sept. 18, 2015, 12:20 a.m., Purshotam Shah wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/38474/
> -----------------------------------------------------------
>
> (Updated Sept. 18, 2015, 12:20 a.m.)
>
>
> Review request for oozie.
>
>
> Bugs: OOZIE-1976
> https://issues.apache.org/jira/browse/OOZIE-1976
>
>
> Repository: oozie-git
>
>
> Description
> -------
>
> There are three components in this patch
>
> 1. User interface
> A new tag is added to coordinator.xml
> ex.
> <input-check>
> <or name="test">
> <and>
> <data-in dataset="A"/>"
> <data-in dataset="B"/>
> </and>
> <and>
> <data-in dataset="C"/>
> <data-in dataset="D"/>
> </and>
>
> </or>;
> <input-check>
>
>
> input-check will have nested and/or/combine operation. It can have min and wait at operator or at date-in.
> If input-check tag is missing then it consider to be old approach where all data dependency are needed.
>
> 2. Processing
> input-check is converted into logical expression
> (a&&B)||(c&&d)
> We use jexl to parse the logical expression.
>
> There are three phase in parsing.
> phase 1 : only resolved dataset are parsed ( only current).
> phase 2 : once all current are resolved, then future/latest are parsed.
> phase 3 : Doesn't do any filecheck, just return what is being parsed by phase1 and phase2. Is used for EL functions
>
>
> 3. Storage.
> if inputcheck is enable, push_missing_dependencies and missing_dependencies are serialized and stored in DB.
> If then not then it's old approach, where they are stored in plan text. This is backward compatible.
>
>
> Diffs
> -----
>
> client/src/main/resources/oozie-coordinator-0.5.xsd e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/pom.xml ca40e2e22293a3df2841764ce725420857425139
> core/src/main/java/org/apache/oozie/CoordinatorActionBean.java 188b70e2e76858228b4d42e5798952383719a93d
> core/src/main/java/org/apache/oozie/action/ActionExecutor.java ff836fbbbe95ca03aace1136abea9548306b2cf2
> core/src/main/java/org/apache/oozie/command/coord/CoordActionInputCheckXCommand.java a975f6edd40ef674638d1c32c36d5234b78beb0f
> core/src/main/java/org/apache/oozie/command/coord/CoordActionUpdatePushMissingDependency.java 4e1c5b3392358cb6b1e98e16e469310338f27fed
> core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java ca8175e8b8019a42e2780fba4f8fbd313577494c
> core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java 548946f05beacbba032b1f411fdae17ca7dd1f44
> core/src/main/java/org/apache/oozie/command/coord/CoordPushDependencyCheckXCommand.java cc346274e2f28fd3eb062e8d3550281486c23954
> core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java 8a279c06a62bd1e6502104f0760d7a1a9792b4bd
> core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java 7f59186bb508536e5662bc3ed25c7ec68ba1e2a7
> core/src/main/java/org/apache/oozie/coord/SyncCoordAction.java 44258eb5be40bf6769e32c8780117e8533d80d7e
> core/src/main/java/org/apache/oozie/coord/dependency/CoordDependenciesInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/CoordDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/CoordInputCheckerPhaseOne.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/CoordInputCheckerPhaseOnePush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/CoordInputCheckerPhaseThree.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/CoordInputCheckerPhaseTwo.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/CoordInputCheckerUtility.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/CoordInputDependencies.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/CoordInputInstance.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/CoordPullDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/CoordPushDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/CoordUnResolvedDependency.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/InputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/OozieJexlEngine.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/OozieJexlInterpreter.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/coord/dependency/OozieJexlNode.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/main/java/org/apache/oozie/dependency/DependencyChecker.java a5507575a3403103133f85a78c873c9976419857
> core/src/main/java/org/apache/oozie/service/SchemaService.java 32105857f51eea9b2e4fd4d9d8cb74900fcdbac8
> core/src/main/java/org/apache/oozie/util/Pair.java 1bf45b41edf19ad0a3115a7bafc010daef11b5c1
> core/src/main/resources/oozie-default.xml 19cae9d4e07ea1c3cd3b287d23cacb9b342b406e
> core/src/test/java/org/apache/oozie/command/coord/TestCoordActionInputCheckXCommand.java f79c9a06c8238aa17b7d331afcb3f137f0bb83f9
> core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java 59b8b48b564b04ddd6eb263f8766bf3a8919a429
> core/src/test/java/org/apache/oozie/coord/dependency/TestCoordInputCheckPush.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/dependency/TestCoordinatorInputCheck.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/java/org/apache/oozie/coord/dependency/TestInputLogicParser.java e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-combine.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-hcat.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-range-latest.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck-range.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
> core/src/test/resources/coord-inputcheck.xml e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
>
> Diff: https://reviews.apache.org/r/38474/diff/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Purshotam Shah
>
>