You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@apex.apache.org by th...@apache.org on 2017/03/27 15:19:02 UTC
[01/19] apex-malhar git commit: Fixed checkstyle and added license
headers.
Repository: apex-malhar
Updated Branches:
refs/heads/master be9f3c8dd -> b8a83a451
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/recordReader/src/main/resources/META-INF/CustomerTxnData.txt
----------------------------------------------------------------------
diff --git a/examples/recordReader/src/main/resources/META-INF/CustomerTxnData.txt b/examples/recordReader/src/main/resources/META-INF/CustomerTxnData.txt
new file mode 100644
index 0000000..88322ab
--- /dev/null
+++ b/examples/recordReader/src/main/resources/META-INF/CustomerTxnData.txt
@@ -0,0 +1,100 @@
+Desirae I. Hart|91-756-943-24728|rutrum.magna@dictummagnaUt.net|Delhi|India|7F2BAD18-D1D1-8DB6-A519-70DA658EF6DE|1048.2446157601|245407|2016-06-17 21:19:02|56095.4691099477
+Fay B. Page|91-952-299-03140|lobortis.risus@eutelluseu.com|Durg|India|5E8E12D3-34CC-53E1-7622-BC533542B930|939.9385330598|245408|2016-06-17 21:19:02|93013.7443360696
+Herrod F. Mcknight|91-609-146-21345|nec@egetmagna.co.uk|Haridwar|India|F589AF28-C10C-3EF1-E9DA-7CAAAD8E453D|1009.5714354316|245409|2016-06-17 21:19:03|57130.4933466919
+Suki K. Hutchinson|91-235-565-54483|justo.Praesent@nonquam.ca|Panjim|India|A263A773-E228-0C5B-D9CC-A5AE37A9F3F5|990.5143009573|245410|2016-06-17 21:19:03|88348.7049292828
+Tucker H. Strickland|91-353-281-78811|Suspendisse@felisNulla.edu|Aizwal|India|748AAD2B-8CB4-6010-2C80-F3B0FE107B4E|1027.2343235232|245411|2016-06-17 21:19:03|63602.53265577
+Carly L. Alvarado|91-364-259-91891|blandit.mattis.Cras@Nullam.co.uk|Mandya|India|292345E9-45E1-AE6C-9051-79681BE10E0A|964.9602866876|245412|2016-06-17 21:19:03|63777.4412787906
+Signe H. Carlson|91-736-760-21206|Curabitur@ac.edu|Basirhat|India|ABFCB28C-98B0-93E4-C5E9-D572648FE467|967.6980193409|245413|2016-06-17 21:19:04|81425.8673128708
+Timothy U. Cantrell|91-266-620-64253|ac.mattis@congueIn.com|Dibrugarh|India|46ED3B8F-5D19-D3FD-2833-7E6D956090D6|990.0113497158|245414|2016-06-17 21:19:04|20765.2379785903
+Todd M. Hunter|91-131-862-91105|vel@per.org|Pondicherry|India|73DDE1E7-7B86-53A0-8296-8DDCC0A8022A|999.2001773265|245416|2016-06-17 21:19:04|28079.5354148245
+Martin J. Joyce|91-312-843-82341|magnis.dis.parturient@penatibusetmagnis.co.uk|Gandhidham|India|46247FA6-883D-2DAA-78E4-626F41937905|974.8521165154|245415|2016-06-17 21:19:04|98219.3968546873
+Cruz D. Slater|91-399-580-01355|ante.dictum@tempor.co.uk|Kalyan|India|7A0AE886-CDBF-98FF-1808-1A977B06F939|989.5735205715|245418|2016-06-17 21:19:05|69472.6529645815
+Keefe M. Hodge|91-123-766-63270|molestie.arcu.Sed@eratnonummy.ca|Bokaro Steel City|India|5488D1DE-E840-827F-2D45-347A58ECF72C|1059.6676733575|245417|2016-06-17 21:19:05|60538.9043471269
+Minerva R. Velez|91-550-563-14205|dapibus.quam.quis@sitamet.co.uk|Bhusawal|India|C2EA3D6F-EEBE-7E7D-4DFD-DA90FAAAC04F|965.3100959637|245419|2016-06-17 21:19:05|22378.4665981816
+Gillian P. Bates|91-530-835-01656|In.nec@sedduiFusce.org|Kohima|India|8882E4F0-0AB1-6098-EEF8-0230BBB931B2|978.3566576884|245420|2016-06-17 21:19:05|57058.528246018
+Jael I. Humphrey|91-815-502-83243|dignissim@quisturpis.co.uk|Chandigarh|India|0D8ADC96-B858-60A9-4F65-3BACADC808A8|1047.1902156238|245421|2016-06-17 21:19:06|71348.2114533201
+Eleanor D. Hart|91-351-706-26576|nonummy@sitametdapibus.ca|Imphal|India|6DE39307-4D0B-77B0-8DAF-36417261DA2B|1021.3673266291|245422|2016-06-17 21:19:06|60474.8184384689
+Kaitlin U. Higgins|91-738-770-10870|sodales.at.velit@Inscelerisque.org|Kohima|India|C383AF31-636B-7897-6CF3-6B261BCDE018|1036.0644769672|245424|2016-06-17 21:19:06|48378.6503012718
+Giacomo A. Puckett|91-859-732-83951|sit.amet@atpedeCras.com|Silchar|India|5E41A5A4-A4B3-17FE-F83D-6F6BF36B0F2D|979.2148681363|245423|2016-06-17 21:19:06|33445.0439158336
+Jamalia Z. Hensley|91-395-592-56855|interdum@cursus.co.uk|Munger|India|7B98AA41-BC35-497E-6F2E-BBA815F84B28|1033.5638665941|245425|2016-06-17 21:19:07|63406.9037231582
+Raja L. Barrett|91-213-236-37863|tristique.senectus@orciUtsemper.com|Agartala|India|730F660A-4A6E-65B3-D64D-6A0AD4BD16F2|1051.51196186|245426|2016-06-17 21:19:07|16656.8948714763
+Audra D. Haley|91-909-551-98035|nostra@velitjusto.net|Shimla|India|5F6BD395-CFB1-86A3-232B-3B461C11B032|985.3099956934|245427|2016-06-17 21:19:07|42576.4326009901
+Burke G. Velez|91-480-634-47735|dictum@eulacusQuisque.com|Shimla|India|C0D51785-79E5-86C1-2F60-6E2B4BC368E9|1014.3374004511|245428|2016-06-17 21:19:07|68099.3435062983
+Helen O. Watson|91-388-550-33596|accumsan.neque.et@sagittis.ca|Rewa|India|267EA362-E266-01FC-2371-065D4B150D6B|995.395922684|245429|2016-06-17 21:19:08|80594.4138667748
+Dillon E. Manning|91-412-483-18804|vulputate.eu@diam.net|Kohima|India|08BD8752-4C34-1C2F-7EC5-063BE9C0025F|988.0527516287|245430|2016-06-17 21:19:08|34997.7855937537
+Darryl H. Preston|91-900-557-62018|consequat@blanditatnisi.ca|Srinagar|India|BAE0CB80-33BC-23AD-4F23-6EED717A1FA9|1005.2310251378|245432|2016-06-17 21:19:08|90995.8552344935
+Teagan L. Merritt|91-755-577-05229|nec.luctus.felis@ornarelibero.ca|Daman|India|A65E89EC-BD39-F826-BF3A-2E4DFD1FFE96|1031.6923612578|245431|2016-06-17 21:19:08|76129.6754380442
+Blythe V. Hester|91-143-693-74169|faucibus@convallisconvallis.net|Nagarcoil|India|457DA7DF-8473-FDA2-DFA8-9CA6FE70AF53|980.0187576543|245433|2016-06-17 21:19:09|1792.7823874316
+Felicia Z. Maynard|91-177-925-06452|facilisis@nequetellus.edu|Jammu|India|7511020D-252A-60E3-0259-B55174DB2BE0|962.3470002516|245434|2016-06-17 21:19:09|27442.1888130468
+Sopoline B. Swanson|91-512-190-23174|mauris.aliquam@vitae.edu|Agartala|India|92A84BDE-A231-715C-5B0A-A2C52FE4C726|1000.4777986036|245436|2016-06-17 21:19:09|32101.1072345695
+Christen T. Cabrera|91-956-276-72371|a.dui@sit.org|Tiruvannamalai|India|1B06945D-2595-5445-2694-12341158C45B|997.3852528283|245435|2016-06-17 21:19:09|86664.8353687926
+Chancellor C. Villarreal|91-882-782-21856|Nullam.ut.nisi@Nam.edu|Pondicherry|India|33EBFDA7-8EEF-48A5-B323-AF7A7A57C4F9|984.5578096612|245437|2016-06-17 21:19:10|46584.2856419789
+Adara A. Price|91-334-227-26172|elementum@aliquameuaccumsan.ca|Shahjahanpur|India|82F22680-CD3B-F1E7-45F9-8E2ADAA5ED42|988.0619905918|245438|2016-06-17 21:19:10|21626.2511858243
+Dana B. Lucas|91-781-212-83146|scelerisque.mollis.Phasellus@Sedeunibh.co.uk|Gangtok|India|88223C00-DCFA-F40B-AC4C-97FA79062789|976.7241552536|245440|2016-06-17 21:19:10|76481.528985468
+Alma X. Ingram|91-944-103-76290|senectus.et.netus@orci.net|Jammu|India|4C0C1266-DE7B-6FE7-6FFF-E0263987C15B|975.6210776138|245439|2016-06-17 21:19:10|64898.0024641107
+Cullen P. Glenn|91-388-717-53901|magna.Praesent@loremlorem.co.uk|Kohima|India|BA6BF2F4-8F94-C4C2-740E-2A3B587C4B3A|1031.9092818916|245442|2016-06-17 21:19:11|9437.7993039509
+Darius S. Allen|91-978-355-75025|mi.lacinia.mattis@enimSuspendissealiquet.co.uk|Trivandrum|India|D59B0CBB-A08D-DE3B-DE5F-426C0C96CC59|1032.6395532022|245441|2016-06-17 21:19:11|12454.7012410075
+Jordan J. Frye|91-399-558-07454|ridiculus.mus@idlibero.ca|Chandigarh|India|EBCC5349-6EC4-356A-F897-20BFBF220087|1019.8537249814|245444|2016-06-17 21:19:11|32040.2035007451
+August N. Chase|91-447-831-17890|libero.at.auctor@Vivamuseuismodurna.org|Agartala|India|8871CD66-C471-B709-61C4-DA8E11395864|1016.3890473484|245443|2016-06-17 21:19:11|45318.9872586436
+Teegan Z. House|91-834-850-62568|nunc.est.mollis@Nunc.edu|Aizwal|India|13B85BBD-EA23-70B2-1723-9F1D6046B42E|959.962281601|245445|2016-06-17 21:19:12|88784.9082302192
+Aquila B. Moran|91-851-475-57283|sapien.Cras.dolor@Suspendissetristiqueneque.com|Thalassery|India|823590C6-3381-1E11-8303-C3DA2A0E1821|993.9503426924|245446|2016-06-17 21:19:12|85906.741529358
+Dominic T. Dalton|91-504-884-93067|Ut@congue.co.uk|Rohtak|India|4EC579B1-4095-421D-27D5-95BD94551B81|988.6711045096|245448|2016-06-17 21:19:12|96034.4758503845
+Francis C. Gay|91-155-131-31144|elementum@CuraeDonec.co.uk|Panjim|India|BEA0E8B8-2153-55C3-9FAC-B645F75DC11F|1041.9520701363|245447|2016-06-17 21:19:12|28285.6913337028
+Florence K. Cox|91-509-817-87497|adipiscing.enim@ornarelectusjusto.edu|Bhilai|India|C613A6CF-0DDF-457B-1611-048AF476CD9F|984.7873386645|245449|2016-06-17 21:19:13|8045.8599610703
+Desirae F. Mooney|91-146-543-19767|ultricies@scelerisquenequeNullam.edu|Gangtok|India|58A86FB7-1F20-2C33-2682-9B20165B5CF9|965.4688425994|245450|2016-06-17 21:19:13|25829.5686136197
+Virginia O. Welch|91-338-235-55295|ultrices.posuere.cubilia@massanonante.co.uk|Srinagar|India|350C6D1C-8C2D-BE4A-EAB5-8098C2BC8054|981.8861112513|245451|2016-06-17 21:19:13|55435.8774902126
+Simon P. Estes|91-101-148-34019|a.aliquet@tortordictum.edu|Kavaratti|India|D42E0FB2-6093-FD6F-EA2E-DC470647340F|961.8009052379|245452|2016-06-17 21:19:13|26959.0061609105
+Garth M. Fuentes|91-401-871-57104|gravida@semperegestasurna.net|Vejalpur|India|1450A844-53D1-6315-4415-5C75B08FC228|974.9367040591|245454|2016-06-17 21:19:14|93043.0249827518
+Jaquelyn D. Rodgers|91-913-783-42186|lobortis.nisi.nibh@Donec.edu|Nagpur|India|D6CE7D75-0659-454A-40DF-3AF296DD4C58|976.7352615741|245453|2016-06-17 21:19:14|59609.6850497366
+Kasper F. Hatfield|91-632-379-95960|posuere@vulputatenisisem.ca|Korba|India|D59B8BE9-DA20-7B57-9942-2C15BCF28603|1000.4924219911|245456|2016-06-17 21:19:14|3275.3243581371
+Isaiah A. Warner|91-626-859-53011|diam.Sed@facilisisfacilisismagna.edu|Daman|India|20B5FE0D-0965-1E2D-681B-5E688DA1CA65|1001.1159994583|245455|2016-06-17 21:19:14|76836.6239820194
+Keaton Y. Solis|91-301-918-77318|sit.amet.nulla@elitelitfermentum.co.uk|Rajkot|India|16CFA137-3FD3-8D3D-7607-54577A6A18AB|987.5936149956|245458|2016-06-17 21:19:15|89465.3972332774
+Regan D. Underwood|91-869-304-48740|ornare@Curabituregestas.org|Daman|India|241FD5E5-01D8-7BEA-BE3A-3435B1D6577F|967.2325392222|245457|2016-06-17 21:19:15|81430.0742569319
+Laurel K. Hardy|91-535-589-44540|rutrum@penatibuset.edu|Shimla|India|46689123-0B18-3A05-2E9D-1995EABE674F|947.0982666227|245459|2016-06-17 21:19:15|14451.1023722487
+Vielka J. Roberson|91-545-218-53793|scelerisque.lorem.ipsum@Sed.org|Srinagar|India|E6384D6E-2AC4-3AF7-8C1C-A37DD4B013C6|933.9068440201|245460|2016-06-17 21:19:15|75600.2030709746
+Brynn S. Morrison|91-977-938-06240|natoque@enimgravidasit.edu|Daman|India|859E9B35-3A21-F7FD-A9A5-167F7FE6A7F5|1002.7243083243|245461|2016-06-17 21:19:16|19882.1569818146
+Graiden T. Jones|91-868-733-90988|lobortis.ultrices.Vivamus@faucibus.org|Nellore|India|C133E546-1E36-A6EF-9980-BB635F247290|973.7286142544|245462|2016-06-17 21:19:16|4393.9931397479
+Tanya X. Dominguez|91-553-110-11147|odio.Nam@antedictum.ca|Anantapur|India|DEF2CE91-CAF2-88BA-C50B-AB7EA1D73C63|959.1678617895|245464|2016-06-17 21:19:16|42536.8767788127
+Anne D. Pace|91-188-230-70605|pede.et.risus@Ut.co.uk|Agartala|India|32FC1F14-21F7-E940-7C9D-B80F44EE4063|1021.5754177485|245463|2016-06-17 21:19:16|32755.1536052658
+Hanna I. Sellers|91-322-901-42448|tincidunt.dui.augue@augue.org|Barddhaman|India|3D4F2FBC-FFCE-EE5C-5D2C-C60856D33B7D|993.2866819089|245466|2016-06-17 21:19:17|75662.1622559809
+Christine R. Gross|91-380-558-08928|convallis.convallis@convalliserateget.org|Shillong|India|DD81CFAC-DE87-E8A1-2850-E2637912543C|951.970651195|245465|2016-06-17 21:19:17|80146.9990770126
+Kirby A. Morse|91-299-538-45635|elit.Nulla.facilisi@acmattisvelit.net|Itanagar|India|CFF08336-B5C9-EECC-1F2D-07ADE42587FE|1009.5054026816|245467|2016-06-17 21:19:17|72602.1251286646
+Ryder B. Vega|91-320-455-77145|Aliquam.gravida@elit.co.uk|Gondiya|India|15F3478F-4D7E-562C-E189-945E655641E9|1006.8962892921|245468|2016-06-17 21:19:17|8089.8058610084
+Demetrius Q. Pope|91-179-308-28350|tempus@commodotinciduntnibh.ca|Imphal|India|CBF9C119-3E02-13F0-842A-BAF0AB46382D|1042.9498091927|245470|2016-06-17 21:19:18|83959.394047181
+Bryar M. Hendricks|91-617-948-40647|dui.Fusce.aliquam@loremvitaeodio.net|Kanchrapara|India|64188EFF-B7B1-AF75-EA3D-013CCA945F81|968.5880217334|245469|2016-06-17 21:19:18|85996.1671440447
+Ann P. Mann|91-395-300-37963|Cum@gravidanon.edu|Imphal|India|12818782-A86C-63A2-AB08-A5F281DA954B|937.3545836376|245472|2016-06-17 21:19:18|83236.1732288256
+Tyrone N. Bender|91-515-367-84014|at@fringillaporttitorvulputate.com|Chandigarh|India|D5109D0E-7DB6-A084-F0F9-E2BEF2B10E4A|1013.1764242663|245471|2016-06-17 21:19:18|36038.3036732134
+Miranda W. Sykes|91-774-614-92485|amet@accumsan.com|Gangtok|India|6EC7240D-5331-5F34-D1B8-38E2024A1925|992.518070982|245473|2016-06-17 21:19:19|97197.8457445721
+Ralph F. Walker|91-815-285-72212|molestie.in@vestibulumlorem.ca|Farrukhabad-cum-Fatehgarh|India|B7982CE7-B815-2794-504B-A8C3ECF0B592|992.3386856766|245474|2016-06-17 21:19:19|60698.1579345159
+Kelsie J. Mack|91-112-975-50329|pede.ac.urna@ligula.com|Hazaribag|India|5CACD398-932B-449D-80AC-5302AE35E027|1027.0394067971|245475|2016-06-17 21:19:19|48589.914064424
+Zenia F. Bond|91-445-745-77345|pede.Cum.sociis@Pellentesquetincidunt.com|Port Blair|India|68EDFE31-8E62-FF3B-3FBA-49B436D7142C|996.5071564618|245476|2016-06-17 21:19:19|40633.7414556007
+Todd V. Barton|91-312-429-52379|mauris.a@tristique.org|Dehradun|India|7D22103A-1DA0-2E4E-BFD7-8117133A983A|977.4969081079|245477|2016-06-17 21:19:20|49497.7649558266
+Kerry O. Franco|91-802-106-72427|Ut.sagittis.lobortis@dictumeueleifend.ca|Kohima|India|FC6049B3-9FD4-9A53-1A7E-C50394D4F4CD|1004.0387939017|245478|2016-06-17 21:19:20|43754.1503070853
+April I. Clarke|91-785-400-08200|sapien@mattisCras.edu|Daman|India|363E468F-BD44-6607-CDD6-226A6FB48B89|990.0985191984|245479|2016-06-17 21:19:20|90752.7220658782
+Randall B. Workman|91-722-286-44955|ut@Integervitae.net|Gujrat|India|1A402674-FD62-8F1B-7AE1-CFAA7E85006A|1012.6352839308|245480|2016-06-17 21:19:20|1815.8797213967
+Blaine J. Martinez|91-987-901-22060|libero.mauris.aliquam@egetlaoreet.co.uk|Kohima|India|7FDD906D-AB41-89EC-D546-B1E04D31FE49|1020.339672989|245481|2016-06-17 21:19:21|51409.0038738681
+May B. Schroeder|91-123-334-23814|aliquet@leo.edu|Orai|India|2E83862F-C3A4-4A4B-4D9C-C60AC4435A04|982.3424542846|245482|2016-06-17 21:19:21|89159.6292602544
+Whitney N. Tran|91-923-716-39943|Aliquam@leoMorbi.edu|Purulia|India|FEE99492-4B48-533F-EB36-5EA3528A121C|1007.9029941609|245483|2016-06-17 21:19:21|24692.473001632
+Ingrid A. Head|91-596-380-43300|ac@montes.ca|Port Blair|India|735007BF-1F13-95E9-7C3E-536C0452714D|995.4883512745|245484|2016-06-17 21:19:21|54890.8030672057
+Berk E. Gould|91-110-540-75246|ipsum@Nulla.org|Alwar|India|FB0297F7-8BC1-86F0-8CCD-C01538B7E5BE|984.2152917929|245486|2016-06-17 21:19:22|70311.7302535036
+Meredith G. Parks|91-154-792-20142|et.eros@auctor.com|Chandigarh|India|CD2691BE-ADB4-87DF-CA3B-88512330E745|960.8832140455|245485|2016-06-17 21:19:22|95293.2215958629
+Cora K. Carrillo|91-786-268-62522|elementum.lorem.ut@eutelluseu.ca|Dindigul|India|CAA843B7-F7C8-AD16-3F05-89DADB6EB288|1001.8761512414|245488|2016-06-17 21:19:22|59127.2684732433
+Galena D. Farmer|91-951-409-02162|sem@dictum.co.uk|Shillong|India|A46AF538-4BEF-D24F-870F-DD4CD20F4DC6|1022.2222771884|245487|2016-06-17 21:19:22|36667.9833050979
+Mollie E. Snow|91-816-823-87138|lectus@pharetra.org|Tiruvannamalai|India|A66AFB86-5297-61AC-2A2A-87C23F0E96E1|966.8626148916|245490|2016-06-17 21:19:23|28859.7500576673
+Abra E. Mccray|91-100-997-13066|Morbi.quis@risusDuis.net|Delhi|India|E49C0B30-586F-ECAA-FCBD-8F2898AA7771|1043.6824086302|245489|2016-06-17 21:19:23|79521.2870470628
+Andrew I. Lowe|91-859-767-29836|sed.turpis@Sed.co.uk|Aizwal|India|A928DE5C-6C18-9B9B-C4F0-3165F2E53C19|955.4198061291|245491|2016-06-17 21:19:23|34946.6776395577
+Zachery B. Marshall|91-103-978-47466|nonummy.Fusce@dui.co.uk|Vijayawada|India|E0D79F5F-2CB6-B9FC-B917-2D627141423B|969.8127419963|245492|2016-06-17 21:19:23|74671.8685208157
+Miranda M. Madden|91-310-815-30233|id@pharetranibh.co.uk|Brahmapur|India|55E04315-DE60-1C1D-78D6-878C9488BA06|963.5417067954|245493|2016-06-17 21:19:24|42089.8687934832
+Ian N. Johnson|91-651-594-79593|elit@pellentesquemassa.ca|Agartala|India|EDBCCD62-4871-4938-60C5-0CD3023CF333|1012.7053923589|245494|2016-06-17 21:19:24|97376.7240252239
+Macey V. Hinton|91-358-832-49086|eget.odio@Nunclectuspede.net|Sambalpur|India|49E4842E-0EB8-D48A-C1EC-A3F0FBAC92B4|982.9530728621|245496|2016-06-17 21:19:24|61850.2465402803
+Talon M. Calhoun|91-280-397-90948|lacinia.Sed@tinciduntvehicularisus.net|Pallavaram|India|E743767A-B1FF-C71C-B0D4-4C106D65AE3B|1040.9448702398|245495|2016-06-17 21:19:24|49928.6243188531
+Dillon A. Reese|91-900-545-84875|Mauris.vestibulum@sagittis.edu|Itanagar|India|9429B72A-36AA-8C78-2988-7FF48315094A|1029.5299833946|245498|2016-06-17 21:19:25|86990.7236411972
+Boris E. Boyd|91-480-944-74996|luctus.Curabitur.egestas@utmolestie.co.uk|Panjim|India|37962E2E-4B3D-27DD-1213-4FF1B5551CB4|969.407095963|245497|2016-06-17 21:19:25|32987.5284224486
+Inez U. Higgins|91-258-514-16019|Aliquam@utlacus.co.uk|Itanagar|India|D48C171C-A009-3813-5951-50A526BE6B58|1005.0207448325|245500|2016-06-17 21:19:25|68069.3550977988
+Leroy N. Snyder|91-682-436-52541|nisl.Maecenas@quam.com|Itanagar|India|BE155410-1F6D-AC60-C0B9-290E9426D5CF|993.1732235668|245499|2016-06-17 21:19:25|64376.3818134466
+George M. Case|91-760-901-60977|at.iaculis.quis@auctorvelit.com|Pondicherry|India|B924E744-29E3-8B6E-3EE9-A48A3BC9072B|987.476080645|245501|2016-06-17 21:19:26|97859.0229750841
+Kane E. Simon|91-461-207-64947|adipiscing.lobortis@tellus.ca|Haridwar|India|13C1AD7F-48D5-C6E5-892B-0345E7E070DB|983.5254219297|245502|2016-06-17 21:19:26|21912.3442110069
+Wanda U. Talley|91-398-803-35370|ipsum.Curabitur@eleifendnecmalesuada.com|Gangtok|India|5E2ED32D-621C-0ACE-71B6-F2F9746DD0C9|994.6344833977|245503|2016-06-17 21:19:26|91796.2772584239
+Asher U. Mccullough|91-149-955-32468|eget.massa.Suspendisse@Pellentesqueultricies.edu|Itanagar|India|3FA3514B-6974-06E3-EF7B-4841487A40D2|985.0811909206|245504|2016-06-17 21:19:26|71603.8572439688
+Malik E. Keller|91-391-507-50089|nec@Quisqueporttitor.net|Daman|India|05CDC866-65A3-E55C-EEE8-C73217F2071A|986.9798931399|245505|2016-06-17 21:19:27|65100.1941324091
+Yasir V. Howell|91-901-430-36468|nec.cursus.a@Cumsociisnatoque.ca|Mandya|India|F90D3C87-4C53-A5E9-D3F8-692C068CBCED|997.1935636906|245506|2016-06-17 21:19:27|35511.2898055096
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/recordReader/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/recordReader/src/main/resources/META-INF/properties.xml b/examples/recordReader/src/main/resources/META-INF/properties.xml
index 612e3fa..3870de2 100644
--- a/examples/recordReader/src/main/resources/META-INF/properties.xml
+++ b/examples/recordReader/src/main/resources/META-INF/properties.xml
@@ -1,6 +1,26 @@
<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<configuration>
- <!--
+ <!--
<property>
<name>dt.application.{appName}.operator.{opName}.prop.{propName}</name>
<value>some-default-value (if value is not specified, it is required from the user or custom config when launching)</value>
@@ -14,7 +34,7 @@
-->
<property>
<name>dt.application.RecordReaderExample.operator.recordReader.prop.files</name>
- <value>src/main/resources/META-INF/CustomerTxnData</value>
+ <value>src/main/resources/META-INF/CustomerTxnData.txt</value>
</property>
<property>
<name>dt.application.RecordReaderExample.attr.CHECKPOINT_WINDOW_COUNT</name>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/recordReader/src/test/java/org/apache/apex/examples/recordReader/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/recordReader/src/test/java/org/apache/apex/examples/recordReader/ApplicationTest.java b/examples/recordReader/src/test/java/org/apache/apex/examples/recordReader/ApplicationTest.java
index 7e133ff..c704d73 100644
--- a/examples/recordReader/src/test/java/org/apache/apex/examples/recordReader/ApplicationTest.java
+++ b/examples/recordReader/src/test/java/org/apache/apex/examples/recordReader/ApplicationTest.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.recordReader;
import java.io.File;
@@ -23,7 +42,7 @@ import com.datatorrent.api.LocalMode;
public class ApplicationTest
{
private String outputDir;
-
+
public static class TestMeta extends TestWatcher
{
public String baseDirectory;
@@ -33,7 +52,7 @@ public class ApplicationTest
{
this.baseDirectory = "target/" + description.getClassName() + "/" + description.getMethodName();
}
-
+
@Override
protected void finished(Description description)
{
@@ -46,16 +65,16 @@ public class ApplicationTest
}
}
-
+
@Rule
public TestMeta testMeta = new TestMeta();
-
+
@Before
public void setup() throws Exception
{
outputDir = testMeta.baseDirectory + File.separator + "output";
}
-
+
@Test
public void testApplication() throws IOException, Exception
{
@@ -65,23 +84,21 @@ public class ApplicationTest
conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
conf.set("dt.application.RecordReaderExample.operator.fileOutput.prop.filePath", outputDir);
File outputfile = FileUtils.getFile(outputDir, "output.txt_5.0");
-
+
lma.prepareDAG(new Application(), conf);
LocalMode.Controller lc = lma.getController();
lc.runAsync();
// wait for tuples to show up
while (!outputfile.exists()) {
- System.out.println("Sleeping ....");
Thread.sleep(1000);
}
-
+
lc.shutdown();
- Assert.assertTrue(
- FileUtils.contentEquals(
- FileUtils.getFile(
- conf.get("dt.application.RecordReaderExample.operator.recordReader.prop.files")
- ),outputfile));
+ Assert.assertTrue(FileUtils.contentEquals(
+ FileUtils.getFile(
+ conf.get("dt.application.RecordReaderExample.operator.recordReader.prop.files")),
+ outputfile));
} catch (ConstraintViolationException e) {
Assert.fail("constraint violations: " + e.getConstraintViolations());
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/recordReader/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/recordReader/src/test/resources/log4j.properties b/examples/recordReader/src/test/resources/log4j.properties
index 3bfcdc5..451cff3 100644
--- a/examples/recordReader/src/test/resources/log4j.properties
+++ b/examples/recordReader/src/test/resources/log4j.properties
@@ -1,3 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
log4j.rootLogger=DEBUG,CONSOLE
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/throttle/XmlJavadocCommentsExtractor.xsl
----------------------------------------------------------------------
diff --git a/examples/throttle/XmlJavadocCommentsExtractor.xsl b/examples/throttle/XmlJavadocCommentsExtractor.xsl
deleted file mode 100644
index 08075a9..0000000
--- a/examples/throttle/XmlJavadocCommentsExtractor.xsl
+++ /dev/null
@@ -1,44 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
--->
-
-<!--
- Document : XmlJavadocCommentsExtractor.xsl
- Created on : September 16, 2014, 11:30 AM
- Description:
- The transformation strips off all information except for comments and tags from xml javadoc generated by xml-doclet.
--->
-
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
- <xsl:output method="xml" standalone="yes"/>
-
- <!-- copy xml by selecting only the following nodes, attributes and text -->
- <xsl:template match="node()|text()|@*">
- <xsl:copy>
- <xsl:apply-templates select="root|package|class|interface|method|field|type|comment|tag|text()|@name|@qualified|@text"/>
- </xsl:copy>
- </xsl:template>
-
- <!-- Strip off the following paths from the selected xml -->
- <xsl:template match="//root/package/interface/interface
- |//root/package/interface/method/@qualified
- |//root/package/class/interface
- |//root/package/class/class
- |//root/package/class/method/@qualified
- |//root/package/class/field/@qualified" />
-
- <xsl:strip-space elements="*"/>
-</xsl:stylesheet>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/throttle/pom.xml
----------------------------------------------------------------------
diff --git a/examples/throttle/pom.xml b/examples/throttle/pom.xml
index d312268..df0a8fd 100644
--- a/examples/throttle/pom.xml
+++ b/examples/throttle/pom.xml
@@ -1,4 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@@ -7,12 +27,12 @@
<artifactId>malhar-examples</artifactId>
<version>3.7.0-SNAPSHOT</version>
</parent>
-
+
<artifactId>malhar-examples-throttle</artifactId>
<packaging>jar</packaging>
<!-- change these to the appropriate values -->
- <name>Throttle Application</name>
+ <name>Apache Apex Malhar Throttle Example</name>
<description>Application demonstrating throttling input when downstream is slower</description>
<dependencies>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/throttle/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/throttle/src/assemble/appPackage.xml b/examples/throttle/src/assemble/appPackage.xml
index 7ad071c..a870807 100644
--- a/examples/throttle/src/assemble/appPackage.xml
+++ b/examples/throttle/src/assemble/appPackage.xml
@@ -1,3 +1,23 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/throttle/src/main/java/org/apache/apex/examples/throttle/Application.java
----------------------------------------------------------------------
diff --git a/examples/throttle/src/main/java/org/apache/apex/examples/throttle/Application.java b/examples/throttle/src/main/java/org/apache/apex/examples/throttle/Application.java
index 9adad32..9571456 100644
--- a/examples/throttle/src/main/java/org/apache/apex/examples/throttle/Application.java
+++ b/examples/throttle/src/main/java/org/apache/apex/examples/throttle/Application.java
@@ -1,6 +1,22 @@
/**
- * Put your copyright and license info here.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
+
package org.apache.apex.examples.throttle;
import java.util.Collection;
@@ -15,7 +31,7 @@ import com.datatorrent.api.StatsListener;
import com.datatorrent.api.StreamingApplication;
import com.datatorrent.api.annotation.ApplicationAnnotation;
-@ApplicationAnnotation(name="ThrottleApplication")
+@ApplicationAnnotation(name = "ThrottleApplication")
public class Application implements StreamingApplication
{
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/throttle/src/main/java/org/apache/apex/examples/throttle/PassThroughOperator.java
----------------------------------------------------------------------
diff --git a/examples/throttle/src/main/java/org/apache/apex/examples/throttle/PassThroughOperator.java b/examples/throttle/src/main/java/org/apache/apex/examples/throttle/PassThroughOperator.java
index bf0fbce..52d3b1b 100644
--- a/examples/throttle/src/main/java/org/apache/apex/examples/throttle/PassThroughOperator.java
+++ b/examples/throttle/src/main/java/org/apache/apex/examples/throttle/PassThroughOperator.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.throttle;
import com.datatorrent.api.DefaultInputPort;
@@ -7,14 +26,16 @@ import com.datatorrent.common.util.BaseOperator;
/**
* Created by pramod on 9/27/16.
*/
-public class PassThroughOperator<T> extends BaseOperator {
-
- public transient final DefaultInputPort<T> input = new DefaultInputPort<T>() {
- @Override
- public void process(T t) {
- output.emit(t);
- }
- };
+public class PassThroughOperator<T> extends BaseOperator
+{
+ public final transient DefaultInputPort<T> input = new DefaultInputPort<T>()
+ {
+ @Override
+ public void process(T t)
+ {
+ output.emit(t);
+ }
+ };
- public transient final DefaultOutputPort<T> output = new DefaultOutputPort<>();
+ public final transient DefaultOutputPort<T> output = new DefaultOutputPort<>();
}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/throttle/src/main/java/org/apache/apex/examples/throttle/RandomNumberGenerator.java
----------------------------------------------------------------------
diff --git a/examples/throttle/src/main/java/org/apache/apex/examples/throttle/RandomNumberGenerator.java b/examples/throttle/src/main/java/org/apache/apex/examples/throttle/RandomNumberGenerator.java
index 6825340..e24b63e 100644
--- a/examples/throttle/src/main/java/org/apache/apex/examples/throttle/RandomNumberGenerator.java
+++ b/examples/throttle/src/main/java/org/apache/apex/examples/throttle/RandomNumberGenerator.java
@@ -1,13 +1,30 @@
/**
- * Put your copyright and license info here.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
+
package org.apache.apex.examples.throttle;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import com.datatorrent.api.DefaultOutputPort;
import com.datatorrent.api.InputOperator;
import com.datatorrent.common.util.BaseOperator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* This is a simple operator that emits random number.
@@ -37,12 +54,14 @@ public class RandomNumberGenerator extends BaseOperator implements InputOperator
}
// Simple suspend and
- public void suspend() {
+ public void suspend()
+ {
logger.debug("Slowing down");
numTuples = 0;
}
- public void normal() {
+ public void normal()
+ {
logger.debug("Normal");
numTuples = origNumTuples;
}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/throttle/src/main/java/org/apache/apex/examples/throttle/SlowDevNullOperator.java
----------------------------------------------------------------------
diff --git a/examples/throttle/src/main/java/org/apache/apex/examples/throttle/SlowDevNullOperator.java b/examples/throttle/src/main/java/org/apache/apex/examples/throttle/SlowDevNullOperator.java
index b49b3c8..e830013 100644
--- a/examples/throttle/src/main/java/org/apache/apex/examples/throttle/SlowDevNullOperator.java
+++ b/examples/throttle/src/main/java/org/apache/apex/examples/throttle/SlowDevNullOperator.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.throttle;
import com.google.common.base.Throwables;
@@ -8,28 +27,32 @@ import com.datatorrent.common.util.BaseOperator;
/**
* Created by pramod on 9/27/16.
*/
-public class SlowDevNullOperator<T> extends BaseOperator {
-
- // Modify sleep time dynamically while app is running to increase and decrease sleep time
- long sleepTime = 1;
+public class SlowDevNullOperator<T> extends BaseOperator
+{
+ // Modify sleep time dynamically while app is running to increase and decrease sleep time
+ long sleepTime = 1;
- public transient final DefaultInputPort<T> input = new DefaultInputPort<T>() {
- @Override
- public void process(T t) {
- // Introduce an artificial delay for every tuple
- try {
- Thread.sleep(sleepTime);
- } catch (InterruptedException e) {
- throw Throwables.propagate(e);
- }
- }
- };
-
- public long getSleepTime() {
- return sleepTime;
+ public final transient DefaultInputPort<T> input = new DefaultInputPort<T>()
+ {
+ @Override
+ public void process(T t)
+ {
+ // Introduce an artificial delay for every tuple
+ try {
+ Thread.sleep(sleepTime);
+ } catch (InterruptedException e) {
+ throw Throwables.propagate(e);
+ }
}
+ };
- public void setSleepTime(long sleepTime) {
- this.sleepTime = sleepTime;
- }
+ public long getSleepTime()
+ {
+ return sleepTime;
+ }
+
+ public void setSleepTime(long sleepTime)
+ {
+ this.sleepTime = sleepTime;
+ }
}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/throttle/src/main/java/org/apache/apex/examples/throttle/ThrottlingStatsListener.java
----------------------------------------------------------------------
diff --git a/examples/throttle/src/main/java/org/apache/apex/examples/throttle/ThrottlingStatsListener.java b/examples/throttle/src/main/java/org/apache/apex/examples/throttle/ThrottlingStatsListener.java
index 7e9e70f..5504332 100644
--- a/examples/throttle/src/main/java/org/apache/apex/examples/throttle/ThrottlingStatsListener.java
+++ b/examples/throttle/src/main/java/org/apache/apex/examples/throttle/ThrottlingStatsListener.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.throttle;
import java.io.IOException;
@@ -17,134 +36,141 @@ import com.datatorrent.api.StatsListener;
/**
* Created by pramod on 9/27/16.
*/
-public class ThrottlingStatsListener implements StatsListener, Serializable {
-
- private static final Logger logger = LoggerFactory.getLogger(ThrottlingStatsListener.class);
-
- // Slowdown input if the window difference between operators increases beyond this value
- long maxThreshold = 100;
- // restore input operator to normal speed if the window difference falls below this threshold
- long minThreshold = 100;
-
- Map<Integer, ThrottleState> throttleStates = Maps.newHashMap();
-
- static class ThrottleState {
- // The current state of the operator, normal or throttled
- boolean normal = true;
- //The latest window id for which stats were received for the operator
- long currentWindowId;
- }
-
- // This method runs on the app master side and is called whenever new stats are received from the operators
- @Override
- public Response processStats(BatchedOperatorStats batchedOperatorStats)
- {
- Response response = new Response();
- int operatorId = batchedOperatorStats.getOperatorId();
-
- ThrottleState throttleState = throttleStates.get(operatorId);
- if (throttleState == null) {
- throttleState = new ThrottleState();
- throttleStates.put(operatorId, throttleState);
- }
-
- long windowId = batchedOperatorStats.getCurrentWindowId();
- throttleState.currentWindowId = windowId;
-
- // Find min and max window to compute difference
- long minWindow = Long.MAX_VALUE;
- long maxWindow = Long.MIN_VALUE;
- for (ThrottleState state : throttleStates.values()) {
- if (state.currentWindowId < minWindow) minWindow = state.currentWindowId;
- if (state.currentWindowId > maxWindow) maxWindow = state.currentWindowId;
- }
- logger.debug("Operator {} min window {} max window {}", operatorId, minWindow, maxWindow);
-
- if (throttleState.normal && ((maxWindow - minWindow) > maxThreshold)) {
- // Send request to operator to slow down
- logger.info("Sending suspend request");
- List<OperatorRequest> operatorRequests = new ArrayList<OperatorRequest>();
- operatorRequests.add(new InputSlowdownRequest());
- response.operatorRequests = operatorRequests;
- //logger.info("Setting suspend");
- throttleState.normal = false;
- } else if (!throttleState.normal && ((maxWindow - minWindow) <= minThreshold)) {
- // Send request to operator to get back to normal
- logger.info("Sending normal request");
- List<OperatorRequest> operatorRequests = new ArrayList<OperatorRequest>();
- operatorRequests.add(new InputNormalRequest());
- response.operatorRequests = operatorRequests;
- //logger.info("Setting normal");
- throttleState.normal = true;
- }
-
- return response;
+public class ThrottlingStatsListener implements StatsListener, Serializable
+{
+
+ private static final Logger logger = LoggerFactory.getLogger(ThrottlingStatsListener.class);
+
+ // Slowdown input if the window difference between operators increases beyond this value
+ long maxThreshold = 100;
+ // restore input operator to normal speed if the window difference falls below this threshold
+ long minThreshold = 100;
+
+ Map<Integer, ThrottleState> throttleStates = Maps.newHashMap();
+
+ static class ThrottleState
+ {
+ // The current state of the operator, normal or throttled
+ boolean normal = true;
+ //The latest window id for which stats were received for the operator
+ long currentWindowId;
+ }
+
+ // This method runs on the app master side and is called whenever new stats are received from the operators
+ @Override
+ public Response processStats(BatchedOperatorStats batchedOperatorStats)
+ {
+ Response response = new Response();
+ int operatorId = batchedOperatorStats.getOperatorId();
+
+ ThrottleState throttleState = throttleStates.get(operatorId);
+ if (throttleState == null) {
+ throttleState = new ThrottleState();
+ throttleStates.put(operatorId, throttleState);
}
- // This runs on the operator side
- public static class InputSlowdownRequest implements OperatorRequest, Serializable
- {
- private static final Logger logger = LoggerFactory.getLogger(InputSlowdownRequest.class);
-
- @Override
- public OperatorResponse execute(Operator operator, int operatorId, long windowId) throws IOException
- {
- logger.debug("Received slowdown operator {} operatorId {} windowId {}", operator, operatorId, windowId);
- if (operator instanceof RandomNumberGenerator) {
- RandomNumberGenerator generator = (RandomNumberGenerator)operator;
- generator.suspend();
- }
- return new InputOperatorResponse();
- }
+ long windowId = batchedOperatorStats.getCurrentWindowId();
+ throttleState.currentWindowId = windowId;
+
+ // Find min and max window to compute difference
+ long minWindow = Long.MAX_VALUE;
+ long maxWindow = Long.MIN_VALUE;
+ for (ThrottleState state : throttleStates.values()) {
+ if (state.currentWindowId < minWindow) {
+ minWindow = state.currentWindowId;
+ }
+ if (state.currentWindowId > maxWindow) {
+ maxWindow = state.currentWindowId;
+ }
}
-
- public static class InputNormalRequest implements OperatorRequest, Serializable
- {
- private static final Logger logger = LoggerFactory.getLogger(InputNormalRequest.class);
-
- @Override
- public OperatorResponse execute(Operator operator, int operatorId, long windowId) throws IOException
- {
- logger.debug("Received normal operator {} operatorId {} windowId {}", operator, operatorId, windowId);
- if (operator instanceof RandomNumberGenerator) {
- RandomNumberGenerator generator = (RandomNumberGenerator)operator;
- generator.normal();
- }
- return new InputOperatorResponse();
- }
+ logger.debug("Operator {} min window {} max window {}", operatorId, minWindow, maxWindow);
+
+ if (throttleState.normal && ((maxWindow - minWindow) > maxThreshold)) {
+ // Send request to operator to slow down
+ logger.info("Sending suspend request");
+ List<OperatorRequest> operatorRequests = new ArrayList<OperatorRequest>();
+ operatorRequests.add(new InputSlowdownRequest());
+ response.operatorRequests = operatorRequests;
+ //logger.info("Setting suspend");
+ throttleState.normal = false;
+ } else if (!throttleState.normal && ((maxWindow - minWindow) <= minThreshold)) {
+ // Send request to operator to get back to normal
+ logger.info("Sending normal request");
+ List<OperatorRequest> operatorRequests = new ArrayList<OperatorRequest>();
+ operatorRequests.add(new InputNormalRequest());
+ response.operatorRequests = operatorRequests;
+ //logger.info("Setting normal");
+ throttleState.normal = true;
}
- public static class InputOperatorResponse implements OperatorResponse, Serializable
- {
-
- @Override
- public Object getResponseId() {
- return 1;
- }
+ return response;
+ }
- @Override
- public Object getResponse() {
- return "";
- }
- }
+ // This runs on the operator side
+ public static class InputSlowdownRequest implements OperatorRequest, Serializable
+ {
+ private static final Logger logger = LoggerFactory.getLogger(InputSlowdownRequest.class);
- public long getMaxThreshold()
+ @Override
+ public OperatorResponse execute(Operator operator, int operatorId, long windowId) throws IOException
{
- return maxThreshold;
+ logger.debug("Received slowdown operator {} operatorId {} windowId {}", operator, operatorId, windowId);
+ if (operator instanceof RandomNumberGenerator) {
+ RandomNumberGenerator generator = (RandomNumberGenerator)operator;
+ generator.suspend();
+ }
+ return new InputOperatorResponse();
}
+ }
+
+ public static class InputNormalRequest implements OperatorRequest, Serializable
+ {
+ private static final Logger logger = LoggerFactory.getLogger(InputNormalRequest.class);
- public void setMaxThreshold(long maxThreshold)
+ @Override
+ public OperatorResponse execute(Operator operator, int operatorId, long windowId) throws IOException
{
- this.maxThreshold = maxThreshold;
+ logger.debug("Received normal operator {} operatorId {} windowId {}", operator, operatorId, windowId);
+ if (operator instanceof RandomNumberGenerator) {
+ RandomNumberGenerator generator = (RandomNumberGenerator)operator;
+ generator.normal();
+ }
+ return new InputOperatorResponse();
}
+ }
- public long getMinThreshold()
+ public static class InputOperatorResponse implements OperatorResponse, Serializable
+ {
+ @Override
+ public Object getResponseId()
{
- return minThreshold;
+ return 1;
}
- public void setMinThreshold(long minThreshold)
+ @Override
+ public Object getResponse()
{
- this.minThreshold = minThreshold;
+ return "";
}
+ }
+
+ public long getMaxThreshold()
+ {
+ return maxThreshold;
+ }
+
+ public void setMaxThreshold(long maxThreshold)
+ {
+ this.maxThreshold = maxThreshold;
+ }
+
+ public long getMinThreshold()
+ {
+ return minThreshold;
+ }
+
+ public void setMinThreshold(long minThreshold)
+ {
+ this.minThreshold = minThreshold;
+ }
}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/throttle/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/throttle/src/main/resources/META-INF/properties.xml b/examples/throttle/src/main/resources/META-INF/properties.xml
index a6ddc4c..92e20f1 100644
--- a/examples/throttle/src/main/resources/META-INF/properties.xml
+++ b/examples/throttle/src/main/resources/META-INF/properties.xml
@@ -1,4 +1,24 @@
<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<configuration>
<!--
<property>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/throttle/src/test/java/org/apache/apex/examples/throttle/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/throttle/src/test/java/org/apache/apex/examples/throttle/ApplicationTest.java b/examples/throttle/src/test/java/org/apache/apex/examples/throttle/ApplicationTest.java
index e85c5c5..77e5c51 100644
--- a/examples/throttle/src/test/java/org/apache/apex/examples/throttle/ApplicationTest.java
+++ b/examples/throttle/src/test/java/org/apache/apex/examples/throttle/ApplicationTest.java
@@ -1,5 +1,20 @@
/**
- * Put your copyright and license info here.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
package org.apache.apex.examples.throttle;
@@ -8,19 +23,21 @@ import java.io.IOException;
import javax.validation.ConstraintViolationException;
import org.junit.Assert;
+import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
-import org.junit.Test;
import com.datatorrent.api.LocalMode;
/**
* Test the DAG declaration in local mode.
*/
-public class ApplicationTest {
+public class ApplicationTest
+{
@Test
- public void testApplication() throws IOException, Exception {
+ public void testApplication() throws IOException, Exception
+ {
try {
LocalMode lma = LocalMode.newInstance();
Configuration conf = new Configuration(false);
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/throttle/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/throttle/src/test/resources/log4j.properties b/examples/throttle/src/test/resources/log4j.properties
index 98544e8..1c9776b 100644
--- a/examples/throttle/src/test/resources/log4j.properties
+++ b/examples/throttle/src/test/resources/log4j.properties
@@ -1,3 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
log4j.rootLogger=DEBUG,CONSOLE
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/transform/XmlJavadocCommentsExtractor.xsl
----------------------------------------------------------------------
diff --git a/examples/transform/XmlJavadocCommentsExtractor.xsl b/examples/transform/XmlJavadocCommentsExtractor.xsl
deleted file mode 100644
index 08075a9..0000000
--- a/examples/transform/XmlJavadocCommentsExtractor.xsl
+++ /dev/null
@@ -1,44 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
--->
-
-<!--
- Document : XmlJavadocCommentsExtractor.xsl
- Created on : September 16, 2014, 11:30 AM
- Description:
- The transformation strips off all information except for comments and tags from xml javadoc generated by xml-doclet.
--->
-
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
- <xsl:output method="xml" standalone="yes"/>
-
- <!-- copy xml by selecting only the following nodes, attributes and text -->
- <xsl:template match="node()|text()|@*">
- <xsl:copy>
- <xsl:apply-templates select="root|package|class|interface|method|field|type|comment|tag|text()|@name|@qualified|@text"/>
- </xsl:copy>
- </xsl:template>
-
- <!-- Strip off the following paths from the selected xml -->
- <xsl:template match="//root/package/interface/interface
- |//root/package/interface/method/@qualified
- |//root/package/class/interface
- |//root/package/class/class
- |//root/package/class/method/@qualified
- |//root/package/class/field/@qualified" />
-
- <xsl:strip-space elements="*"/>
-</xsl:stylesheet>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/transform/pom.xml
----------------------------------------------------------------------
diff --git a/examples/transform/pom.xml b/examples/transform/pom.xml
index 93dbad1..b2354f9 100644
--- a/examples/transform/pom.xml
+++ b/examples/transform/pom.xml
@@ -1,4 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@@ -11,7 +31,7 @@
<artifactId>malhar-examples-transform</artifactId>
<packaging>jar</packaging>
- <name>Transform Application</name>
+ <name>Apache Apex Malhar Transform Example</name>
<description>Sample application for transform operator</description>
<dependencies>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/transform/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/transform/src/assemble/appPackage.xml b/examples/transform/src/assemble/appPackage.xml
index 7ad071c..2adeec0 100644
--- a/examples/transform/src/assemble/appPackage.xml
+++ b/examples/transform/src/assemble/appPackage.xml
@@ -1,3 +1,24 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/transform/src/main/java/org/apache/apex/examples/transform/Application.java
----------------------------------------------------------------------
diff --git a/examples/transform/src/main/java/org/apache/apex/examples/transform/Application.java b/examples/transform/src/main/java/org/apache/apex/examples/transform/Application.java
index e6a0419..f35a591 100644
--- a/examples/transform/src/main/java/org/apache/apex/examples/transform/Application.java
+++ b/examples/transform/src/main/java/org/apache/apex/examples/transform/Application.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.transform;
import java.util.HashMap;
@@ -13,7 +32,7 @@ import com.datatorrent.common.partitioner.StatelessPartitioner;
import com.datatorrent.lib.io.ConsoleOutputOperator;
import com.datatorrent.lib.transform.TransformOperator;
-@ApplicationAnnotation(name="TransformExample")
+@ApplicationAnnotation(name = "TransformExample")
public class Application implements StreamingApplication
{
@Override
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/transform/src/main/java/org/apache/apex/examples/transform/CustomerEvent.java
----------------------------------------------------------------------
diff --git a/examples/transform/src/main/java/org/apache/apex/examples/transform/CustomerEvent.java b/examples/transform/src/main/java/org/apache/apex/examples/transform/CustomerEvent.java
index 8011e12..bfc9660 100644
--- a/examples/transform/src/main/java/org/apache/apex/examples/transform/CustomerEvent.java
+++ b/examples/transform/src/main/java/org/apache/apex/examples/transform/CustomerEvent.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.transform;
import java.util.Date;
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/transform/src/main/java/org/apache/apex/examples/transform/CustomerInfo.java
----------------------------------------------------------------------
diff --git a/examples/transform/src/main/java/org/apache/apex/examples/transform/CustomerInfo.java b/examples/transform/src/main/java/org/apache/apex/examples/transform/CustomerInfo.java
index ece76a6..1dc347b 100644
--- a/examples/transform/src/main/java/org/apache/apex/examples/transform/CustomerInfo.java
+++ b/examples/transform/src/main/java/org/apache/apex/examples/transform/CustomerInfo.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.transform;
public class CustomerInfo
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/transform/src/main/java/org/apache/apex/examples/transform/DynamicTransformApplication.java
----------------------------------------------------------------------
diff --git a/examples/transform/src/main/java/org/apache/apex/examples/transform/DynamicTransformApplication.java b/examples/transform/src/main/java/org/apache/apex/examples/transform/DynamicTransformApplication.java
index 01bc446..4063cce 100644
--- a/examples/transform/src/main/java/org/apache/apex/examples/transform/DynamicTransformApplication.java
+++ b/examples/transform/src/main/java/org/apache/apex/examples/transform/DynamicTransformApplication.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.transform;
import java.util.Arrays;
@@ -15,7 +34,7 @@ import com.datatorrent.lib.io.ConsoleOutputOperator;
import com.datatorrent.lib.partitioner.StatelessThroughputBasedPartitioner;
import com.datatorrent.lib.transform.TransformOperator;
-@ApplicationAnnotation(name="DynamicTransformApp")
+@ApplicationAnnotation(name = "DynamicTransformApp")
public class DynamicTransformApplication implements StreamingApplication
{
private static String COOL_DOWN_MILLIS = "dt.cooldown";
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/transform/src/main/java/org/apache/apex/examples/transform/POJOGenerator.java
----------------------------------------------------------------------
diff --git a/examples/transform/src/main/java/org/apache/apex/examples/transform/POJOGenerator.java b/examples/transform/src/main/java/org/apache/apex/examples/transform/POJOGenerator.java
index f8f3b22..1b4f3ef 100644
--- a/examples/transform/src/main/java/org/apache/apex/examples/transform/POJOGenerator.java
+++ b/examples/transform/src/main/java/org/apache/apex/examples/transform/POJOGenerator.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.transform;
import java.util.Date;
@@ -52,8 +71,8 @@ public class POJOGenerator implements InputOperator
}
- CustomerEvent generateCustomersEvent() throws Exception {
-
+ CustomerEvent generateCustomersEvent() throws Exception
+ {
CustomerEvent customerEvent = new CustomerEvent();
customerEvent.setCustomerId(randomId(maxCustomerId));
customerEvent.setFirstName(rRandom.randomAlphabetic(randomId(maxNameLength)));
@@ -65,8 +84,11 @@ public class POJOGenerator implements InputOperator
return customerEvent;
}
- private int randomId(int max) {
- if (max < 1) return 1;
+ private int randomId(int max)
+ {
+ if (max < 1) {
+ return 1;
+ }
return 1 + random.nextInt(max);
}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/transform/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/transform/src/main/resources/META-INF/properties.xml b/examples/transform/src/main/resources/META-INF/properties.xml
index f885664..175b5ab 100644
--- a/examples/transform/src/main/resources/META-INF/properties.xml
+++ b/examples/transform/src/main/resources/META-INF/properties.xml
@@ -1,4 +1,25 @@
<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
<configuration>
<property>
<name>dt.application.*.operator.transform.attr.MEMORY_MB</name>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/transform/src/test/java/org/apache/apex/examples/transform/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/transform/src/test/java/org/apache/apex/examples/transform/ApplicationTest.java b/examples/transform/src/test/java/org/apache/apex/examples/transform/ApplicationTest.java
index aab8af8..181a10b 100644
--- a/examples/transform/src/test/java/org/apache/apex/examples/transform/ApplicationTest.java
+++ b/examples/transform/src/test/java/org/apache/apex/examples/transform/ApplicationTest.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.transform;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/transform/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/transform/src/test/resources/log4j.properties b/examples/transform/src/test/resources/log4j.properties
index 98544e8..1c9776b 100644
--- a/examples/transform/src/test/resources/log4j.properties
+++ b/examples/transform/src/test/resources/log4j.properties
@@ -1,3 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
log4j.rootLogger=DEBUG,CONSOLE
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
[16/19] apex-malhar git commit: APEXMALHAR-2233 Updated the examples
to follow the structure of apex-malhar examples. Specified dependencies in
pom.xmls of individual examples correctly.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/innerjoin/src/main/java/com/example/join/POJOGenerator.java
----------------------------------------------------------------------
diff --git a/examples/innerjoin/src/main/java/com/example/join/POJOGenerator.java b/examples/innerjoin/src/main/java/com/example/join/POJOGenerator.java
deleted file mode 100644
index 613c80e..0000000
--- a/examples/innerjoin/src/main/java/com/example/join/POJOGenerator.java
+++ /dev/null
@@ -1,260 +0,0 @@
-package com.example.join;
-
-import java.util.Random;
-
-import javax.validation.constraints.Min;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.InputOperator;
-
-/**
- * Generates and emits the SalesEvent/ProductEvent based on isSalesEvent.
- */
-public class POJOGenerator implements InputOperator
-{
- @Min(1)
- private int maxProductId = 100000;
- @Min(1)
- private int maxCustomerId = 100000;
- @Min(1)
- private int maxProductCategories = 100;
- private double maxAmount = 100.0;
- private long tuplesCounter;
- private long time;
- private long timeIncrement;
- private boolean isSalesEvent = true;
- // Limit number of emitted tuples per window
- @Min(0)
- private long maxTuplesPerWindow = 100;
- private final Random random = new Random();
- public final transient DefaultOutputPort<Object> output = new DefaultOutputPort<>();
-
- @Override
- public void beginWindow(long windowId)
- {
- tuplesCounter = 0;
- }
-
- @Override
- public void endWindow()
- {
- time += timeIncrement;
- }
-
- @Override
- public void setup(Context.OperatorContext context)
- {
- time = System.currentTimeMillis();
- timeIncrement = context.getValue(Context.OperatorContext.APPLICATION_WINDOW_COUNT) *
- context.getValue(Context.DAGContext.STREAMING_WINDOW_SIZE_MILLIS);
- }
-
- @Override
- public void teardown()
- {
-
- }
-
- SalesEvent generateSalesEvent() throws Exception {
-
- SalesEvent salesEvent = new SalesEvent();
- salesEvent.productId = randomId(maxProductId);
- salesEvent.customerId = randomId(maxCustomerId);
- salesEvent.amount = randomAmount();
- salesEvent.timestamp = time;
- return salesEvent;
- }
-
- ProductEvent generateProductEvent() throws Exception {
- ProductEvent productEvent = new ProductEvent();
- productEvent.productId = randomId(maxProductId);
- productEvent.productCategory = randomId(maxProductCategories);
- productEvent.timestamp = time;
- return productEvent;
- }
-
- private int randomId(int max) {
- if (max < 1) return 1;
- return 1 + random.nextInt(max);
- }
-
- private double randomAmount() {
- return maxAmount * random.nextDouble();
- }
-
- @Override
- public void emitTuples()
- {
- while (tuplesCounter++ < maxTuplesPerWindow) {
- try {
- if (isSalesEvent) {
- SalesEvent event = generateSalesEvent();
- this.output.emit(event);
- } else {
- ProductEvent event = generateProductEvent();
- this.output.emit(event);
- }
-
- } catch (Exception ex) {
- throw new RuntimeException(ex);
- }
- }
- }
-
- public static class SalesEvent
- {
- public int customerId;
- public int productId;
- public int productCategory;
- public double amount;
- public long timestamp;
-
- public int getCustomerId()
- {
- return customerId;
- }
-
- public void setCustomerId(int customerId)
- {
- this.customerId = customerId;
- }
-
- public int getProductId()
- {
- return productId;
- }
-
- public void setProductId(int productId)
- {
- this.productId = productId;
- }
-
- public int getProductCategory()
- {
- return productCategory;
- }
-
- public void setProductCategory(int productCategory)
- {
- this.productCategory = productCategory;
- }
-
- public double getAmount()
- {
- return amount;
- }
-
- public void setAmount(double amount)
- {
- this.amount = amount;
- }
-
- public long getTimestamp()
- {
- return timestamp;
- }
-
- public void setTimestamp(long timestamp)
- {
- this.timestamp = timestamp;
- }
- }
-
- public static class ProductEvent
- {
- public int productId;
- public int productCategory;
- public long timestamp;
-
- public int getProductId()
- {
- return productId;
- }
-
- public void setProductId(int productId)
- {
- this.productId = productId;
- }
-
- public int getProductCategory()
- {
- return productCategory;
- }
-
- public void setProductCategory(int productCategory)
- {
- this.productCategory = productCategory;
- }
-
- public long getTimestamp()
- {
- return timestamp;
- }
-
- public void setTimestamp(long timestamp)
- {
- this.timestamp = timestamp;
- }
- }
-
- public int getMaxProductId()
- {
- return maxProductId;
- }
-
- public void setMaxProductId(int maxProductId)
- {
- this.maxProductId = maxProductId;
- }
-
- public int getMaxCustomerId()
- {
- return maxCustomerId;
- }
-
- public void setMaxCustomerId(int maxCustomerId)
- {
- this.maxCustomerId = maxCustomerId;
- }
-
- public int getMaxProductCategories()
- {
- return maxProductCategories;
- }
-
- public void setMaxProductCategories(int maxProductCategories)
- {
- this.maxProductCategories = maxProductCategories;
- }
-
- public double getMaxAmount()
- {
- return maxAmount;
- }
-
- public void setMaxAmount(double maxAmount)
- {
- this.maxAmount = maxAmount;
- }
-
- public boolean isSalesEvent()
- {
- return isSalesEvent;
- }
-
- public void setSalesEvent(boolean salesEvent)
- {
- isSalesEvent = salesEvent;
- }
-
- public long getMaxTuplesPerWindow()
- {
- return maxTuplesPerWindow;
- }
-
- public void setMaxTuplesPerWindow(long maxTuplesPerWindow)
- {
- this.maxTuplesPerWindow = maxTuplesPerWindow;
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/innerjoin/src/main/java/org/apache/apex/examples/innerjoin/InnerJoinApplication.java
----------------------------------------------------------------------
diff --git a/examples/innerjoin/src/main/java/org/apache/apex/examples/innerjoin/InnerJoinApplication.java b/examples/innerjoin/src/main/java/org/apache/apex/examples/innerjoin/InnerJoinApplication.java
new file mode 100644
index 0000000..3e7c8d5
--- /dev/null
+++ b/examples/innerjoin/src/main/java/org/apache/apex/examples/innerjoin/InnerJoinApplication.java
@@ -0,0 +1,38 @@
+package org.apache.apex.examples.innerjoin;
+
+import org.apache.apex.malhar.lib.join.POJOInnerJoinOperator;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+
+@ApplicationAnnotation(name="InnerJoinExample")
+public class InnerJoinApplication implements StreamingApplication
+{
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ // SalesEvent Generator
+ POJOGenerator salesGenerator = dag.addOperator("Input1", new POJOGenerator());
+ // ProductEvent Generator
+ POJOGenerator productGenerator = dag.addOperator("Input2", new POJOGenerator());
+ productGenerator.setSalesEvent(false);
+
+ // Inner join Operator
+ POJOInnerJoinOperator join = dag.addOperator("Join", new POJOInnerJoinOperator());
+ ConsoleOutputOperator output = dag.addOperator("Output", new ConsoleOutputOperator());
+
+ // Streams
+ dag.addStream("SalesToJoin", salesGenerator.output, join.input1);
+ dag.addStream("ProductToJoin", productGenerator.output, join.input2);
+ dag.addStream("JoinToConsole", join.outputPort, output.input);
+
+ // Setting tuple class properties to the ports of join operator
+ dag.setInputPortAttribute(join.input1, Context.PortContext.TUPLE_CLASS, POJOGenerator.SalesEvent.class);
+ dag.setInputPortAttribute(join.input2, Context.PortContext.TUPLE_CLASS, POJOGenerator.ProductEvent.class);
+ dag.setOutputPortAttribute(join.outputPort,Context.PortContext.TUPLE_CLASS, POJOGenerator.SalesEvent.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/innerjoin/src/main/java/org/apache/apex/examples/innerjoin/POJOGenerator.java
----------------------------------------------------------------------
diff --git a/examples/innerjoin/src/main/java/org/apache/apex/examples/innerjoin/POJOGenerator.java b/examples/innerjoin/src/main/java/org/apache/apex/examples/innerjoin/POJOGenerator.java
new file mode 100644
index 0000000..2a736c5
--- /dev/null
+++ b/examples/innerjoin/src/main/java/org/apache/apex/examples/innerjoin/POJOGenerator.java
@@ -0,0 +1,260 @@
+package org.apache.apex.examples.innerjoin;
+
+import java.util.Random;
+
+import javax.validation.constraints.Min;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+
+/**
+ * Generates and emits the SalesEvent/ProductEvent based on isSalesEvent.
+ */
+public class POJOGenerator implements InputOperator
+{
+ @Min(1)
+ private int maxProductId = 100000;
+ @Min(1)
+ private int maxCustomerId = 100000;
+ @Min(1)
+ private int maxProductCategories = 100;
+ private double maxAmount = 100.0;
+ private long tuplesCounter;
+ private long time;
+ private long timeIncrement;
+ private boolean isSalesEvent = true;
+ // Limit number of emitted tuples per window
+ @Min(0)
+ private long maxTuplesPerWindow = 100;
+ private final Random random = new Random();
+ public final transient DefaultOutputPort<Object> output = new DefaultOutputPort<>();
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ tuplesCounter = 0;
+ }
+
+ @Override
+ public void endWindow()
+ {
+ time += timeIncrement;
+ }
+
+ @Override
+ public void setup(Context.OperatorContext context)
+ {
+ time = System.currentTimeMillis();
+ timeIncrement = context.getValue(Context.OperatorContext.APPLICATION_WINDOW_COUNT) *
+ context.getValue(Context.DAGContext.STREAMING_WINDOW_SIZE_MILLIS);
+ }
+
+ @Override
+ public void teardown()
+ {
+
+ }
+
+ SalesEvent generateSalesEvent() throws Exception {
+
+ SalesEvent salesEvent = new SalesEvent();
+ salesEvent.productId = randomId(maxProductId);
+ salesEvent.customerId = randomId(maxCustomerId);
+ salesEvent.amount = randomAmount();
+ salesEvent.timestamp = time;
+ return salesEvent;
+ }
+
+ ProductEvent generateProductEvent() throws Exception {
+ ProductEvent productEvent = new ProductEvent();
+ productEvent.productId = randomId(maxProductId);
+ productEvent.productCategory = randomId(maxProductCategories);
+ productEvent.timestamp = time;
+ return productEvent;
+ }
+
+ private int randomId(int max) {
+ if (max < 1) return 1;
+ return 1 + random.nextInt(max);
+ }
+
+ private double randomAmount() {
+ return maxAmount * random.nextDouble();
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ while (tuplesCounter++ < maxTuplesPerWindow) {
+ try {
+ if (isSalesEvent) {
+ SalesEvent event = generateSalesEvent();
+ this.output.emit(event);
+ } else {
+ ProductEvent event = generateProductEvent();
+ this.output.emit(event);
+ }
+
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+ }
+
+ public static class SalesEvent
+ {
+ public int customerId;
+ public int productId;
+ public int productCategory;
+ public double amount;
+ public long timestamp;
+
+ public int getCustomerId()
+ {
+ return customerId;
+ }
+
+ public void setCustomerId(int customerId)
+ {
+ this.customerId = customerId;
+ }
+
+ public int getProductId()
+ {
+ return productId;
+ }
+
+ public void setProductId(int productId)
+ {
+ this.productId = productId;
+ }
+
+ public int getProductCategory()
+ {
+ return productCategory;
+ }
+
+ public void setProductCategory(int productCategory)
+ {
+ this.productCategory = productCategory;
+ }
+
+ public double getAmount()
+ {
+ return amount;
+ }
+
+ public void setAmount(double amount)
+ {
+ this.amount = amount;
+ }
+
+ public long getTimestamp()
+ {
+ return timestamp;
+ }
+
+ public void setTimestamp(long timestamp)
+ {
+ this.timestamp = timestamp;
+ }
+ }
+
+ public static class ProductEvent
+ {
+ public int productId;
+ public int productCategory;
+ public long timestamp;
+
+ public int getProductId()
+ {
+ return productId;
+ }
+
+ public void setProductId(int productId)
+ {
+ this.productId = productId;
+ }
+
+ public int getProductCategory()
+ {
+ return productCategory;
+ }
+
+ public void setProductCategory(int productCategory)
+ {
+ this.productCategory = productCategory;
+ }
+
+ public long getTimestamp()
+ {
+ return timestamp;
+ }
+
+ public void setTimestamp(long timestamp)
+ {
+ this.timestamp = timestamp;
+ }
+ }
+
+ public int getMaxProductId()
+ {
+ return maxProductId;
+ }
+
+ public void setMaxProductId(int maxProductId)
+ {
+ this.maxProductId = maxProductId;
+ }
+
+ public int getMaxCustomerId()
+ {
+ return maxCustomerId;
+ }
+
+ public void setMaxCustomerId(int maxCustomerId)
+ {
+ this.maxCustomerId = maxCustomerId;
+ }
+
+ public int getMaxProductCategories()
+ {
+ return maxProductCategories;
+ }
+
+ public void setMaxProductCategories(int maxProductCategories)
+ {
+ this.maxProductCategories = maxProductCategories;
+ }
+
+ public double getMaxAmount()
+ {
+ return maxAmount;
+ }
+
+ public void setMaxAmount(double maxAmount)
+ {
+ this.maxAmount = maxAmount;
+ }
+
+ public boolean isSalesEvent()
+ {
+ return isSalesEvent;
+ }
+
+ public void setSalesEvent(boolean salesEvent)
+ {
+ isSalesEvent = salesEvent;
+ }
+
+ public long getMaxTuplesPerWindow()
+ {
+ return maxTuplesPerWindow;
+ }
+
+ public void setMaxTuplesPerWindow(long maxTuplesPerWindow)
+ {
+ this.maxTuplesPerWindow = maxTuplesPerWindow;
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/innerjoin/src/test/java/com/example/join/InnerJoinApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/innerjoin/src/test/java/com/example/join/InnerJoinApplicationTest.java b/examples/innerjoin/src/test/java/com/example/join/InnerJoinApplicationTest.java
deleted file mode 100644
index 6da3908..0000000
--- a/examples/innerjoin/src/test/java/com/example/join/InnerJoinApplicationTest.java
+++ /dev/null
@@ -1,21 +0,0 @@
-package com.example.join;
-
-import org.junit.Test;
-import org.apache.hadoop.conf.Configuration;
-import com.datatorrent.api.LocalMode;
-
-public class InnerJoinApplicationTest
-{
- @Test
- public void testApplication() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
- lma.prepareDAG(new InnerJoinApplication(), conf);
- LocalMode.Controller lc = lma.getController();
- lc.runAsync();
- Thread.sleep(10 * 1000);
- lc.shutdown();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/innerjoin/src/test/java/org/apache/apex/examples/innerjoin/InnerJoinApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/innerjoin/src/test/java/org/apache/apex/examples/innerjoin/InnerJoinApplicationTest.java b/examples/innerjoin/src/test/java/org/apache/apex/examples/innerjoin/InnerJoinApplicationTest.java
new file mode 100644
index 0000000..81ae442
--- /dev/null
+++ b/examples/innerjoin/src/test/java/org/apache/apex/examples/innerjoin/InnerJoinApplicationTest.java
@@ -0,0 +1,21 @@
+package org.apache.apex.examples.innerjoin;
+
+import org.junit.Test;
+import org.apache.hadoop.conf.Configuration;
+import com.datatorrent.api.LocalMode;
+
+public class InnerJoinApplicationTest
+{
+ @Test
+ public void testApplication() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ lma.prepareDAG(new InnerJoinApplication(), conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.runAsync();
+ Thread.sleep(10 * 1000);
+ lc.shutdown();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/parser/pom.xml
----------------------------------------------------------------------
diff --git a/examples/parser/pom.xml b/examples/parser/pom.xml
index 4eefe75..ec03cae 100644
--- a/examples/parser/pom.xml
+++ b/examples/parser/pom.xml
@@ -1,261 +1,50 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
-
- <groupId>com.example</groupId>
- <version>1.0-SNAPSHOT</version>
- <artifactId>parser</artifactId>
+
+ <parent>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-examples</artifactId>
+ <version>3.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>malhar-examples-parser</artifactId>
<packaging>jar</packaging>
<!-- change these to the appropriate values -->
<name>Parser Apps</name>
<description>Applications to showcase different parsers</description>
- <properties>
- <!-- change this if you desire to use a different version of Apex Core -->
- <apex.version>3.5.0</apex.version>
- <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath>
- <malhar.version>3.7.0-SNAPSHOT</malhar.version>
- </properties>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-eclipse-plugin</artifactId>
- <version>2.9</version>
- <configuration>
- <downloadSources>true</downloadSources>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>3.3</version>
- <configuration>
- <encoding>UTF-8</encoding>
- <source>1.7</source>
- <target>1.7</target>
- <debug>true</debug>
- <optimize>false</optimize>
- <showDeprecation>true</showDeprecation>
- <showWarnings>true</showWarnings>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <version>2.8</version>
- <executions>
- <execution>
- <id>copy-dependencies</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>copy-dependencies</goal>
- </goals>
- <configuration>
- <outputDirectory>target/deps</outputDirectory>
- <includeScope>runtime</includeScope>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <executions>
- <execution>
- <id>app-package-assembly</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <finalName>${project.artifactId}-${project.version}-apexapp</finalName>
- <appendAssemblyId>false</appendAssemblyId>
- <descriptors>
- <descriptor>src/assemble/appPackage.xml</descriptor>
- </descriptors>
- <archiverConfig>
- <defaultDirectoryMode>0755</defaultDirectoryMode>
- </archiverConfig>
- <archive>
- <manifestEntries>
- <Class-Path>${apex.apppackage.classpath}</Class-Path>
- <DT-Engine-Version>${apex.version}</DT-Engine-Version>
- <DT-App-Package-Group-Id>${project.groupId}</DT-App-Package-Group-Id>
- <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name>
- <DT-App-Package-Version>${project.version}</DT-App-Package-Version>
- <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name>
- <DT-App-Package-Description>${project.description}</DT-App-Package-Description>
- </manifestEntries>
- </archive>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <artifactId>maven-antrun-plugin</artifactId>
- <version>1.7</version>
- <executions>
- <execution>
- <phase>package</phase>
- <configuration>
- <target>
- <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar"
- tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa" />
- </target>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- <execution>
- <!-- create resource directory for xml javadoc-->
- <id>createJavadocDirectory</id>
- <phase>generate-resources</phase>
- <configuration>
- <tasks>
- <delete dir="${project.build.directory}/generated-resources/xml-javadoc"/>
- <mkdir dir="${project.build.directory}/generated-resources/xml-javadoc"/>
- </tasks>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>1.9.1</version>
- <executions>
- <execution>
- <id>attach-artifacts</id>
- <phase>package</phase>
- <goals>
- <goal>attach-artifact</goal>
- </goals>
- <configuration>
- <artifacts>
- <artifact>
- <file>target/${project.artifactId}-${project.version}.apa</file>
- <type>apa</type>
- </artifact>
- </artifacts>
- <skipAttach>false</skipAttach>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <!-- generate javdoc -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <executions>
- <!-- generate xml javadoc -->
- <execution>
- <id>xml-doclet</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>javadoc</goal>
- </goals>
- <configuration>
- <doclet>com.github.markusbernhardt.xmldoclet.XmlDoclet</doclet>
- <additionalparam>-d ${project.build.directory}/generated-resources/xml-javadoc -filename ${project.artifactId}-${project.version}-javadoc.xml</additionalparam>
- <useStandardDocletOptions>false</useStandardDocletOptions>
- <docletArtifact>
- <groupId>com.github.markusbernhardt</groupId>
- <artifactId>xml-doclet</artifactId>
- <version>1.0.4</version>
- </docletArtifact>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <!-- Transform xml javadoc to stripped down version containing only class/interface comments and tags-->
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>xml-maven-plugin</artifactId>
- <version>1.0</version>
- <executions>
- <execution>
- <id>transform-xmljavadoc</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>transform</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <transformationSets>
- <transformationSet>
- <dir>${project.build.directory}/generated-resources/xml-javadoc</dir>
- <includes>
- <include>${pmalhar.versioroject.artifactId}-${project.version}-javadoc.xml</include>
- </includes>
- <stylesheet>XmlJavadocCommentsExtractor.xsl</stylesheet>
- <outputDir>${project.build.directory}/generated-resources/xml-javadoc</outputDir>
- </transformationSet>
- </transformationSets>
- </configuration>
- </plugin>
- <!-- copy xml javadoc to class jar -->
- <plugin>
- <artifactId>maven-resources-plugin</artifactId>
- <version>2.6</version>
- <executions>
- <execution>
- <id>copy-resources</id>
- <phase>process-resources</phase>
- <goals>
- <goal>copy-resources</goal>
- </goals>
- <configuration>
- <outputDirectory>${basedir}/target/classes</outputDirectory>
- <resources>
- <resource>
- <directory>${project.build.directory}/generated-resources/xml-javadoc</directory>
- <includes>
- <include>${project.artifactId}-${project.version}-javadoc.xml</include>
- </includes>
- <filtering>true</filtering>
- </resource>
- </resources>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
-
- </build>
-
<dependencies>
- <!-- add your dependencies here -->
<dependency>
<groupId>org.apache.apex</groupId>
<artifactId>malhar-library</artifactId>
- <version>${malhar.version}</version>
- <!--
+ <version>${project.version}</version>
+ <!--
If you know that your application does not need transitive dependencies pulled in by malhar-library,
uncomment the following to reduce the size of your app package.
-->
- <!--
<exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
- -->
</dependency>
<dependency>
<groupId>org.apache.apex</groupId>
<artifactId>malhar-contrib</artifactId>
- <version>${malhar.version}</version>
+ <version>${project.version}</version>
+ <!--
+ If you know that your application does not need transitive dependencies pulled in by malhar-library,
+ uncomment the following to reduce the size of your app package.
+ -->
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
@@ -288,22 +77,9 @@
</dependency>
<dependency>
<groupId>org.apache.apex</groupId>
- <artifactId>apex-common</artifactId>
- <version>${apex.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.10</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.apex</groupId>
<artifactId>apex-engine</artifactId>
- <version>${apex.version}</version>
+ <version>${apex.core.version}</version>
<scope>provided</scope>
</dependency>
</dependencies>
-
</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/Application.java
----------------------------------------------------------------------
diff --git a/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/Application.java b/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/Application.java
deleted file mode 100644
index e75541d..0000000
--- a/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/Application.java
+++ /dev/null
@@ -1,35 +0,0 @@
-package com.datatorrent.tutorial.jsonparser;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.contrib.parser.JsonParser;
-import com.datatorrent.lib.formatter.JsonFormatter;
-import com.datatorrent.lib.io.ConsoleOutputOperator;
-
-@ApplicationAnnotation(name = "JsonProcessor")
-public class Application implements StreamingApplication
-{
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
-
- JsonGenerator generator = dag.addOperator("JsonGenerator", JsonGenerator.class);
- JsonParser parser = dag.addOperator("JsonParser", JsonParser.class);
- JsonFormatter formatter = dag.addOperator("JsonFormatter", JsonFormatter.class);
-
- ConsoleOutputOperator jsonString = dag.addOperator("JsonString", ConsoleOutputOperator.class);
- ConsoleOutputOperator jsonObject = dag.addOperator("JsonObject", ConsoleOutputOperator.class);
- ConsoleOutputOperator error = dag.addOperator("Error", ConsoleOutputOperator.class);
-
- dag.addStream("json", generator.out, parser.in);
- dag.addStream("pojo", parser.out, formatter.in);
- dag.addStream("jsonString", formatter.out, jsonString.input);
- dag.addStream("jsonObject", parser.parsedOutput, jsonObject.input);
- dag.addStream("error", parser.err, error.input);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/Campaign.java
----------------------------------------------------------------------
diff --git a/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/Campaign.java b/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/Campaign.java
deleted file mode 100644
index f0245eb..0000000
--- a/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/Campaign.java
+++ /dev/null
@@ -1,74 +0,0 @@
-package com.datatorrent.tutorial.jsonparser;
-
-import java.util.Date;
-
-import com.fasterxml.jackson.annotation.JsonFormat;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-public class Campaign
-{
- private int adId;
- private String campaignName;
- @JsonProperty("budget")
- private double campaignBudget;
- private boolean weatherTargeting;
- @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy/MM/dd")
- private Date startDate;
-
- public int getAdId()
- {
- return adId;
- }
-
- public void setAdId(int adId)
- {
- this.adId = adId;
- }
-
- public String getCampaignName()
- {
- return campaignName;
- }
-
- public void setCampaignName(String campaignName)
- {
- this.campaignName = campaignName;
- }
-
- public double getCampaignBudget()
- {
- return campaignBudget;
- }
-
- public void setCampaignBudget(double campaignBudget)
- {
- this.campaignBudget = campaignBudget;
- }
-
- public boolean isWeatherTargeting()
- {
- return weatherTargeting;
- }
-
- public void setWeatherTargeting(boolean weatherTargeting)
- {
- this.weatherTargeting = weatherTargeting;
- }
-
- public Date getStartDate()
- {
- return startDate;
- }
-
- public void setStartDate(Date startDate)
- {
- this.startDate = startDate;
- }
-
- @Override
- public String toString()
- {
- return "Campaign [adId=" + adId + ", campaignName=" + campaignName + ", campaignBudget=" + campaignBudget
- + ", weatherTargeting=" + weatherTargeting + ", startDate=" + startDate + "]";
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/JsonGenerator.java
----------------------------------------------------------------------
diff --git a/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/JsonGenerator.java b/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/JsonGenerator.java
deleted file mode 100644
index 90beb13..0000000
--- a/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/JsonGenerator.java
+++ /dev/null
@@ -1,83 +0,0 @@
-package com.datatorrent.tutorial.jsonparser;
-
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.Random;
-
-import javax.validation.constraints.Min;
-
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.InputOperator;
-import com.datatorrent.common.util.BaseOperator;
-
-public class JsonGenerator extends BaseOperator implements InputOperator
-{
-
- private static final Logger LOG = LoggerFactory.getLogger(JsonGenerator.class);
-
- @Min(1)
- private int numTuples = 20;
- private transient int count = 0;
-
- public static Random rand = new Random();
- public static SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd");
- public static int[] adId = { 1, 2, 3, 4, 5 };
- public static String[] campaignName = { "cmp1", "cmp2", "cmp3", "cmp4" };
- public static double[] campaignBudget = { 10000.0, 20000.0, 300000.0 };
- public static boolean[] weatherTargeting = { true, false };
- private int sleepTime;
-
- public final transient DefaultOutputPort<byte[]> out = new DefaultOutputPort<byte[]>();
-
- private static String getNext(int num)
- {
-
- JSONObject obj = new JSONObject();
- try {
- obj.put("adId", adId[num % adId.length]);
- obj.put("campaignName", campaignName[num % campaignName.length]);
- obj.put("campaignBudget", campaignBudget[num % campaignBudget.length]);
- obj.put("weatherTargeting", weatherTargeting[num % weatherTargeting.length]);
- obj.put("startDate", sdf.format(new Date()));
- } catch (JSONException e) {
- return null;
- }
- return obj.toString();
- }
-
- @Override
- public void beginWindow(long windowId)
- {
- count = 0;
- }
-
- @Override
- public void emitTuples()
- {
- if (count++ < numTuples) {
- out.emit(getNext(rand.nextInt(numTuples) + 1).getBytes());
- } else {
- try {
- Thread.sleep(sleepTime);
- } catch (InterruptedException e) {
- LOG.info("Sleep interrupted");
- }
- }
- }
-
- public int getNumTuples()
- {
- return numTuples;
- }
-
- public void setNumTuples(int numTuples)
- {
- this.numTuples = numTuples;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/parser/src/main/java/org/apache/apex/examples/parser/jsonparser/Application.java
----------------------------------------------------------------------
diff --git a/examples/parser/src/main/java/org/apache/apex/examples/parser/jsonparser/Application.java b/examples/parser/src/main/java/org/apache/apex/examples/parser/jsonparser/Application.java
new file mode 100644
index 0000000..34b64a3
--- /dev/null
+++ b/examples/parser/src/main/java/org/apache/apex/examples/parser/jsonparser/Application.java
@@ -0,0 +1,35 @@
+package org.apache.apex.examples.parser.jsonparser;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.contrib.parser.JsonParser;
+import com.datatorrent.lib.formatter.JsonFormatter;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+
+@ApplicationAnnotation(name = "JsonProcessor")
+public class Application implements StreamingApplication
+{
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+
+ JsonGenerator generator = dag.addOperator("JsonGenerator", JsonGenerator.class);
+ JsonParser parser = dag.addOperator("JsonParser", JsonParser.class);
+ JsonFormatter formatter = dag.addOperator("JsonFormatter", JsonFormatter.class);
+
+ ConsoleOutputOperator jsonString = dag.addOperator("JsonString", ConsoleOutputOperator.class);
+ ConsoleOutputOperator jsonObject = dag.addOperator("JsonObject", ConsoleOutputOperator.class);
+ ConsoleOutputOperator error = dag.addOperator("Error", ConsoleOutputOperator.class);
+
+ dag.addStream("json", generator.out, parser.in);
+ dag.addStream("pojo", parser.out, formatter.in);
+ dag.addStream("jsonString", formatter.out, jsonString.input);
+ dag.addStream("jsonObject", parser.parsedOutput, jsonObject.input);
+ dag.addStream("error", parser.err, error.input);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/parser/src/main/java/org/apache/apex/examples/parser/jsonparser/Campaign.java
----------------------------------------------------------------------
diff --git a/examples/parser/src/main/java/org/apache/apex/examples/parser/jsonparser/Campaign.java b/examples/parser/src/main/java/org/apache/apex/examples/parser/jsonparser/Campaign.java
new file mode 100644
index 0000000..8946f5d
--- /dev/null
+++ b/examples/parser/src/main/java/org/apache/apex/examples/parser/jsonparser/Campaign.java
@@ -0,0 +1,74 @@
+package org.apache.apex.examples.parser.jsonparser;
+
+import java.util.Date;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class Campaign
+{
+ private int adId;
+ private String campaignName;
+ @JsonProperty("budget")
+ private double campaignBudget;
+ private boolean weatherTargeting;
+ @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy/MM/dd")
+ private Date startDate;
+
+ public int getAdId()
+ {
+ return adId;
+ }
+
+ public void setAdId(int adId)
+ {
+ this.adId = adId;
+ }
+
+ public String getCampaignName()
+ {
+ return campaignName;
+ }
+
+ public void setCampaignName(String campaignName)
+ {
+ this.campaignName = campaignName;
+ }
+
+ public double getCampaignBudget()
+ {
+ return campaignBudget;
+ }
+
+ public void setCampaignBudget(double campaignBudget)
+ {
+ this.campaignBudget = campaignBudget;
+ }
+
+ public boolean isWeatherTargeting()
+ {
+ return weatherTargeting;
+ }
+
+ public void setWeatherTargeting(boolean weatherTargeting)
+ {
+ this.weatherTargeting = weatherTargeting;
+ }
+
+ public Date getStartDate()
+ {
+ return startDate;
+ }
+
+ public void setStartDate(Date startDate)
+ {
+ this.startDate = startDate;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "Campaign [adId=" + adId + ", campaignName=" + campaignName + ", campaignBudget=" + campaignBudget
+ + ", weatherTargeting=" + weatherTargeting + ", startDate=" + startDate + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/parser/src/main/java/org/apache/apex/examples/parser/jsonparser/JsonGenerator.java
----------------------------------------------------------------------
diff --git a/examples/parser/src/main/java/org/apache/apex/examples/parser/jsonparser/JsonGenerator.java b/examples/parser/src/main/java/org/apache/apex/examples/parser/jsonparser/JsonGenerator.java
new file mode 100644
index 0000000..a0e3611
--- /dev/null
+++ b/examples/parser/src/main/java/org/apache/apex/examples/parser/jsonparser/JsonGenerator.java
@@ -0,0 +1,83 @@
+package org.apache.apex.examples.parser.jsonparser;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Random;
+
+import javax.validation.constraints.Min;
+
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.common.util.BaseOperator;
+
+public class JsonGenerator extends BaseOperator implements InputOperator
+{
+
+ private static final Logger LOG = LoggerFactory.getLogger(JsonGenerator.class);
+
+ @Min(1)
+ private int numTuples = 20;
+ private transient int count = 0;
+
+ public static Random rand = new Random();
+ public static SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd");
+ public static int[] adId = { 1, 2, 3, 4, 5 };
+ public static String[] campaignName = { "cmp1", "cmp2", "cmp3", "cmp4" };
+ public static double[] campaignBudget = { 10000.0, 20000.0, 300000.0 };
+ public static boolean[] weatherTargeting = { true, false };
+ private int sleepTime;
+
+ public final transient DefaultOutputPort<byte[]> out = new DefaultOutputPort<byte[]>();
+
+ private static String getNext(int num)
+ {
+
+ JSONObject obj = new JSONObject();
+ try {
+ obj.put("adId", adId[num % adId.length]);
+ obj.put("campaignName", campaignName[num % campaignName.length]);
+ obj.put("campaignBudget", campaignBudget[num % campaignBudget.length]);
+ obj.put("weatherTargeting", weatherTargeting[num % weatherTargeting.length]);
+ obj.put("startDate", sdf.format(new Date()));
+ } catch (JSONException e) {
+ return null;
+ }
+ return obj.toString();
+ }
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ count = 0;
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ if (count++ < numTuples) {
+ out.emit(getNext(rand.nextInt(numTuples) + 1).getBytes());
+ } else {
+ try {
+ Thread.sleep(sleepTime);
+ } catch (InterruptedException e) {
+ LOG.info("Sleep interrupted");
+ }
+ }
+ }
+
+ public int getNumTuples()
+ {
+ return numTuples;
+ }
+
+ public void setNumTuples(int numTuples)
+ {
+ this.numTuples = numTuples;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/parser/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/parser/src/main/resources/META-INF/properties.xml b/examples/parser/src/main/resources/META-INF/properties.xml
index d8ac178..2cd3c04 100644
--- a/examples/parser/src/main/resources/META-INF/properties.xml
+++ b/examples/parser/src/main/resources/META-INF/properties.xml
@@ -15,12 +15,12 @@
<property>
<name>dt.application.JsonProcessor.operator.JsonParser.port.out.attr.TUPLE_CLASS
</name>
- <value>com.datatorrent.tutorial.jsonparser.Campaign</value>
+ <value>org.apache.apex.examples.parser.jsonparser.Campaign</value>
</property>
<property>
<name>dt.application.JsonProcessor.operator.JsonFormatter.port.in.attr.TUPLE_CLASS
</name>
- <value>com.datatorrent.tutorial.jsonparser.Campaign</value>
+ <value>org.apache.apex.examples.parser.jsonparser.Campaign</value>
</property>
<property>
<name>dt.application.JsonProcessor.operator.JsonString.prop.stringFormat
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/parser/src/test/java/com/datatorrent/tutorial/jsonparser/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/parser/src/test/java/com/datatorrent/tutorial/jsonparser/ApplicationTest.java b/examples/parser/src/test/java/com/datatorrent/tutorial/jsonparser/ApplicationTest.java
deleted file mode 100644
index 390d4cb..0000000
--- a/examples/parser/src/test/java/com/datatorrent/tutorial/jsonparser/ApplicationTest.java
+++ /dev/null
@@ -1,36 +0,0 @@
-package com.datatorrent.tutorial.jsonparser;
-
-import java.io.IOException;
-
-import javax.validation.ConstraintViolationException;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.LocalMode;
-import com.datatorrent.tutorial.jsonparser.Application;
-
-/**
- * Test the DAG declaration in local mode.
- */
-public class ApplicationTest
-{
-
- @Test
- public void testApplication() throws IOException, Exception
- {
-
- try {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
- lma.prepareDAG(new Application(), conf);
- LocalMode.Controller lc = lma.getController();
- lc.run(10 * 1000); // runs for 30 seconds and quits
- } catch (ConstraintViolationException e) {
- Assert.fail("constraint violations: " + e.getConstraintViolations());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/parser/src/test/java/org/apache/apex/examples/parser/jsonparser/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/parser/src/test/java/org/apache/apex/examples/parser/jsonparser/ApplicationTest.java b/examples/parser/src/test/java/org/apache/apex/examples/parser/jsonparser/ApplicationTest.java
new file mode 100644
index 0000000..6f1a6f6
--- /dev/null
+++ b/examples/parser/src/test/java/org/apache/apex/examples/parser/jsonparser/ApplicationTest.java
@@ -0,0 +1,35 @@
+package org.apache.apex.examples.parser.jsonparser;
+
+import java.io.IOException;
+
+import javax.validation.ConstraintViolationException;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.LocalMode;
+
+/**
+ * Test the DAG declaration in local mode.
+ */
+public class ApplicationTest
+{
+
+ @Test
+ public void testApplication() throws IOException, Exception
+ {
+
+ try {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ lma.prepareDAG(new Application(), conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.run(10 * 1000); // runs for 30 seconds and quits
+ } catch (ConstraintViolationException e) {
+ Assert.fail("constraint violations: " + e.getConstraintViolations());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/partition/pom.xml
----------------------------------------------------------------------
diff --git a/examples/partition/pom.xml b/examples/partition/pom.xml
index ac15981..cafdcd7 100644
--- a/examples/partition/pom.xml
+++ b/examples/partition/pom.xml
@@ -1,274 +1,18 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
-
- <groupId>com.example</groupId>
- <version>1.0-SNAPSHOT</version>
- <artifactId>Test-Ram</artifactId>
- <packaging>jar</packaging>
-
- <!-- change these to the appropriate values -->
- <name>Test_ram</name>
- <description>Test_ram</description>
-
- <properties>
- <!-- change this if you desire to use a different version of Apex core -->
- <apex.version>3.5.0</apex.version>
- <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath>
- </properties>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-eclipse-plugin</artifactId>
- <version>2.9</version>
- <configuration>
- <downloadSources>true</downloadSources>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>3.3</version>
- <configuration>
- <encoding>UTF-8</encoding>
- <source>1.7</source>
- <target>1.7</target>
- <debug>true</debug>
- <optimize>false</optimize>
- <showDeprecation>true</showDeprecation>
- <showWarnings>true</showWarnings>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <version>2.8</version>
- <executions>
- <execution>
- <id>copy-dependencies</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>copy-dependencies</goal>
- </goals>
- <configuration>
- <outputDirectory>target/deps</outputDirectory>
- <includeScope>runtime</includeScope>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <executions>
- <execution>
- <id>app-package-assembly</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <finalName>${project.artifactId}-${project.version}-apexapp</finalName>
- <appendAssemblyId>false</appendAssemblyId>
- <descriptors>
- <descriptor>src/assemble/appPackage.xml</descriptor>
- </descriptors>
- <archiverConfig>
- <defaultDirectoryMode>0755</defaultDirectoryMode>
- </archiverConfig>
- <archive>
- <manifestEntries>
- <Class-Path>${apex.apppackage.classpath}</Class-Path>
- <DT-Engine-Version>${apex.version}</DT-Engine-Version>
- <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name>
- <DT-App-Package-Version>${project.version}</DT-App-Package-Version>
- <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name>
- <DT-App-Package-Description>${project.description}</DT-App-Package-Description>
- </manifestEntries>
- </archive>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <artifactId>maven-antrun-plugin</artifactId>
- <version>1.7</version>
- <executions>
- <execution>
- <phase>package</phase>
- <configuration>
- <target>
- <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar"
- tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa" />
- </target>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- <execution>
- <!-- create resource directory for xml javadoc-->
- <id>createJavadocDirectory</id>
- <phase>generate-resources</phase>
- <configuration>
- <tasks>
- <delete dir="${project.build.directory}/generated-resources/xml-javadoc"/>
- <mkdir dir="${project.build.directory}/generated-resources/xml-javadoc"/>
- </tasks>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
+ <parent>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-examples</artifactId>
+ <version>3.7.0-SNAPSHOT</version>
+ </parent>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>1.9.1</version>
- <executions>
- <execution>
- <id>attach-artifacts</id>
- <phase>package</phase>
- <goals>
- <goal>attach-artifact</goal>
- </goals>
- <configuration>
- <artifacts>
- <artifact>
- <file>target/${project.artifactId}-${project.version}.apa</file>
- <type>apa</type>
- </artifact>
- </artifacts>
- <skipAttach>false</skipAttach>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <!-- generate javdoc -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <executions>
- <!-- generate xml javadoc -->
- <execution>
- <id>xml-doclet</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>javadoc</goal>
- </goals>
- <configuration>
- <doclet>com.github.markusbernhardt.xmldoclet.XmlDoclet</doclet>
- <additionalparam>-d ${project.build.directory}/generated-resources/xml-javadoc -filename ${project.artifactId}-${project.version}-javadoc.xml</additionalparam>
- <useStandardDocletOptions>false</useStandardDocletOptions>
- <docletArtifact>
- <groupId>com.github.markusbernhardt</groupId>
- <artifactId>xml-doclet</artifactId>
- <version>1.0.4</version>
- </docletArtifact>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <!-- Transform xml javadoc to stripped down version containing only class/interface comments and tags-->
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>xml-maven-plugin</artifactId>
- <version>1.0</version>
- <executions>
- <execution>
- <id>transform-xmljavadoc</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>transform</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <transformationSets>
- <transformationSet>
- <dir>${project.build.directory}/generated-resources/xml-javadoc</dir>
- <includes>
- <include>${project.artifactId}-${project.version}-javadoc.xml</include>
- </includes>
- <stylesheet>XmlJavadocCommentsExtractor.xsl</stylesheet>
- <outputDir>${project.build.directory}/generated-resources/xml-javadoc</outputDir>
- </transformationSet>
- </transformationSets>
- </configuration>
- </plugin>
- <!-- copy xml javadoc to class jar -->
- <plugin>
- <artifactId>maven-resources-plugin</artifactId>
- <version>2.6</version>
- <executions>
- <execution>
- <id>copy-resources</id>
- <phase>process-resources</phase>
- <goals>
- <goal>copy-resources</goal>
- </goals>
- <configuration>
- <outputDirectory>${basedir}/target/classes</outputDirectory>
- <resources>
- <resource>
- <directory>${project.build.directory}/generated-resources/xml-javadoc</directory>
- <includes>
- <include>${project.artifactId}-${project.version}-javadoc.xml</include>
- </includes>
- <filtering>true</filtering>
- </resource>
- </resources>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
-
- </build>
-
- <dependencies>
- <!-- add your dependencies here -->
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-library</artifactId>
- <version>3.6.0</version>
+ <artifactId>malhar-examples-partition</artifactId>
+ <packaging>jar</packaging>
- <!--
- If you know that your application does not need transitive dependencies pulled in by malhar-library,
- uncomment the following to reduce the size of your app package.
- -->
- <!--
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- -->
- </dependency>
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>apex-common</artifactId>
- <version>${apex.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.10</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>apex-engine</artifactId>
- <version>${apex.version}</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
+ <!-- change these to the appropriate values -->
+ <name>Partition</name>
+ <description>malhar-examples-partition</description>
</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/partition/src/main/java/com/example/myapexapp/Application.java
----------------------------------------------------------------------
diff --git a/examples/partition/src/main/java/com/example/myapexapp/Application.java b/examples/partition/src/main/java/com/example/myapexapp/Application.java
deleted file mode 100644
index e1ca2ff..0000000
--- a/examples/partition/src/main/java/com/example/myapexapp/Application.java
+++ /dev/null
@@ -1,27 +0,0 @@
-package com.example.myapexapp;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.Context.PortContext;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.DAG.Locality;
-import com.datatorrent.lib.io.ConsoleOutputOperator;
-
-@ApplicationAnnotation(name="TestStuff")
-public class Application implements StreamingApplication
-{
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- RandomNumberGenerator random = dag.addOperator("randomInt", RandomNumberGenerator.class);
- TestPartition testPartition = dag.addOperator("testPartition", TestPartition.class);
- Codec3 codec = new Codec3();
- dag.setInputPortAttribute(testPartition.in, PortContext.STREAM_CODEC, codec);
-
- //Add locality if needed, e.g.: .setLocality(Locality.CONTAINER_LOCAL);
- dag.addStream("randomData", random.out, testPartition.in);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/partition/src/main/java/com/example/myapexapp/Codec3.java
----------------------------------------------------------------------
diff --git a/examples/partition/src/main/java/com/example/myapexapp/Codec3.java b/examples/partition/src/main/java/com/example/myapexapp/Codec3.java
deleted file mode 100644
index 2754e9b..0000000
--- a/examples/partition/src/main/java/com/example/myapexapp/Codec3.java
+++ /dev/null
@@ -1,13 +0,0 @@
-package com.example.myapexapp;
-
-import com.datatorrent.lib.codec.KryoSerializableStreamCodec;
-
-public class Codec3 extends KryoSerializableStreamCodec<Integer> {
- @Override
- public int getPartition(Integer tuple) {
- final int v = tuple;
- return (1 == (v & 1)) ? 0 // odd
- : (0 == (v & 3)) ? 1 // divisible by 4
- : 2; // divisible by 2 but not 4
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/partition/src/main/java/com/example/myapexapp/RandomNumberGenerator.java
----------------------------------------------------------------------
diff --git a/examples/partition/src/main/java/com/example/myapexapp/RandomNumberGenerator.java b/examples/partition/src/main/java/com/example/myapexapp/RandomNumberGenerator.java
deleted file mode 100644
index de2797b..0000000
--- a/examples/partition/src/main/java/com/example/myapexapp/RandomNumberGenerator.java
+++ /dev/null
@@ -1,83 +0,0 @@
-package com.example.myapexapp;
-
-import java.util.Random;
-
-import javax.validation.constraints.Min;
-import javax.validation.constraints.Size;
-import javax.validation.ConstraintViolation;
-import javax.validation.ValidatorFactory;
-import javax.validation.Validator;
-import javax.validation.Validation;
-
-import com.datatorrent.api.Attribute;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.InputOperator;
-import com.datatorrent.common.util.BaseOperator;
-import com.datatorrent.api.Context;
-import com.datatorrent.api.Context.OperatorContext;
-
-/**
- * This is a simple operator that emits random integer.
- */
-public class RandomNumberGenerator extends BaseOperator implements InputOperator
-{
- private static final Logger LOG = LoggerFactory.getLogger(RandomNumberGenerator.class);
-
- @Min(1)
- private int numTuples = 20;
- private transient int count = 0;
-
- private int sleepTime;
- private transient long curWindowId;
- private transient Random random = new Random();
-
- public final transient DefaultOutputPort<Integer> out = new DefaultOutputPort<Integer>();
-
- @Override
- public void setup(Context.OperatorContext context)
- {
- super.setup(context);
-
- long appWindowId = context.getValue(context.ACTIVATION_WINDOW_ID);
- sleepTime = context.getValue(context.SPIN_MILLIS);
- LOG.debug("Started setup, appWindowId = {}, sleepTime = {}", appWindowId, sleepTime);
- }
-
- @Override
- public void beginWindow(long windowId)
- {
- count = 0;
- LOG.debug("beginWindow: windowId = {}", windowId);
- }
-
- @Override
- public void emitTuples()
- {
- if (count++ < numTuples) {
- out.emit(random.nextInt());
- } else {
- LOG.debug("count = {}, time = {}", count, System.currentTimeMillis());
-
- try {
- // avoid repeated calls to this function
- Thread.sleep(sleepTime);
- } catch (InterruptedException e) {
- LOG.info("Sleep interrupted");
- }
- }
- }
-
- public int getNumTuples()
- {
- return numTuples;
- }
-
- public void setNumTuples(int numTuples)
- {
- this.numTuples = numTuples;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/partition/src/main/java/com/example/myapexapp/TestPartition.java
----------------------------------------------------------------------
diff --git a/examples/partition/src/main/java/com/example/myapexapp/TestPartition.java b/examples/partition/src/main/java/com/example/myapexapp/TestPartition.java
deleted file mode 100644
index 1f77e72..0000000
--- a/examples/partition/src/main/java/com/example/myapexapp/TestPartition.java
+++ /dev/null
@@ -1,164 +0,0 @@
-package com.example.myapexapp;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Random;
-
-import javax.validation.constraints.Max;
-import javax.validation.constraints.Min;
-import javax.validation.constraints.Size;
-import javax.validation.ConstraintViolation;
-import javax.validation.ValidatorFactory;
-import javax.validation.Validator;
-import javax.validation.Validation;
-
-import com.datatorrent.api.Attribute;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.DefaultPartition;
-import com.datatorrent.api.InputOperator;
-import com.datatorrent.api.Partitioner;
-import com.datatorrent.api.Partitioner.Partition;
-import com.datatorrent.api.Partitioner.PartitionKeys;
-import com.datatorrent.api.Partitioner.PartitioningContext;
-
-import com.datatorrent.common.util.BaseOperator;
-
-/**
- * Simple operator to test partitioning
- */
-public class TestPartition extends BaseOperator implements Partitioner<TestPartition>
-{
- private static final Logger LOG = LoggerFactory.getLogger(TestPartition.class);
-
- private transient int id; // operator/partition id
- private transient long curWindowId; // current window id
- private transient long cnt; // per-window tuple count
-
- @Min(1) @Max(20)
- private int nPartitions = 3;
-
- public final transient DefaultInputPort<Integer> in = new DefaultInputPort<Integer>() {
- @Override
- public void process(Integer tuple)
- {
- LOG.debug("{}: tuple = {}, operator id = {}", cnt, tuple, id);
- ++cnt;
- }
- };
-
- //public final transient DefaultOutputPort<Integer> out = new DefaultOutputPort<Integer>();
-
- @Override
- public void setup(Context.OperatorContext context)
- {
- super.setup(context);
-
- long appWindowId = context.getValue(context.ACTIVATION_WINDOW_ID);
- id = context.getId();
- LOG.debug("Started setup, appWindowId = {}, operator id = {}", appWindowId, id);
- }
-
- @Override
- public void beginWindow(long windowId)
- {
- cnt = 0;
- curWindowId = windowId;
- LOG.debug("window id = {}, operator id = {}", curWindowId, id);
- }
-
- @Override
- public void endWindow()
- {
- LOG.debug("window id = {}, operator id = {}, cnt = {}", curWindowId, id, cnt);
- }
-
- @Override
- public void partitioned(Map<Integer, Partition<TestPartition>> partitions)
- {
- //Do nothing
- }
-
- @Override
- public Collection<Partition<TestPartition>> definePartitions(
- Collection<Partition<TestPartition>> partitions,
- PartitioningContext context)
- {
- int oldSize = partitions.size();
- LOG.debug("partitionCount: current = {} requested = {}", oldSize, nPartitions);
-
- // each partition i in 0...nPartitions receives tuples divisible by i but not by any other
- // j in that range; all other tuples ignored
- //
- if (3 != nPartitions) return getPartitions(partitions, context);
-
- // special case of 3 partitions: All odd numbers to partition 0; even numbers divisible
- // by 4 to partition 1, those divisible by 2 but not 4 to partition 2.
-
- // mask used to extract discriminant from tuple hashcode
- int mask = 0x03;
-
- Partition<TestPartition>[] newPartitions = new Partition[] {
- new DefaultPartition<TestPartition>(new TestPartition()),
- new DefaultPartition<TestPartition>(new TestPartition()),
- new DefaultPartition<TestPartition>(new TestPartition()) };
-
- HashSet<Integer>[] set
- = new HashSet[] {new HashSet<>(), new HashSet<>(), new HashSet<>()};
- set[0].add(0);
- set[1].add(1);
- set[2].add(2);
-
- PartitionKeys[] keys = {
- new PartitionKeys(mask, set[0]),
- new PartitionKeys(mask, set[1]),
- new PartitionKeys(mask, set[2]) };
-
- for (int i = 0; i < 3; ++i ) {
- Partition<TestPartition> partition = newPartitions[i];
- partition.getPartitionKeys().put(in, keys[i]);
- }
-
- return new ArrayList<Partition<TestPartition>>(Arrays.asList(newPartitions));
- } // definePartitions
-
- private Collection<Partition<TestPartition>> getPartitions(
- Collection<Partition<TestPartition>> partitions,
- PartitioningContext context)
- {
- // create array of partitions to return
- Collection<Partition<TestPartition>> result
- = new ArrayList<Partition<TestPartition>>(nPartitions);
-
- int mask = getMask(nPartitions);
- for (int i = 0; i < nPartitions; ++i) {
- HashSet<Integer> set = new HashSet<>();
- set.add(i);
- PartitionKeys keys = new PartitionKeys(mask, set);
- Partition partition = new DefaultPartition<TestPartition>(new TestPartition());
- partition.getPartitionKeys().put(in, keys);
- }
-
- return result;
- } // getPartitions
-
- // return mask with bits 0..N set where N is the highest set bit of argument
- private int getMask(final int n) {
- return -1 >>> Integer.numberOfLeadingZeros(n);
- } // getMask
-
- // accessors
- public int getNPartitions() { return nPartitions; }
- public void setNPartitions(int v) { nPartitions = v; }
-}
-
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/partition/src/main/java/org/apache/apex/examples/partition/Application.java
----------------------------------------------------------------------
diff --git a/examples/partition/src/main/java/org/apache/apex/examples/partition/Application.java b/examples/partition/src/main/java/org/apache/apex/examples/partition/Application.java
new file mode 100644
index 0000000..eb9a7c2
--- /dev/null
+++ b/examples/partition/src/main/java/org/apache/apex/examples/partition/Application.java
@@ -0,0 +1,25 @@
+package org.apache.apex.examples.partition;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.Context.PortContext;
+import com.datatorrent.api.DAG;
+
+@ApplicationAnnotation(name="TestStuff")
+public class Application implements StreamingApplication
+{
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ RandomNumberGenerator random = dag.addOperator("randomInt", RandomNumberGenerator.class);
+ TestPartition testPartition = dag.addOperator("testPartition", TestPartition.class);
+ Codec3 codec = new Codec3();
+ dag.setInputPortAttribute(testPartition.in, PortContext.STREAM_CODEC, codec);
+
+ //Add locality if needed, e.g.: .setLocality(Locality.CONTAINER_LOCAL);
+ dag.addStream("randomData", random.out, testPartition.in);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/partition/src/main/java/org/apache/apex/examples/partition/Codec3.java
----------------------------------------------------------------------
diff --git a/examples/partition/src/main/java/org/apache/apex/examples/partition/Codec3.java b/examples/partition/src/main/java/org/apache/apex/examples/partition/Codec3.java
new file mode 100644
index 0000000..daa0e05
--- /dev/null
+++ b/examples/partition/src/main/java/org/apache/apex/examples/partition/Codec3.java
@@ -0,0 +1,13 @@
+package org.apache.apex.examples.partition;
+
+import com.datatorrent.lib.codec.KryoSerializableStreamCodec;
+
+public class Codec3 extends KryoSerializableStreamCodec<Integer> {
+ @Override
+ public int getPartition(Integer tuple) {
+ final int v = tuple;
+ return (1 == (v & 1)) ? 0 // odd
+ : (0 == (v & 3)) ? 1 // divisible by 4
+ : 2; // divisible by 2 but not 4
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/partition/src/main/java/org/apache/apex/examples/partition/RandomNumberGenerator.java
----------------------------------------------------------------------
diff --git a/examples/partition/src/main/java/org/apache/apex/examples/partition/RandomNumberGenerator.java b/examples/partition/src/main/java/org/apache/apex/examples/partition/RandomNumberGenerator.java
new file mode 100644
index 0000000..e3959d1
--- /dev/null
+++ b/examples/partition/src/main/java/org/apache/apex/examples/partition/RandomNumberGenerator.java
@@ -0,0 +1,76 @@
+package org.apache.apex.examples.partition;
+
+import java.util.Random;
+
+import javax.validation.constraints.Min;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.common.util.BaseOperator;
+import com.datatorrent.api.Context;
+
+/**
+ * This is a simple operator that emits random integer.
+ */
+public class RandomNumberGenerator extends BaseOperator implements InputOperator
+{
+ private static final Logger LOG = LoggerFactory.getLogger(RandomNumberGenerator.class);
+
+ @Min(1)
+ private int numTuples = 20;
+ private transient int count = 0;
+
+ private int sleepTime;
+ private transient long curWindowId;
+ private transient Random random = new Random();
+
+ public final transient DefaultOutputPort<Integer> out = new DefaultOutputPort<Integer>();
+
+ @Override
+ public void setup(Context.OperatorContext context)
+ {
+ super.setup(context);
+
+ long appWindowId = context.getValue(context.ACTIVATION_WINDOW_ID);
+ sleepTime = context.getValue(context.SPIN_MILLIS);
+ LOG.debug("Started setup, appWindowId = {}, sleepTime = {}", appWindowId, sleepTime);
+ }
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ count = 0;
+ LOG.debug("beginWindow: windowId = {}", windowId);
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ if (count++ < numTuples) {
+ out.emit(random.nextInt());
+ } else {
+ LOG.debug("count = {}, time = {}", count, System.currentTimeMillis());
+
+ try {
+ // avoid repeated calls to this function
+ Thread.sleep(sleepTime);
+ } catch (InterruptedException e) {
+ LOG.info("Sleep interrupted");
+ }
+ }
+ }
+
+ public int getNumTuples()
+ {
+ return numTuples;
+ }
+
+ public void setNumTuples(int numTuples)
+ {
+ this.numTuples = numTuples;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/partition/src/main/java/org/apache/apex/examples/partition/TestPartition.java
----------------------------------------------------------------------
diff --git a/examples/partition/src/main/java/org/apache/apex/examples/partition/TestPartition.java b/examples/partition/src/main/java/org/apache/apex/examples/partition/TestPartition.java
new file mode 100644
index 0000000..5b4e693
--- /dev/null
+++ b/examples/partition/src/main/java/org/apache/apex/examples/partition/TestPartition.java
@@ -0,0 +1,149 @@
+package org.apache.apex.examples.partition;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Map;
+
+import javax.validation.constraints.Max;
+import javax.validation.constraints.Min;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultPartition;
+import com.datatorrent.api.Partitioner;
+
+import com.datatorrent.common.util.BaseOperator;
+
+/**
+ * Simple operator to test partitioning
+ */
+public class TestPartition extends BaseOperator implements Partitioner<TestPartition>
+{
+ private static final Logger LOG = LoggerFactory.getLogger(TestPartition.class);
+
+ private transient int id; // operator/partition id
+ private transient long curWindowId; // current window id
+ private transient long cnt; // per-window tuple count
+
+ @Min(1) @Max(20)
+ private int nPartitions = 3;
+
+ public final transient DefaultInputPort<Integer> in = new DefaultInputPort<Integer>() {
+ @Override
+ public void process(Integer tuple)
+ {
+ LOG.debug("{}: tuple = {}, operator id = {}", cnt, tuple, id);
+ ++cnt;
+ }
+ };
+
+ //public final transient DefaultOutputPort<Integer> out = new DefaultOutputPort<Integer>();
+
+ @Override
+ public void setup(Context.OperatorContext context)
+ {
+ super.setup(context);
+
+ long appWindowId = context.getValue(context.ACTIVATION_WINDOW_ID);
+ id = context.getId();
+ LOG.debug("Started setup, appWindowId = {}, operator id = {}", appWindowId, id);
+ }
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ cnt = 0;
+ curWindowId = windowId;
+ LOG.debug("window id = {}, operator id = {}", curWindowId, id);
+ }
+
+ @Override
+ public void endWindow()
+ {
+ LOG.debug("window id = {}, operator id = {}, cnt = {}", curWindowId, id, cnt);
+ }
+
+ @Override
+ public void partitioned(Map<Integer, Partition<TestPartition>> partitions)
+ {
+ //Do nothing
+ }
+
+ @Override
+ public Collection<Partition<TestPartition>> definePartitions(
+ Collection<Partition<TestPartition>> partitions,
+ PartitioningContext context)
+ {
+ int oldSize = partitions.size();
+ LOG.debug("partitionCount: current = {} requested = {}", oldSize, nPartitions);
+
+ // each partition i in 0...nPartitions receives tuples divisible by i but not by any other
+ // j in that range; all other tuples ignored
+ //
+ if (3 != nPartitions) return getPartitions(partitions, context);
+
+ // special case of 3 partitions: All odd numbers to partition 0; even numbers divisible
+ // by 4 to partition 1, those divisible by 2 but not 4 to partition 2.
+
+ // mask used to extract discriminant from tuple hashcode
+ int mask = 0x03;
+
+ Partition<TestPartition>[] newPartitions = new Partition[] {
+ new DefaultPartition<TestPartition>(new TestPartition()),
+ new DefaultPartition<TestPartition>(new TestPartition()),
+ new DefaultPartition<TestPartition>(new TestPartition()) };
+
+ HashSet<Integer>[] set
+ = new HashSet[] {new HashSet<>(), new HashSet<>(), new HashSet<>()};
+ set[0].add(0);
+ set[1].add(1);
+ set[2].add(2);
+
+ PartitionKeys[] keys = {
+ new PartitionKeys(mask, set[0]),
+ new PartitionKeys(mask, set[1]),
+ new PartitionKeys(mask, set[2]) };
+
+ for (int i = 0; i < 3; ++i ) {
+ Partition<TestPartition> partition = newPartitions[i];
+ partition.getPartitionKeys().put(in, keys[i]);
+ }
+
+ return new ArrayList<Partition<TestPartition>>(Arrays.asList(newPartitions));
+ } // definePartitions
+
+ private Collection<Partition<TestPartition>> getPartitions(
+ Collection<Partition<TestPartition>> partitions,
+ PartitioningContext context)
+ {
+ // create array of partitions to return
+ Collection<Partition<TestPartition>> result
+ = new ArrayList<Partition<TestPartition>>(nPartitions);
+
+ int mask = getMask(nPartitions);
+ for (int i = 0; i < nPartitions; ++i) {
+ HashSet<Integer> set = new HashSet<>();
+ set.add(i);
+ PartitionKeys keys = new PartitionKeys(mask, set);
+ Partition partition = new DefaultPartition<TestPartition>(new TestPartition());
+ partition.getPartitionKeys().put(in, keys);
+ }
+
+ return result;
+ } // getPartitions
+
+ // return mask with bits 0..N set where N is the highest set bit of argument
+ private int getMask(final int n) {
+ return -1 >>> Integer.numberOfLeadingZeros(n);
+ } // getMask
+
+ // accessors
+ public int getNPartitions() { return nPartitions; }
+ public void setNPartitions(int v) { nPartitions = v; }
+}
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/partition/src/main/resources/my-log4j.properties
----------------------------------------------------------------------
diff --git a/examples/partition/src/main/resources/my-log4j.properties b/examples/partition/src/main/resources/my-log4j.properties
index 21ead89..8949bfa 100644
--- a/examples/partition/src/main/resources/my-log4j.properties
+++ b/examples/partition/src/main/resources/my-log4j.properties
@@ -13,4 +13,4 @@ log4j.logger.org=INFO
log4j.logger.org.apache=INFO
log4j.logger.com.datatorrent=INFO
-#log4j.logger.com.example.myapexapp.TestPartition=DEBUG, CONSOLE
+#log4j.logger.TestPartition=DEBUG, CONSOLE
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/partition/src/test/java/com/example/myapexapp/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/partition/src/test/java/com/example/myapexapp/ApplicationTest.java b/examples/partition/src/test/java/com/example/myapexapp/ApplicationTest.java
deleted file mode 100644
index 5f490d8..0000000
--- a/examples/partition/src/test/java/com/example/myapexapp/ApplicationTest.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Put your copyright and license info here.
- */
-package com.example.myapexapp;
-
-import java.io.IOException;
-
-import javax.validation.ConstraintViolationException;
-
-import org.junit.Assert;
-
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Test;
-
-import com.datatorrent.api.LocalMode;
-import com.example.myapexapp.Application;
-
-/**
- * Test the DAG declaration in local mode.
- */
-public class ApplicationTest {
-
- @Test
- public void testApplication() throws IOException, Exception {
- try {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
- lma.prepareDAG(new Application(), conf);
- LocalMode.Controller lc = lma.getController();
- lc.run(5000); // runs for 5 seconds and quits
- } catch (ConstraintViolationException e) {
- Assert.fail("constraint violations: " + e.getConstraintViolations());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/partition/src/test/java/org/apache/apex/examples/partition/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/partition/src/test/java/org/apache/apex/examples/partition/ApplicationTest.java b/examples/partition/src/test/java/org/apache/apex/examples/partition/ApplicationTest.java
new file mode 100644
index 0000000..4cfb3c2
--- /dev/null
+++ b/examples/partition/src/test/java/org/apache/apex/examples/partition/ApplicationTest.java
@@ -0,0 +1,36 @@
+/**
+ * Put your copyright and license info here.
+ */
+package org.apache.apex.examples.partition;
+
+import java.io.IOException;
+
+import javax.validation.ConstraintViolationException;
+
+import org.junit.Assert;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+import com.datatorrent.api.LocalMode;
+
+/**
+ * Test the DAG declaration in local mode.
+ */
+public class ApplicationTest {
+
+ @Test
+ public void testApplication() throws IOException, Exception {
+ try {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ lma.prepareDAG(new Application(), conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.run(5000); // runs for 5 seconds and quits
+ } catch (ConstraintViolationException e) {
+ Assert.fail("constraint violations: " + e.getConstraintViolations());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/pom.xml
----------------------------------------------------------------------
diff --git a/examples/pom.xml b/examples/pom.xml
index 30ce061..be0f0d3 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -187,6 +187,16 @@
<module>r</module>
<module>echoserver</module>
<module>iteration</module>
+ <module>csvformatter</module>
+ <module>dedup</module>
+ <module>dynamic-partition</module>
+ <module>enricher</module>
+ <module>filter</module>
+ <module>innerjoin</module>
+ <module>partition</module>
+ <module>recordReader</module>
+ <module>throttle</module>
+ <module>transform</module>
</modules>
<dependencies>
[13/19] apex-malhar git commit: SPOI-8656 filter example app
Posted by th...@apache.org.
SPOI-8656 filter example app
Incorporating review comments
1. Updating README
2. added log4j properties
3. Updated test cases.
Project: http://git-wip-us.apache.org/repos/asf/apex-malhar/repo
Commit: http://git-wip-us.apache.org/repos/asf/apex-malhar/commit/0590c258
Tree: http://git-wip-us.apache.org/repos/asf/apex-malhar/tree/0590c258
Diff: http://git-wip-us.apache.org/repos/asf/apex-malhar/diff/0590c258
Branch: refs/heads/master
Commit: 0590c25820b89b5616156587c0df107a6e871c2a
Parents: f7c7b7c
Author: yogidevendra <de...@datatorrent.com>
Authored: Mon Jul 18 11:55:22 2016 +0530
Committer: Lakshmi Prasanna Velineni <la...@datatorrent.com>
Committed: Sun Mar 26 11:43:48 2017 -0700
----------------------------------------------------------------------
examples/filter/README.md | 20 +
examples/filter/XmlJavadocCommentsExtractor.xsl | 44 ++
examples/filter/pom.xml | 312 +++++++++++++++
examples/filter/src/assemble/appPackage.xml | 43 ++
.../tutorial/filter/Application.java | 49 +++
.../tutorial/filter/TransactionPOJO.java | 64 +++
.../src/main/resources/META-INF/input.txt | 401 +++++++++++++++++++
.../src/main/resources/META-INF/properties.xml | 95 +++++
.../main/resources/META-INF/rejected_output.txt | 326 +++++++++++++++
.../main/resources/META-INF/selected_output.txt | 74 ++++
.../tutorial/filter/ApplicationTest.java | 111 +++++
.../filter/src/test/resources/log4j.properties | 22 +
12 files changed, 1561 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/0590c258/examples/filter/README.md
----------------------------------------------------------------------
diff --git a/examples/filter/README.md b/examples/filter/README.md
new file mode 100644
index 0000000..367495a
--- /dev/null
+++ b/examples/filter/README.md
@@ -0,0 +1,20 @@
+# Filter operator example
+
+Sample application to show how to use the filter operator.
+
+The application reads transaction records from a csv file using `FSRecordReaderModule`. Then converts these records into plain old java objects (POJO) using CSVParser. These POJOs are filtered based on given condition. POJOs meeting the filter criteria are written to `selected.txt`. POJOs not meeting filter criteria are written to `rejected.txt`. Writing POJO output to file is done using `CSVFormatter` and `StringFileOutputOperator`
+
+### How to configure
+The properties file META-INF/properties.xml shows how to configure the respective operators.
+
+### How to compile
+`shell> mvn clean package`
+
+This will generate application package filter-1.0-SNAPSHOT.apa inside target directory.
+
+### How to run
+Use the application package generated above to launch the application from UI console(if available) or apex command line interface.
+
+`apex> launch target/filter-1.0-SNAPSHOT.apa`
+
+You may also the run the application in local mode within your IDE by simply running the method ApplicationTest.testApplication().
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/0590c258/examples/filter/XmlJavadocCommentsExtractor.xsl
----------------------------------------------------------------------
diff --git a/examples/filter/XmlJavadocCommentsExtractor.xsl b/examples/filter/XmlJavadocCommentsExtractor.xsl
new file mode 100644
index 0000000..08075a9
--- /dev/null
+++ b/examples/filter/XmlJavadocCommentsExtractor.xsl
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+
+<!--
+ Document : XmlJavadocCommentsExtractor.xsl
+ Created on : September 16, 2014, 11:30 AM
+ Description:
+ The transformation strips off all information except for comments and tags from xml javadoc generated by xml-doclet.
+-->
+
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+ <xsl:output method="xml" standalone="yes"/>
+
+ <!-- copy xml by selecting only the following nodes, attributes and text -->
+ <xsl:template match="node()|text()|@*">
+ <xsl:copy>
+ <xsl:apply-templates select="root|package|class|interface|method|field|type|comment|tag|text()|@name|@qualified|@text"/>
+ </xsl:copy>
+ </xsl:template>
+
+ <!-- Strip off the following paths from the selected xml -->
+ <xsl:template match="//root/package/interface/interface
+ |//root/package/interface/method/@qualified
+ |//root/package/class/interface
+ |//root/package/class/class
+ |//root/package/class/method/@qualified
+ |//root/package/class/field/@qualified" />
+
+ <xsl:strip-space elements="*"/>
+</xsl:stylesheet>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/0590c258/examples/filter/pom.xml
----------------------------------------------------------------------
diff --git a/examples/filter/pom.xml b/examples/filter/pom.xml
new file mode 100644
index 0000000..9407818
--- /dev/null
+++ b/examples/filter/pom.xml
@@ -0,0 +1,312 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>com.datatorrent.tutorial</groupId>
+ <version>1.0-SNAPSHOT</version>
+ <artifactId>filter</artifactId>
+ <packaging>jar</packaging>
+
+ <!-- change these to the appropriate values -->
+ <name>Filter Operator</name>
+ <description>Apex application demonstrating filter operator</description>
+
+ <properties>
+ <!-- change this if you desire to use a different version of Apex Core -->
+ <apex.version>3.5.0</apex.version>
+ <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath>
+ <malhar.version>3.6.0</malhar.version>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-eclipse-plugin</artifactId>
+ <version>2.9</version>
+ <configuration>
+ <downloadSources>true</downloadSources>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.3</version>
+ <configuration>
+ <encoding>UTF-8</encoding>
+ <source>1.7</source>
+ <target>1.7</target>
+ <debug>true</debug>
+ <optimize>false</optimize>
+ <showDeprecation>true</showDeprecation>
+ <showWarnings>true</showWarnings>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.8</version>
+ <executions>
+ <execution>
+ <id>copy-dependencies</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>target/deps</outputDirectory>
+ <includeScope>runtime</includeScope>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>app-package-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <finalName>${project.artifactId}-${project.version}-apexapp</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <descriptors>
+ <descriptor>src/assemble/appPackage.xml</descriptor>
+ </descriptors>
+ <archiverConfig>
+ <defaultDirectoryMode>0755</defaultDirectoryMode>
+ </archiverConfig>
+ <archive>
+ <manifestEntries>
+ <Class-Path>${apex.apppackage.classpath}</Class-Path>
+ <DT-Engine-Version>${apex.version}</DT-Engine-Version>
+ <DT-App-Package-Group-Id>${project.groupId}</DT-App-Package-Group-Id>
+ <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name>
+ <DT-App-Package-Version>${project.version}</DT-App-Package-Version>
+ <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name>
+ <DT-App-Package-Description>${project.description}</DT-App-Package-Description>
+ </manifestEntries>
+ </archive>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <configuration>
+ <target>
+ <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar"
+ tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa" />
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ <execution>
+ <!-- create resource directory for xml javadoc-->
+ <id>createJavadocDirectory</id>
+ <phase>generate-resources</phase>
+ <configuration>
+ <tasks>
+ <delete dir="${project.build.directory}/generated-resources/xml-javadoc"/>
+ <mkdir dir="${project.build.directory}/generated-resources/xml-javadoc"/>
+ </tasks>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.9.1</version>
+ <executions>
+ <execution>
+ <id>attach-artifacts</id>
+ <phase>package</phase>
+ <goals>
+ <goal>attach-artifact</goal>
+ </goals>
+ <configuration>
+ <artifacts>
+ <artifact>
+ <file>target/${project.artifactId}-${project.version}.apa</file>
+ <type>apa</type>
+ </artifact>
+ </artifacts>
+ <skipAttach>false</skipAttach>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- generate javdoc -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <executions>
+ <!-- generate xml javadoc -->
+ <execution>
+ <id>xml-doclet</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>javadoc</goal>
+ </goals>
+ <configuration>
+ <doclet>com.github.markusbernhardt.xmldoclet.XmlDoclet</doclet>
+ <additionalparam>-d ${project.build.directory}/generated-resources/xml-javadoc -filename ${project.artifactId}-${project.version}-javadoc.xml</additionalparam>
+ <useStandardDocletOptions>false</useStandardDocletOptions>
+ <docletArtifact>
+ <groupId>com.github.markusbernhardt</groupId>
+ <artifactId>xml-doclet</artifactId>
+ <version>1.0.4</version>
+ </docletArtifact>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <!-- Transform xml javadoc to stripped down version containing only class/interface comments and tags-->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>xml-maven-plugin</artifactId>
+ <version>1.0</version>
+ <executions>
+ <execution>
+ <id>transform-xmljavadoc</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>transform</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <transformationSets>
+ <transformationSet>
+ <dir>${project.build.directory}/generated-resources/xml-javadoc</dir>
+ <includes>
+ <include>${project.artifactId}-${project.version}-javadoc.xml</include>
+ </includes>
+ <stylesheet>XmlJavadocCommentsExtractor.xsl</stylesheet>
+ <outputDir>${project.build.directory}/generated-resources/xml-javadoc</outputDir>
+ </transformationSet>
+ </transformationSets>
+ </configuration>
+ </plugin>
+ <!-- copy xml javadoc to class jar -->
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>2.6</version>
+ <executions>
+ <execution>
+ <id>copy-resources</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${basedir}/target/classes</outputDirectory>
+ <resources>
+ <resource>
+ <directory>${project.build.directory}/generated-resources/xml-javadoc</directory>
+ <includes>
+ <include>${project.artifactId}-${project.version}-javadoc.xml</include>
+ </includes>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+
+ </build>
+
+ <dependencies>
+ <!-- add your dependencies here -->
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-library</artifactId>
+ <version>${malhar.version}</version>
+ <!--
+ If you know that your application does not need transitive dependencies pulled in by malhar-library,
+ uncomment the following to reduce the size of your app package.
+ -->
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>3.1</version>
+ <type>jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-contrib</artifactId>
+ <version>${malhar.version}</version>
+ <!--
+ If you know that your application does not need transitive dependencies pulled in by malhar-library,
+ uncomment the following to reduce the size of your app package.
+ -->
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>net.sf.supercsv</groupId>
+ <artifactId>super-csv</artifactId>
+ <version>2.4.0</version>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>apex-common</artifactId>
+ <version>${apex.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.10</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>apex-engine</artifactId>
+ <version>${apex.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.janino</groupId>
+ <artifactId>commons-compiler</artifactId>
+ <version>2.7.8</version>
+ <type>jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.janino</groupId>
+ <artifactId>janino</artifactId>
+ <version>2.7.8</version>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/0590c258/examples/filter/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/filter/src/assemble/appPackage.xml b/examples/filter/src/assemble/appPackage.xml
new file mode 100644
index 0000000..7ad071c
--- /dev/null
+++ b/examples/filter/src/assemble/appPackage.xml
@@ -0,0 +1,43 @@
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>appPackage</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/target/</directory>
+ <outputDirectory>/app</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/target/deps</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/site/conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/META-INF</directory>
+ <outputDirectory>/META-INF</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/app</directory>
+ <outputDirectory>/app</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/resources</directory>
+ <outputDirectory>/resources</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/0590c258/examples/filter/src/main/java/com/datatorrent/tutorial/filter/Application.java
----------------------------------------------------------------------
diff --git a/examples/filter/src/main/java/com/datatorrent/tutorial/filter/Application.java b/examples/filter/src/main/java/com/datatorrent/tutorial/filter/Application.java
new file mode 100644
index 0000000..4ebb153
--- /dev/null
+++ b/examples/filter/src/main/java/com/datatorrent/tutorial/filter/Application.java
@@ -0,0 +1,49 @@
+/**
+ * Copyright (c) 2016 DataTorrent, Inc.
+ * All rights reserved.
+ */
+
+package com.datatorrent.tutorial.filter;
+
+import org.apache.apex.malhar.lib.fs.FSRecordReaderModule;
+import org.apache.apex.malhar.lib.fs.GenericFileOutputOperator.StringFileOutputOperator;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.contrib.formatter.CsvFormatter;
+import com.datatorrent.contrib.parser.CsvParser;
+import com.datatorrent.lib.filter.FilterOperator;
+
+/**
+ * Simple application illustrating filter operator
+ */
+@ApplicationAnnotation(name="FilterExample")
+public class Application implements StreamingApplication
+{
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+
+ FSRecordReaderModule recordReader = dag.addModule("recordReader", FSRecordReaderModule.class);
+ CsvParser csvParser = dag.addOperator("csvParser", CsvParser.class);
+ FilterOperator filterOperator = dag.addOperator("filterOperator", new FilterOperator());
+
+ CsvFormatter selectedFormatter = dag.addOperator("selectedFormatter", new CsvFormatter());
+ CsvFormatter rejectedFormatter = dag.addOperator("rejectedFormatter", new CsvFormatter());
+
+ StringFileOutputOperator selectedOutput = dag.addOperator("selectedOutput", new StringFileOutputOperator());
+ StringFileOutputOperator rejectedOutput = dag.addOperator("rejectedOutput", new StringFileOutputOperator());
+
+ dag.addStream("record", recordReader.records, csvParser.in);
+ dag.addStream("pojo", csvParser.out, filterOperator.input);
+
+ dag.addStream("pojoSelected", filterOperator.truePort, selectedFormatter.in);
+ dag.addStream("pojoRejected", filterOperator.falsePort, rejectedFormatter.in);
+
+ dag.addStream("csvSelected", selectedFormatter.out, selectedOutput.input);
+ dag.addStream("csvRejected", rejectedFormatter.out, rejectedOutput.input);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/0590c258/examples/filter/src/main/java/com/datatorrent/tutorial/filter/TransactionPOJO.java
----------------------------------------------------------------------
diff --git a/examples/filter/src/main/java/com/datatorrent/tutorial/filter/TransactionPOJO.java b/examples/filter/src/main/java/com/datatorrent/tutorial/filter/TransactionPOJO.java
new file mode 100644
index 0000000..c0d8817
--- /dev/null
+++ b/examples/filter/src/main/java/com/datatorrent/tutorial/filter/TransactionPOJO.java
@@ -0,0 +1,64 @@
+/**
+ * Copyright (c) 2016 DataTorrent, Inc.
+ * All rights reserved.
+ */
+
+package com.datatorrent.tutorial.filter;
+
+public class TransactionPOJO
+{
+
+ private long trasactionId;
+ private double amount;
+ private long accountNumber;
+ private String type;
+
+ public long getTrasactionId()
+ {
+ return trasactionId;
+ }
+
+ public void setTrasactionId(long trasactionId)
+ {
+ this.trasactionId = trasactionId;
+ }
+
+ public double getAmount()
+ {
+ return amount;
+ }
+
+ public void setAmount(double amount)
+ {
+ this.amount = amount;
+ }
+
+ public long getAccountNumber()
+ {
+ return accountNumber;
+ }
+
+ public void setAccountNumber(long accountNumber)
+ {
+ this.accountNumber = accountNumber;
+ }
+
+ public String getType()
+ {
+ return type;
+ }
+
+ public void setType(String type)
+ {
+ this.type = type;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "TransactionPOJO [trasactionId=" + trasactionId + ", amount=" + amount + ", accountNumber=" + accountNumber
+ + ", type=" + type + "]";
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/0590c258/examples/filter/src/main/resources/META-INF/input.txt
----------------------------------------------------------------------
diff --git a/examples/filter/src/main/resources/META-INF/input.txt b/examples/filter/src/main/resources/META-INF/input.txt
new file mode 100644
index 0000000..f87a772
--- /dev/null
+++ b/examples/filter/src/main/resources/META-INF/input.txt
@@ -0,0 +1,401 @@
+10001|84804533|51.0|DEBIT
+10002|529683455|8637.0|CREDIT
+10003|34646572|7443.0|CREDIT
+10004|967805584|7342.0|CREDIT
+10005|89940328|5100.0|DEBIT
+10006|484971385|4559.0|DEBIT
+10007|168472218|3539.0|CREDIT
+10008|560263718|1624.0|CREDIT
+10009|27097195|2471.0|DEBIT
+10010|562944412|5143.0|DEBIT
+10011|228473379|8336.0|DEBIT
+10012|138798397|5595.0|CREDIT
+10013|522515858|6054.0|CREDIT
+10014|358415062|1886.0|CREDIT
+10015|596239488|9936.0|DEBIT
+10016|378386029|4623.0|DEBIT
+10017|329310325|9064.0|CREDIT
+10018|652183672|2748.0|DEBIT
+10019|92850307|7752.0|DEBIT
+10020|608582237|2329.0|CREDIT
+10021|50283707|9987.0|DEBIT
+10022|540706222|8639.0|CREDIT
+10023|373122065|4834.0|DEBIT
+10024|215520537|5684.0|DEBIT
+10025|766304418|3561.0|CREDIT
+10026|467948355|2802.0|CREDIT
+10027|180522562|8662.0|DEBIT
+10028|444946324|7783.0|CREDIT
+10029|994074259|7125.0|CREDIT
+10030|182740938|3113.0|CREDIT
+10031|614671827|24979.0|CREDIT
+10032|88726012|3603.0|CREDIT
+10033|571353545|4157.0|CREDIT
+10034|748258207|5329.0|CREDIT
+10035|228328193|6883.0|DEBIT
+10036|953766573|22516.0|DEBIT
+10037|461283441|431.0|CREDIT
+10038|518514310|1586.0|CREDIT
+10039|381773726|8684.0|CREDIT
+10040|680364582|4221.0|CREDIT
+10041|728080172|23189.0|DEBIT
+10042|468126452|4990.0|DEBIT
+10043|459970527|6188.0|CREDIT
+10044|100934948|3422.0|CREDIT
+10045|31728881|435.0|DEBIT
+10046|906868986|26145.0|CREDIT
+10047|950687958|5002.0|CREDIT
+10048|164869066|1487.0|CREDIT
+10049|892759267|8102.0|DEBIT
+10050|69946718|4464.0|CREDIT
+10051|732833481|22186.0|DEBIT
+10052|222384809|196.0|CREDIT
+10053|619012622|5892.0|DEBIT
+10054|524593769|8796.0|CREDIT
+10055|360301464|9846.0|CREDIT
+10056|76837426|21464.0|DEBIT
+10057|894461469|3232.0|CREDIT
+10058|905824283|9614.0|DEBIT
+10059|201267525|6839.0|DEBIT
+10060|650778374|3956.0|CREDIT
+10061|539521241|26964.0|DEBIT
+10062|257584130|2458.0|DEBIT
+10063|185240027|4987.0|DEBIT
+10064|340187828|6919.0|CREDIT
+10065|167114400|758.0|DEBIT
+10066|645317265|22942.0|CREDIT
+10067|834054944|558.0|CREDIT
+10068|4848158|2972.0|CREDIT
+10069|611089049|3830.0|DEBIT
+10070|574929455|7585.0|CREDIT
+10071|23360143|22515.0|CREDIT
+10072|678734877|1593.0|DEBIT
+10073|465428415|4876.0|CREDIT
+10074|631548543|4502.0|CREDIT
+10075|708806494|954.0|DEBIT
+10076|656870588|26892.0|DEBIT
+10077|919361479|7612.0|DEBIT
+10078|831562086|9544.0|DEBIT
+10079|564566420|5850.0|DEBIT
+10080|732363069|1502.0|CREDIT
+10081|817401315|20419.0|CREDIT
+10082|498367448|6829.0|CREDIT
+10083|387288387|4453.0|DEBIT
+10084|719081200|2961.0|CREDIT
+10085|735565137|7400.0|DEBIT
+10086|973661932|24358.0|CREDIT
+10087|371001709|8504.0|CREDIT
+10088|793778660|6867.0|DEBIT
+10089|620050993|1582.0|CREDIT
+10090|367541932|2858.0|DEBIT
+10091|146470368|27897.0|DEBIT
+10092|534083940|9915.0|CREDIT
+10093|171218647|9262.0|DEBIT
+10094|566590232|1904.0|DEBIT
+10095|940994162|2697.0|CREDIT
+10096|683965112|29656.0|CREDIT
+10097|599783690|7551.0|CREDIT
+10098|472038054|7107.0|CREDIT
+10099|462438976|8978.0|CREDIT
+10100|419462514|5799.0|CREDIT
+10101|318680824|20128.0|CREDIT
+10102|209434250|4813.0|DEBIT
+10103|869224102|6504.0|DEBIT
+10104|725686791|11.0|CREDIT
+10105|781644695|1705.0|DEBIT
+10106|48125174|22815.0|DEBIT
+10107|857136119|2809.0|CREDIT
+10108|290114536|7476.0|CREDIT
+10109|90800192|2512.0|DEBIT
+10110|884687225|1396.0|CREDIT
+10111|780970052|24959.0|DEBIT
+10112|976868472|8363.0|DEBIT
+10113|570179873|7231.0|CREDIT
+10114|399863103|4291.0|DEBIT
+10115|358440857|5322.0|CREDIT
+10116|713800593|21282.0|CREDIT
+10117|50382939|6544.0|CREDIT
+10118|42616440|5372.0|CREDIT
+10119|220722302|3191.0|DEBIT
+10120|676925763|9413.0|CREDIT
+10121|66746026|21667.0|DEBIT
+10122|412477439|9943.0|DEBIT
+10123|405618613|2350.0|DEBIT
+10124|415714190|9245.0|CREDIT
+10125|563760753|7092.0|CREDIT
+10126|737861484|23370.0|CREDIT
+10127|434529942|7629.0|CREDIT
+10128|514574310|3897.0|DEBIT
+10129|27419748|8440.0|CREDIT
+10130|165864941|9762.0|DEBIT
+10131|571929475|26224.0|DEBIT
+10132|52768757|3941.0|CREDIT
+10133|734355265|621.0|CREDIT
+10134|126968199|1286.0|DEBIT
+10135|585441472|1770.0|DEBIT
+10136|893117252|27637.0|CREDIT
+10137|346242993|1487.0|CREDIT
+10138|490611170|6440.0|DEBIT
+10139|801216167|1515.0|DEBIT
+10140|659265220|4552.0|CREDIT
+10141|953467032|23295.0|CREDIT
+10142|245703913|3093.0|DEBIT
+10143|2247312|5209.0|DEBIT
+10144|519787641|9527.0|DEBIT
+10145|912166127|2172.0|DEBIT
+10146|95502398|28322.0|DEBIT
+10147|582275337|8712.0|DEBIT
+10148|259281594|5252.0|CREDIT
+10149|694423378|2060.0|DEBIT
+10150|629554298|5967.0|CREDIT
+10151|323058477|27765.0|DEBIT
+10152|53101351|4382.0|CREDIT
+10153|323164961|5328.0|CREDIT
+10154|179064326|4341.0|DEBIT
+10155|52433644|7812.0|DEBIT
+10156|3744164|22345.0|DEBIT
+10157|869913610|5813.0|CREDIT
+10158|347708099|6201.0|CREDIT
+10159|423486262|4871.0|DEBIT
+10160|461540100|9256.0|DEBIT
+10161|278026136|25472.0|CREDIT
+10162|798468803|9224.0|DEBIT
+10163|418877075|519.0|DEBIT
+10164|40368770|245.0|DEBIT
+10165|463671480|5775.0|CREDIT
+10166|410182527|24299.0|DEBIT
+10167|256176469|8602.0|DEBIT
+10168|739490131|3012.0|CREDIT
+10169|845152384|8188.0|DEBIT
+10170|425841395|2578.0|DEBIT
+10171|676685046|24621.0|CREDIT
+10172|226331065|6325.0|DEBIT
+10173|244147717|6282.0|DEBIT
+10174|838618959|2970.0|CREDIT
+10175|966466342|6090.0|DEBIT
+10176|347315258|22401.0|CREDIT
+10177|960290189|2392.0|DEBIT
+10178|665504299|5721.0|DEBIT
+10179|207166700|4961.0|DEBIT
+10180|910728570|7626.0|CREDIT
+10181|495476490|23835.0|CREDIT
+10182|163142417|1630.0|CREDIT
+10183|662313571|6927.0|DEBIT
+10184|105583140|7757.0|DEBIT
+10185|639328437|7461.0|DEBIT
+10186|353744821|24419.0|CREDIT
+10187|922940528|6379.0|CREDIT
+10188|3320970|48.0|DEBIT
+10189|217812412|7170.0|DEBIT
+10190|660688167|8493.0|CREDIT
+10191|319041088|27622.0|CREDIT
+10192|29286525|4470.0|CREDIT
+10193|170950644|5463.0|DEBIT
+10194|813477150|376.0|CREDIT
+10195|180786030|6536.0|DEBIT
+10196|200090142|24312.0|CREDIT
+10197|517613853|4244.0|CREDIT
+10198|142109351|4045.0|CREDIT
+10199|120408343|1224.0|CREDIT
+10200|372437363|2587.0|CREDIT
+10201|988342695|28896.0|DEBIT
+10202|691849636|4285.0|CREDIT
+10203|654204917|3074.0|DEBIT
+10204|373303209|3566.0|CREDIT
+10205|28536699|3681.0|CREDIT
+10206|332098186|20324.0|DEBIT
+10207|703547342|6351.0|DEBIT
+10208|117707726|8618.0|CREDIT
+10209|240155449|5987.0|DEBIT
+10210|539240301|6073.0|CREDIT
+10211|304868598|29587.0|DEBIT
+10212|120948449|7938.0|CREDIT
+10213|968947065|2045.0|DEBIT
+10214|951562769|1738.0|CREDIT
+10215|136201581|1112.0|DEBIT
+10216|446120114|27378.0|CREDIT
+10217|863802751|6610.0|DEBIT
+10218|855970512|354.0|CREDIT
+10219|339900628|4677.0|CREDIT
+10220|706630053|3682.0|DEBIT
+10221|904298621|27819.0|CREDIT
+10222|174008949|3797.0|CREDIT
+10223|546023627|7130.0|CREDIT
+10224|138693364|4482.0|CREDIT
+10225|698667700|1055.0|CREDIT
+10226|286039255|23856.0|CREDIT
+10227|597215884|6851.0|DEBIT
+10228|841851199|3901.0|CREDIT
+10229|658855748|4700.0|DEBIT
+10230|741198758|8011.0|CREDIT
+10231|659386308|29955.0|DEBIT
+10232|130516026|8200.0|CREDIT
+10233|255513743|877.0|CREDIT
+10234|235986880|5608.0|DEBIT
+10235|755745324|4888.0|CREDIT
+10236|864054476|25873.0|DEBIT
+10237|117830905|4932.0|CREDIT
+10238|436598482|3043.0|DEBIT
+10239|415570756|5563.0|DEBIT
+10240|160904677|5698.0|DEBIT
+10241|12110227|25870.0|CREDIT
+10242|325540634|3633.0|CREDIT
+10243|934407707|9198.0|DEBIT
+10244|970044154|253.0|CREDIT
+10245|276211772|2484.0|DEBIT
+10246|632081618|22307.0|DEBIT
+10247|515392682|3085.0|DEBIT
+10248|106926205|9599.0|CREDIT
+10249|403051823|6598.0|DEBIT
+10250|739699283|2898.0|CREDIT
+10251|913373304|25760.0|CREDIT
+10252|588855751|4159.0|CREDIT
+10253|433188311|8895.0|CREDIT
+10254|613207771|2386.0|CREDIT
+10255|529952498|2469.0|CREDIT
+10256|487018926|27501.0|DEBIT
+10257|788255680|4566.0|CREDIT
+10258|177961056|4417.0|DEBIT
+10259|856270849|7959.0|DEBIT
+10260|362871038|2661.0|DEBIT
+10261|552260828|20328.0|DEBIT
+10262|51438711|7238.0|CREDIT
+10263|550177844|6044.0|DEBIT
+10264|796710293|3723.0|CREDIT
+10265|747419280|353.0|CREDIT
+10266|625233460|29396.0|DEBIT
+10267|502287841|7040.0|DEBIT
+10268|868155169|8175.0|DEBIT
+10269|217771255|6130.0|DEBIT
+10270|983601385|9798.0|CREDIT
+10271|676948266|29532.0|DEBIT
+10272|931578961|6792.0|DEBIT
+10273|627410737|6313.0|CREDIT
+10274|227280958|6610.0|DEBIT
+10275|814312724|2083.0|DEBIT
+10276|614444626|24522.0|CREDIT
+10277|734388101|3870.0|CREDIT
+10278|969611064|344.0|DEBIT
+10279|955987400|2897.0|DEBIT
+10280|742505356|8610.0|CREDIT
+10281|522681680|22593.0|CREDIT
+10282|381786126|4002.0|CREDIT
+10283|466001485|8284.0|CREDIT
+10284|310654102|4678.0|DEBIT
+10285|437156002|7311.0|CREDIT
+10286|495903944|27075.0|DEBIT
+10287|503495897|6757.0|CREDIT
+10288|411513405|263.0|CREDIT
+10289|143598653|1820.0|DEBIT
+10290|702368818|4650.0|CREDIT
+10291|248658152|28021.0|DEBIT
+10292|545322914|7008.0|DEBIT
+10293|176940492|584.0|CREDIT
+10294|592922722|5374.0|DEBIT
+10295|255199017|2002.0|DEBIT
+10296|858007111|24374.0|CREDIT
+10297|103227713|6796.0|DEBIT
+10298|366884380|2496.0|DEBIT
+10299|147239319|9209.0|CREDIT
+10300|137542451|6119.0|DEBIT
+10301|794527271|26028.0|DEBIT
+10302|399318593|4910.0|DEBIT
+10303|568931105|7331.0|CREDIT
+10304|144716426|1469.0|CREDIT
+10305|603015441|4052.0|CREDIT
+10306|293223789|20876.0|CREDIT
+10307|606444995|7826.0|DEBIT
+10308|400150434|746.0|DEBIT
+10309|767683056|3244.0|DEBIT
+10310|876098616|6538.0|DEBIT
+10311|74081563|25362.0|DEBIT
+10312|711049958|2091.0|DEBIT
+10313|657718602|5381.0|CREDIT
+10314|912202073|6467.0|DEBIT
+10315|654099636|2938.0|DEBIT
+10316|241063831|21767.0|DEBIT
+10317|858815265|6203.0|CREDIT
+10318|939864847|9961.0|CREDIT
+10319|949717040|5242.0|DEBIT
+10320|204238472|9615.0|CREDIT
+10321|866409326|22291.0|DEBIT
+10322|949996114|7868.0|DEBIT
+10323|422911083|5645.0|CREDIT
+10324|322526780|3789.0|DEBIT
+10325|133951099|5756.0|DEBIT
+10326|591884483|29458.0|CREDIT
+10327|695253171|3670.0|CREDIT
+10328|215281673|5174.0|DEBIT
+10329|436311786|5423.0|CREDIT
+10330|518432518|9037.0|CREDIT
+10331|425349532|26548.0|DEBIT
+10332|837790968|8858.0|CREDIT
+10333|887625960|6337.0|CREDIT
+10334|757883576|6954.0|CREDIT
+10335|459944677|3951.0|DEBIT
+10336|801117092|22688.0|CREDIT
+10337|516885021|9358.0|DEBIT
+10338|232560679|8366.0|CREDIT
+10339|611885533|4186.0|DEBIT
+10340|689162123|7565.0|CREDIT
+10341|501197134|25917.0|DEBIT
+10342|334174445|617.0|CREDIT
+10343|180187633|4945.0|CREDIT
+10344|887467274|5195.0|DEBIT
+10345|808750192|1136.0|DEBIT
+10346|13639891|21796.0|CREDIT
+10347|638906135|8026.0|DEBIT
+10348|834147190|1571.0|DEBIT
+10349|22124303|4402.0|CREDIT
+10350|235076790|2772.0|CREDIT
+10351|552645711|21462.0|DEBIT
+10352|389278238|9606.0|CREDIT
+10353|895551786|3928.0|DEBIT
+10354|584140777|8246.0|CREDIT
+10355|593368904|6750.0|CREDIT
+10356|329314918|27229.0|DEBIT
+10357|483588760|5955.0|DEBIT
+10358|827410725|2563.0|DEBIT
+10359|932975398|3319.0|DEBIT
+10360|976026166|5742.0|DEBIT
+10361|823102894|29658.0|CREDIT
+10362|711407791|2859.0|DEBIT
+10363|815218938|5248.0|DEBIT
+10364|406037352|9096.0|DEBIT
+10365|653047392|1314.0|CREDIT
+10366|625297640|25451.0|CREDIT
+10367|726603183|97.0|CREDIT
+10368|530543948|2115.0|CREDIT
+10369|935474118|2615.0|CREDIT
+10370|455235000|1673.0|CREDIT
+10371|24584042|24892.0|CREDIT
+10372|827942510|9186.0|CREDIT
+10373|999662321|3634.0|DEBIT
+10374|634102829|324.0|CREDIT
+10375|57740902|1549.0|DEBIT
+10376|908335261|29778.0|CREDIT
+10377|208503014|5634.0|DEBIT
+10378|886641515|9910.0|DEBIT
+10379|899684657|5835.0|DEBIT
+10380|953557084|3530.0|CREDIT
+10381|6573510|28709.0|DEBIT
+10382|7477110|835.0|DEBIT
+10383|347810136|3494.0|DEBIT
+10384|434772334|4891.0|DEBIT
+10385|33987507|4190.0|DEBIT
+10386|347989457|28234.0|CREDIT
+10387|595056330|5993.0|CREDIT
+10388|770726182|8415.0|CREDIT
+10389|358147886|7501.0|CREDIT
+10390|722150002|6400.0|DEBIT
+10391|275383311|27191.0|DEBIT
+10392|230539494|8684.0|CREDIT
+10393|326949401|2621.0|CREDIT
+10394|514140841|5321.0|DEBIT
+10395|117582460|4431.0|DEBIT
+10396|401338491|24864.0|CREDIT
+10397|661347779|8126.0|CREDIT
+10398|512723286|5761.0|DEBIT
+10399|435455695|8425.0|DEBIT
+10400|379646166|7155.0|DEBIT
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/0590c258/examples/filter/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/filter/src/main/resources/META-INF/properties.xml b/examples/filter/src/main/resources/META-INF/properties.xml
new file mode 100644
index 0000000..079fb3f
--- /dev/null
+++ b/examples/filter/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,95 @@
+<?xml version="1.0"?>
+<configuration>
+ <!--
+ <property>
+ <name>dt.application.{appName}.operator.{opName}.prop.{propName}</name>
+ <value>some-default-value (if value is not specified, it is required from the user or custom config when launching)</value>
+ </property>
+ -->
+ <!-- memory assigned to app master
+ <property>
+ <name>dt.attr.MASTER_MEMORY_MB</name>
+ <value>1024</value>
+ </property>
+ -->
+
+ <!-- these values override those set in populateDAG -->
+
+ <property>
+ <name>dt.application.FilterExample.operator.*.attr.MEMORY_MB</name>
+ <value>750</value>
+ </property>
+ <property>
+ <name>dt.application.FilterExample.attr.CHECKPOINT_WINDOW_COUNT</name>
+ <value>10</value>
+ </property>
+ <property>
+ <name>dt.application.FilterExample.operator.recordReader.prop.files</name>
+ <value>src/main/resources/META-INF/input.txt</value>
+ </property>
+ <property>
+ <name>dt.application.FilterExample.operator.csvParser.prop.schema</name>
+ <value>{"separator": "|","quoteChar":"\"","lineDelimiter":"","fields": [{"name": "trasactionId","type": "long"},{"name": "accountNumber","type": "long"},{"name": "amount","type": "double"},{"name": "type","type": "String"}]}</value>
+ </property>
+ <property>
+ <name>dt.application.FilterExample.operator.csvParser.port.out.attr.TUPLE_CLASS</name>
+ <value>com.datatorrent.tutorial.filter.TransactionPOJO</value>
+ </property>
+ <property>
+ <name>dt.application.FilterExample.operator.selectedOutput.prop.maxIdleWindows</name>
+ <value>20</value>
+ </property>
+ <property>
+ <name>dt.application.FilterExample.operator.rejectedOutput.prop.maxIdleWindows</name>
+ <value>20</value>
+ </property>
+ <property>
+ <name>dt.application.FilterExample.operator.filterOperator.port.input.attr.TUPLE_CLASS</name>
+ <value>com.datatorrent.tutorial.filter.TransactionPOJO</value>
+ </property>
+ <property>
+ <name>dt.application.FilterExample.operator.selectedFormatter.port.in.attr.TUPLE_CLASS</name>
+ <value>com.datatorrent.tutorial.filter.TransactionPOJO</value>
+ </property>
+ <property>
+ <name>dt.application.FilterExample.operator.rejectedFormatter.port.in.attr.TUPLE_CLASS</name>
+ <value>com.datatorrent.tutorial.filter.TransactionPOJO</value>
+ </property>
+ <property>
+ <name>dt.application.FilterExample.operator.filterOperator.prop.condition</name>
+ <value>({$}.getAmount() >= 20000)</value>
+ </property>
+ <property>
+ <name>dt.application.FilterExample.operator.filterOperator.prop.expressionFunctionsItem[5]</name>
+ <value>org.apache.commons.lang3.BooleanUtils.*</value>
+ </property>
+ <property>
+ <name>dt.application.FilterExample.operator.selectedFormatter.prop.schema</name>
+ <value>{"separator": "|","quoteChar":"\"","lineDelimiter":"","fields": [{"name": "trasactionId","type": "long"},{"name": "accountNumber","type": "long"},{"name": "amount","type": "double"},{"name": "type","type": "String"}]}</value>
+ </property>
+ <property>
+ <name>dt.application.FilterExample.operator.rejectedFormatter.prop.schema</name>
+ <value>{"separator": "|","quoteChar":"\"","lineDelimiter":"","fields": [{"name": "trasactionId","type": "long"},{"name": "accountNumber","type": "long"},{"name": "amount","type": "double"},{"name": "type","type": "String"}]}</value>
+ </property>
+ <property>
+ <name>dt.application.FilterExample.operator.selectedOutput.prop.filePath</name>
+ <value>file:///tmp</value>
+ </property>
+ <property>
+ <name>dt.application.FilterExample.operator.selectedOutput.prop.outputFileName</name>
+ <value>selected.txt</value>
+ </property>
+ <property>
+ <name>dt.application.FilterExample.operator.rejectedOutput.prop.filePath</name>
+ <value>file:///tmp</value>
+ </property>
+ <property>
+ <name>dt.application.FilterExample.operator.rejectedOutput.prop.outputFileName</name>
+ <value>rejected.txt</value>
+ </property>
+ <property>
+ <name>dt.loggers.level</name>
+ <value>com.datatorrent.*:DEBUG,org.apache.apex.malhar.*:DEBUG</value>
+ </property>
+</configuration>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/0590c258/examples/filter/src/main/resources/META-INF/rejected_output.txt
----------------------------------------------------------------------
diff --git a/examples/filter/src/main/resources/META-INF/rejected_output.txt b/examples/filter/src/main/resources/META-INF/rejected_output.txt
new file mode 100644
index 0000000..d83b7a9
--- /dev/null
+++ b/examples/filter/src/main/resources/META-INF/rejected_output.txt
@@ -0,0 +1,326 @@
+10001|84804533|51.0|DEBIT
+10002|529683455|8637.0|CREDIT
+10003|34646572|7443.0|CREDIT
+10004|967805584|7342.0|CREDIT
+10005|89940328|5100.0|DEBIT
+10006|484971385|4559.0|DEBIT
+10007|168472218|3539.0|CREDIT
+10008|560263718|1624.0|CREDIT
+10009|27097195|2471.0|DEBIT
+10010|562944412|5143.0|DEBIT
+10011|228473379|8336.0|DEBIT
+10012|138798397|5595.0|CREDIT
+10013|522515858|6054.0|CREDIT
+10014|358415062|1886.0|CREDIT
+10015|596239488|9936.0|DEBIT
+10016|378386029|4623.0|DEBIT
+10017|329310325|9064.0|CREDIT
+10018|652183672|2748.0|DEBIT
+10019|92850307|7752.0|DEBIT
+10020|608582237|2329.0|CREDIT
+10021|50283707|9987.0|DEBIT
+10022|540706222|8639.0|CREDIT
+10023|373122065|4834.0|DEBIT
+10024|215520537|5684.0|DEBIT
+10025|766304418|3561.0|CREDIT
+10026|467948355|2802.0|CREDIT
+10027|180522562|8662.0|DEBIT
+10028|444946324|7783.0|CREDIT
+10029|994074259|7125.0|CREDIT
+10030|182740938|3113.0|CREDIT
+10032|88726012|3603.0|CREDIT
+10033|571353545|4157.0|CREDIT
+10034|748258207|5329.0|CREDIT
+10035|228328193|6883.0|DEBIT
+10037|461283441|431.0|CREDIT
+10038|518514310|1586.0|CREDIT
+10039|381773726|8684.0|CREDIT
+10040|680364582|4221.0|CREDIT
+10042|468126452|4990.0|DEBIT
+10043|459970527|6188.0|CREDIT
+10044|100934948|3422.0|CREDIT
+10045|31728881|435.0|DEBIT
+10047|950687958|5002.0|CREDIT
+10048|164869066|1487.0|CREDIT
+10049|892759267|8102.0|DEBIT
+10050|69946718|4464.0|CREDIT
+10052|222384809|196.0|CREDIT
+10053|619012622|5892.0|DEBIT
+10054|524593769|8796.0|CREDIT
+10055|360301464|9846.0|CREDIT
+10057|894461469|3232.0|CREDIT
+10058|905824283|9614.0|DEBIT
+10059|201267525|6839.0|DEBIT
+10060|650778374|3956.0|CREDIT
+10062|257584130|2458.0|DEBIT
+10063|185240027|4987.0|DEBIT
+10064|340187828|6919.0|CREDIT
+10065|167114400|758.0|DEBIT
+10067|834054944|558.0|CREDIT
+10068|4848158|2972.0|CREDIT
+10069|611089049|3830.0|DEBIT
+10070|574929455|7585.0|CREDIT
+10072|678734877|1593.0|DEBIT
+10073|465428415|4876.0|CREDIT
+10074|631548543|4502.0|CREDIT
+10075|708806494|954.0|DEBIT
+10077|919361479|7612.0|DEBIT
+10078|831562086|9544.0|DEBIT
+10079|564566420|5850.0|DEBIT
+10080|732363069|1502.0|CREDIT
+10082|498367448|6829.0|CREDIT
+10083|387288387|4453.0|DEBIT
+10084|719081200|2961.0|CREDIT
+10085|735565137|7400.0|DEBIT
+10087|371001709|8504.0|CREDIT
+10088|793778660|6867.0|DEBIT
+10089|620050993|1582.0|CREDIT
+10090|367541932|2858.0|DEBIT
+10092|534083940|9915.0|CREDIT
+10093|171218647|9262.0|DEBIT
+10094|566590232|1904.0|DEBIT
+10095|940994162|2697.0|CREDIT
+10097|599783690|7551.0|CREDIT
+10098|472038054|7107.0|CREDIT
+10099|462438976|8978.0|CREDIT
+10100|419462514|5799.0|CREDIT
+10102|209434250|4813.0|DEBIT
+10103|869224102|6504.0|DEBIT
+10104|725686791|11.0|CREDIT
+10105|781644695|1705.0|DEBIT
+10107|857136119|2809.0|CREDIT
+10108|290114536|7476.0|CREDIT
+10109|90800192|2512.0|DEBIT
+10110|884687225|1396.0|CREDIT
+10112|976868472|8363.0|DEBIT
+10113|570179873|7231.0|CREDIT
+10114|399863103|4291.0|DEBIT
+10115|358440857|5322.0|CREDIT
+10117|50382939|6544.0|CREDIT
+10118|42616440|5372.0|CREDIT
+10119|220722302|3191.0|DEBIT
+10120|676925763|9413.0|CREDIT
+10122|412477439|9943.0|DEBIT
+10123|405618613|2350.0|DEBIT
+10124|415714190|9245.0|CREDIT
+10125|563760753|7092.0|CREDIT
+10127|434529942|7629.0|CREDIT
+10128|514574310|3897.0|DEBIT
+10129|27419748|8440.0|CREDIT
+10130|165864941|9762.0|DEBIT
+10132|52768757|3941.0|CREDIT
+10133|734355265|621.0|CREDIT
+10134|126968199|1286.0|DEBIT
+10135|585441472|1770.0|DEBIT
+10137|346242993|1487.0|CREDIT
+10138|490611170|6440.0|DEBIT
+10139|801216167|1515.0|DEBIT
+10140|659265220|4552.0|CREDIT
+10142|245703913|3093.0|DEBIT
+10143|2247312|5209.0|DEBIT
+10144|519787641|9527.0|DEBIT
+10145|912166127|2172.0|DEBIT
+10147|582275337|8712.0|DEBIT
+10148|259281594|5252.0|CREDIT
+10149|694423378|2060.0|DEBIT
+10150|629554298|5967.0|CREDIT
+10152|53101351|4382.0|CREDIT
+10153|323164961|5328.0|CREDIT
+10154|179064326|4341.0|DEBIT
+10155|52433644|7812.0|DEBIT
+10157|869913610|5813.0|CREDIT
+10158|347708099|6201.0|CREDIT
+10159|423486262|4871.0|DEBIT
+10160|461540100|9256.0|DEBIT
+10162|798468803|9224.0|DEBIT
+10163|418877075|519.0|DEBIT
+10164|40368770|245.0|DEBIT
+10165|463671480|5775.0|CREDIT
+10167|256176469|8602.0|DEBIT
+10168|739490131|3012.0|CREDIT
+10169|845152384|8188.0|DEBIT
+10170|425841395|2578.0|DEBIT
+10172|226331065|6325.0|DEBIT
+10173|244147717|6282.0|DEBIT
+10174|838618959|2970.0|CREDIT
+10175|966466342|6090.0|DEBIT
+10177|960290189|2392.0|DEBIT
+10178|665504299|5721.0|DEBIT
+10179|207166700|4961.0|DEBIT
+10180|910728570|7626.0|CREDIT
+10182|163142417|1630.0|CREDIT
+10183|662313571|6927.0|DEBIT
+10184|105583140|7757.0|DEBIT
+10185|639328437|7461.0|DEBIT
+10187|922940528|6379.0|CREDIT
+10188|3320970|48.0|DEBIT
+10189|217812412|7170.0|DEBIT
+10190|660688167|8493.0|CREDIT
+10192|29286525|4470.0|CREDIT
+10193|170950644|5463.0|DEBIT
+10194|813477150|376.0|CREDIT
+10195|180786030|6536.0|DEBIT
+10197|517613853|4244.0|CREDIT
+10198|142109351|4045.0|CREDIT
+10199|120408343|1224.0|CREDIT
+10200|372437363|2587.0|CREDIT
+10202|691849636|4285.0|CREDIT
+10203|654204917|3074.0|DEBIT
+10204|373303209|3566.0|CREDIT
+10205|28536699|3681.0|CREDIT
+10207|703547342|6351.0|DEBIT
+10208|117707726|8618.0|CREDIT
+10209|240155449|5987.0|DEBIT
+10210|539240301|6073.0|CREDIT
+10212|120948449|7938.0|CREDIT
+10213|968947065|2045.0|DEBIT
+10214|951562769|1738.0|CREDIT
+10215|136201581|1112.0|DEBIT
+10217|863802751|6610.0|DEBIT
+10218|855970512|354.0|CREDIT
+10219|339900628|4677.0|CREDIT
+10220|706630053|3682.0|DEBIT
+10222|174008949|3797.0|CREDIT
+10223|546023627|7130.0|CREDIT
+10224|138693364|4482.0|CREDIT
+10225|698667700|1055.0|CREDIT
+10227|597215884|6851.0|DEBIT
+10228|841851199|3901.0|CREDIT
+10229|658855748|4700.0|DEBIT
+10230|741198758|8011.0|CREDIT
+10232|130516026|8200.0|CREDIT
+10233|255513743|877.0|CREDIT
+10234|235986880|5608.0|DEBIT
+10235|755745324|4888.0|CREDIT
+10237|117830905|4932.0|CREDIT
+10238|436598482|3043.0|DEBIT
+10239|415570756|5563.0|DEBIT
+10240|160904677|5698.0|DEBIT
+10242|325540634|3633.0|CREDIT
+10243|934407707|9198.0|DEBIT
+10244|970044154|253.0|CREDIT
+10245|276211772|2484.0|DEBIT
+10247|515392682|3085.0|DEBIT
+10248|106926205|9599.0|CREDIT
+10249|403051823|6598.0|DEBIT
+10250|739699283|2898.0|CREDIT
+10252|588855751|4159.0|CREDIT
+10253|433188311|8895.0|CREDIT
+10254|613207771|2386.0|CREDIT
+10255|529952498|2469.0|CREDIT
+10257|788255680|4566.0|CREDIT
+10258|177961056|4417.0|DEBIT
+10259|856270849|7959.0|DEBIT
+10260|362871038|2661.0|DEBIT
+10262|51438711|7238.0|CREDIT
+10263|550177844|6044.0|DEBIT
+10264|796710293|3723.0|CREDIT
+10265|747419280|353.0|CREDIT
+10267|502287841|7040.0|DEBIT
+10268|868155169|8175.0|DEBIT
+10269|217771255|6130.0|DEBIT
+10270|983601385|9798.0|CREDIT
+10272|931578961|6792.0|DEBIT
+10273|627410737|6313.0|CREDIT
+10274|227280958|6610.0|DEBIT
+10275|814312724|2083.0|DEBIT
+10277|734388101|3870.0|CREDIT
+10278|969611064|344.0|DEBIT
+10279|955987400|2897.0|DEBIT
+10280|742505356|8610.0|CREDIT
+10282|381786126|4002.0|CREDIT
+10283|466001485|8284.0|CREDIT
+10284|310654102|4678.0|DEBIT
+10285|437156002|7311.0|CREDIT
+10287|503495897|6757.0|CREDIT
+10288|411513405|263.0|CREDIT
+10289|143598653|1820.0|DEBIT
+10290|702368818|4650.0|CREDIT
+10292|545322914|7008.0|DEBIT
+10293|176940492|584.0|CREDIT
+10294|592922722|5374.0|DEBIT
+10295|255199017|2002.0|DEBIT
+10297|103227713|6796.0|DEBIT
+10298|366884380|2496.0|DEBIT
+10299|147239319|9209.0|CREDIT
+10300|137542451|6119.0|DEBIT
+10302|399318593|4910.0|DEBIT
+10303|568931105|7331.0|CREDIT
+10304|144716426|1469.0|CREDIT
+10305|603015441|4052.0|CREDIT
+10307|606444995|7826.0|DEBIT
+10308|400150434|746.0|DEBIT
+10309|767683056|3244.0|DEBIT
+10310|876098616|6538.0|DEBIT
+10312|711049958|2091.0|DEBIT
+10313|657718602|5381.0|CREDIT
+10314|912202073|6467.0|DEBIT
+10315|654099636|2938.0|DEBIT
+10317|858815265|6203.0|CREDIT
+10318|939864847|9961.0|CREDIT
+10319|949717040|5242.0|DEBIT
+10320|204238472|9615.0|CREDIT
+10322|949996114|7868.0|DEBIT
+10323|422911083|5645.0|CREDIT
+10324|322526780|3789.0|DEBIT
+10325|133951099|5756.0|DEBIT
+10327|695253171|3670.0|CREDIT
+10328|215281673|5174.0|DEBIT
+10329|436311786|5423.0|CREDIT
+10330|518432518|9037.0|CREDIT
+10332|837790968|8858.0|CREDIT
+10333|887625960|6337.0|CREDIT
+10334|757883576|6954.0|CREDIT
+10335|459944677|3951.0|DEBIT
+10337|516885021|9358.0|DEBIT
+10338|232560679|8366.0|CREDIT
+10339|611885533|4186.0|DEBIT
+10340|689162123|7565.0|CREDIT
+10342|334174445|617.0|CREDIT
+10343|180187633|4945.0|CREDIT
+10344|887467274|5195.0|DEBIT
+10345|808750192|1136.0|DEBIT
+10347|638906135|8026.0|DEBIT
+10348|834147190|1571.0|DEBIT
+10349|22124303|4402.0|CREDIT
+10350|235076790|2772.0|CREDIT
+10352|389278238|9606.0|CREDIT
+10353|895551786|3928.0|DEBIT
+10354|584140777|8246.0|CREDIT
+10355|593368904|6750.0|CREDIT
+10357|483588760|5955.0|DEBIT
+10358|827410725|2563.0|DEBIT
+10359|932975398|3319.0|DEBIT
+10360|976026166|5742.0|DEBIT
+10362|711407791|2859.0|DEBIT
+10363|815218938|5248.0|DEBIT
+10364|406037352|9096.0|DEBIT
+10365|653047392|1314.0|CREDIT
+10367|726603183|97.0|CREDIT
+10368|530543948|2115.0|CREDIT
+10369|935474118|2615.0|CREDIT
+10370|455235000|1673.0|CREDIT
+10372|827942510|9186.0|CREDIT
+10373|999662321|3634.0|DEBIT
+10374|634102829|324.0|CREDIT
+10375|57740902|1549.0|DEBIT
+10377|208503014|5634.0|DEBIT
+10378|886641515|9910.0|DEBIT
+10379|899684657|5835.0|DEBIT
+10380|953557084|3530.0|CREDIT
+10382|7477110|835.0|DEBIT
+10383|347810136|3494.0|DEBIT
+10384|434772334|4891.0|DEBIT
+10385|33987507|4190.0|DEBIT
+10387|595056330|5993.0|CREDIT
+10388|770726182|8415.0|CREDIT
+10389|358147886|7501.0|CREDIT
+10390|722150002|6400.0|DEBIT
+10392|230539494|8684.0|CREDIT
+10393|326949401|2621.0|CREDIT
+10394|514140841|5321.0|DEBIT
+10395|117582460|4431.0|DEBIT
+10397|661347779|8126.0|CREDIT
+10398|512723286|5761.0|DEBIT
+10399|435455695|8425.0|DEBIT
+10400|379646166|7155.0|DEBIT
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/0590c258/examples/filter/src/main/resources/META-INF/selected_output.txt
----------------------------------------------------------------------
diff --git a/examples/filter/src/main/resources/META-INF/selected_output.txt b/examples/filter/src/main/resources/META-INF/selected_output.txt
new file mode 100644
index 0000000..887d721
--- /dev/null
+++ b/examples/filter/src/main/resources/META-INF/selected_output.txt
@@ -0,0 +1,74 @@
+10031|614671827|24979.0|CREDIT
+10036|953766573|22516.0|DEBIT
+10041|728080172|23189.0|DEBIT
+10046|906868986|26145.0|CREDIT
+10051|732833481|22186.0|DEBIT
+10056|76837426|21464.0|DEBIT
+10061|539521241|26964.0|DEBIT
+10066|645317265|22942.0|CREDIT
+10071|23360143|22515.0|CREDIT
+10076|656870588|26892.0|DEBIT
+10081|817401315|20419.0|CREDIT
+10086|973661932|24358.0|CREDIT
+10091|146470368|27897.0|DEBIT
+10096|683965112|29656.0|CREDIT
+10101|318680824|20128.0|CREDIT
+10106|48125174|22815.0|DEBIT
+10111|780970052|24959.0|DEBIT
+10116|713800593|21282.0|CREDIT
+10121|66746026|21667.0|DEBIT
+10126|737861484|23370.0|CREDIT
+10131|571929475|26224.0|DEBIT
+10136|893117252|27637.0|CREDIT
+10141|953467032|23295.0|CREDIT
+10146|95502398|28322.0|DEBIT
+10151|323058477|27765.0|DEBIT
+10156|3744164|22345.0|DEBIT
+10161|278026136|25472.0|CREDIT
+10166|410182527|24299.0|DEBIT
+10171|676685046|24621.0|CREDIT
+10176|347315258|22401.0|CREDIT
+10181|495476490|23835.0|CREDIT
+10186|353744821|24419.0|CREDIT
+10191|319041088|27622.0|CREDIT
+10196|200090142|24312.0|CREDIT
+10201|988342695|28896.0|DEBIT
+10206|332098186|20324.0|DEBIT
+10211|304868598|29587.0|DEBIT
+10216|446120114|27378.0|CREDIT
+10221|904298621|27819.0|CREDIT
+10226|286039255|23856.0|CREDIT
+10231|659386308|29955.0|DEBIT
+10236|864054476|25873.0|DEBIT
+10241|12110227|25870.0|CREDIT
+10246|632081618|22307.0|DEBIT
+10251|913373304|25760.0|CREDIT
+10256|487018926|27501.0|DEBIT
+10261|552260828|20328.0|DEBIT
+10266|625233460|29396.0|DEBIT
+10271|676948266|29532.0|DEBIT
+10276|614444626|24522.0|CREDIT
+10281|522681680|22593.0|CREDIT
+10286|495903944|27075.0|DEBIT
+10291|248658152|28021.0|DEBIT
+10296|858007111|24374.0|CREDIT
+10301|794527271|26028.0|DEBIT
+10306|293223789|20876.0|CREDIT
+10311|74081563|25362.0|DEBIT
+10316|241063831|21767.0|DEBIT
+10321|866409326|22291.0|DEBIT
+10326|591884483|29458.0|CREDIT
+10331|425349532|26548.0|DEBIT
+10336|801117092|22688.0|CREDIT
+10341|501197134|25917.0|DEBIT
+10346|13639891|21796.0|CREDIT
+10351|552645711|21462.0|DEBIT
+10356|329314918|27229.0|DEBIT
+10361|823102894|29658.0|CREDIT
+10366|625297640|25451.0|CREDIT
+10371|24584042|24892.0|CREDIT
+10376|908335261|29778.0|CREDIT
+10381|6573510|28709.0|DEBIT
+10386|347989457|28234.0|CREDIT
+10391|275383311|27191.0|DEBIT
+10396|401338491|24864.0|CREDIT
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/0590c258/examples/filter/src/test/java/com/datatorrent/tutorial/filter/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/filter/src/test/java/com/datatorrent/tutorial/filter/ApplicationTest.java b/examples/filter/src/test/java/com/datatorrent/tutorial/filter/ApplicationTest.java
new file mode 100644
index 0000000..a90e822
--- /dev/null
+++ b/examples/filter/src/test/java/com/datatorrent/tutorial/filter/ApplicationTest.java
@@ -0,0 +1,111 @@
+/**
+ * Copyright (c) 2016 DataTorrent, Inc.
+ * All rights reserved.
+ */
+
+package com.datatorrent.tutorial.filter;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.concurrent.Callable;
+
+import javax.validation.ConstraintViolationException;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.LocalMode;
+import com.datatorrent.stram.StramLocalCluster;
+
+/**
+ * Test the DAG declaration in local mode.
+ */
+public class ApplicationTest
+{
+ private String outputDir;
+
+ public static class TestMeta extends TestWatcher
+ {
+ public String baseDirectory;
+
+ @Override
+ protected void starting(org.junit.runner.Description description)
+ {
+ this.baseDirectory = "target/" + description.getClassName() + "/" + description.getMethodName();
+ }
+
+ @Override
+ protected void finished(Description description)
+ {
+ super.finished(description);
+ try {
+ FileUtils.forceDelete(new File(baseDirectory));
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ }
+
+ @Rule
+ public TestMeta testMeta = new TestMeta();
+
+ @Before
+ public void setup() throws Exception
+ {
+ outputDir = testMeta.baseDirectory + File.separator + "output";
+ }
+
+ @Test
+ public void testApplication() throws IOException, Exception
+ {
+
+ try {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ conf.set("dt.application.FilterExample.operator.selectedOutput.prop.filePath", outputDir);
+ conf.set("dt.application.FilterExample.operator.rejectedOutput.prop.filePath", outputDir);
+ final File selectedfile = FileUtils.getFile(outputDir, "selected.txt_8.0");
+ final File rejectedfile = FileUtils.getFile(outputDir, "rejected.txt_6.0");
+
+ lma.prepareDAG(new Application(), conf);
+ LocalMode.Controller lc = lma.getController();
+
+ ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>()
+ {
+ @Override
+ public Boolean call() throws Exception
+ {
+ if (selectedfile.exists() && rejectedfile.exists()) {
+ return true;
+ }
+ return false;
+ }
+ });
+
+ lc.run(40000);
+ Assert.assertTrue(
+ FileUtils.contentEquals(
+ FileUtils.getFile(
+ "src/main/resources/META-INF/selected_output.txt"
+ ),selectedfile));
+
+ Assert.assertTrue(
+ FileUtils.contentEquals(
+ FileUtils.getFile(
+ "src/main/resources/META-INF/rejected_output.txt"
+ ),rejectedfile));
+
+ } catch (ConstraintViolationException e) {
+ Assert.fail("constraint violations: " + e.getConstraintViolations());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/0590c258/examples/filter/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/filter/src/test/resources/log4j.properties b/examples/filter/src/test/resources/log4j.properties
new file mode 100644
index 0000000..98544e8
--- /dev/null
+++ b/examples/filter/src/test/resources/log4j.properties
@@ -0,0 +1,22 @@
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=debug
+log4j.logger.org.apache.apex=debug
+log4j.logger.org=info
[14/19] apex-malhar git commit: APEXMALHAR-2233 Updated the examples
to follow the structure of apex-malhar examples. Specified dependencies in
pom.xmls of individual examples correctly.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/transform/src/main/java/com/example/transform/DynamicTransformApplication.java
----------------------------------------------------------------------
diff --git a/examples/transform/src/main/java/com/example/transform/DynamicTransformApplication.java b/examples/transform/src/main/java/com/example/transform/DynamicTransformApplication.java
deleted file mode 100644
index a769016..0000000
--- a/examples/transform/src/main/java/com/example/transform/DynamicTransformApplication.java
+++ /dev/null
@@ -1,52 +0,0 @@
-package com.example.transform;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.StatsListener;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.common.partitioner.StatelessPartitioner;
-import com.datatorrent.lib.io.ConsoleOutputOperator;
-import com.datatorrent.lib.partitioner.StatelessThroughputBasedPartitioner;
-import com.datatorrent.lib.transform.TransformOperator;
-
-@ApplicationAnnotation(name="DynamicTransformApp")
-public class DynamicTransformApplication implements StreamingApplication
-{
- private static String COOL_DOWN_MILLIS = "dt.cooldown";
- private static String MAX_THROUGHPUT = "dt.maxThroughput";
- private static String MIN_THROUGHPUT = "dt.minThroughput";
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- POJOGenerator input = dag.addOperator("Input", new POJOGenerator());
- TransformOperator transform = dag.addOperator("Process", new TransformOperator());
- // Set expression map
- Map<String, String> expMap = new HashMap<>();
- expMap.put("name", "{$.firstName}.concat(\" \").concat({$.lastName})");
- expMap.put("age", "(new java.util.Date()).getYear() - {$.dateOfBirth}.getYear()");
- expMap.put("address", "{$.address}.toLowerCase()");
- transform.setExpressionMap(expMap);
- ConsoleOutputOperator output = dag.addOperator("Output", new ConsoleOutputOperator());
-
- dag.addStream("InputToTransform", input.output, transform.input);
- dag.addStream("TransformToOutput", transform.output, output.input);
-
- dag.setInputPortAttribute(transform.input, Context.PortContext.TUPLE_CLASS, CustomerEvent.class);
- dag.setOutputPortAttribute(transform.output, Context.PortContext.TUPLE_CLASS, CustomerInfo.class);
-
- StatelessThroughputBasedPartitioner<TransformOperator> partitioner = new StatelessThroughputBasedPartitioner<>();
- partitioner.setCooldownMillis(conf.getLong(COOL_DOWN_MILLIS, 10000));
- partitioner.setMaximumEvents(conf.getLong(MAX_THROUGHPUT, 30000));
- partitioner.setMinimumEvents(conf.getLong(MIN_THROUGHPUT, 10000));
- dag.setAttribute(transform, Context.OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[]{partitioner}));
- dag.setAttribute(transform, Context.OperatorContext.PARTITIONER, partitioner);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/transform/src/main/java/com/example/transform/POJOGenerator.java
----------------------------------------------------------------------
diff --git a/examples/transform/src/main/java/com/example/transform/POJOGenerator.java b/examples/transform/src/main/java/com/example/transform/POJOGenerator.java
deleted file mode 100644
index 9db5fd1..0000000
--- a/examples/transform/src/main/java/com/example/transform/POJOGenerator.java
+++ /dev/null
@@ -1,125 +0,0 @@
-package com.example.transform;
-
-import java.util.Date;
-import java.util.Random;
-
-import javax.validation.constraints.Min;
-
-import org.apache.commons.lang3.RandomStringUtils;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.InputOperator;
-
-/**
- * Generates and emits the CustomerEvent
- */
-public class POJOGenerator implements InputOperator
-{
- @Min(1)
- private int maxCustomerId = 100000;
- @Min(1)
- private int maxNameLength = 10;
- @Min(1)
- private int maxAddressLength = 15;
- private long tuplesCounter;
- // Limit number of emitted tuples per window
- @Min(1)
- private long maxTuplesPerWindow = 100;
- private final Random random = new Random();
- private final RandomStringUtils rRandom = new RandomStringUtils();
- public final transient DefaultOutputPort<Object> output = new DefaultOutputPort<>();
-
- @Override
- public void beginWindow(long windowId)
- {
- tuplesCounter = 0;
- }
-
- @Override
- public void endWindow()
- {
- }
-
- @Override
- public void setup(Context.OperatorContext context)
- {
- }
-
- @Override
- public void teardown()
- {
-
- }
-
- CustomerEvent generateCustomersEvent() throws Exception {
-
- CustomerEvent customerEvent = new CustomerEvent();
- customerEvent.setCustomerId(randomId(maxCustomerId));
- customerEvent.setFirstName(rRandom.randomAlphabetic(randomId(maxNameLength)));
- customerEvent.setLastName(rRandom.randomAlphabetic(randomId(maxNameLength)));
- long val = random.nextLong();
- long diff1 = val % System.currentTimeMillis();
- customerEvent.setDateOfBirth(new Date(diff1));
- customerEvent.setAddress(rRandom.randomAlphabetic(randomId(maxAddressLength)));
- return customerEvent;
- }
-
- private int randomId(int max) {
- if (max < 1) return 1;
- return 1 + random.nextInt(max);
- }
-
- @Override
- public void emitTuples()
- {
- while (tuplesCounter++ < maxTuplesPerWindow) {
- try {
- CustomerEvent event = generateCustomersEvent();
- this.output.emit(event);
- } catch (Exception ex) {
- throw new RuntimeException(ex);
- }
- }
- }
-
- public long getMaxTuplesPerWindow()
- {
- return maxTuplesPerWindow;
- }
-
- public void setMaxTuplesPerWindow(long maxTuplesPerWindow)
- {
- this.maxTuplesPerWindow = maxTuplesPerWindow;
- }
-
- public int getMaxAddressLength()
- {
- return maxAddressLength;
- }
-
- public void setMaxAddressLength(int maxAddressLength)
- {
- this.maxAddressLength = maxAddressLength;
- }
-
- public int getMaxNameLength()
- {
- return maxNameLength;
- }
-
- public void setMaxNameLength(int maxNameLength)
- {
- this.maxNameLength = maxNameLength;
- }
-
- public int getMaxCustomerId()
- {
- return maxCustomerId;
- }
-
- public void setMaxCustomerId(int maxCustomerId)
- {
- this.maxCustomerId = maxCustomerId;
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/transform/src/main/java/org/apache/apex/examples/transform/Application.java
----------------------------------------------------------------------
diff --git a/examples/transform/src/main/java/org/apache/apex/examples/transform/Application.java b/examples/transform/src/main/java/org/apache/apex/examples/transform/Application.java
new file mode 100644
index 0000000..e6a0419
--- /dev/null
+++ b/examples/transform/src/main/java/org/apache/apex/examples/transform/Application.java
@@ -0,0 +1,39 @@
+package org.apache.apex.examples.transform;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.common.partitioner.StatelessPartitioner;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+import com.datatorrent.lib.transform.TransformOperator;
+
+@ApplicationAnnotation(name="TransformExample")
+public class Application implements StreamingApplication
+{
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ POJOGenerator input = dag.addOperator("Input", new POJOGenerator());
+ TransformOperator transform = dag.addOperator("Process", new TransformOperator());
+ // Set expression map
+ Map<String, String> expMap = new HashMap<>();
+ expMap.put("name", "{$.firstName}.concat(\" \").concat({$.lastName})");
+ expMap.put("age", "(new java.util.Date()).getYear() - {$.dateOfBirth}.getYear()");
+ expMap.put("address", "{$.address}.toLowerCase()");
+ transform.setExpressionMap(expMap);
+ ConsoleOutputOperator output = dag.addOperator("Output", new ConsoleOutputOperator());
+
+ dag.addStream("InputToTransform", input.output, transform.input);
+ dag.addStream("TransformToOutput", transform.output, output.input);
+
+ dag.setInputPortAttribute(transform.input, Context.PortContext.TUPLE_CLASS, CustomerEvent.class);
+ dag.setOutputPortAttribute(transform.output, Context.PortContext.TUPLE_CLASS, CustomerInfo.class);
+ dag.setAttribute(transform, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<TransformOperator>(2));
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/transform/src/main/java/org/apache/apex/examples/transform/CustomerEvent.java
----------------------------------------------------------------------
diff --git a/examples/transform/src/main/java/org/apache/apex/examples/transform/CustomerEvent.java b/examples/transform/src/main/java/org/apache/apex/examples/transform/CustomerEvent.java
new file mode 100644
index 0000000..8011e12
--- /dev/null
+++ b/examples/transform/src/main/java/org/apache/apex/examples/transform/CustomerEvent.java
@@ -0,0 +1,74 @@
+package org.apache.apex.examples.transform;
+
+import java.util.Date;
+
+public class CustomerEvent
+{
+ private int customerId;
+ private String firstName;
+ private String lastName;
+ private Date dateOfBirth;
+ private String address;
+
+ public int getCustomerId()
+ {
+ return customerId;
+ }
+
+ public void setCustomerId(int customerId)
+ {
+ this.customerId = customerId;
+ }
+
+ public String getFirstName()
+ {
+ return firstName;
+ }
+
+ public void setFirstName(String firstName)
+ {
+ this.firstName = firstName;
+ }
+
+ public String getLastName()
+ {
+ return lastName;
+ }
+
+ public void setLastName(String lastName)
+ {
+ this.lastName = lastName;
+ }
+
+ public Date getDateOfBirth()
+ {
+ return dateOfBirth;
+ }
+
+ public void setDateOfBirth(Date dateOfBirth)
+ {
+ this.dateOfBirth = dateOfBirth;
+ }
+
+ public String getAddress()
+ {
+ return address;
+ }
+
+ public void setAddress(String address)
+ {
+ this.address = address;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "CustomerEvent{" +
+ "customerId=" + customerId +
+ ", firstName='" + firstName + '\'' +
+ ", lastName='" + lastName + '\'' +
+ ", dateOfBirth=" + dateOfBirth +
+ ", address='" + address + '\'' +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/transform/src/main/java/org/apache/apex/examples/transform/CustomerInfo.java
----------------------------------------------------------------------
diff --git a/examples/transform/src/main/java/org/apache/apex/examples/transform/CustomerInfo.java b/examples/transform/src/main/java/org/apache/apex/examples/transform/CustomerInfo.java
new file mode 100644
index 0000000..ece76a6
--- /dev/null
+++ b/examples/transform/src/main/java/org/apache/apex/examples/transform/CustomerInfo.java
@@ -0,0 +1,60 @@
+package org.apache.apex.examples.transform;
+
+public class CustomerInfo
+{
+ private int customerId;
+ private String name;
+ private int age;
+ private String address;
+
+ public int getCustomerId()
+ {
+ return customerId;
+ }
+
+ public void setCustomerId(int customerId)
+ {
+ this.customerId = customerId;
+ }
+
+ public String getName()
+ {
+ return name;
+ }
+
+ public void setName(String name)
+ {
+ this.name = name;
+ }
+
+ public int getAge()
+ {
+ return age;
+ }
+
+ public void setAge(int age)
+ {
+ this.age = age;
+ }
+
+ public String getAddress()
+ {
+ return address;
+ }
+
+ public void setAddress(String address)
+ {
+ this.address = address;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "CustomerInfo{" +
+ "customerId=" + customerId +
+ ", name='" + name + '\'' +
+ ", age=" + age +
+ ", address='" + address + '\'' +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/transform/src/main/java/org/apache/apex/examples/transform/DynamicTransformApplication.java
----------------------------------------------------------------------
diff --git a/examples/transform/src/main/java/org/apache/apex/examples/transform/DynamicTransformApplication.java b/examples/transform/src/main/java/org/apache/apex/examples/transform/DynamicTransformApplication.java
new file mode 100644
index 0000000..01bc446
--- /dev/null
+++ b/examples/transform/src/main/java/org/apache/apex/examples/transform/DynamicTransformApplication.java
@@ -0,0 +1,51 @@
+package org.apache.apex.examples.transform;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StatsListener;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+import com.datatorrent.lib.partitioner.StatelessThroughputBasedPartitioner;
+import com.datatorrent.lib.transform.TransformOperator;
+
+@ApplicationAnnotation(name="DynamicTransformApp")
+public class DynamicTransformApplication implements StreamingApplication
+{
+ private static String COOL_DOWN_MILLIS = "dt.cooldown";
+ private static String MAX_THROUGHPUT = "dt.maxThroughput";
+ private static String MIN_THROUGHPUT = "dt.minThroughput";
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ POJOGenerator input = dag.addOperator("Input", new POJOGenerator());
+ TransformOperator transform = dag.addOperator("Process", new TransformOperator());
+ // Set expression map
+ Map<String, String> expMap = new HashMap<>();
+ expMap.put("name", "{$.firstName}.concat(\" \").concat({$.lastName})");
+ expMap.put("age", "(new java.util.Date()).getYear() - {$.dateOfBirth}.getYear()");
+ expMap.put("address", "{$.address}.toLowerCase()");
+ transform.setExpressionMap(expMap);
+ ConsoleOutputOperator output = dag.addOperator("Output", new ConsoleOutputOperator());
+
+ dag.addStream("InputToTransform", input.output, transform.input);
+ dag.addStream("TransformToOutput", transform.output, output.input);
+
+ dag.setInputPortAttribute(transform.input, Context.PortContext.TUPLE_CLASS, CustomerEvent.class);
+ dag.setOutputPortAttribute(transform.output, Context.PortContext.TUPLE_CLASS, CustomerInfo.class);
+
+ StatelessThroughputBasedPartitioner<TransformOperator> partitioner = new StatelessThroughputBasedPartitioner<>();
+ partitioner.setCooldownMillis(conf.getLong(COOL_DOWN_MILLIS, 10000));
+ partitioner.setMaximumEvents(conf.getLong(MAX_THROUGHPUT, 30000));
+ partitioner.setMinimumEvents(conf.getLong(MIN_THROUGHPUT, 10000));
+ dag.setAttribute(transform, Context.OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[]{partitioner}));
+ dag.setAttribute(transform, Context.OperatorContext.PARTITIONER, partitioner);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/transform/src/main/java/org/apache/apex/examples/transform/POJOGenerator.java
----------------------------------------------------------------------
diff --git a/examples/transform/src/main/java/org/apache/apex/examples/transform/POJOGenerator.java b/examples/transform/src/main/java/org/apache/apex/examples/transform/POJOGenerator.java
new file mode 100644
index 0000000..f8f3b22
--- /dev/null
+++ b/examples/transform/src/main/java/org/apache/apex/examples/transform/POJOGenerator.java
@@ -0,0 +1,125 @@
+package org.apache.apex.examples.transform;
+
+import java.util.Date;
+import java.util.Random;
+
+import javax.validation.constraints.Min;
+
+import org.apache.commons.lang3.RandomStringUtils;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+
+/**
+ * Generates and emits the CustomerEvent
+ */
+public class POJOGenerator implements InputOperator
+{
+ @Min(1)
+ private int maxCustomerId = 100000;
+ @Min(1)
+ private int maxNameLength = 10;
+ @Min(1)
+ private int maxAddressLength = 15;
+ private long tuplesCounter;
+ // Limit number of emitted tuples per window
+ @Min(1)
+ private long maxTuplesPerWindow = 100;
+ private final Random random = new Random();
+ private final RandomStringUtils rRandom = new RandomStringUtils();
+ public final transient DefaultOutputPort<Object> output = new DefaultOutputPort<>();
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ tuplesCounter = 0;
+ }
+
+ @Override
+ public void endWindow()
+ {
+ }
+
+ @Override
+ public void setup(Context.OperatorContext context)
+ {
+ }
+
+ @Override
+ public void teardown()
+ {
+
+ }
+
+ CustomerEvent generateCustomersEvent() throws Exception {
+
+ CustomerEvent customerEvent = new CustomerEvent();
+ customerEvent.setCustomerId(randomId(maxCustomerId));
+ customerEvent.setFirstName(rRandom.randomAlphabetic(randomId(maxNameLength)));
+ customerEvent.setLastName(rRandom.randomAlphabetic(randomId(maxNameLength)));
+ long val = random.nextLong();
+ long diff1 = val % System.currentTimeMillis();
+ customerEvent.setDateOfBirth(new Date(diff1));
+ customerEvent.setAddress(rRandom.randomAlphabetic(randomId(maxAddressLength)));
+ return customerEvent;
+ }
+
+ private int randomId(int max) {
+ if (max < 1) return 1;
+ return 1 + random.nextInt(max);
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ while (tuplesCounter++ < maxTuplesPerWindow) {
+ try {
+ CustomerEvent event = generateCustomersEvent();
+ this.output.emit(event);
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+ }
+
+ public long getMaxTuplesPerWindow()
+ {
+ return maxTuplesPerWindow;
+ }
+
+ public void setMaxTuplesPerWindow(long maxTuplesPerWindow)
+ {
+ this.maxTuplesPerWindow = maxTuplesPerWindow;
+ }
+
+ public int getMaxAddressLength()
+ {
+ return maxAddressLength;
+ }
+
+ public void setMaxAddressLength(int maxAddressLength)
+ {
+ this.maxAddressLength = maxAddressLength;
+ }
+
+ public int getMaxNameLength()
+ {
+ return maxNameLength;
+ }
+
+ public void setMaxNameLength(int maxNameLength)
+ {
+ this.maxNameLength = maxNameLength;
+ }
+
+ public int getMaxCustomerId()
+ {
+ return maxCustomerId;
+ }
+
+ public void setMaxCustomerId(int maxCustomerId)
+ {
+ this.maxCustomerId = maxCustomerId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/transform/src/test/java/com/example/transform/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/transform/src/test/java/com/example/transform/ApplicationTest.java b/examples/transform/src/test/java/com/example/transform/ApplicationTest.java
deleted file mode 100644
index 2d331d2..0000000
--- a/examples/transform/src/test/java/com/example/transform/ApplicationTest.java
+++ /dev/null
@@ -1,21 +0,0 @@
-package com.example.transform;
-
-import org.junit.Test;
-import org.apache.hadoop.conf.Configuration;
-import com.datatorrent.api.LocalMode;
-
-public class ApplicationTest
-{
- @Test
- public void testApplication() throws Exception
- {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
- lma.prepareDAG(new Application(), conf);
- LocalMode.Controller lc = lma.getController();
- lc.runAsync();
- Thread.sleep(10 * 1000);
- lc.shutdown();
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/transform/src/test/java/org/apache/apex/examples/transform/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/transform/src/test/java/org/apache/apex/examples/transform/ApplicationTest.java b/examples/transform/src/test/java/org/apache/apex/examples/transform/ApplicationTest.java
new file mode 100644
index 0000000..aab8af8
--- /dev/null
+++ b/examples/transform/src/test/java/org/apache/apex/examples/transform/ApplicationTest.java
@@ -0,0 +1,21 @@
+package org.apache.apex.examples.transform;
+
+import org.junit.Test;
+import org.apache.hadoop.conf.Configuration;
+import com.datatorrent.api.LocalMode;
+
+public class ApplicationTest
+{
+ @Test
+ public void testApplication() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ lma.prepareDAG(new Application(), conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.runAsync();
+ Thread.sleep(10 * 1000);
+ lc.shutdown();
+ }
+}
[05/19] apex-malhar git commit: Add dynamic partition example Update
index and revise some README files
Posted by th...@apache.org.
Add dynamic partition example
Update index and revise some README files
Project: http://git-wip-us.apache.org/repos/asf/apex-malhar/repo
Commit: http://git-wip-us.apache.org/repos/asf/apex-malhar/commit/041af061
Tree: http://git-wip-us.apache.org/repos/asf/apex-malhar/tree/041af061
Diff: http://git-wip-us.apache.org/repos/asf/apex-malhar/diff/041af061
Branch: refs/heads/master
Commit: 041af061bbce5f5d498f658b5b152f1a6c337ce6
Parents: a492e22
Author: Munagala V. Ramanath <ra...@datatorrent.com>
Authored: Tue Jun 7 09:27:51 2016 -0700
Committer: Lakshmi Prasanna Velineni <la...@datatorrent.com>
Committed: Sun Mar 26 11:43:48 2017 -0700
----------------------------------------------------------------------
examples/dynamic-partition/README.md | 8 +
.../XmlJavadocCommentsExtractor.xsl | 44 +++
examples/dynamic-partition/pom.xml | 274 +++++++++++++++++++
.../src/assemble/appPackage.xml | 43 +++
.../src/main/java/com/example/dynamic/App.java | 23 ++
.../src/main/java/com/example/dynamic/Gen.java | 169 ++++++++++++
.../src/main/resources/META-INF/properties.xml | 25 ++
.../src/main/resources/unused-log4j.properties | 9 +
.../com/example/dynamic/ApplicationTest.java | 34 +++
.../src/test/resources/log4j.properties | 21 ++
10 files changed, 650 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/041af061/examples/dynamic-partition/README.md
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/README.md b/examples/dynamic-partition/README.md
new file mode 100644
index 0000000..f82aaf5
--- /dev/null
+++ b/examples/dynamic-partition/README.md
@@ -0,0 +1,8 @@
+This example illustrates the use of dynamic partitioning of an operator.
+
+It uses an input operator that generates random numbers and outputs them to a
+`DevNull` operator (which, as the name suggests, simply discards them).
+
+The input operator starts out with two partitions; after some tuples have been
+processed, a dynamic repartition is triggered via the `StatsListener` interface
+to increase the number of partitions to four.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/041af061/examples/dynamic-partition/XmlJavadocCommentsExtractor.xsl
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/XmlJavadocCommentsExtractor.xsl b/examples/dynamic-partition/XmlJavadocCommentsExtractor.xsl
new file mode 100644
index 0000000..08075a9
--- /dev/null
+++ b/examples/dynamic-partition/XmlJavadocCommentsExtractor.xsl
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+
+<!--
+ Document : XmlJavadocCommentsExtractor.xsl
+ Created on : September 16, 2014, 11:30 AM
+ Description:
+ The transformation strips off all information except for comments and tags from xml javadoc generated by xml-doclet.
+-->
+
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+ <xsl:output method="xml" standalone="yes"/>
+
+ <!-- copy xml by selecting only the following nodes, attributes and text -->
+ <xsl:template match="node()|text()|@*">
+ <xsl:copy>
+ <xsl:apply-templates select="root|package|class|interface|method|field|type|comment|tag|text()|@name|@qualified|@text"/>
+ </xsl:copy>
+ </xsl:template>
+
+ <!-- Strip off the following paths from the selected xml -->
+ <xsl:template match="//root/package/interface/interface
+ |//root/package/interface/method/@qualified
+ |//root/package/class/interface
+ |//root/package/class/class
+ |//root/package/class/method/@qualified
+ |//root/package/class/field/@qualified" />
+
+ <xsl:strip-space elements="*"/>
+</xsl:stylesheet>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/041af061/examples/dynamic-partition/pom.xml
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/pom.xml b/examples/dynamic-partition/pom.xml
new file mode 100644
index 0000000..34e91ee
--- /dev/null
+++ b/examples/dynamic-partition/pom.xml
@@ -0,0 +1,274 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>com.example</groupId>
+ <version>1.0-SNAPSHOT</version>
+ <artifactId>dynamic-partition</artifactId>
+ <packaging>jar</packaging>
+
+ <!-- change these to the appropriate values -->
+ <name>Dynamic Partitioning</name>
+ <description>Example showing dynamic partitioning</description>
+
+ <properties>
+ <!-- change this if you desire to use a different version of Apex Core -->
+ <apex.version>3.5.0</apex.version>
+ <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-eclipse-plugin</artifactId>
+ <version>2.9</version>
+ <configuration>
+ <downloadSources>true</downloadSources>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.3</version>
+ <configuration>
+ <encoding>UTF-8</encoding>
+ <source>1.7</source>
+ <target>1.7</target>
+ <debug>true</debug>
+ <optimize>false</optimize>
+ <showDeprecation>true</showDeprecation>
+ <showWarnings>true</showWarnings>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.8</version>
+ <executions>
+ <execution>
+ <id>copy-dependencies</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>target/deps</outputDirectory>
+ <includeScope>runtime</includeScope>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>app-package-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <finalName>${project.artifactId}-${project.version}-apexapp</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <descriptors>
+ <descriptor>src/assemble/appPackage.xml</descriptor>
+ </descriptors>
+ <archiverConfig>
+ <defaultDirectoryMode>0755</defaultDirectoryMode>
+ </archiverConfig>
+ <archive>
+ <manifestEntries>
+ <Class-Path>${apex.apppackage.classpath}</Class-Path>
+ <DT-Engine-Version>${apex.version}</DT-Engine-Version>
+ <DT-App-Package-Group-Id>${project.groupId}</DT-App-Package-Group-Id>
+ <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name>
+ <DT-App-Package-Version>${project.version}</DT-App-Package-Version>
+ <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name>
+ <DT-App-Package-Description>${project.description}</DT-App-Package-Description>
+ </manifestEntries>
+ </archive>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <configuration>
+ <target>
+ <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar"
+ tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa" />
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ <execution>
+ <!-- create resource directory for xml javadoc-->
+ <id>createJavadocDirectory</id>
+ <phase>generate-resources</phase>
+ <configuration>
+ <tasks>
+ <delete dir="${project.build.directory}/generated-resources/xml-javadoc"/>
+ <mkdir dir="${project.build.directory}/generated-resources/xml-javadoc"/>
+ </tasks>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.9.1</version>
+ <executions>
+ <execution>
+ <id>attach-artifacts</id>
+ <phase>package</phase>
+ <goals>
+ <goal>attach-artifact</goal>
+ </goals>
+ <configuration>
+ <artifacts>
+ <artifact>
+ <file>target/${project.artifactId}-${project.version}.apa</file>
+ <type>apa</type>
+ </artifact>
+ </artifacts>
+ <skipAttach>false</skipAttach>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- generate javdoc -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <executions>
+ <!-- generate xml javadoc -->
+ <execution>
+ <id>xml-doclet</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>javadoc</goal>
+ </goals>
+ <configuration>
+ <doclet>com.github.markusbernhardt.xmldoclet.XmlDoclet</doclet>
+ <additionalparam>-d ${project.build.directory}/generated-resources/xml-javadoc -filename ${project.artifactId}-${project.version}-javadoc.xml</additionalparam>
+ <useStandardDocletOptions>false</useStandardDocletOptions>
+ <docletArtifact>
+ <groupId>com.github.markusbernhardt</groupId>
+ <artifactId>xml-doclet</artifactId>
+ <version>1.0.4</version>
+ </docletArtifact>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <!-- Transform xml javadoc to stripped down version containing only class/interface comments and tags-->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>xml-maven-plugin</artifactId>
+ <version>1.0</version>
+ <executions>
+ <execution>
+ <id>transform-xmljavadoc</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>transform</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <transformationSets>
+ <transformationSet>
+ <dir>${project.build.directory}/generated-resources/xml-javadoc</dir>
+ <includes>
+ <include>${project.artifactId}-${project.version}-javadoc.xml</include>
+ </includes>
+ <stylesheet>XmlJavadocCommentsExtractor.xsl</stylesheet>
+ <outputDir>${project.build.directory}/generated-resources/xml-javadoc</outputDir>
+ </transformationSet>
+ </transformationSets>
+ </configuration>
+ </plugin>
+ <!-- copy xml javadoc to class jar -->
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>2.6</version>
+ <executions>
+ <execution>
+ <id>copy-resources</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${basedir}/target/classes</outputDirectory>
+ <resources>
+ <resource>
+ <directory>${project.build.directory}/generated-resources/xml-javadoc</directory>
+ <includes>
+ <include>${project.artifactId}-${project.version}-javadoc.xml</include>
+ </includes>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+
+ </build>
+
+ <dependencies>
+ <!-- add your dependencies here -->
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-library</artifactId>
+ <version>3.6.0</version>
+ <!--
+ If you know that your application does not need transitive dependencies pulled in by malhar-library,
+ uncomment the following to reduce the size of your app package.
+ -->
+ <!--
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ -->
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>apex-common</artifactId>
+ <version>${apex.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.10</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>apex-engine</artifactId>
+ <version>${apex.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/041af061/examples/dynamic-partition/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/src/assemble/appPackage.xml b/examples/dynamic-partition/src/assemble/appPackage.xml
new file mode 100644
index 0000000..7ad071c
--- /dev/null
+++ b/examples/dynamic-partition/src/assemble/appPackage.xml
@@ -0,0 +1,43 @@
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>appPackage</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/target/</directory>
+ <outputDirectory>/app</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/target/deps</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/site/conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/META-INF</directory>
+ <outputDirectory>/META-INF</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/app</directory>
+ <outputDirectory>/app</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/resources</directory>
+ <outputDirectory>/resources</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/041af061/examples/dynamic-partition/src/main/java/com/example/dynamic/App.java
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/src/main/java/com/example/dynamic/App.java b/examples/dynamic-partition/src/main/java/com/example/dynamic/App.java
new file mode 100644
index 0000000..9eec263
--- /dev/null
+++ b/examples/dynamic-partition/src/main/java/com/example/dynamic/App.java
@@ -0,0 +1,23 @@
+package com.example.dynamic;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.DAG;
+
+import com.datatorrent.lib.stream.DevNull;
+
+@ApplicationAnnotation(name="Dyn")
+public class App implements StreamingApplication
+{
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ Gen gen = dag.addOperator("gen", Gen.class);
+ DevNull devNull = dag.addOperator("devNull", DevNull.class);
+
+ dag.addStream("data", gen.out, devNull.data);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/041af061/examples/dynamic-partition/src/main/java/com/example/dynamic/Gen.java
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/src/main/java/com/example/dynamic/Gen.java b/examples/dynamic-partition/src/main/java/com/example/dynamic/Gen.java
new file mode 100644
index 0000000..4cccd23
--- /dev/null
+++ b/examples/dynamic-partition/src/main/java/com/example/dynamic/Gen.java
@@ -0,0 +1,169 @@
+package com.example.dynamic;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+import com.google.common.collect.Lists;
+import org.apache.commons.io.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.validation.constraints.NotNull;
+import java.io.ByteArrayOutputStream;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import com.datatorrent.api.DefaultPartition;
+import com.datatorrent.api.Partitioner;
+import com.datatorrent.api.StatsListener;
+
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.common.util.BaseOperator;
+
+/**
+ * Operator that dynamically partitions itself after 500 tuples have been emitted
+ */
+public class Gen extends BaseOperator implements InputOperator, Partitioner<Gen>, StatsListener
+{
+ private static final Logger LOG = LoggerFactory.getLogger(Gen.class);
+
+ private static final int MAX_PARTITIONS = 4; // maximum number of partitions
+
+ private int partitions = 2; // initial number of partitions
+
+ @NotNull
+ private int numTuples; // number of tuples to emit per window
+
+ private transient int count = 0;
+
+ public final transient DefaultOutputPort<Double> out = new DefaultOutputPort<Double>();
+
+ @Override
+ public void partitioned(Map<Integer, Partition<Gen>> map)
+ {
+ if (partitions != map.size()) {
+ String msg = String.format("partitions = %d, map.size = %d%n", partitions, map.size());
+ throw new RuntimeException(msg);
+ }
+ }
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ count = 0;
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ if (count < numTuples) {
+ ++count;
+ out.emit(Math.random());
+ }
+ }
+
+ public int getNumTuples()
+ {
+ return numTuples;
+ }
+
+ /**
+ * Sets the number of tuples to be emitted every window.
+ * @param numTuples number of tuples
+ */
+ public void setNumTuples(int numTuples)
+ {
+ this.numTuples = numTuples;
+ }
+
+ @Override
+ public Response processStats(BatchedOperatorStats batchedOperatorStats) {
+
+ final long emittedCount = batchedOperatorStats.getTuplesEmittedPSMA();
+
+ // we only perform a single dynamic repartition
+ Response res = new Response();
+ res.repartitionRequired = false;
+ if (emittedCount > 500 && partitions < MAX_PARTITIONS) {
+ LOG.info("processStats: trying repartition of input operator current {} required {}",
+ partitions, MAX_PARTITIONS);
+ LOG.info("**** operator id = {}, window id = {}, tuplesProcessedPSMA = {}, tuplesEmittedPSMA = {}",
+ batchedOperatorStats.getOperatorId(),
+ batchedOperatorStats.getCurrentWindowId(),
+ batchedOperatorStats.getTuplesProcessedPSMA(),
+ emittedCount);
+ partitions = MAX_PARTITIONS;
+ res.repartitionRequired = true;
+ }
+
+ return res;
+ } // processStats
+
+ /**
+ * Clone object by serializing and deserializing using Kryo.
+ * Note this is different from using {@link Kryo#copy(Object)}, which will attempt to also clone transient fields.
+ *
+ * @param kryo kryo object used to clone objects
+ * @param src src object that copy from
+ * @return cloned object
+ */
+ @SuppressWarnings("unchecked")
+ private static <SRC> SRC cloneObject(Kryo kryo, SRC src)
+ {
+ kryo.setClassLoader(src.getClass().getClassLoader());
+ ByteArrayOutputStream bos = null;
+ Output output;
+ Input input = null;
+ try {
+ bos = new ByteArrayOutputStream();
+ output = new Output(bos);
+ kryo.writeObject(output, src);
+ output.close();
+ input = new Input(bos.toByteArray());
+ return (SRC)kryo.readObject(input, src.getClass());
+ } finally {
+ IOUtils.closeQuietly(input);
+ IOUtils.closeQuietly(bos);
+ }
+ }
+
+ @Override
+ public Collection<Partition<Gen>> definePartitions(
+ Collection<Partition<Gen>> list, PartitioningContext context)
+ {
+ if (partitions < 0) { // error
+ String msg = String.format("Error: Bad value: partitions = %d%n", partitions);
+ LOG.error(msg);
+ throw new RuntimeException(msg);
+ }
+
+ final int prevCount = list.size();
+ if (1 == prevCount) { // initial call
+ LOG.info("definePartitions: First call, prevCount = {}, partitions = {}",
+ prevCount, partitions);
+ }
+
+ if (prevCount == partitions) {
+ LOG.info("definePartitions: Nothing to do in definePartitions");
+ return list; // nothing to do
+ }
+
+ LOG.debug("definePartitions: Repartitioning from {} to {}", prevCount, partitions);
+
+ Kryo kryo = new Kryo();
+
+ // return value: new list of partitions (includes old list)
+ List<Partition<Gen>> newPartitions = Lists.newArrayListWithExpectedSize(partitions);
+
+ for (int i = 0; i < partitions; i++) {
+ Gen oper = cloneObject(kryo, this);
+ newPartitions.add(new DefaultPartition<>(oper));
+ }
+
+ LOG.info("definePartition: returning {} partitions", newPartitions.size());
+ return newPartitions;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/041af061/examples/dynamic-partition/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/src/main/resources/META-INF/properties.xml b/examples/dynamic-partition/src/main/resources/META-INF/properties.xml
new file mode 100644
index 0000000..5bbb4b0
--- /dev/null
+++ b/examples/dynamic-partition/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,25 @@
+<?xml version="1.0"?>
+<configuration>
+ <!--
+ <property>
+ <name>dt.application.{appName}.operator.{opName}.prop.{propName}</name>
+ <value>some-default-value (if value is not specified, it is required from the user or custom config when launching)</value>
+ </property>
+ -->
+ <!-- memory assigned to app master
+ <property>
+ <name>dt.attr.MASTER_MEMORY_MB</name>
+ <value>1024</value>
+ </property>
+ -->
+
+ <property>
+ <name>dt.application.*.operator.*.attr.MEMORY_MB</name>
+ <value>200</value>
+ </property> <property>
+ <name>dt.application.Dyn.operator.gen.prop.numTuples</name>
+ <value>1000</value>
+ </property>
+
+</configuration>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/041af061/examples/dynamic-partition/src/main/resources/unused-log4j.properties
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/src/main/resources/unused-log4j.properties b/examples/dynamic-partition/src/main/resources/unused-log4j.properties
new file mode 100644
index 0000000..a83a30b
--- /dev/null
+++ b/examples/dynamic-partition/src/main/resources/unused-log4j.properties
@@ -0,0 +1,9 @@
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+
+log4j.logger.org=info
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=info
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/041af061/examples/dynamic-partition/src/test/java/com/example/dynamic/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/src/test/java/com/example/dynamic/ApplicationTest.java b/examples/dynamic-partition/src/test/java/com/example/dynamic/ApplicationTest.java
new file mode 100644
index 0000000..788b9d3
--- /dev/null
+++ b/examples/dynamic-partition/src/test/java/com/example/dynamic/ApplicationTest.java
@@ -0,0 +1,34 @@
+package com.example.dynamic;
+
+import java.io.IOException;
+
+import javax.validation.ConstraintViolationException;
+
+import org.junit.Assert;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+import com.datatorrent.api.LocalMode;
+//import com.example.myapexapp.Application;
+
+/**
+ * Test the DAG declaration in local mode.
+ */
+public class ApplicationTest {
+
+ @Test
+ public void testApplication() throws IOException, Exception {
+ try {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ lma.prepareDAG(new App(), conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.run(10000); // runs for 10 seconds and quits
+ } catch (ConstraintViolationException e) {
+ Assert.fail("constraint violations: " + e.getConstraintViolations());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/041af061/examples/dynamic-partition/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/src/test/resources/log4j.properties b/examples/dynamic-partition/src/test/resources/log4j.properties
new file mode 100644
index 0000000..58d6913
--- /dev/null
+++ b/examples/dynamic-partition/src/test/resources/log4j.properties
@@ -0,0 +1,21 @@
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+log4j.logger.org=info
+#log4j.logger.org.apache.commons.beanutils=warn
+#log4j.logger.com.datatorrent=debug
[12/19] apex-malhar git commit: SPOI-9079 Added sample application
for transform operator
Posted by th...@apache.org.
SPOI-9079 Added sample application for transform operator
Project: http://git-wip-us.apache.org/repos/asf/apex-malhar/repo
Commit: http://git-wip-us.apache.org/repos/asf/apex-malhar/commit/a492e22f
Tree: http://git-wip-us.apache.org/repos/asf/apex-malhar/tree/a492e22f
Diff: http://git-wip-us.apache.org/repos/asf/apex-malhar/diff/a492e22f
Branch: refs/heads/master
Commit: a492e22fe2a0a18fdfde53f19550a13664b97231
Parents: ca12bca
Author: chaitanya <ch...@apache.org>
Authored: Mon Oct 10 17:03:40 2016 +0530
Committer: Lakshmi Prasanna Velineni <la...@datatorrent.com>
Committed: Sun Mar 26 11:43:48 2017 -0700
----------------------------------------------------------------------
examples/transform/README.md | 6 +
.../transform/XmlJavadocCommentsExtractor.xsl | 44 +++
examples/transform/pom.xml | 267 +++++++++++++++++++
examples/transform/src/assemble/appPackage.xml | 43 +++
.../java/com/example/transform/Application.java | 39 +++
.../com/example/transform/CustomerEvent.java | 74 +++++
.../com/example/transform/CustomerInfo.java | 60 +++++
.../transform/DynamicTransformApplication.java | 52 ++++
.../com/example/transform/POJOGenerator.java | 125 +++++++++
.../src/main/resources/META-INF/properties.xml | 19 ++
.../com/example/transform/ApplicationTest.java | 21 ++
.../src/test/resources/log4j.properties | 22 ++
12 files changed, 772 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/a492e22f/examples/transform/README.md
----------------------------------------------------------------------
diff --git a/examples/transform/README.md b/examples/transform/README.md
new file mode 100644
index 0000000..af016ff
--- /dev/null
+++ b/examples/transform/README.md
@@ -0,0 +1,6 @@
+Sample application to show how to use the TransformOperator to transform the input POJO using given expressions.
+
+Operators in sample application are as follows:
+1) POJOGenerator which generates and emits the CustomerEvent POJO.
+2) TransformOperator which transforms the input POJO using provided expressions and emits the transformed POJO.
+3) ConsoleOutputOperator which writes transformed POJO to stdout.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/a492e22f/examples/transform/XmlJavadocCommentsExtractor.xsl
----------------------------------------------------------------------
diff --git a/examples/transform/XmlJavadocCommentsExtractor.xsl b/examples/transform/XmlJavadocCommentsExtractor.xsl
new file mode 100644
index 0000000..08075a9
--- /dev/null
+++ b/examples/transform/XmlJavadocCommentsExtractor.xsl
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+
+<!--
+ Document : XmlJavadocCommentsExtractor.xsl
+ Created on : September 16, 2014, 11:30 AM
+ Description:
+ The transformation strips off all information except for comments and tags from xml javadoc generated by xml-doclet.
+-->
+
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+ <xsl:output method="xml" standalone="yes"/>
+
+ <!-- copy xml by selecting only the following nodes, attributes and text -->
+ <xsl:template match="node()|text()|@*">
+ <xsl:copy>
+ <xsl:apply-templates select="root|package|class|interface|method|field|type|comment|tag|text()|@name|@qualified|@text"/>
+ </xsl:copy>
+ </xsl:template>
+
+ <!-- Strip off the following paths from the selected xml -->
+ <xsl:template match="//root/package/interface/interface
+ |//root/package/interface/method/@qualified
+ |//root/package/class/interface
+ |//root/package/class/class
+ |//root/package/class/method/@qualified
+ |//root/package/class/field/@qualified" />
+
+ <xsl:strip-space elements="*"/>
+</xsl:stylesheet>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/a492e22f/examples/transform/pom.xml
----------------------------------------------------------------------
diff --git a/examples/transform/pom.xml b/examples/transform/pom.xml
new file mode 100644
index 0000000..e8846d3
--- /dev/null
+++ b/examples/transform/pom.xml
@@ -0,0 +1,267 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>com.example</groupId>
+ <version>1.0-SNAPSHOT</version>
+ <artifactId>transform</artifactId>
+ <packaging>jar</packaging>
+
+ <name>Transform Application</name>
+ <description>Sample application for transform operator</description>
+
+ <properties>
+ <!-- change this if you desire to use a different version of Apex Core -->
+ <apex.version>3.5.0</apex.version>
+ <malhar.version>3.6.0</malhar.version>
+ <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-eclipse-plugin</artifactId>
+ <version>2.9</version>
+ <configuration>
+ <downloadSources>true</downloadSources>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.3</version>
+ <configuration>
+ <encoding>UTF-8</encoding>
+ <source>1.7</source>
+ <target>1.7</target>
+ <debug>true</debug>
+ <optimize>false</optimize>
+ <showDeprecation>true</showDeprecation>
+ <showWarnings>true</showWarnings>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.8</version>
+ <executions>
+ <execution>
+ <id>copy-dependencies</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>target/deps</outputDirectory>
+ <includeScope>runtime</includeScope>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>app-package-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <finalName>${project.artifactId}-${project.version}-apexapp</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <descriptors>
+ <descriptor>src/assemble/appPackage.xml</descriptor>
+ </descriptors>
+ <archiverConfig>
+ <defaultDirectoryMode>0755</defaultDirectoryMode>
+ </archiverConfig>
+ <archive>
+ <manifestEntries>
+ <Class-Path>${apex.apppackage.classpath}</Class-Path>
+ <DT-Engine-Version>${apex.version}</DT-Engine-Version>
+ <DT-App-Package-Group-Id>${project.groupId}</DT-App-Package-Group-Id>
+ <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name>
+ <DT-App-Package-Version>${project.version}</DT-App-Package-Version>
+ <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name>
+ <DT-App-Package-Description>${project.description}</DT-App-Package-Description>
+ </manifestEntries>
+ </archive>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <configuration>
+ <target>
+ <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar"
+ tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa" />
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ <execution>
+ <!-- create resource directory for xml javadoc-->
+ <id>createJavadocDirectory</id>
+ <phase>generate-resources</phase>
+ <configuration>
+ <tasks>
+ <delete dir="${project.build.directory}/generated-resources/xml-javadoc"/>
+ <mkdir dir="${project.build.directory}/generated-resources/xml-javadoc"/>
+ </tasks>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.9.1</version>
+ <executions>
+ <execution>
+ <id>attach-artifacts</id>
+ <phase>package</phase>
+ <goals>
+ <goal>attach-artifact</goal>
+ </goals>
+ <configuration>
+ <artifacts>
+ <artifact>
+ <file>target/${project.artifactId}-${project.version}.apa</file>
+ <type>apa</type>
+ </artifact>
+ </artifacts>
+ <skipAttach>false</skipAttach>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- generate javdoc -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <executions>
+ <!-- generate xml javadoc -->
+ <execution>
+ <id>xml-doclet</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>javadoc</goal>
+ </goals>
+ <configuration>
+ <doclet>com.github.markusbernhardt.xmldoclet.XmlDoclet</doclet>
+ <additionalparam>-d ${project.build.directory}/generated-resources/xml-javadoc -filename ${project.artifactId}-${project.version}-javadoc.xml</additionalparam>
+ <useStandardDocletOptions>false</useStandardDocletOptions>
+ <docletArtifact>
+ <groupId>com.github.markusbernhardt</groupId>
+ <artifactId>xml-doclet</artifactId>
+ <version>1.0.4</version>
+ </docletArtifact>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <!-- Transform xml javadoc to stripped down version containing only class/interface comments and tags-->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>xml-maven-plugin</artifactId>
+ <version>1.0</version>
+ <executions>
+ <execution>
+ <id>transform-xmljavadoc</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>transform</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <transformationSets>
+ <transformationSet>
+ <dir>${project.build.directory}/generated-resources/xml-javadoc</dir>
+ <includes>
+ <include>${project.artifactId}-${project.version}-javadoc.xml</include>
+ </includes>
+ <stylesheet>XmlJavadocCommentsExtractor.xsl</stylesheet>
+ <outputDir>${project.build.directory}/generated-resources/xml-javadoc</outputDir>
+ </transformationSet>
+ </transformationSets>
+ </configuration>
+ </plugin>
+ <!-- copy xml javadoc to class jar -->
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>2.6</version>
+ <executions>
+ <execution>
+ <id>copy-resources</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${basedir}/target/classes</outputDirectory>
+ <resources>
+ <resource>
+ <directory>${project.build.directory}/generated-resources/xml-javadoc</directory>
+ <includes>
+ <include>${project.artifactId}-${project.version}-javadoc.xml</include>
+ </includes>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+
+ </build>
+
+ <dependencies>
+ <!-- add your dependencies here -->
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-library</artifactId>
+ <version>${malhar.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>apex-common</artifactId>
+ <version>${apex.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.10</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>apex-engine</artifactId>
+ <version>${apex.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.janino</groupId>
+ <artifactId>janino</artifactId>
+ <version>2.7.8</version>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/a492e22f/examples/transform/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/transform/src/assemble/appPackage.xml b/examples/transform/src/assemble/appPackage.xml
new file mode 100644
index 0000000..7ad071c
--- /dev/null
+++ b/examples/transform/src/assemble/appPackage.xml
@@ -0,0 +1,43 @@
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>appPackage</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/target/</directory>
+ <outputDirectory>/app</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/target/deps</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/site/conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/META-INF</directory>
+ <outputDirectory>/META-INF</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/app</directory>
+ <outputDirectory>/app</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/resources</directory>
+ <outputDirectory>/resources</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/a492e22f/examples/transform/src/main/java/com/example/transform/Application.java
----------------------------------------------------------------------
diff --git a/examples/transform/src/main/java/com/example/transform/Application.java b/examples/transform/src/main/java/com/example/transform/Application.java
new file mode 100644
index 0000000..d73b47d
--- /dev/null
+++ b/examples/transform/src/main/java/com/example/transform/Application.java
@@ -0,0 +1,39 @@
+package com.example.transform;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.common.partitioner.StatelessPartitioner;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+import com.datatorrent.lib.transform.TransformOperator;
+
+@ApplicationAnnotation(name="TransformExample")
+public class Application implements StreamingApplication
+{
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ POJOGenerator input = dag.addOperator("Input", new POJOGenerator());
+ TransformOperator transform = dag.addOperator("Process", new TransformOperator());
+ // Set expression map
+ Map<String, String> expMap = new HashMap<>();
+ expMap.put("name", "{$.firstName}.concat(\" \").concat({$.lastName})");
+ expMap.put("age", "(new java.util.Date()).getYear() - {$.dateOfBirth}.getYear()");
+ expMap.put("address", "{$.address}.toLowerCase()");
+ transform.setExpressionMap(expMap);
+ ConsoleOutputOperator output = dag.addOperator("Output", new ConsoleOutputOperator());
+
+ dag.addStream("InputToTransform", input.output, transform.input);
+ dag.addStream("TransformToOutput", transform.output, output.input);
+
+ dag.setInputPortAttribute(transform.input, Context.PortContext.TUPLE_CLASS, CustomerEvent.class);
+ dag.setOutputPortAttribute(transform.output, Context.PortContext.TUPLE_CLASS, CustomerInfo.class);
+ dag.setAttribute(transform, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<TransformOperator>(2));
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/a492e22f/examples/transform/src/main/java/com/example/transform/CustomerEvent.java
----------------------------------------------------------------------
diff --git a/examples/transform/src/main/java/com/example/transform/CustomerEvent.java b/examples/transform/src/main/java/com/example/transform/CustomerEvent.java
new file mode 100644
index 0000000..14a9c82
--- /dev/null
+++ b/examples/transform/src/main/java/com/example/transform/CustomerEvent.java
@@ -0,0 +1,74 @@
+package com.example.transform;
+
+import java.util.Date;
+
+public class CustomerEvent
+{
+ private int customerId;
+ private String firstName;
+ private String lastName;
+ private Date dateOfBirth;
+ private String address;
+
+ public int getCustomerId()
+ {
+ return customerId;
+ }
+
+ public void setCustomerId(int customerId)
+ {
+ this.customerId = customerId;
+ }
+
+ public String getFirstName()
+ {
+ return firstName;
+ }
+
+ public void setFirstName(String firstName)
+ {
+ this.firstName = firstName;
+ }
+
+ public String getLastName()
+ {
+ return lastName;
+ }
+
+ public void setLastName(String lastName)
+ {
+ this.lastName = lastName;
+ }
+
+ public Date getDateOfBirth()
+ {
+ return dateOfBirth;
+ }
+
+ public void setDateOfBirth(Date dateOfBirth)
+ {
+ this.dateOfBirth = dateOfBirth;
+ }
+
+ public String getAddress()
+ {
+ return address;
+ }
+
+ public void setAddress(String address)
+ {
+ this.address = address;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "CustomerEvent{" +
+ "customerId=" + customerId +
+ ", firstName='" + firstName + '\'' +
+ ", lastName='" + lastName + '\'' +
+ ", dateOfBirth=" + dateOfBirth +
+ ", address='" + address + '\'' +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/a492e22f/examples/transform/src/main/java/com/example/transform/CustomerInfo.java
----------------------------------------------------------------------
diff --git a/examples/transform/src/main/java/com/example/transform/CustomerInfo.java b/examples/transform/src/main/java/com/example/transform/CustomerInfo.java
new file mode 100644
index 0000000..28a9ccb
--- /dev/null
+++ b/examples/transform/src/main/java/com/example/transform/CustomerInfo.java
@@ -0,0 +1,60 @@
+package com.example.transform;
+
+public class CustomerInfo
+{
+ private int customerId;
+ private String name;
+ private int age;
+ private String address;
+
+ public int getCustomerId()
+ {
+ return customerId;
+ }
+
+ public void setCustomerId(int customerId)
+ {
+ this.customerId = customerId;
+ }
+
+ public String getName()
+ {
+ return name;
+ }
+
+ public void setName(String name)
+ {
+ this.name = name;
+ }
+
+ public int getAge()
+ {
+ return age;
+ }
+
+ public void setAge(int age)
+ {
+ this.age = age;
+ }
+
+ public String getAddress()
+ {
+ return address;
+ }
+
+ public void setAddress(String address)
+ {
+ this.address = address;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "CustomerInfo{" +
+ "customerId=" + customerId +
+ ", name='" + name + '\'' +
+ ", age=" + age +
+ ", address='" + address + '\'' +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/a492e22f/examples/transform/src/main/java/com/example/transform/DynamicTransformApplication.java
----------------------------------------------------------------------
diff --git a/examples/transform/src/main/java/com/example/transform/DynamicTransformApplication.java b/examples/transform/src/main/java/com/example/transform/DynamicTransformApplication.java
new file mode 100644
index 0000000..a769016
--- /dev/null
+++ b/examples/transform/src/main/java/com/example/transform/DynamicTransformApplication.java
@@ -0,0 +1,52 @@
+package com.example.transform;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StatsListener;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.common.partitioner.StatelessPartitioner;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+import com.datatorrent.lib.partitioner.StatelessThroughputBasedPartitioner;
+import com.datatorrent.lib.transform.TransformOperator;
+
+@ApplicationAnnotation(name="DynamicTransformApp")
+public class DynamicTransformApplication implements StreamingApplication
+{
+ private static String COOL_DOWN_MILLIS = "dt.cooldown";
+ private static String MAX_THROUGHPUT = "dt.maxThroughput";
+ private static String MIN_THROUGHPUT = "dt.minThroughput";
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ POJOGenerator input = dag.addOperator("Input", new POJOGenerator());
+ TransformOperator transform = dag.addOperator("Process", new TransformOperator());
+ // Set expression map
+ Map<String, String> expMap = new HashMap<>();
+ expMap.put("name", "{$.firstName}.concat(\" \").concat({$.lastName})");
+ expMap.put("age", "(new java.util.Date()).getYear() - {$.dateOfBirth}.getYear()");
+ expMap.put("address", "{$.address}.toLowerCase()");
+ transform.setExpressionMap(expMap);
+ ConsoleOutputOperator output = dag.addOperator("Output", new ConsoleOutputOperator());
+
+ dag.addStream("InputToTransform", input.output, transform.input);
+ dag.addStream("TransformToOutput", transform.output, output.input);
+
+ dag.setInputPortAttribute(transform.input, Context.PortContext.TUPLE_CLASS, CustomerEvent.class);
+ dag.setOutputPortAttribute(transform.output, Context.PortContext.TUPLE_CLASS, CustomerInfo.class);
+
+ StatelessThroughputBasedPartitioner<TransformOperator> partitioner = new StatelessThroughputBasedPartitioner<>();
+ partitioner.setCooldownMillis(conf.getLong(COOL_DOWN_MILLIS, 10000));
+ partitioner.setMaximumEvents(conf.getLong(MAX_THROUGHPUT, 30000));
+ partitioner.setMinimumEvents(conf.getLong(MIN_THROUGHPUT, 10000));
+ dag.setAttribute(transform, Context.OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[]{partitioner}));
+ dag.setAttribute(transform, Context.OperatorContext.PARTITIONER, partitioner);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/a492e22f/examples/transform/src/main/java/com/example/transform/POJOGenerator.java
----------------------------------------------------------------------
diff --git a/examples/transform/src/main/java/com/example/transform/POJOGenerator.java b/examples/transform/src/main/java/com/example/transform/POJOGenerator.java
new file mode 100644
index 0000000..9db5fd1
--- /dev/null
+++ b/examples/transform/src/main/java/com/example/transform/POJOGenerator.java
@@ -0,0 +1,125 @@
+package com.example.transform;
+
+import java.util.Date;
+import java.util.Random;
+
+import javax.validation.constraints.Min;
+
+import org.apache.commons.lang3.RandomStringUtils;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+
+/**
+ * Generates and emits the CustomerEvent
+ */
+public class POJOGenerator implements InputOperator
+{
+ @Min(1)
+ private int maxCustomerId = 100000;
+ @Min(1)
+ private int maxNameLength = 10;
+ @Min(1)
+ private int maxAddressLength = 15;
+ private long tuplesCounter;
+ // Limit number of emitted tuples per window
+ @Min(1)
+ private long maxTuplesPerWindow = 100;
+ private final Random random = new Random();
+ private final RandomStringUtils rRandom = new RandomStringUtils();
+ public final transient DefaultOutputPort<Object> output = new DefaultOutputPort<>();
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ tuplesCounter = 0;
+ }
+
+ @Override
+ public void endWindow()
+ {
+ }
+
+ @Override
+ public void setup(Context.OperatorContext context)
+ {
+ }
+
+ @Override
+ public void teardown()
+ {
+
+ }
+
+ CustomerEvent generateCustomersEvent() throws Exception {
+
+ CustomerEvent customerEvent = new CustomerEvent();
+ customerEvent.setCustomerId(randomId(maxCustomerId));
+ customerEvent.setFirstName(rRandom.randomAlphabetic(randomId(maxNameLength)));
+ customerEvent.setLastName(rRandom.randomAlphabetic(randomId(maxNameLength)));
+ long val = random.nextLong();
+ long diff1 = val % System.currentTimeMillis();
+ customerEvent.setDateOfBirth(new Date(diff1));
+ customerEvent.setAddress(rRandom.randomAlphabetic(randomId(maxAddressLength)));
+ return customerEvent;
+ }
+
+ private int randomId(int max) {
+ if (max < 1) return 1;
+ return 1 + random.nextInt(max);
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ while (tuplesCounter++ < maxTuplesPerWindow) {
+ try {
+ CustomerEvent event = generateCustomersEvent();
+ this.output.emit(event);
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+ }
+
+ public long getMaxTuplesPerWindow()
+ {
+ return maxTuplesPerWindow;
+ }
+
+ public void setMaxTuplesPerWindow(long maxTuplesPerWindow)
+ {
+ this.maxTuplesPerWindow = maxTuplesPerWindow;
+ }
+
+ public int getMaxAddressLength()
+ {
+ return maxAddressLength;
+ }
+
+ public void setMaxAddressLength(int maxAddressLength)
+ {
+ this.maxAddressLength = maxAddressLength;
+ }
+
+ public int getMaxNameLength()
+ {
+ return maxNameLength;
+ }
+
+ public void setMaxNameLength(int maxNameLength)
+ {
+ this.maxNameLength = maxNameLength;
+ }
+
+ public int getMaxCustomerId()
+ {
+ return maxCustomerId;
+ }
+
+ public void setMaxCustomerId(int maxCustomerId)
+ {
+ this.maxCustomerId = maxCustomerId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/a492e22f/examples/transform/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/transform/src/main/resources/META-INF/properties.xml b/examples/transform/src/main/resources/META-INF/properties.xml
new file mode 100644
index 0000000..f885664
--- /dev/null
+++ b/examples/transform/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,19 @@
+<?xml version="1.0"?>
+<configuration>
+ <property>
+ <name>dt.application.*.operator.transform.attr.MEMORY_MB</name>
+ <value>1024</value>
+ </property>
+ <property>
+ <name>dt.maxThroughput</name>
+ <value>150</value>
+ </property>
+ <property>
+ <name>dt.minThroughput</name>
+ <value>50</value>
+ </property>
+ <property>
+ <name>dt.cooldown</name>
+ <value>1800000</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/a492e22f/examples/transform/src/test/java/com/example/transform/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/transform/src/test/java/com/example/transform/ApplicationTest.java b/examples/transform/src/test/java/com/example/transform/ApplicationTest.java
new file mode 100644
index 0000000..2d331d2
--- /dev/null
+++ b/examples/transform/src/test/java/com/example/transform/ApplicationTest.java
@@ -0,0 +1,21 @@
+package com.example.transform;
+
+import org.junit.Test;
+import org.apache.hadoop.conf.Configuration;
+import com.datatorrent.api.LocalMode;
+
+public class ApplicationTest
+{
+ @Test
+ public void testApplication() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ lma.prepareDAG(new Application(), conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.runAsync();
+ Thread.sleep(10 * 1000);
+ lc.shutdown();
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/a492e22f/examples/transform/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/transform/src/test/resources/log4j.properties b/examples/transform/src/test/resources/log4j.properties
new file mode 100644
index 0000000..98544e8
--- /dev/null
+++ b/examples/transform/src/test/resources/log4j.properties
@@ -0,0 +1,22 @@
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=debug
+log4j.logger.org.apache.apex=debug
+log4j.logger.org=info
[08/19] apex-malhar git commit: SPOI-8652 Example for Enricher
Posted by th...@apache.org.
SPOI-8652 Example for Enricher
Project: http://git-wip-us.apache.org/repos/asf/apex-malhar/repo
Commit: http://git-wip-us.apache.org/repos/asf/apex-malhar/commit/dd509f2a
Tree: http://git-wip-us.apache.org/repos/asf/apex-malhar/tree/dd509f2a
Diff: http://git-wip-us.apache.org/repos/asf/apex-malhar/diff/dd509f2a
Branch: refs/heads/master
Commit: dd509f2a17b4139b335949c305566a5ca48ee8e0
Parents: 4573528
Author: Chinmay <ch...@apache.org>
Authored: Wed Jul 20 20:35:01 2016 +0530
Committer: Lakshmi Prasanna Velineni <la...@datatorrent.com>
Committed: Sun Mar 26 11:43:48 2017 -0700
----------------------------------------------------------------------
examples/enricher/README.md | 13 +
.../enricher/XmlJavadocCommentsExtractor.xsl | 44 +++
examples/enricher/pom.xml | 308 +++++++++++++++++++
examples/enricher/src/assemble/appPackage.xml | 43 +++
.../com/example/myapexapp/DataGenerator.java | 94 ++++++
.../myapexapp/EnricherAppWithJSONFile.java | 47 +++
.../example/myapexapp/LineOutputOperator.java | 34 ++
.../main/java/com/example/myapexapp/POJO.java | 49 +++
.../com/example/myapexapp/POJOEnriched.java | 71 +++++
.../src/main/resources/META-INF/properties.xml | 41 +++
.../src/main/resources/circleMapping.txt | 10 +
.../com/example/myapexapp/ApplicationTest.java | 31 ++
.../src/test/resources/log4j.properties | 21 ++
13 files changed, 806 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/dd509f2a/examples/enricher/README.md
----------------------------------------------------------------------
diff --git a/examples/enricher/README.md b/examples/enricher/README.md
new file mode 100644
index 0000000..a987a0b
--- /dev/null
+++ b/examples/enricher/README.md
@@ -0,0 +1,13 @@
+This sample application show how to use POJOEnricher to enrich streaming data using
+external source.
+The operators in order as as follows:
+1. Random data generator which emits data in JSON string format
+2. JSON Parser which takes JSON string and emits POJO
+3. POJO Enricher which enriches input using file and emits output POJO
+4. Line Output Operator which emits line by line output to File System
+The output files start out with a `.tmp` extension and get renamed when they reach the
+size bound.
+
+Similar to FSLoader JDBCLoader can be used when JDBC backend is required.
+
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/dd509f2a/examples/enricher/XmlJavadocCommentsExtractor.xsl
----------------------------------------------------------------------
diff --git a/examples/enricher/XmlJavadocCommentsExtractor.xsl b/examples/enricher/XmlJavadocCommentsExtractor.xsl
new file mode 100644
index 0000000..08075a9
--- /dev/null
+++ b/examples/enricher/XmlJavadocCommentsExtractor.xsl
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+
+<!--
+ Document : XmlJavadocCommentsExtractor.xsl
+ Created on : September 16, 2014, 11:30 AM
+ Description:
+ The transformation strips off all information except for comments and tags from xml javadoc generated by xml-doclet.
+-->
+
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+ <xsl:output method="xml" standalone="yes"/>
+
+ <!-- copy xml by selecting only the following nodes, attributes and text -->
+ <xsl:template match="node()|text()|@*">
+ <xsl:copy>
+ <xsl:apply-templates select="root|package|class|interface|method|field|type|comment|tag|text()|@name|@qualified|@text"/>
+ </xsl:copy>
+ </xsl:template>
+
+ <!-- Strip off the following paths from the selected xml -->
+ <xsl:template match="//root/package/interface/interface
+ |//root/package/interface/method/@qualified
+ |//root/package/class/interface
+ |//root/package/class/class
+ |//root/package/class/method/@qualified
+ |//root/package/class/field/@qualified" />
+
+ <xsl:strip-space elements="*"/>
+</xsl:stylesheet>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/dd509f2a/examples/enricher/pom.xml
----------------------------------------------------------------------
diff --git a/examples/enricher/pom.xml b/examples/enricher/pom.xml
new file mode 100644
index 0000000..a93bcf5
--- /dev/null
+++ b/examples/enricher/pom.xml
@@ -0,0 +1,308 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>com.example</groupId>
+ <version>1.0-SNAPSHOT</version>
+ <artifactId>enricher</artifactId>
+ <packaging>jar</packaging>
+
+ <!-- change these to the appropriate values -->
+ <name>Enricher</name>
+ <description>Example Use of POJO Enricher</description>
+
+ <properties>
+ <!-- change this if you desire to use a different version of Apex Core -->
+ <apex.version>3.5.0</apex.version>
+ <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath>
+ <malhar.version>3.6.0</malhar.version>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-eclipse-plugin</artifactId>
+ <version>2.9</version>
+ <configuration>
+ <downloadSources>true</downloadSources>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.3</version>
+ <configuration>
+ <encoding>UTF-8</encoding>
+ <source>1.7</source>
+ <target>1.7</target>
+ <debug>true</debug>
+ <optimize>false</optimize>
+ <showDeprecation>true</showDeprecation>
+ <showWarnings>true</showWarnings>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.8</version>
+ <executions>
+ <execution>
+ <id>copy-dependencies</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>target/deps</outputDirectory>
+ <includeScope>runtime</includeScope>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>app-package-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <finalName>${project.artifactId}-${project.version}-apexapp</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <descriptors>
+ <descriptor>src/assemble/appPackage.xml</descriptor>
+ </descriptors>
+ <archiverConfig>
+ <defaultDirectoryMode>0755</defaultDirectoryMode>
+ </archiverConfig>
+ <archive>
+ <manifestEntries>
+ <Class-Path>${apex.apppackage.classpath}</Class-Path>
+ <DT-Engine-Version>${apex.version}</DT-Engine-Version>
+ <DT-App-Package-Group-Id>${project.groupId}</DT-App-Package-Group-Id>
+ <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name>
+ <DT-App-Package-Version>${project.version}</DT-App-Package-Version>
+ <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name>
+ <DT-App-Package-Description>${project.description}</DT-App-Package-Description>
+ </manifestEntries>
+ </archive>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <configuration>
+ <target>
+ <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar"
+ tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa" />
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ <execution>
+ <!-- create resource directory for xml javadoc-->
+ <id>createJavadocDirectory</id>
+ <phase>generate-resources</phase>
+ <configuration>
+ <tasks>
+ <delete dir="${project.build.directory}/generated-resources/xml-javadoc"/>
+ <mkdir dir="${project.build.directory}/generated-resources/xml-javadoc"/>
+ </tasks>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.9.1</version>
+ <executions>
+ <execution>
+ <id>attach-artifacts</id>
+ <phase>package</phase>
+ <goals>
+ <goal>attach-artifact</goal>
+ </goals>
+ <configuration>
+ <artifacts>
+ <artifact>
+ <file>target/${project.artifactId}-${project.version}.apa</file>
+ <type>apa</type>
+ </artifact>
+ </artifacts>
+ <skipAttach>false</skipAttach>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- generate javdoc -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <executions>
+ <!-- generate xml javadoc -->
+ <execution>
+ <id>xml-doclet</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>javadoc</goal>
+ </goals>
+ <configuration>
+ <doclet>com.github.markusbernhardt.xmldoclet.XmlDoclet</doclet>
+ <additionalparam>-d ${project.build.directory}/generated-resources/xml-javadoc -filename ${project.artifactId}-${project.version}-javadoc.xml</additionalparam>
+ <useStandardDocletOptions>false</useStandardDocletOptions>
+ <docletArtifact>
+ <groupId>com.github.markusbernhardt</groupId>
+ <artifactId>xml-doclet</artifactId>
+ <version>1.0.4</version>
+ </docletArtifact>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <!-- Transform xml javadoc to stripped down version containing only class/interface comments and tags-->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>xml-maven-plugin</artifactId>
+ <version>1.0</version>
+ <executions>
+ <execution>
+ <id>transform-xmljavadoc</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>transform</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <transformationSets>
+ <transformationSet>
+ <dir>${project.build.directory}/generated-resources/xml-javadoc</dir>
+ <includes>
+ <include>${project.artifactId}-${project.version}-javadoc.xml</include>
+ </includes>
+ <stylesheet>XmlJavadocCommentsExtractor.xsl</stylesheet>
+ <outputDir>${project.build.directory}/generated-resources/xml-javadoc</outputDir>
+ </transformationSet>
+ </transformationSets>
+ </configuration>
+ </plugin>
+ <!-- copy xml javadoc to class jar -->
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>2.6</version>
+ <executions>
+ <execution>
+ <id>copy-resources</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${basedir}/target/classes</outputDirectory>
+ <resources>
+ <resource>
+ <directory>${project.build.directory}/generated-resources/xml-javadoc</directory>
+ <includes>
+ <include>${project.artifactId}-${project.version}-javadoc.xml</include>
+ </includes>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+
+ </build>
+
+ <dependencies>
+ <!-- add your dependencies here -->
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-library</artifactId>
+ <version>${malhar.version}</version>
+ <!--
+ If you know that your application does not need transitive dependencies pulled in by malhar-library,
+ uncomment the following to reduce the size of your app package.
+ -->
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-contrib</artifactId>
+ <version>${malhar.version}</version>
+ <!--
+ If you know that your application does not need transitive dependencies pulled in by malhar-library,
+ uncomment the following to reduce the size of your app package.
+ -->
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>apex-common</artifactId>
+ <version>${apex.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.10</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>apex-engine</artifactId>
+ <version>${apex.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.github.fge</groupId>
+ <artifactId>json-schema-validator</artifactId>
+ <version>2.0.1</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.janino</groupId>
+ <artifactId>commons-compiler</artifactId>
+ <version>2.7.8</version>
+ <type>jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.janino</groupId>
+ <artifactId>janino</artifactId>
+ <version>2.7.8</version>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/dd509f2a/examples/enricher/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/enricher/src/assemble/appPackage.xml b/examples/enricher/src/assemble/appPackage.xml
new file mode 100644
index 0000000..7ad071c
--- /dev/null
+++ b/examples/enricher/src/assemble/appPackage.xml
@@ -0,0 +1,43 @@
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>appPackage</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/target/</directory>
+ <outputDirectory>/app</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/target/deps</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/site/conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/META-INF</directory>
+ <outputDirectory>/META-INF</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/app</directory>
+ <outputDirectory>/app</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/resources</directory>
+ <outputDirectory>/resources</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/dd509f2a/examples/enricher/src/main/java/com/example/myapexapp/DataGenerator.java
----------------------------------------------------------------------
diff --git a/examples/enricher/src/main/java/com/example/myapexapp/DataGenerator.java b/examples/enricher/src/main/java/com/example/myapexapp/DataGenerator.java
new file mode 100644
index 0000000..3afbb87
--- /dev/null
+++ b/examples/enricher/src/main/java/com/example/myapexapp/DataGenerator.java
@@ -0,0 +1,94 @@
+package com.example.myapexapp;
+
+import java.util.Random;
+
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.common.util.BaseOperator;
+
+/**
+ * Generates Subscriber Data:
+ * A Party Phone
+ * A Party IMEI
+ * A Party IMSI
+ * Circle Id
+ */
+public class DataGenerator extends BaseOperator implements InputOperator
+{
+ public static int NUM_CIRCLES = 10;
+
+ private Random r;
+ private int count = 0;
+ private int limit = 1000;
+
+ public final transient DefaultOutputPort<byte[]> output = new DefaultOutputPort<>();
+
+ @Override
+ public void setup(OperatorContext context)
+ {
+ r = new Random(System.currentTimeMillis());
+ }
+
+ @Override
+ public void beginWindow(long windowId) {
+ super.beginWindow(windowId);
+ count = 0;
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ if(count++ < limit) {
+ output.emit(getRecord());
+ }
+ }
+
+ private byte[] getRecord()
+ {
+ String phone = getRandomNumber(10);
+ String imsi = getHashInRange(phone, 15);
+ String imei = getHashInRange(imsi, 15);
+ String circleId = Math.abs(phone.hashCode()) % NUM_CIRCLES + "";
+// String record = MessageFormat.format(baseDataTemplate, phone, imsi, imei, circleId);
+ String record = "{" +
+ "\"phone\":\"" + phone + "\"," +
+ "\"imei\":\"" + imei+ "\"," +
+ "\"imsi\":\"" + imsi+ "\"," +
+ "\"circleId\":" + circleId +
+ "}";
+ return record.getBytes();
+ }
+
+ private String getRandomNumber(int numDigits)
+ {
+ String retVal = (r.nextInt((9 - 1) + 1) + 1) + "";
+
+ for (int i = 0; i < numDigits - 1; i++) {
+ retVal += (r.nextInt((9 - 0) + 1) + 0);
+ }
+ return retVal;
+ }
+
+ private String getHashInRange(String s, int n)
+ {
+ StringBuilder retVal = new StringBuilder();
+ for (int i = 0, j = 0; i < n && j < s.length(); i++, j++) {
+ retVal.append(Math.abs(s.charAt(j) + "".hashCode()) % 10);
+ if (j == s.length() - 1) {
+ j = -1;
+ }
+ }
+ return retVal.toString();
+ }
+
+ public int getLimit()
+ {
+ return limit;
+ }
+
+ public void setLimit(int limit)
+ {
+ this.limit = limit;
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/dd509f2a/examples/enricher/src/main/java/com/example/myapexapp/EnricherAppWithJSONFile.java
----------------------------------------------------------------------
diff --git a/examples/enricher/src/main/java/com/example/myapexapp/EnricherAppWithJSONFile.java b/examples/enricher/src/main/java/com/example/myapexapp/EnricherAppWithJSONFile.java
new file mode 100644
index 0000000..a0dab64
--- /dev/null
+++ b/examples/enricher/src/main/java/com/example/myapexapp/EnricherAppWithJSONFile.java
@@ -0,0 +1,47 @@
+package com.example.myapexapp;
+
+import java.util.ArrayList;
+
+import com.datatorrent.contrib.enrich.JsonFSLoader;
+import com.datatorrent.contrib.enrich.POJOEnricher;
+import com.datatorrent.contrib.parser.JsonParser;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+
+@ApplicationAnnotation(name="EnricherAppWithJSONFile")
+public class EnricherAppWithJSONFile implements StreamingApplication
+{
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ DataGenerator dataGenerator = dag.addOperator("DataGenerator", DataGenerator.class);
+ JsonParser parser = dag.addOperator("Parser", JsonParser.class);
+
+ /**
+ * FSLoader is used to configure Enricher backend. Property of FSLoader file which is fileName is set in
+ * properties.xml file.
+ * The format that is used to read the file is present as an example in resources/circleMapping.txt file.
+ */
+ JsonFSLoader fsLoader = new JsonFSLoader();
+ POJOEnricher enrich = dag.addOperator("Enrich", POJOEnricher.class);
+ enrich.setStore(fsLoader);
+
+ ArrayList includeFields = new ArrayList();
+ includeFields.add("circleName");
+ ArrayList lookupFields = new ArrayList();
+ lookupFields.add("circleId");
+
+ enrich.setIncludeFields(includeFields);
+ enrich.setLookupFields(lookupFields);
+
+ ConsoleOutputOperator console = dag.addOperator("Console", ConsoleOutputOperator.class);
+
+ dag.addStream("Parse", dataGenerator.output, parser.in);
+ dag.addStream("Enrich", parser.out, enrich.input);
+ dag.addStream("Console", enrich.output, console.input);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/dd509f2a/examples/enricher/src/main/java/com/example/myapexapp/LineOutputOperator.java
----------------------------------------------------------------------
diff --git a/examples/enricher/src/main/java/com/example/myapexapp/LineOutputOperator.java b/examples/enricher/src/main/java/com/example/myapexapp/LineOutputOperator.java
new file mode 100644
index 0000000..3b7a298
--- /dev/null
+++ b/examples/enricher/src/main/java/com/example/myapexapp/LineOutputOperator.java
@@ -0,0 +1,34 @@
+package com.example.myapexapp;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+
+import javax.validation.constraints.NotNull;
+
+import com.datatorrent.lib.io.fs.AbstractFileOutputOperator;
+
+/**
+ * Converts each tuple to a string and writes it as a new line to the output file
+ */
+public class LineOutputOperator extends AbstractFileOutputOperator<Object>
+{
+ private static final String NL = System.lineSeparator();
+ private static final Charset CS = StandardCharsets.UTF_8;
+
+ @NotNull
+ private String baseName;
+
+ @Override
+ public byte[] getBytesForTuple(Object t) {
+ String result = new String(t.toString().getBytes(), CS) + NL;
+ return result.getBytes(CS);
+ }
+
+ @Override
+ protected String getFileName(Object tuple) {
+ return baseName;
+ }
+
+ public String getBaseName() { return baseName; }
+ public void setBaseName(String v) { baseName = v; }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/dd509f2a/examples/enricher/src/main/java/com/example/myapexapp/POJO.java
----------------------------------------------------------------------
diff --git a/examples/enricher/src/main/java/com/example/myapexapp/POJO.java b/examples/enricher/src/main/java/com/example/myapexapp/POJO.java
new file mode 100644
index 0000000..32845e8
--- /dev/null
+++ b/examples/enricher/src/main/java/com/example/myapexapp/POJO.java
@@ -0,0 +1,49 @@
+package com.example.myapexapp;
+
+public class POJO
+{
+ private String phone;
+ private String imei;
+ private String imsi;
+ private int circleId;
+
+ public String getPhone()
+ {
+ return phone;
+ }
+
+ public void setPhone(String phone)
+ {
+ this.phone = phone;
+ }
+
+ public String getImei()
+ {
+ return imei;
+ }
+
+ public void setImei(String imei)
+ {
+ this.imei = imei;
+ }
+
+ public String getImsi()
+ {
+ return imsi;
+ }
+
+ public void setImsi(String imsi)
+ {
+ this.imsi = imsi;
+ }
+
+ public int getCircleId()
+ {
+ return circleId;
+ }
+
+ public void setCircleId(int circleId)
+ {
+ this.circleId = circleId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/dd509f2a/examples/enricher/src/main/java/com/example/myapexapp/POJOEnriched.java
----------------------------------------------------------------------
diff --git a/examples/enricher/src/main/java/com/example/myapexapp/POJOEnriched.java b/examples/enricher/src/main/java/com/example/myapexapp/POJOEnriched.java
new file mode 100644
index 0000000..bed2cfb
--- /dev/null
+++ b/examples/enricher/src/main/java/com/example/myapexapp/POJOEnriched.java
@@ -0,0 +1,71 @@
+package com.example.myapexapp;
+
+public class POJOEnriched
+{
+ private String phone;
+ private String imei;
+ private String imsi;
+ private int circleId;
+ private String circleName;
+
+ public String getPhone()
+ {
+ return phone;
+ }
+
+ public void setPhone(String phone)
+ {
+ this.phone = phone;
+ }
+
+ public String getImei()
+ {
+ return imei;
+ }
+
+ public void setImei(String imei)
+ {
+ this.imei = imei;
+ }
+
+ public String getImsi()
+ {
+ return imsi;
+ }
+
+ public void setImsi(String imsi)
+ {
+ this.imsi = imsi;
+ }
+
+ public int getCircleId()
+ {
+ return circleId;
+ }
+
+ public void setCircleId(int circleId)
+ {
+ this.circleId = circleId;
+ }
+
+ public String getCircleName()
+ {
+ return circleName;
+ }
+
+ public void setCircleName(String circleName)
+ {
+ this.circleName = circleName;
+ }
+
+ @Override public String toString()
+ {
+ return "POJOEnriched{" +
+ "phone='" + phone + '\'' +
+ ", imei='" + imei + '\'' +
+ ", imsi='" + imsi + '\'' +
+ ", circleId=" + circleId +
+ ", circleName='" + circleName + '\'' +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/dd509f2a/examples/enricher/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/enricher/src/main/resources/META-INF/properties.xml b/examples/enricher/src/main/resources/META-INF/properties.xml
new file mode 100644
index 0000000..9ecf899
--- /dev/null
+++ b/examples/enricher/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0"?>
+<configuration>
+ <!-- Parser -->
+ <property>
+ <name>dt.application.EnricherAppWithJSONFile.operator.Parser.port.out.attr.TUPLE_CLASS</name>
+ <value>com.example.myapexapp.POJO</value>
+ </property>
+
+ <!-- Enrich -->
+ <property>
+ <name>dt.application.EnricherAppWithJSONFile.operator.Enrich.port.input.attr.TUPLE_CLASS</name>
+ <value>com.example.myapexapp.POJO</value>
+ </property>
+ <property>
+ <name>dt.application.EnricherAppWithJSONFile.operator.Enrich.port.output.attr.TUPLE_CLASS</name>
+ <value>com.example.myapexapp.POJOEnriched</value>
+ </property>
+ <property>
+ <name>dt.application.EnricherAppWithJSONFile.operator.Enrich.prop.store.fileName</name>
+ <value>/tmp/circleMapping.txt</value>
+ </property>
+
+ <!-- file output operator -->
+ <property>
+ <name>dt.application.EnricherAppWithJSONFile.operator.fileOut.prop.filePath</name>
+ <value>/tmp/enrichedData</value>
+ </property>
+ <property>
+ <name>dt.application.EnricherAppWithJSONFile.operator.fileOut.prop.baseName</name>
+ <value>test</value>
+ </property>
+ <property>
+ <name>dt.application.EnricherAppWithJSONFile.operator.fileOut.prop.maxLength</name>
+ <value>1024</value>
+ </property>
+ <property>
+ <name>dt.application.EnricherAppWithJSONFile.operator.fileOut.prop.rotationWindows</name>
+ <value>4</value>
+ </property>
+</configuration>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/dd509f2a/examples/enricher/src/main/resources/circleMapping.txt
----------------------------------------------------------------------
diff --git a/examples/enricher/src/main/resources/circleMapping.txt b/examples/enricher/src/main/resources/circleMapping.txt
new file mode 100644
index 0000000..a9db91c
--- /dev/null
+++ b/examples/enricher/src/main/resources/circleMapping.txt
@@ -0,0 +1,10 @@
+{"circleId":0, "circleName":"A"}
+{"circleId":1, "circleName":"B"}
+{"circleId":2, "circleName":"C"}
+{"circleId":3, "circleName":"D"}
+{"circleId":4, "circleName":"E"}
+{"circleId":5, "circleName":"F"}
+{"circleId":6, "circleName":"G"}
+{"circleId":7, "circleName":"H"}
+{"circleId":8, "circleName":"I"}
+{"circleId":9, "circleName":"J"}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/dd509f2a/examples/enricher/src/test/java/com/example/myapexapp/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/enricher/src/test/java/com/example/myapexapp/ApplicationTest.java b/examples/enricher/src/test/java/com/example/myapexapp/ApplicationTest.java
new file mode 100644
index 0000000..4b04603
--- /dev/null
+++ b/examples/enricher/src/test/java/com/example/myapexapp/ApplicationTest.java
@@ -0,0 +1,31 @@
+package com.example.myapexapp;
+
+import javax.validation.ConstraintViolationException;
+
+import org.junit.Assert;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+import com.datatorrent.api.LocalMode;
+
+/**
+ * Test the DAG declaration in local mode.
+ */
+public class ApplicationTest {
+
+ @Test
+ public void testApplication() throws Exception {
+ try {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ lma.prepareDAG(new EnricherAppWithJSONFile(), conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.run(10000); // runs for 10 seconds and quits
+ } catch (ConstraintViolationException e) {
+ Assert.fail("constraint violations: " + e.getConstraintViolations());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/dd509f2a/examples/enricher/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/enricher/src/test/resources/log4j.properties b/examples/enricher/src/test/resources/log4j.properties
new file mode 100644
index 0000000..3bfcdc5
--- /dev/null
+++ b/examples/enricher/src/test/resources/log4j.properties
@@ -0,0 +1,21 @@
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+log4j.logger.org=info
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=debug
[06/19] apex-malhar git commit: SPOI-8793 Added formatter example
description Rename Readme.md to README.md Updated csvformatter schema
definition removed properties and files that are not needed for csvformatter
Updated review comments for csvformatter
Posted by th...@apache.org.
SPOI-8793
Added formatter example description
Rename Readme.md to README.md
Updated csvformatter schema definition
removed properties and files that are not needed for csvformatter
Updated review comments for csvformatter example
Updated the example with Review comments
Updated csvformatter example with PR comments
changed the name from formatter to csvformatter
Incorporated review comments
Updated the format of the Readme file
changed the input JSON generation to static values and updated the test case
updated the readme format
Project: http://git-wip-us.apache.org/repos/asf/apex-malhar/repo
Commit: http://git-wip-us.apache.org/repos/asf/apex-malhar/commit/63df474b
Tree: http://git-wip-us.apache.org/repos/asf/apex-malhar/tree/63df474b
Diff: http://git-wip-us.apache.org/repos/asf/apex-malhar/diff/63df474b
Branch: refs/heads/master
Commit: 63df474b19283ab1cbd00d9e1aecbed7a7827baa
Parents: be9f3c8
Author: venkateshDT <ve...@datatorrent.com>
Authored: Wed Aug 17 22:10:09 2016 -0700
Committer: Lakshmi Prasanna Velineni <la...@datatorrent.com>
Committed: Sun Mar 26 11:43:48 2017 -0700
----------------------------------------------------------------------
examples/csvformatter/README.md | 64 ++++
.../XmlJavadocCommentsExtractor.xsl | 44 +++
examples/csvformatter/pom.xml | 297 +++++++++++++++++++
.../csvformatter/src/assemble/appPackage.xml | 43 +++
.../java/com/demo/myapexapp/Application.java | 45 +++
.../com/demo/myapexapp/HDFSOutputOperator.java | 87 ++++++
.../java/com/demo/myapexapp/JsonGenerator.java | 78 +++++
.../main/java/com/demo/myapexapp/PojoEvent.java | 141 +++++++++
.../src/main/resources/META-INF/properties.xml | 40 +++
.../csvformatter/src/main/resources/schema.json | 60 ++++
.../com/demo/myapexapp/ApplicationTest.java | 67 +++++
.../src/test/resources/log4j.properties | 22 ++
12 files changed, 988 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/63df474b/examples/csvformatter/README.md
----------------------------------------------------------------------
diff --git a/examples/csvformatter/README.md b/examples/csvformatter/README.md
new file mode 100644
index 0000000..332818c
--- /dev/null
+++ b/examples/csvformatter/README.md
@@ -0,0 +1,64 @@
+## CsvFormatter Example
+
+`CsvFormatter` converts the incoming POJO to CSV(by default) or custom delimiter('|', ':' etc.) separated string.
+
+Accordingly, we have additional config file for formatting preferences (`/src/main/resources/schema.json`) besides the common properties file (`/src/main/resources/META-INF/properties.xml`).
+
+Users can choose the application and additional configuration file to use during launch time. In this example, we use the files mentioned above to customise the schema and configure the operator properties.
+
+
+#### **Update Properties from properties.xml - This is needed to run the example:**
+
+- Update these common properties in the file `/src/main/resources/META-INF/properties.xml`:
+
+| Property Name | Description |
+| ------------- | ----------- |
+| dt.application.CustomOutputFormatter.operator.jsonParser.prop.sleepTime | sleep time for the container |
+| dt.application.CustomOutputFormatter.operator.jsonParser.port.out.attr.TUPLE_CLASS | expected POJO object output of the JSONParser operator |
+| dt.application.CustomOutputFormatter.operator.HDFSOutputOperator.prop.filePath | output file path for the records after formatting |
+| dt.application.CustomOutputFormatter.operator.HDFSOutputOperator.prop.outFileName | output file name for the records to be written after formatting |
+
+
+#### **Update Properties from Application.java - This is needed to customise Formatter:**
+
+```
+ CsvFormatter formatter = dag.addOperator("formatter", CsvFormatter.class);
+ formatter.setSchema(SchemaUtils.jarResourceFileToString(filename));
+ dag.setInputPortAttribute(formatter.in, PortContext.TUPLE_CLASS, PojoEvent.class);
+```
+
+'filename' above is the variable for storing schema file in the example.
+
+The input port attribute can be configured using properties.xml by setting the below property as well.
+
+`dt.application.CustomOutputFormatter.operator.formatter.port.input.attr.TUPLE_CLASS`
+
+
+
+
+#### **Sample Run:**
+
+- This example generates the JsonData, inputs the generated data to JsonParser which creates the POJO that will be passed to CsvFormatter operator and the formatted output is written to a hdfs location.
+
+- Configure JsonGenerator.java to generate needed data and define the schema of the POJO class PojoEvent.java which is input to the CsvFormatter.
+
+- The schema defined in the `/src/main/resources/schema.json` will be used to evaluate the fields from the object in the CsvFormatter, the field name should match with the field name from the schema.json string.
+
+- The formatting order depends on the order defined in the schema.json string.
+
+- You can build the project and run the example as it is once you configure properties.xml. You can also customise the same app to the schema needed by configuring the PojoEvent.java and schema.json.
+
+
+#### **Sample Output:**
+
+- After running successfully, verify that the hdfs files has the similar output :
+
+```
+ 1234|SimpleCsvFormatterExample|10000.0|false|APEX
+ 1234|SimpleCsvFormatterExample|10000.0|false|APEX
+ 1234|SimpleCsvFormatterExample|10000.0|false|APEX
+ 1234|SimpleCsvFormatterExample|10000.0|false|APEX
+ 1234|SimpleCsvFormatterExample|10000.0|false|APEX
+ ```
+
+In case you have issues configuring the operator or running the application, please send an email to users@apache.apex.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/63df474b/examples/csvformatter/XmlJavadocCommentsExtractor.xsl
----------------------------------------------------------------------
diff --git a/examples/csvformatter/XmlJavadocCommentsExtractor.xsl b/examples/csvformatter/XmlJavadocCommentsExtractor.xsl
new file mode 100644
index 0000000..08075a9
--- /dev/null
+++ b/examples/csvformatter/XmlJavadocCommentsExtractor.xsl
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+
+<!--
+ Document : XmlJavadocCommentsExtractor.xsl
+ Created on : September 16, 2014, 11:30 AM
+ Description:
+ The transformation strips off all information except for comments and tags from xml javadoc generated by xml-doclet.
+-->
+
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+ <xsl:output method="xml" standalone="yes"/>
+
+ <!-- copy xml by selecting only the following nodes, attributes and text -->
+ <xsl:template match="node()|text()|@*">
+ <xsl:copy>
+ <xsl:apply-templates select="root|package|class|interface|method|field|type|comment|tag|text()|@name|@qualified|@text"/>
+ </xsl:copy>
+ </xsl:template>
+
+ <!-- Strip off the following paths from the selected xml -->
+ <xsl:template match="//root/package/interface/interface
+ |//root/package/interface/method/@qualified
+ |//root/package/class/interface
+ |//root/package/class/class
+ |//root/package/class/method/@qualified
+ |//root/package/class/field/@qualified" />
+
+ <xsl:strip-space elements="*"/>
+</xsl:stylesheet>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/63df474b/examples/csvformatter/pom.xml
----------------------------------------------------------------------
diff --git a/examples/csvformatter/pom.xml b/examples/csvformatter/pom.xml
new file mode 100644
index 0000000..9033db5
--- /dev/null
+++ b/examples/csvformatter/pom.xml
@@ -0,0 +1,297 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>com.example</groupId>
+ <version>1.0-SNAPSHOT</version>
+ <artifactId>formatter</artifactId>
+ <packaging>jar</packaging>
+
+ <!-- change these to the appropriate values -->
+ <name>Formatter Apps</name>
+ <description>Applications to showcase different formatters</description>
+
+ <properties>
+ <!-- change this if you desire to use a different version of Apex Core -->
+ <apex.version>3.5.0</apex.version>
+ <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath>
+ <malhar.version>3.6.0</malhar.version>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-eclipse-plugin</artifactId>
+ <version>2.9</version>
+ <configuration>
+ <downloadSources>true</downloadSources>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.3</version>
+ <configuration>
+ <encoding>UTF-8</encoding>
+ <source>1.7</source>
+ <target>1.7</target>
+ <debug>true</debug>
+ <optimize>false</optimize>
+ <showDeprecation>true</showDeprecation>
+ <showWarnings>true</showWarnings>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.8</version>
+ <executions>
+ <execution>
+ <id>copy-dependencies</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>target/deps</outputDirectory>
+ <includeScope>runtime</includeScope>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>app-package-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <finalName>${project.artifactId}-${project.version}-apexapp</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <descriptors>
+ <descriptor>src/assemble/appPackage.xml</descriptor>
+ </descriptors>
+ <archiverConfig>
+ <defaultDirectoryMode>0755</defaultDirectoryMode>
+ </archiverConfig>
+ <archive>
+ <manifestEntries>
+ <Class-Path>${apex.apppackage.classpath}</Class-Path>
+ <DT-Engine-Version>${apex.version}</DT-Engine-Version>
+ <DT-App-Package-Group-Id>${project.groupId}</DT-App-Package-Group-Id>
+ <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name>
+ <DT-App-Package-Version>${project.version}</DT-App-Package-Version>
+ <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name>
+ <DT-App-Package-Description>${project.description}</DT-App-Package-Description>
+ </manifestEntries>
+ </archive>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <configuration>
+ <target>
+ <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar"
+ tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa" />
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ <execution>
+ <!-- create resource directory for xml javadoc-->
+ <id>createJavadocDirectory</id>
+ <phase>generate-resources</phase>
+ <configuration>
+ <tasks>
+ <delete dir="${project.build.directory}/generated-resources/xml-javadoc"/>
+ <mkdir dir="${project.build.directory}/generated-resources/xml-javadoc"/>
+ </tasks>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.9.1</version>
+ <executions>
+ <execution>
+ <id>attach-artifacts</id>
+ <phase>package</phase>
+ <goals>
+ <goal>attach-artifact</goal>
+ </goals>
+ <configuration>
+ <artifacts>
+ <artifact>
+ <file>target/${project.artifactId}-${project.version}.apa</file>
+ <type>apa</type>
+ </artifact>
+ </artifacts>
+ <skipAttach>false</skipAttach>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- generate javdoc -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <executions>
+ <!-- generate xml javadoc -->
+ <execution>
+ <id>xml-doclet</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>javadoc</goal>
+ </goals>
+ <configuration>
+ <doclet>com.github.markusbernhardt.xmldoclet.XmlDoclet</doclet>
+ <additionalparam>-d ${project.build.directory}/generated-resources/xml-javadoc -filename ${project.artifactId}-${project.version}-javadoc.xml</additionalparam>
+ <useStandardDocletOptions>false</useStandardDocletOptions>
+ <docletArtifact>
+ <groupId>com.github.markusbernhardt</groupId>
+ <artifactId>xml-doclet</artifactId>
+ <version>1.0.4</version>
+ </docletArtifact>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <!-- Transform xml javadoc to stripped down version containing only class/interface comments and tags-->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>xml-maven-plugin</artifactId>
+ <version>1.0</version>
+ <executions>
+ <execution>
+ <id>transform-xmljavadoc</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>transform</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <transformationSets>
+ <transformationSet>
+ <dir>${project.build.directory}/generated-resources/xml-javadoc</dir>
+ <includes>
+ <include>${project.artifactId}-${project.version}-javadoc.xml</include>
+ </includes>
+ <stylesheet>XmlJavadocCommentsExtractor.xsl</stylesheet>
+ <outputDir>${project.build.directory}/generated-resources/xml-javadoc</outputDir>
+ </transformationSet>
+ </transformationSets>
+ </configuration>
+ </plugin>
+ <!-- copy xml javadoc to class jar -->
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>2.6</version>
+ <executions>
+ <execution>
+ <id>copy-resources</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${basedir}/target/classes</outputDirectory>
+ <resources>
+ <resource>
+ <directory>${project.build.directory}/generated-resources/xml-javadoc</directory>
+ <includes>
+ <include>${project.artifactId}-${project.version}-javadoc.xml</include>
+ </includes>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+
+ </build>
+
+ <dependencies>
+ <!-- add your dependencies here -->
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-library</artifactId>
+ <version>${malhar.version}</version>
+ <!--
+ If you know that your application does not need transitive dependencies pulled in by malhar-library,
+ uncomment the following to reduce the size of your app package.
+ -->
+ <!--
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ -->
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-contrib</artifactId>
+ <version>${malhar.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>2.5.4</version>
+ </dependency>
+ <dependency>
+ <groupId>com.github.fge</groupId>
+ <artifactId>json-schema-validator</artifactId>
+ <version>2.0.1</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>apex-common</artifactId>
+ <version>${apex.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.10</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>apex-engine</artifactId>
+ <version>${apex.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>net.sf.supercsv</groupId>
+ <artifactId>super-csv</artifactId>
+ <version>2.4.0</version>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/63df474b/examples/csvformatter/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/csvformatter/src/assemble/appPackage.xml b/examples/csvformatter/src/assemble/appPackage.xml
new file mode 100644
index 0000000..7ad071c
--- /dev/null
+++ b/examples/csvformatter/src/assemble/appPackage.xml
@@ -0,0 +1,43 @@
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>appPackage</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/target/</directory>
+ <outputDirectory>/app</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/target/deps</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/site/conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/META-INF</directory>
+ <outputDirectory>/META-INF</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/app</directory>
+ <outputDirectory>/app</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/resources</directory>
+ <outputDirectory>/resources</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/63df474b/examples/csvformatter/src/main/java/com/demo/myapexapp/Application.java
----------------------------------------------------------------------
diff --git a/examples/csvformatter/src/main/java/com/demo/myapexapp/Application.java b/examples/csvformatter/src/main/java/com/demo/myapexapp/Application.java
new file mode 100644
index 0000000..a4ff06f
--- /dev/null
+++ b/examples/csvformatter/src/main/java/com/demo/myapexapp/Application.java
@@ -0,0 +1,45 @@
+package com.demo.myapexapp;
+
+import java.util.Arrays;
+
+import com.datatorrent.contrib.parser.JsonParser;
+
+import org.apache.apex.malhar.contrib.parser.StreamingJsonParser;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.Context.PortContext;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StatsListener;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.contrib.formatter.CsvFormatter;
+import com.datatorrent.lib.appdata.schemas.SchemaUtils;
+import com.datatorrent.lib.io.fs.AbstractFileInputOperator.DirectoryScanner;
+import com.datatorrent.lib.partitioner.StatelessThroughputBasedPartitioner;
+
+@ApplicationAnnotation(name = "CustomOutputFormatter")
+public class Application implements StreamingApplication
+{
+ //Set the delimiters and schema structure for the custom output in schema.json
+ private static final String filename = "schema.json";
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ JsonGenerator generator = dag.addOperator("JsonGenerator", JsonGenerator.class);
+ JsonParser jsonParser = dag.addOperator("jsonParser", JsonParser.class);
+
+ CsvFormatter formatter = dag.addOperator("formatter", CsvFormatter.class);
+ formatter.setSchema(SchemaUtils.jarResourceFileToString(filename));
+ dag.setInputPortAttribute(formatter.in, PortContext.TUPLE_CLASS, PojoEvent.class);
+
+ HDFSOutputOperator<String> hdfsOutput = dag.addOperator("HDFSOutputOperator", HDFSOutputOperator.class);
+ hdfsOutput.setLineDelimiter("");
+
+ dag.addStream("parserStream", generator.out, jsonParser.in);
+ dag.addStream("formatterStream", jsonParser.out, formatter.in);
+ dag.addStream("outputStream", formatter.out, hdfsOutput.input);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/63df474b/examples/csvformatter/src/main/java/com/demo/myapexapp/HDFSOutputOperator.java
----------------------------------------------------------------------
diff --git a/examples/csvformatter/src/main/java/com/demo/myapexapp/HDFSOutputOperator.java b/examples/csvformatter/src/main/java/com/demo/myapexapp/HDFSOutputOperator.java
new file mode 100644
index 0000000..5cb162c
--- /dev/null
+++ b/examples/csvformatter/src/main/java/com/demo/myapexapp/HDFSOutputOperator.java
@@ -0,0 +1,87 @@
+package com.demo.myapexapp;
+
+import javax.validation.constraints.NotNull;
+
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.lib.io.fs.AbstractFileOutputOperator;
+
+/**
+ * HDFSoutput operator with implementation to write Objects to HDFS
+ *
+ * @param <T>
+ */
+public class HDFSOutputOperator<T> extends AbstractFileOutputOperator<T>
+{
+
+ @NotNull
+ String outFileName;
+
+ //setting default value
+ String lineDelimiter = "\n";
+
+ //Switch to write the files to HDFS - set to false to diable writes
+ private boolean writeFilesFlag = true;
+
+ int id;
+
+ @Override
+ public void setup(OperatorContext context)
+ {
+ super.setup(context);
+ id = context.getId();
+ }
+
+ public boolean isWriteFilesFlag()
+ {
+ return writeFilesFlag;
+ }
+
+ public void setWriteFilesFlag(boolean writeFilesFlag)
+ {
+ this.writeFilesFlag = writeFilesFlag;
+ }
+
+ public String getOutFileName()
+ {
+ return outFileName;
+ }
+
+ public void setOutFileName(String outFileName)
+ {
+ this.outFileName = outFileName;
+ }
+
+ @Override
+ protected String getFileName(T tuple)
+ {
+ return getOutFileName() + id;
+ }
+
+ public String getLineDelimiter()
+ {
+ return lineDelimiter;
+ }
+
+ public void setLineDelimiter(String lineDelimiter)
+ {
+ this.lineDelimiter = lineDelimiter;
+ }
+
+ @Override
+ protected byte[] getBytesForTuple(T tuple)
+ {
+ String temp = tuple.toString().concat(String.valueOf(lineDelimiter));
+ byte[] theByteArray = temp.getBytes();
+
+ return theByteArray;
+ }
+
+ @Override
+ protected void processTuple(T tuple)
+ {
+ if (writeFilesFlag) {
+ }
+ super.processTuple(tuple);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/63df474b/examples/csvformatter/src/main/java/com/demo/myapexapp/JsonGenerator.java
----------------------------------------------------------------------
diff --git a/examples/csvformatter/src/main/java/com/demo/myapexapp/JsonGenerator.java b/examples/csvformatter/src/main/java/com/demo/myapexapp/JsonGenerator.java
new file mode 100644
index 0000000..f50f300
--- /dev/null
+++ b/examples/csvformatter/src/main/java/com/demo/myapexapp/JsonGenerator.java
@@ -0,0 +1,78 @@
+package com.demo.myapexapp;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Random;
+
+import javax.validation.constraints.Min;
+
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.common.util.BaseOperator;
+
+public class JsonGenerator extends BaseOperator implements InputOperator
+{
+
+ private static final Logger LOG = LoggerFactory.getLogger(JsonGenerator.class);
+
+ @Min(1)
+ private int numTuples = 20;
+ private transient int count = 0;
+
+ public static Random rand = new Random();
+ private int sleepTime=5;
+
+ public final transient DefaultOutputPort<byte[]> out = new DefaultOutputPort<byte[]>();
+
+ private static String getJson()
+ {
+
+ JSONObject obj = new JSONObject();
+ try {
+ obj.put("campaignId", 1234);
+ obj.put("campaignName", "SimpleCsvFormatterExample");
+ obj.put("campaignBudget", 10000.0);
+ obj.put("weatherTargeting", "false");
+ obj.put("securityCode", "APEX");
+ } catch (JSONException e) {
+ return null;
+ }
+ return obj.toString();
+ }
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ count = 0;
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ if (count++ < numTuples) {
+ out.emit(getJson().getBytes());
+ } else {
+ try {
+ Thread.sleep(sleepTime);
+ } catch (InterruptedException e) {
+ LOG.info("Sleep interrupted");
+ }
+ }
+ }
+
+ public int getNumTuples()
+ {
+ return numTuples;
+ }
+
+ public void setNumTuples(int numTuples)
+ {
+ this.numTuples = numTuples;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/63df474b/examples/csvformatter/src/main/java/com/demo/myapexapp/PojoEvent.java
----------------------------------------------------------------------
diff --git a/examples/csvformatter/src/main/java/com/demo/myapexapp/PojoEvent.java b/examples/csvformatter/src/main/java/com/demo/myapexapp/PojoEvent.java
new file mode 100644
index 0000000..8514856
--- /dev/null
+++ b/examples/csvformatter/src/main/java/com/demo/myapexapp/PojoEvent.java
@@ -0,0 +1,141 @@
+package com.demo.myapexapp;
+
+import java.util.Date;
+
+public class PojoEvent
+{
+
+ private int advId;
+ private int campaignId;
+ private String campaignName;
+ private double campaignBudget;
+ private Date startDate;
+ private Date endDate;
+ private String securityCode;
+ private boolean weatherTargeting;
+ private boolean optimized;
+ private String parentCampaign;
+ private Character weatherTargeted;
+ private String valid;
+
+ public int getAdvId()
+ {
+ return advId;
+ }
+
+ public void setAdvId(int AdId)
+ {
+ this.advId = advId;
+ }
+
+ public int getCampaignId()
+ {
+ return campaignId;
+ }
+
+ public void setCampaignId(int campaignId)
+ {
+ this.campaignId = campaignId;
+ }
+
+ public String getCampaignName()
+ {
+ return campaignName;
+ }
+
+ public void setCampaignName(String campaignName)
+ {
+ this.campaignName = campaignName;
+ }
+
+ public double getCampaignBudget()
+ {
+ return campaignBudget;
+ }
+
+ public void setCampaignBudget(double campaignBudget)
+ {
+ this.campaignBudget = campaignBudget;
+ }
+
+ public Date getStartDate()
+ {
+ return startDate;
+ }
+
+ public void setStartDate(Date startDate)
+ {
+ this.startDate = startDate;
+ }
+
+ public Date getEndDate()
+ {
+ return endDate;
+ }
+
+ public void setEndDate(Date endDate)
+ {
+ this.endDate = endDate;
+ }
+
+ public String getSecurityCode()
+ {
+ return securityCode;
+ }
+
+ public void setSecurityCode(String securityCode)
+ {
+ this.securityCode = securityCode;
+ }
+
+ public boolean isWeatherTargeting()
+ {
+ return weatherTargeting;
+ }
+
+ public void setWeatherTargeting(boolean weatherTargeting)
+ {
+ this.weatherTargeting = weatherTargeting;
+ }
+
+ public boolean isOptimized()
+ {
+ return optimized;
+ }
+
+ public void setOptimized(boolean optimized)
+ {
+ this.optimized = optimized;
+ }
+
+ public String getParentCampaign()
+ {
+ return parentCampaign;
+ }
+
+ public void setParentCampaign(String parentCampaign)
+ {
+ this.parentCampaign = parentCampaign;
+ }
+
+ public Character getWeatherTargeted()
+ {
+ return weatherTargeted;
+ }
+
+ public void setWeatherTargeted(Character weatherTargeted)
+ {
+ this.weatherTargeted = weatherTargeted;
+ }
+
+ public String getValid()
+ {
+ return valid;
+ }
+
+ public void setValid(String valid)
+ {
+ this.valid = valid;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/63df474b/examples/csvformatter/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/csvformatter/src/main/resources/META-INF/properties.xml b/examples/csvformatter/src/main/resources/META-INF/properties.xml
new file mode 100644
index 0000000..ed2b5ce
--- /dev/null
+++ b/examples/csvformatter/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0"?>
+<configuration>
+ <!--
+ <property>
+ <name>dt.application.{appName}.operator.{opName}.prop.{propName}</name>
+ <value>some-default-value (if value is not specified, it is required from the user or custom config when launching)</value>
+ </property>
+ -->
+ <!-- memory assigned to app master
+ <property>
+ <name>dt.attr.MASTER_MEMORY_MB</name>
+ <value>1024</value>
+ </property>
+ -->
+ <property>
+ <name>dt.application.CustomOutputFormatter.operator.jsonParser.prop.sleepTime
+ </name>
+ <value>100</value>
+ </property>
+ <property>
+ <name>dt.application.CustomOutputFormatter.operator.jsonParser.port.out.attr.TUPLE_CLASS
+ </name>
+ <value>com.demo.myapexapp.PojoEvent</value>
+ </property>
+
+
+ <property>
+ <name>dt.application.CustomOutputFormatter.operator.HDFSOutputOperator.prop.filePath
+ </name>
+ <value>/tmp/formatterApp</value>
+ </property>
+
+ <property>
+ <name>dt.application.CustomOutputFormatter.operator.HDFSOutputOperator.prop.outFileName
+ </name>
+ <value>customFormatResult</value>
+ </property>
+
+</configuration>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/63df474b/examples/csvformatter/src/main/resources/schema.json
----------------------------------------------------------------------
diff --git a/examples/csvformatter/src/main/resources/schema.json b/examples/csvformatter/src/main/resources/schema.json
new file mode 100644
index 0000000..9209469
--- /dev/null
+++ b/examples/csvformatter/src/main/resources/schema.json
@@ -0,0 +1,60 @@
+{
+ "separator": "|",
+ "quoteChar": "\"",
+ "lineDelimiter": "\n",
+ "fields": [
+ {
+ "name": "campaignId",
+ "type": "Integer"
+ },
+ {
+ "name": "advId",
+ "type": "Integer"
+ },
+ {
+ "name": "campaignName",
+ "type": "String"
+ },
+ {
+ "name": "campaignBudget",
+ "type": "Double",
+ "constraints": {
+ "maxValue": 30000
+ }
+ },
+ {
+ "name": "startDate",
+ "type": "Date",
+ "constraints": {
+ "format": "yyyy-MM-dd"
+ }
+ },
+ {
+ "name": "endDate",
+ "type": "Date",
+ "constraints": {
+ "format": "dd/MM/yyyy"
+ }
+ },
+ {
+ "name": "securityCode",
+ "type": "String"
+ },
+ {
+ "name": "weatherTargeting",
+ "type": "Boolean"
+ },
+ {
+ "name": "optimized",
+ "type": "Boolean"
+ },
+ {
+ "name": "parentCampaign",
+ "type": "String"
+ },
+ {
+ "name": "weatherTargeted",
+ "type": "Character"
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/63df474b/examples/csvformatter/src/test/java/com/demo/myapexapp/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/csvformatter/src/test/java/com/demo/myapexapp/ApplicationTest.java b/examples/csvformatter/src/test/java/com/demo/myapexapp/ApplicationTest.java
new file mode 100644
index 0000000..efe5946
--- /dev/null
+++ b/examples/csvformatter/src/test/java/com/demo/myapexapp/ApplicationTest.java
@@ -0,0 +1,67 @@
+package com.demo.myapexapp;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+
+import javax.validation.ConstraintViolationException;
+
+import org.apache.commons.io.FileUtils;
+
+import org.junit.AfterClass;
+import org.junit.Assert;
+
+import org.apache.hadoop.conf.Configuration;
+
+import org.junit.Test;
+
+import com.datatorrent.api.LocalMode;
+
+/**
+ * Test the DAG declaration in local mode.
+ */
+public class ApplicationTest
+{
+
+ private static final String FILE_NAME = "/tmp/formatterApp";
+
+ @AfterClass
+ public static void cleanup()
+ {
+ try {
+ FileUtils.deleteDirectory(new File(FILE_NAME));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Test
+ public void testApplication() throws Exception
+ {
+ try {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ lma.prepareDAG(new Application(), conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.runAsync();
+
+ // wait for output files to roll
+ Thread.sleep(5000);
+
+ String[] extensions = {"dat.0", "tmp"};
+ Collection<File> list = FileUtils.listFiles(new File(FILE_NAME), extensions, false);
+
+ for (File file : list) {
+ for (String line : FileUtils.readLines(file)) {
+ Assert.assertEquals("Delimiter in record", true, (line.equals(
+ "1234|0|SimpleCsvFormatterExample|10000.0|||APEX|false|false||")));
+ }
+ }
+
+ } catch (ConstraintViolationException e) {
+ Assert.fail("constraint violations: " + e.getConstraintViolations());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/63df474b/examples/csvformatter/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/csvformatter/src/test/resources/log4j.properties b/examples/csvformatter/src/test/resources/log4j.properties
new file mode 100644
index 0000000..98544e8
--- /dev/null
+++ b/examples/csvformatter/src/test/resources/log4j.properties
@@ -0,0 +1,22 @@
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=debug
+log4j.logger.org.apache.apex=debug
+log4j.logger.org=info
[02/19] apex-malhar git commit: Fixed checkstyle and added license
headers.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/innerjoin/XmlJavadocCommentsExtractor.xsl
----------------------------------------------------------------------
diff --git a/examples/innerjoin/XmlJavadocCommentsExtractor.xsl b/examples/innerjoin/XmlJavadocCommentsExtractor.xsl
deleted file mode 100644
index 08075a9..0000000
--- a/examples/innerjoin/XmlJavadocCommentsExtractor.xsl
+++ /dev/null
@@ -1,44 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
--->
-
-<!--
- Document : XmlJavadocCommentsExtractor.xsl
- Created on : September 16, 2014, 11:30 AM
- Description:
- The transformation strips off all information except for comments and tags from xml javadoc generated by xml-doclet.
--->
-
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
- <xsl:output method="xml" standalone="yes"/>
-
- <!-- copy xml by selecting only the following nodes, attributes and text -->
- <xsl:template match="node()|text()|@*">
- <xsl:copy>
- <xsl:apply-templates select="root|package|class|interface|method|field|type|comment|tag|text()|@name|@qualified|@text"/>
- </xsl:copy>
- </xsl:template>
-
- <!-- Strip off the following paths from the selected xml -->
- <xsl:template match="//root/package/interface/interface
- |//root/package/interface/method/@qualified
- |//root/package/class/interface
- |//root/package/class/class
- |//root/package/class/method/@qualified
- |//root/package/class/field/@qualified" />
-
- <xsl:strip-space elements="*"/>
-</xsl:stylesheet>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/innerjoin/pom.xml
----------------------------------------------------------------------
diff --git a/examples/innerjoin/pom.xml b/examples/innerjoin/pom.xml
index 78193ca..ef2a12c 100644
--- a/examples/innerjoin/pom.xml
+++ b/examples/innerjoin/pom.xml
@@ -1,4 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@@ -11,7 +31,7 @@
<artifactId>malhar-examples-innerjoin</artifactId>
<packaging>jar</packaging>
- <name>Inner Join Application</name>
+ <name>Apache Apex Malhar Inner Join Example</name>
<description>Sample Application for Inner Join</description>
<dependencies>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/innerjoin/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/innerjoin/src/assemble/appPackage.xml b/examples/innerjoin/src/assemble/appPackage.xml
index 7ad071c..a870807 100644
--- a/examples/innerjoin/src/assemble/appPackage.xml
+++ b/examples/innerjoin/src/assemble/appPackage.xml
@@ -1,3 +1,23 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/innerjoin/src/main/java/org/apache/apex/examples/innerjoin/InnerJoinApplication.java
----------------------------------------------------------------------
diff --git a/examples/innerjoin/src/main/java/org/apache/apex/examples/innerjoin/InnerJoinApplication.java b/examples/innerjoin/src/main/java/org/apache/apex/examples/innerjoin/InnerJoinApplication.java
index 3e7c8d5..ae62016 100644
--- a/examples/innerjoin/src/main/java/org/apache/apex/examples/innerjoin/InnerJoinApplication.java
+++ b/examples/innerjoin/src/main/java/org/apache/apex/examples/innerjoin/InnerJoinApplication.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.innerjoin;
import org.apache.apex.malhar.lib.join.POJOInnerJoinOperator;
@@ -9,7 +28,7 @@ import com.datatorrent.api.StreamingApplication;
import com.datatorrent.api.annotation.ApplicationAnnotation;
import com.datatorrent.lib.io.ConsoleOutputOperator;
-@ApplicationAnnotation(name="InnerJoinExample")
+@ApplicationAnnotation(name = "InnerJoinExample")
public class InnerJoinApplication implements StreamingApplication
{
@Override
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/innerjoin/src/main/java/org/apache/apex/examples/innerjoin/POJOGenerator.java
----------------------------------------------------------------------
diff --git a/examples/innerjoin/src/main/java/org/apache/apex/examples/innerjoin/POJOGenerator.java b/examples/innerjoin/src/main/java/org/apache/apex/examples/innerjoin/POJOGenerator.java
index 2a736c5..0b57c07 100644
--- a/examples/innerjoin/src/main/java/org/apache/apex/examples/innerjoin/POJOGenerator.java
+++ b/examples/innerjoin/src/main/java/org/apache/apex/examples/innerjoin/POJOGenerator.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.innerjoin;
import java.util.Random;
@@ -56,8 +75,8 @@ public class POJOGenerator implements InputOperator
}
- SalesEvent generateSalesEvent() throws Exception {
-
+ SalesEvent generateSalesEvent() throws Exception
+ {
SalesEvent salesEvent = new SalesEvent();
salesEvent.productId = randomId(maxProductId);
salesEvent.customerId = randomId(maxCustomerId);
@@ -66,7 +85,8 @@ public class POJOGenerator implements InputOperator
return salesEvent;
}
- ProductEvent generateProductEvent() throws Exception {
+ ProductEvent generateProductEvent() throws Exception
+ {
ProductEvent productEvent = new ProductEvent();
productEvent.productId = randomId(maxProductId);
productEvent.productCategory = randomId(maxProductCategories);
@@ -74,12 +94,16 @@ public class POJOGenerator implements InputOperator
return productEvent;
}
- private int randomId(int max) {
- if (max < 1) return 1;
+ private int randomId(int max)
+ {
+ if (max < 1) {
+ return 1;
+ }
return 1 + random.nextInt(max);
}
- private double randomAmount() {
+ private double randomAmount()
+ {
return maxAmount * random.nextDouble();
}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/innerjoin/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/innerjoin/src/main/resources/META-INF/properties.xml b/examples/innerjoin/src/main/resources/META-INF/properties.xml
index 5c9cb75..879d010 100644
--- a/examples/innerjoin/src/main/resources/META-INF/properties.xml
+++ b/examples/innerjoin/src/main/resources/META-INF/properties.xml
@@ -1,4 +1,24 @@
<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<configuration>
<property>
<name>dt.application.InnerJoinExample.operator.Join.attr.MEMORY_MB</name>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/innerjoin/src/test/java/org/apache/apex/examples/innerjoin/InnerJoinApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/innerjoin/src/test/java/org/apache/apex/examples/innerjoin/InnerJoinApplicationTest.java b/examples/innerjoin/src/test/java/org/apache/apex/examples/innerjoin/InnerJoinApplicationTest.java
index 81ae442..03152a4 100644
--- a/examples/innerjoin/src/test/java/org/apache/apex/examples/innerjoin/InnerJoinApplicationTest.java
+++ b/examples/innerjoin/src/test/java/org/apache/apex/examples/innerjoin/InnerJoinApplicationTest.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.innerjoin;
import org.junit.Test;
@@ -18,4 +37,4 @@ public class InnerJoinApplicationTest
Thread.sleep(10 * 1000);
lc.shutdown();
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/innerjoin/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/innerjoin/src/test/resources/log4j.properties b/examples/innerjoin/src/test/resources/log4j.properties
index 98544e8..1c9776b 100644
--- a/examples/innerjoin/src/test/resources/log4j.properties
+++ b/examples/innerjoin/src/test/resources/log4j.properties
@@ -1,3 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
log4j.rootLogger=DEBUG,CONSOLE
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/parser/README.md
----------------------------------------------------------------------
diff --git a/examples/parser/README.md b/examples/parser/README.md
index da230e3..51052b9 100644
--- a/examples/parser/README.md
+++ b/examples/parser/README.md
@@ -5,7 +5,7 @@ This app showcases **Json Parser**. Data generator sends Json data to the Json P
* **Csv Parser App**
-This application showcases how to use [CsvParser](https://datatorrent.com/docs/apidocs/com/datatorrent/contrib/parser/CsvParser.html) from [Apex Malhar](https://github.com/apache/apex-malhar) library. The CsvParser converts your delimited data to a key-value pair map or concrete java class also know as [POJO](https://en.wikipedia.org/wiki/Plain_Old_Java_Object). The parser emits key-value pair map on *parsedOutput* port. It emits POJO on *out* and error records on *err* port.
+This application showcases how to use CsvParser from [Apex Malhar](https://github.com/apache/apex-malhar) library. The CsvParser converts your delimited data to a key-value pair map or concrete java class also know as [POJO](https://en.wikipedia.org/wiki/Plain_Old_Java_Object). The parser emits key-value pair map on *parsedOutput* port. It emits POJO on *out* and error records on *err* port.
Follow these steps to run this application:
@@ -21,8 +21,7 @@ that the output by checking hdfs file path configured in properties-csvParseAppl
* **Xml Parser App**
-This application showcases how to use [XmlParser](https://datatorrent.com/docs/apidocs/com/datatorrent/lib/parser/XmlParser.html)
-from [Apex Malhar](https://github.com/apache/apex-malhar) library. The XmlParser Operator converts XML string to POJO.
+This application showcases how to use XmlParser from [Apex Malhar](https://github.com/apache/apex-malhar) library. The XmlParser Operator converts XML string to POJO.
The parser emits dom based Document on *parsedOutput* port. It emits POJO on *out* and error records on *err* port.
Follow these steps to run this application:
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/parser/XmlJavadocCommentsExtractor.xsl
----------------------------------------------------------------------
diff --git a/examples/parser/XmlJavadocCommentsExtractor.xsl b/examples/parser/XmlJavadocCommentsExtractor.xsl
deleted file mode 100644
index 08075a9..0000000
--- a/examples/parser/XmlJavadocCommentsExtractor.xsl
+++ /dev/null
@@ -1,44 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
--->
-
-<!--
- Document : XmlJavadocCommentsExtractor.xsl
- Created on : September 16, 2014, 11:30 AM
- Description:
- The transformation strips off all information except for comments and tags from xml javadoc generated by xml-doclet.
--->
-
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
- <xsl:output method="xml" standalone="yes"/>
-
- <!-- copy xml by selecting only the following nodes, attributes and text -->
- <xsl:template match="node()|text()|@*">
- <xsl:copy>
- <xsl:apply-templates select="root|package|class|interface|method|field|type|comment|tag|text()|@name|@qualified|@text"/>
- </xsl:copy>
- </xsl:template>
-
- <!-- Strip off the following paths from the selected xml -->
- <xsl:template match="//root/package/interface/interface
- |//root/package/interface/method/@qualified
- |//root/package/class/interface
- |//root/package/class/class
- |//root/package/class/method/@qualified
- |//root/package/class/field/@qualified" />
-
- <xsl:strip-space elements="*"/>
-</xsl:stylesheet>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/parser/pom.xml
----------------------------------------------------------------------
diff --git a/examples/parser/pom.xml b/examples/parser/pom.xml
index ec03cae..b90b06e 100644
--- a/examples/parser/pom.xml
+++ b/examples/parser/pom.xml
@@ -1,4 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@@ -12,7 +32,7 @@
<packaging>jar</packaging>
<!-- change these to the appropriate values -->
- <name>Parser Apps</name>
+ <name>Apache Apex Malhar Parsers Example</name>
<description>Applications to showcase different parsers</description>
<dependencies>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/parser/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/parser/src/assemble/appPackage.xml b/examples/parser/src/assemble/appPackage.xml
index 7ad071c..a870807 100644
--- a/examples/parser/src/assemble/appPackage.xml
+++ b/examples/parser/src/assemble/appPackage.xml
@@ -1,3 +1,23 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/parser/src/main/java/org/apache/apex/examples/parser/jsonparser/Application.java
----------------------------------------------------------------------
diff --git a/examples/parser/src/main/java/org/apache/apex/examples/parser/jsonparser/Application.java b/examples/parser/src/main/java/org/apache/apex/examples/parser/jsonparser/Application.java
index 34b64a3..cacfadc 100644
--- a/examples/parser/src/main/java/org/apache/apex/examples/parser/jsonparser/Application.java
+++ b/examples/parser/src/main/java/org/apache/apex/examples/parser/jsonparser/Application.java
@@ -1,3 +1,23 @@
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.parser.jsonparser;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/parser/src/main/java/org/apache/apex/examples/parser/jsonparser/Campaign.java
----------------------------------------------------------------------
diff --git a/examples/parser/src/main/java/org/apache/apex/examples/parser/jsonparser/Campaign.java b/examples/parser/src/main/java/org/apache/apex/examples/parser/jsonparser/Campaign.java
index 8946f5d..f31e56d 100644
--- a/examples/parser/src/main/java/org/apache/apex/examples/parser/jsonparser/Campaign.java
+++ b/examples/parser/src/main/java/org/apache/apex/examples/parser/jsonparser/Campaign.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.parser.jsonparser;
import java.util.Date;
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/parser/src/main/java/org/apache/apex/examples/parser/jsonparser/JsonGenerator.java
----------------------------------------------------------------------
diff --git a/examples/parser/src/main/java/org/apache/apex/examples/parser/jsonparser/JsonGenerator.java b/examples/parser/src/main/java/org/apache/apex/examples/parser/jsonparser/JsonGenerator.java
index a0e3611..27416c3 100644
--- a/examples/parser/src/main/java/org/apache/apex/examples/parser/jsonparser/JsonGenerator.java
+++ b/examples/parser/src/main/java/org/apache/apex/examples/parser/jsonparser/JsonGenerator.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.parser.jsonparser;
import java.text.SimpleDateFormat;
@@ -26,10 +45,10 @@ public class JsonGenerator extends BaseOperator implements InputOperator
public static Random rand = new Random();
public static SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd");
- public static int[] adId = { 1, 2, 3, 4, 5 };
- public static String[] campaignName = { "cmp1", "cmp2", "cmp3", "cmp4" };
- public static double[] campaignBudget = { 10000.0, 20000.0, 300000.0 };
- public static boolean[] weatherTargeting = { true, false };
+ public static int[] adId = {1, 2, 3, 4, 5};
+ public static String[] campaignName = {"cmp1", "cmp2", "cmp3", "cmp4"};
+ public static double[] campaignBudget = {10000.0, 20000.0, 300000.0};
+ public static boolean[] weatherTargeting = {true, false};
private int sleepTime;
public final transient DefaultOutputPort<byte[]> out = new DefaultOutputPort<byte[]>();
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/parser/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/parser/src/main/resources/META-INF/properties.xml b/examples/parser/src/main/resources/META-INF/properties.xml
index 2cd3c04..956498e 100644
--- a/examples/parser/src/main/resources/META-INF/properties.xml
+++ b/examples/parser/src/main/resources/META-INF/properties.xml
@@ -1,9 +1,29 @@
<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<configuration>
- <!-- <property> <name>dt.application.{appName}.operator.{opName}.prop.{propName}</name>
- <value>some-default-value (if value is not specified, it is required from
+ <!-- <property> <name>dt.application.{appName}.operator.{opName}.prop.{propName}</name>
+ <value>some-default-value (if value is not specified, it is required from
the user or custom config when launching)</value> </property> -->
- <!-- memory assigned to app master <property> <name>dt.attr.MASTER_MEMORY_MB</name>
+ <!-- memory assigned to app master <property> <name>dt.attr.MASTER_MEMORY_MB</name>
<value>1024</value> </property> -->
<!-- these values override those set in populateDAG -->
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/parser/src/test/java/org/apache/apex/examples/parser/jsonparser/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/parser/src/test/java/org/apache/apex/examples/parser/jsonparser/ApplicationTest.java b/examples/parser/src/test/java/org/apache/apex/examples/parser/jsonparser/ApplicationTest.java
index 6f1a6f6..0a354f6 100644
--- a/examples/parser/src/test/java/org/apache/apex/examples/parser/jsonparser/ApplicationTest.java
+++ b/examples/parser/src/test/java/org/apache/apex/examples/parser/jsonparser/ApplicationTest.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.parser.jsonparser;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/parser/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/parser/src/test/resources/log4j.properties b/examples/parser/src/test/resources/log4j.properties
index 98544e8..1c9776b 100644
--- a/examples/parser/src/test/resources/log4j.properties
+++ b/examples/parser/src/test/resources/log4j.properties
@@ -1,3 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
log4j.rootLogger=DEBUG,CONSOLE
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/partition/XmlJavadocCommentsExtractor.xsl
----------------------------------------------------------------------
diff --git a/examples/partition/XmlJavadocCommentsExtractor.xsl b/examples/partition/XmlJavadocCommentsExtractor.xsl
deleted file mode 100644
index 08075a9..0000000
--- a/examples/partition/XmlJavadocCommentsExtractor.xsl
+++ /dev/null
@@ -1,44 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
--->
-
-<!--
- Document : XmlJavadocCommentsExtractor.xsl
- Created on : September 16, 2014, 11:30 AM
- Description:
- The transformation strips off all information except for comments and tags from xml javadoc generated by xml-doclet.
--->
-
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
- <xsl:output method="xml" standalone="yes"/>
-
- <!-- copy xml by selecting only the following nodes, attributes and text -->
- <xsl:template match="node()|text()|@*">
- <xsl:copy>
- <xsl:apply-templates select="root|package|class|interface|method|field|type|comment|tag|text()|@name|@qualified|@text"/>
- </xsl:copy>
- </xsl:template>
-
- <!-- Strip off the following paths from the selected xml -->
- <xsl:template match="//root/package/interface/interface
- |//root/package/interface/method/@qualified
- |//root/package/class/interface
- |//root/package/class/class
- |//root/package/class/method/@qualified
- |//root/package/class/field/@qualified" />
-
- <xsl:strip-space elements="*"/>
-</xsl:stylesheet>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/partition/pom.xml
----------------------------------------------------------------------
diff --git a/examples/partition/pom.xml b/examples/partition/pom.xml
index cafdcd7..55af32f 100644
--- a/examples/partition/pom.xml
+++ b/examples/partition/pom.xml
@@ -1,4 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@@ -12,7 +32,7 @@
<packaging>jar</packaging>
<!-- change these to the appropriate values -->
- <name>Partition</name>
+ <name>Apache Apex Malhar Partitions Example</name>
<description>malhar-examples-partition</description>
</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/partition/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/partition/src/assemble/appPackage.xml b/examples/partition/src/assemble/appPackage.xml
index 7ad071c..a870807 100644
--- a/examples/partition/src/assemble/appPackage.xml
+++ b/examples/partition/src/assemble/appPackage.xml
@@ -1,3 +1,23 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/partition/src/main/java/org/apache/apex/examples/partition/Application.java
----------------------------------------------------------------------
diff --git a/examples/partition/src/main/java/org/apache/apex/examples/partition/Application.java b/examples/partition/src/main/java/org/apache/apex/examples/partition/Application.java
index eb9a7c2..8d1b3bf 100644
--- a/examples/partition/src/main/java/org/apache/apex/examples/partition/Application.java
+++ b/examples/partition/src/main/java/org/apache/apex/examples/partition/Application.java
@@ -1,13 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.partition;
import org.apache.hadoop.conf.Configuration;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.api.StreamingApplication;
import com.datatorrent.api.Context.PortContext;
import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
-@ApplicationAnnotation(name="TestStuff")
+@ApplicationAnnotation(name = "TestStuff")
public class Application implements StreamingApplication
{
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/partition/src/main/java/org/apache/apex/examples/partition/Codec3.java
----------------------------------------------------------------------
diff --git a/examples/partition/src/main/java/org/apache/apex/examples/partition/Codec3.java b/examples/partition/src/main/java/org/apache/apex/examples/partition/Codec3.java
index daa0e05..b48cd83 100644
--- a/examples/partition/src/main/java/org/apache/apex/examples/partition/Codec3.java
+++ b/examples/partition/src/main/java/org/apache/apex/examples/partition/Codec3.java
@@ -1,13 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.partition;
import com.datatorrent.lib.codec.KryoSerializableStreamCodec;
-public class Codec3 extends KryoSerializableStreamCodec<Integer> {
- @Override
- public int getPartition(Integer tuple) {
- final int v = tuple;
- return (1 == (v & 1)) ? 0 // odd
+public class Codec3 extends KryoSerializableStreamCodec<Integer>
+{
+ @Override
+ public int getPartition(Integer tuple)
+ {
+ final int v = tuple;
+ return (1 == (v & 1)) ? 0 // odd
: (0 == (v & 3)) ? 1 // divisible by 4
: 2; // divisible by 2 but not 4
- }
+ }
}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/partition/src/main/java/org/apache/apex/examples/partition/RandomNumberGenerator.java
----------------------------------------------------------------------
diff --git a/examples/partition/src/main/java/org/apache/apex/examples/partition/RandomNumberGenerator.java b/examples/partition/src/main/java/org/apache/apex/examples/partition/RandomNumberGenerator.java
index e3959d1..40eb041 100644
--- a/examples/partition/src/main/java/org/apache/apex/examples/partition/RandomNumberGenerator.java
+++ b/examples/partition/src/main/java/org/apache/apex/examples/partition/RandomNumberGenerator.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.partition;
import java.util.Random;
@@ -7,10 +26,10 @@ import javax.validation.constraints.Min;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.datatorrent.api.Context;
import com.datatorrent.api.DefaultOutputPort;
import com.datatorrent.api.InputOperator;
import com.datatorrent.common.util.BaseOperator;
-import com.datatorrent.api.Context;
/**
* This is a simple operator that emits random integer.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/partition/src/main/java/org/apache/apex/examples/partition/TestPartition.java
----------------------------------------------------------------------
diff --git a/examples/partition/src/main/java/org/apache/apex/examples/partition/TestPartition.java b/examples/partition/src/main/java/org/apache/apex/examples/partition/TestPartition.java
index 5b4e693..60eb69e 100644
--- a/examples/partition/src/main/java/org/apache/apex/examples/partition/TestPartition.java
+++ b/examples/partition/src/main/java/org/apache/apex/examples/partition/TestPartition.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.partition;
import java.util.ArrayList;
@@ -30,10 +49,12 @@ public class TestPartition extends BaseOperator implements Partitioner<TestParti
private transient long curWindowId; // current window id
private transient long cnt; // per-window tuple count
- @Min(1) @Max(20)
+ @Min(1)
+ @Max(20)
private int nPartitions = 3;
- public final transient DefaultInputPort<Integer> in = new DefaultInputPort<Integer>() {
+ public final transient DefaultInputPort<Integer> in = new DefaultInputPort<Integer>()
+ {
@Override
public void process(Integer tuple)
{
@@ -75,9 +96,7 @@ public class TestPartition extends BaseOperator implements Partitioner<TestParti
}
@Override
- public Collection<Partition<TestPartition>> definePartitions(
- Collection<Partition<TestPartition>> partitions,
- PartitioningContext context)
+ public Collection<Partition<TestPartition>> definePartitions(Collection<Partition<TestPartition>> partitions, PartitioningContext context)
{
int oldSize = partitions.size();
LOG.debug("partitionCount: current = {} requested = {}", oldSize, nPartitions);
@@ -85,7 +104,9 @@ public class TestPartition extends BaseOperator implements Partitioner<TestParti
// each partition i in 0...nPartitions receives tuples divisible by i but not by any other
// j in that range; all other tuples ignored
//
- if (3 != nPartitions) return getPartitions(partitions, context);
+ if (3 != nPartitions) {
+ return getPartitions(partitions, context);
+ }
// special case of 3 partitions: All odd numbers to partition 0; even numbers divisible
// by 4 to partition 1, those divisible by 2 but not 4 to partition 2.
@@ -117,13 +138,10 @@ public class TestPartition extends BaseOperator implements Partitioner<TestParti
return new ArrayList<Partition<TestPartition>>(Arrays.asList(newPartitions));
} // definePartitions
- private Collection<Partition<TestPartition>> getPartitions(
- Collection<Partition<TestPartition>> partitions,
- PartitioningContext context)
+ private Collection<Partition<TestPartition>> getPartitions(Collection<Partition<TestPartition>> partitions, PartitioningContext context)
{
// create array of partitions to return
- Collection<Partition<TestPartition>> result
- = new ArrayList<Partition<TestPartition>>(nPartitions);
+ Collection<Partition<TestPartition>> result = new ArrayList<Partition<TestPartition>>(nPartitions);
int mask = getMask(nPartitions);
for (int i = 0; i < nPartitions; ++i) {
@@ -138,12 +156,20 @@ public class TestPartition extends BaseOperator implements Partitioner<TestParti
} // getPartitions
// return mask with bits 0..N set where N is the highest set bit of argument
- private int getMask(final int n) {
+ private int getMask(final int n)
+ {
return -1 >>> Integer.numberOfLeadingZeros(n);
} // getMask
// accessors
- public int getNPartitions() { return nPartitions; }
- public void setNPartitions(int v) { nPartitions = v; }
+ public int getNPartitions()
+ {
+ return nPartitions;
+ }
+
+ public void setNPartitions(int v)
+ {
+ nPartitions = v;
+ }
}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/partition/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/partition/src/main/resources/META-INF/properties.xml b/examples/partition/src/main/resources/META-INF/properties.xml
index bf30603..d0374f2 100644
--- a/examples/partition/src/main/resources/META-INF/properties.xml
+++ b/examples/partition/src/main/resources/META-INF/properties.xml
@@ -1,6 +1,26 @@
<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<configuration>
- <!--
+ <!--
<property>
<name>dt.application.{appName}.operator.{opName}.prop.{propName}</name>
<value>some-default-value (if value is not specified, it is required from the user or custom config when launching)</value>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/partition/src/main/resources/my-log4j.properties
----------------------------------------------------------------------
diff --git a/examples/partition/src/main/resources/my-log4j.properties b/examples/partition/src/main/resources/my-log4j.properties
index 8949bfa..9c8a662 100644
--- a/examples/partition/src/main/resources/my-log4j.properties
+++ b/examples/partition/src/main/resources/my-log4j.properties
@@ -1,3 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
log4j.rootLogger=DEBUG,CONSOLE
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/partition/src/test/java/org/apache/apex/examples/partition/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/partition/src/test/java/org/apache/apex/examples/partition/ApplicationTest.java b/examples/partition/src/test/java/org/apache/apex/examples/partition/ApplicationTest.java
index 4cfb3c2..22fc4b2 100644
--- a/examples/partition/src/test/java/org/apache/apex/examples/partition/ApplicationTest.java
+++ b/examples/partition/src/test/java/org/apache/apex/examples/partition/ApplicationTest.java
@@ -1,5 +1,20 @@
/**
- * Put your copyright and license info here.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
package org.apache.apex.examples.partition;
@@ -8,19 +23,20 @@ import java.io.IOException;
import javax.validation.ConstraintViolationException;
import org.junit.Assert;
+import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
-import org.junit.Test;
import com.datatorrent.api.LocalMode;
/**
* Test the DAG declaration in local mode.
*/
-public class ApplicationTest {
-
+public class ApplicationTest
+{
@Test
- public void testApplication() throws IOException, Exception {
+ public void testApplication() throws IOException, Exception
+ {
try {
LocalMode lma = LocalMode.newInstance();
Configuration conf = new Configuration(false);
@@ -32,5 +48,5 @@ public class ApplicationTest {
Assert.fail("constraint violations: " + e.getConstraintViolations());
}
}
-
}
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/partition/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/partition/src/test/resources/log4j.properties b/examples/partition/src/test/resources/log4j.properties
index 3bfcdc5..451cff3 100644
--- a/examples/partition/src/test/resources/log4j.properties
+++ b/examples/partition/src/test/resources/log4j.properties
@@ -1,3 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
log4j.rootLogger=DEBUG,CONSOLE
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/recordReader/XmlJavadocCommentsExtractor.xsl
----------------------------------------------------------------------
diff --git a/examples/recordReader/XmlJavadocCommentsExtractor.xsl b/examples/recordReader/XmlJavadocCommentsExtractor.xsl
deleted file mode 100644
index 08075a9..0000000
--- a/examples/recordReader/XmlJavadocCommentsExtractor.xsl
+++ /dev/null
@@ -1,44 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
--->
-
-<!--
- Document : XmlJavadocCommentsExtractor.xsl
- Created on : September 16, 2014, 11:30 AM
- Description:
- The transformation strips off all information except for comments and tags from xml javadoc generated by xml-doclet.
--->
-
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
- <xsl:output method="xml" standalone="yes"/>
-
- <!-- copy xml by selecting only the following nodes, attributes and text -->
- <xsl:template match="node()|text()|@*">
- <xsl:copy>
- <xsl:apply-templates select="root|package|class|interface|method|field|type|comment|tag|text()|@name|@qualified|@text"/>
- </xsl:copy>
- </xsl:template>
-
- <!-- Strip off the following paths from the selected xml -->
- <xsl:template match="//root/package/interface/interface
- |//root/package/interface/method/@qualified
- |//root/package/class/interface
- |//root/package/class/class
- |//root/package/class/method/@qualified
- |//root/package/class/field/@qualified" />
-
- <xsl:strip-space elements="*"/>
-</xsl:stylesheet>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/recordReader/pom.xml
----------------------------------------------------------------------
diff --git a/examples/recordReader/pom.xml b/examples/recordReader/pom.xml
index 25ac191..e264f51 100644
--- a/examples/recordReader/pom.xml
+++ b/examples/recordReader/pom.xml
@@ -1,4 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@@ -11,7 +31,7 @@
<artifactId>malhar-examples-recordReader</artifactId>
<packaging>jar</packaging>
- <name>File Record Reader</name>
+ <name>Apache Apex Malhar File Record Reader Example</name>
<description>Simple application illustrating use of record reader operator</description>
<dependencies>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/recordReader/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/recordReader/src/assemble/appPackage.xml b/examples/recordReader/src/assemble/appPackage.xml
index 7ad071c..a870807 100644
--- a/examples/recordReader/src/assemble/appPackage.xml
+++ b/examples/recordReader/src/assemble/appPackage.xml
@@ -1,3 +1,23 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/recordReader/src/main/java/org/apache/apex/examples/recordReader/Application.java
----------------------------------------------------------------------
diff --git a/examples/recordReader/src/main/java/org/apache/apex/examples/recordReader/Application.java b/examples/recordReader/src/main/java/org/apache/apex/examples/recordReader/Application.java
index 916b061..ed78c17 100644
--- a/examples/recordReader/src/main/java/org/apache/apex/examples/recordReader/Application.java
+++ b/examples/recordReader/src/main/java/org/apache/apex/examples/recordReader/Application.java
@@ -1,5 +1,20 @@
/**
- * Put your copyright and license info here.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
package org.apache.apex.examples.recordReader;
@@ -13,7 +28,7 @@ import com.datatorrent.api.annotation.ApplicationAnnotation;
import com.datatorrent.contrib.formatter.CsvFormatter;
import com.datatorrent.contrib.parser.CsvParser;
-@ApplicationAnnotation(name="RecordReaderExample")
+@ApplicationAnnotation(name = "RecordReaderExample")
public class Application implements StreamingApplication
{
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/recordReader/src/main/java/org/apache/apex/examples/recordReader/TransactionsSchema.java
----------------------------------------------------------------------
diff --git a/examples/recordReader/src/main/java/org/apache/apex/examples/recordReader/TransactionsSchema.java b/examples/recordReader/src/main/java/org/apache/apex/examples/recordReader/TransactionsSchema.java
index 28c90f2..11437df 100644
--- a/examples/recordReader/src/main/java/org/apache/apex/examples/recordReader/TransactionsSchema.java
+++ b/examples/recordReader/src/main/java/org/apache/apex/examples/recordReader/TransactionsSchema.java
@@ -1,6 +1,20 @@
/**
- * Copyright (c) 2016 DataTorrent, Inc.
- * All rights reserved.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
package org.apache.apex.examples.recordReader;
@@ -119,13 +133,17 @@ public class TransactionsSchema
{
this.txAmount = txAmount;
}
-
+
@Override
public String toString()
{
- return "TransactionsSchema [customerName=" + customerName + ", customerPhone=" + customerPhone + ", customerEmail="
- + customerEmail + ", city=" + city + ", country=" + country + ", uid=" + uid + ", accountNumber="
- + accountNumber + ", txId=" + txId + ", txDate=" + txDate + ", txAmount=" + txAmount
+ return "TransactionsSchema [customerName=" + customerName +
+ ", customerPhone=" + customerPhone +
+ ", customerEmail=" + customerEmail +
+ ", city=" + city + ", country=" + country +
+ ", uid=" + uid + ", accountNumber=" + accountNumber +
+ ", txId=" + txId + ", txDate=" + txDate +
+ ", txAmount=" + txAmount
+ "]\n";
}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/recordReader/src/main/resources/META-INF/CustomerTxnData
----------------------------------------------------------------------
diff --git a/examples/recordReader/src/main/resources/META-INF/CustomerTxnData b/examples/recordReader/src/main/resources/META-INF/CustomerTxnData
deleted file mode 100644
index 88322ab..0000000
--- a/examples/recordReader/src/main/resources/META-INF/CustomerTxnData
+++ /dev/null
@@ -1,100 +0,0 @@
-Desirae I. Hart|91-756-943-24728|rutrum.magna@dictummagnaUt.net|Delhi|India|7F2BAD18-D1D1-8DB6-A519-70DA658EF6DE|1048.2446157601|245407|2016-06-17 21:19:02|56095.4691099477
-Fay B. Page|91-952-299-03140|lobortis.risus@eutelluseu.com|Durg|India|5E8E12D3-34CC-53E1-7622-BC533542B930|939.9385330598|245408|2016-06-17 21:19:02|93013.7443360696
-Herrod F. Mcknight|91-609-146-21345|nec@egetmagna.co.uk|Haridwar|India|F589AF28-C10C-3EF1-E9DA-7CAAAD8E453D|1009.5714354316|245409|2016-06-17 21:19:03|57130.4933466919
-Suki K. Hutchinson|91-235-565-54483|justo.Praesent@nonquam.ca|Panjim|India|A263A773-E228-0C5B-D9CC-A5AE37A9F3F5|990.5143009573|245410|2016-06-17 21:19:03|88348.7049292828
-Tucker H. Strickland|91-353-281-78811|Suspendisse@felisNulla.edu|Aizwal|India|748AAD2B-8CB4-6010-2C80-F3B0FE107B4E|1027.2343235232|245411|2016-06-17 21:19:03|63602.53265577
-Carly L. Alvarado|91-364-259-91891|blandit.mattis.Cras@Nullam.co.uk|Mandya|India|292345E9-45E1-AE6C-9051-79681BE10E0A|964.9602866876|245412|2016-06-17 21:19:03|63777.4412787906
-Signe H. Carlson|91-736-760-21206|Curabitur@ac.edu|Basirhat|India|ABFCB28C-98B0-93E4-C5E9-D572648FE467|967.6980193409|245413|2016-06-17 21:19:04|81425.8673128708
-Timothy U. Cantrell|91-266-620-64253|ac.mattis@congueIn.com|Dibrugarh|India|46ED3B8F-5D19-D3FD-2833-7E6D956090D6|990.0113497158|245414|2016-06-17 21:19:04|20765.2379785903
-Todd M. Hunter|91-131-862-91105|vel@per.org|Pondicherry|India|73DDE1E7-7B86-53A0-8296-8DDCC0A8022A|999.2001773265|245416|2016-06-17 21:19:04|28079.5354148245
-Martin J. Joyce|91-312-843-82341|magnis.dis.parturient@penatibusetmagnis.co.uk|Gandhidham|India|46247FA6-883D-2DAA-78E4-626F41937905|974.8521165154|245415|2016-06-17 21:19:04|98219.3968546873
-Cruz D. Slater|91-399-580-01355|ante.dictum@tempor.co.uk|Kalyan|India|7A0AE886-CDBF-98FF-1808-1A977B06F939|989.5735205715|245418|2016-06-17 21:19:05|69472.6529645815
-Keefe M. Hodge|91-123-766-63270|molestie.arcu.Sed@eratnonummy.ca|Bokaro Steel City|India|5488D1DE-E840-827F-2D45-347A58ECF72C|1059.6676733575|245417|2016-06-17 21:19:05|60538.9043471269
-Minerva R. Velez|91-550-563-14205|dapibus.quam.quis@sitamet.co.uk|Bhusawal|India|C2EA3D6F-EEBE-7E7D-4DFD-DA90FAAAC04F|965.3100959637|245419|2016-06-17 21:19:05|22378.4665981816
-Gillian P. Bates|91-530-835-01656|In.nec@sedduiFusce.org|Kohima|India|8882E4F0-0AB1-6098-EEF8-0230BBB931B2|978.3566576884|245420|2016-06-17 21:19:05|57058.528246018
-Jael I. Humphrey|91-815-502-83243|dignissim@quisturpis.co.uk|Chandigarh|India|0D8ADC96-B858-60A9-4F65-3BACADC808A8|1047.1902156238|245421|2016-06-17 21:19:06|71348.2114533201
-Eleanor D. Hart|91-351-706-26576|nonummy@sitametdapibus.ca|Imphal|India|6DE39307-4D0B-77B0-8DAF-36417261DA2B|1021.3673266291|245422|2016-06-17 21:19:06|60474.8184384689
-Kaitlin U. Higgins|91-738-770-10870|sodales.at.velit@Inscelerisque.org|Kohima|India|C383AF31-636B-7897-6CF3-6B261BCDE018|1036.0644769672|245424|2016-06-17 21:19:06|48378.6503012718
-Giacomo A. Puckett|91-859-732-83951|sit.amet@atpedeCras.com|Silchar|India|5E41A5A4-A4B3-17FE-F83D-6F6BF36B0F2D|979.2148681363|245423|2016-06-17 21:19:06|33445.0439158336
-Jamalia Z. Hensley|91-395-592-56855|interdum@cursus.co.uk|Munger|India|7B98AA41-BC35-497E-6F2E-BBA815F84B28|1033.5638665941|245425|2016-06-17 21:19:07|63406.9037231582
-Raja L. Barrett|91-213-236-37863|tristique.senectus@orciUtsemper.com|Agartala|India|730F660A-4A6E-65B3-D64D-6A0AD4BD16F2|1051.51196186|245426|2016-06-17 21:19:07|16656.8948714763
-Audra D. Haley|91-909-551-98035|nostra@velitjusto.net|Shimla|India|5F6BD395-CFB1-86A3-232B-3B461C11B032|985.3099956934|245427|2016-06-17 21:19:07|42576.4326009901
-Burke G. Velez|91-480-634-47735|dictum@eulacusQuisque.com|Shimla|India|C0D51785-79E5-86C1-2F60-6E2B4BC368E9|1014.3374004511|245428|2016-06-17 21:19:07|68099.3435062983
-Helen O. Watson|91-388-550-33596|accumsan.neque.et@sagittis.ca|Rewa|India|267EA362-E266-01FC-2371-065D4B150D6B|995.395922684|245429|2016-06-17 21:19:08|80594.4138667748
-Dillon E. Manning|91-412-483-18804|vulputate.eu@diam.net|Kohima|India|08BD8752-4C34-1C2F-7EC5-063BE9C0025F|988.0527516287|245430|2016-06-17 21:19:08|34997.7855937537
-Darryl H. Preston|91-900-557-62018|consequat@blanditatnisi.ca|Srinagar|India|BAE0CB80-33BC-23AD-4F23-6EED717A1FA9|1005.2310251378|245432|2016-06-17 21:19:08|90995.8552344935
-Teagan L. Merritt|91-755-577-05229|nec.luctus.felis@ornarelibero.ca|Daman|India|A65E89EC-BD39-F826-BF3A-2E4DFD1FFE96|1031.6923612578|245431|2016-06-17 21:19:08|76129.6754380442
-Blythe V. Hester|91-143-693-74169|faucibus@convallisconvallis.net|Nagarcoil|India|457DA7DF-8473-FDA2-DFA8-9CA6FE70AF53|980.0187576543|245433|2016-06-17 21:19:09|1792.7823874316
-Felicia Z. Maynard|91-177-925-06452|facilisis@nequetellus.edu|Jammu|India|7511020D-252A-60E3-0259-B55174DB2BE0|962.3470002516|245434|2016-06-17 21:19:09|27442.1888130468
-Sopoline B. Swanson|91-512-190-23174|mauris.aliquam@vitae.edu|Agartala|India|92A84BDE-A231-715C-5B0A-A2C52FE4C726|1000.4777986036|245436|2016-06-17 21:19:09|32101.1072345695
-Christen T. Cabrera|91-956-276-72371|a.dui@sit.org|Tiruvannamalai|India|1B06945D-2595-5445-2694-12341158C45B|997.3852528283|245435|2016-06-17 21:19:09|86664.8353687926
-Chancellor C. Villarreal|91-882-782-21856|Nullam.ut.nisi@Nam.edu|Pondicherry|India|33EBFDA7-8EEF-48A5-B323-AF7A7A57C4F9|984.5578096612|245437|2016-06-17 21:19:10|46584.2856419789
-Adara A. Price|91-334-227-26172|elementum@aliquameuaccumsan.ca|Shahjahanpur|India|82F22680-CD3B-F1E7-45F9-8E2ADAA5ED42|988.0619905918|245438|2016-06-17 21:19:10|21626.2511858243
-Dana B. Lucas|91-781-212-83146|scelerisque.mollis.Phasellus@Sedeunibh.co.uk|Gangtok|India|88223C00-DCFA-F40B-AC4C-97FA79062789|976.7241552536|245440|2016-06-17 21:19:10|76481.528985468
-Alma X. Ingram|91-944-103-76290|senectus.et.netus@orci.net|Jammu|India|4C0C1266-DE7B-6FE7-6FFF-E0263987C15B|975.6210776138|245439|2016-06-17 21:19:10|64898.0024641107
-Cullen P. Glenn|91-388-717-53901|magna.Praesent@loremlorem.co.uk|Kohima|India|BA6BF2F4-8F94-C4C2-740E-2A3B587C4B3A|1031.9092818916|245442|2016-06-17 21:19:11|9437.7993039509
-Darius S. Allen|91-978-355-75025|mi.lacinia.mattis@enimSuspendissealiquet.co.uk|Trivandrum|India|D59B0CBB-A08D-DE3B-DE5F-426C0C96CC59|1032.6395532022|245441|2016-06-17 21:19:11|12454.7012410075
-Jordan J. Frye|91-399-558-07454|ridiculus.mus@idlibero.ca|Chandigarh|India|EBCC5349-6EC4-356A-F897-20BFBF220087|1019.8537249814|245444|2016-06-17 21:19:11|32040.2035007451
-August N. Chase|91-447-831-17890|libero.at.auctor@Vivamuseuismodurna.org|Agartala|India|8871CD66-C471-B709-61C4-DA8E11395864|1016.3890473484|245443|2016-06-17 21:19:11|45318.9872586436
-Teegan Z. House|91-834-850-62568|nunc.est.mollis@Nunc.edu|Aizwal|India|13B85BBD-EA23-70B2-1723-9F1D6046B42E|959.962281601|245445|2016-06-17 21:19:12|88784.9082302192
-Aquila B. Moran|91-851-475-57283|sapien.Cras.dolor@Suspendissetristiqueneque.com|Thalassery|India|823590C6-3381-1E11-8303-C3DA2A0E1821|993.9503426924|245446|2016-06-17 21:19:12|85906.741529358
-Dominic T. Dalton|91-504-884-93067|Ut@congue.co.uk|Rohtak|India|4EC579B1-4095-421D-27D5-95BD94551B81|988.6711045096|245448|2016-06-17 21:19:12|96034.4758503845
-Francis C. Gay|91-155-131-31144|elementum@CuraeDonec.co.uk|Panjim|India|BEA0E8B8-2153-55C3-9FAC-B645F75DC11F|1041.9520701363|245447|2016-06-17 21:19:12|28285.6913337028
-Florence K. Cox|91-509-817-87497|adipiscing.enim@ornarelectusjusto.edu|Bhilai|India|C613A6CF-0DDF-457B-1611-048AF476CD9F|984.7873386645|245449|2016-06-17 21:19:13|8045.8599610703
-Desirae F. Mooney|91-146-543-19767|ultricies@scelerisquenequeNullam.edu|Gangtok|India|58A86FB7-1F20-2C33-2682-9B20165B5CF9|965.4688425994|245450|2016-06-17 21:19:13|25829.5686136197
-Virginia O. Welch|91-338-235-55295|ultrices.posuere.cubilia@massanonante.co.uk|Srinagar|India|350C6D1C-8C2D-BE4A-EAB5-8098C2BC8054|981.8861112513|245451|2016-06-17 21:19:13|55435.8774902126
-Simon P. Estes|91-101-148-34019|a.aliquet@tortordictum.edu|Kavaratti|India|D42E0FB2-6093-FD6F-EA2E-DC470647340F|961.8009052379|245452|2016-06-17 21:19:13|26959.0061609105
-Garth M. Fuentes|91-401-871-57104|gravida@semperegestasurna.net|Vejalpur|India|1450A844-53D1-6315-4415-5C75B08FC228|974.9367040591|245454|2016-06-17 21:19:14|93043.0249827518
-Jaquelyn D. Rodgers|91-913-783-42186|lobortis.nisi.nibh@Donec.edu|Nagpur|India|D6CE7D75-0659-454A-40DF-3AF296DD4C58|976.7352615741|245453|2016-06-17 21:19:14|59609.6850497366
-Kasper F. Hatfield|91-632-379-95960|posuere@vulputatenisisem.ca|Korba|India|D59B8BE9-DA20-7B57-9942-2C15BCF28603|1000.4924219911|245456|2016-06-17 21:19:14|3275.3243581371
-Isaiah A. Warner|91-626-859-53011|diam.Sed@facilisisfacilisismagna.edu|Daman|India|20B5FE0D-0965-1E2D-681B-5E688DA1CA65|1001.1159994583|245455|2016-06-17 21:19:14|76836.6239820194
-Keaton Y. Solis|91-301-918-77318|sit.amet.nulla@elitelitfermentum.co.uk|Rajkot|India|16CFA137-3FD3-8D3D-7607-54577A6A18AB|987.5936149956|245458|2016-06-17 21:19:15|89465.3972332774
-Regan D. Underwood|91-869-304-48740|ornare@Curabituregestas.org|Daman|India|241FD5E5-01D8-7BEA-BE3A-3435B1D6577F|967.2325392222|245457|2016-06-17 21:19:15|81430.0742569319
-Laurel K. Hardy|91-535-589-44540|rutrum@penatibuset.edu|Shimla|India|46689123-0B18-3A05-2E9D-1995EABE674F|947.0982666227|245459|2016-06-17 21:19:15|14451.1023722487
-Vielka J. Roberson|91-545-218-53793|scelerisque.lorem.ipsum@Sed.org|Srinagar|India|E6384D6E-2AC4-3AF7-8C1C-A37DD4B013C6|933.9068440201|245460|2016-06-17 21:19:15|75600.2030709746
-Brynn S. Morrison|91-977-938-06240|natoque@enimgravidasit.edu|Daman|India|859E9B35-3A21-F7FD-A9A5-167F7FE6A7F5|1002.7243083243|245461|2016-06-17 21:19:16|19882.1569818146
-Graiden T. Jones|91-868-733-90988|lobortis.ultrices.Vivamus@faucibus.org|Nellore|India|C133E546-1E36-A6EF-9980-BB635F247290|973.7286142544|245462|2016-06-17 21:19:16|4393.9931397479
-Tanya X. Dominguez|91-553-110-11147|odio.Nam@antedictum.ca|Anantapur|India|DEF2CE91-CAF2-88BA-C50B-AB7EA1D73C63|959.1678617895|245464|2016-06-17 21:19:16|42536.8767788127
-Anne D. Pace|91-188-230-70605|pede.et.risus@Ut.co.uk|Agartala|India|32FC1F14-21F7-E940-7C9D-B80F44EE4063|1021.5754177485|245463|2016-06-17 21:19:16|32755.1536052658
-Hanna I. Sellers|91-322-901-42448|tincidunt.dui.augue@augue.org|Barddhaman|India|3D4F2FBC-FFCE-EE5C-5D2C-C60856D33B7D|993.2866819089|245466|2016-06-17 21:19:17|75662.1622559809
-Christine R. Gross|91-380-558-08928|convallis.convallis@convalliserateget.org|Shillong|India|DD81CFAC-DE87-E8A1-2850-E2637912543C|951.970651195|245465|2016-06-17 21:19:17|80146.9990770126
-Kirby A. Morse|91-299-538-45635|elit.Nulla.facilisi@acmattisvelit.net|Itanagar|India|CFF08336-B5C9-EECC-1F2D-07ADE42587FE|1009.5054026816|245467|2016-06-17 21:19:17|72602.1251286646
-Ryder B. Vega|91-320-455-77145|Aliquam.gravida@elit.co.uk|Gondiya|India|15F3478F-4D7E-562C-E189-945E655641E9|1006.8962892921|245468|2016-06-17 21:19:17|8089.8058610084
-Demetrius Q. Pope|91-179-308-28350|tempus@commodotinciduntnibh.ca|Imphal|India|CBF9C119-3E02-13F0-842A-BAF0AB46382D|1042.9498091927|245470|2016-06-17 21:19:18|83959.394047181
-Bryar M. Hendricks|91-617-948-40647|dui.Fusce.aliquam@loremvitaeodio.net|Kanchrapara|India|64188EFF-B7B1-AF75-EA3D-013CCA945F81|968.5880217334|245469|2016-06-17 21:19:18|85996.1671440447
-Ann P. Mann|91-395-300-37963|Cum@gravidanon.edu|Imphal|India|12818782-A86C-63A2-AB08-A5F281DA954B|937.3545836376|245472|2016-06-17 21:19:18|83236.1732288256
-Tyrone N. Bender|91-515-367-84014|at@fringillaporttitorvulputate.com|Chandigarh|India|D5109D0E-7DB6-A084-F0F9-E2BEF2B10E4A|1013.1764242663|245471|2016-06-17 21:19:18|36038.3036732134
-Miranda W. Sykes|91-774-614-92485|amet@accumsan.com|Gangtok|India|6EC7240D-5331-5F34-D1B8-38E2024A1925|992.518070982|245473|2016-06-17 21:19:19|97197.8457445721
-Ralph F. Walker|91-815-285-72212|molestie.in@vestibulumlorem.ca|Farrukhabad-cum-Fatehgarh|India|B7982CE7-B815-2794-504B-A8C3ECF0B592|992.3386856766|245474|2016-06-17 21:19:19|60698.1579345159
-Kelsie J. Mack|91-112-975-50329|pede.ac.urna@ligula.com|Hazaribag|India|5CACD398-932B-449D-80AC-5302AE35E027|1027.0394067971|245475|2016-06-17 21:19:19|48589.914064424
-Zenia F. Bond|91-445-745-77345|pede.Cum.sociis@Pellentesquetincidunt.com|Port Blair|India|68EDFE31-8E62-FF3B-3FBA-49B436D7142C|996.5071564618|245476|2016-06-17 21:19:19|40633.7414556007
-Todd V. Barton|91-312-429-52379|mauris.a@tristique.org|Dehradun|India|7D22103A-1DA0-2E4E-BFD7-8117133A983A|977.4969081079|245477|2016-06-17 21:19:20|49497.7649558266
-Kerry O. Franco|91-802-106-72427|Ut.sagittis.lobortis@dictumeueleifend.ca|Kohima|India|FC6049B3-9FD4-9A53-1A7E-C50394D4F4CD|1004.0387939017|245478|2016-06-17 21:19:20|43754.1503070853
-April I. Clarke|91-785-400-08200|sapien@mattisCras.edu|Daman|India|363E468F-BD44-6607-CDD6-226A6FB48B89|990.0985191984|245479|2016-06-17 21:19:20|90752.7220658782
-Randall B. Workman|91-722-286-44955|ut@Integervitae.net|Gujrat|India|1A402674-FD62-8F1B-7AE1-CFAA7E85006A|1012.6352839308|245480|2016-06-17 21:19:20|1815.8797213967
-Blaine J. Martinez|91-987-901-22060|libero.mauris.aliquam@egetlaoreet.co.uk|Kohima|India|7FDD906D-AB41-89EC-D546-B1E04D31FE49|1020.339672989|245481|2016-06-17 21:19:21|51409.0038738681
-May B. Schroeder|91-123-334-23814|aliquet@leo.edu|Orai|India|2E83862F-C3A4-4A4B-4D9C-C60AC4435A04|982.3424542846|245482|2016-06-17 21:19:21|89159.6292602544
-Whitney N. Tran|91-923-716-39943|Aliquam@leoMorbi.edu|Purulia|India|FEE99492-4B48-533F-EB36-5EA3528A121C|1007.9029941609|245483|2016-06-17 21:19:21|24692.473001632
-Ingrid A. Head|91-596-380-43300|ac@montes.ca|Port Blair|India|735007BF-1F13-95E9-7C3E-536C0452714D|995.4883512745|245484|2016-06-17 21:19:21|54890.8030672057
-Berk E. Gould|91-110-540-75246|ipsum@Nulla.org|Alwar|India|FB0297F7-8BC1-86F0-8CCD-C01538B7E5BE|984.2152917929|245486|2016-06-17 21:19:22|70311.7302535036
-Meredith G. Parks|91-154-792-20142|et.eros@auctor.com|Chandigarh|India|CD2691BE-ADB4-87DF-CA3B-88512330E745|960.8832140455|245485|2016-06-17 21:19:22|95293.2215958629
-Cora K. Carrillo|91-786-268-62522|elementum.lorem.ut@eutelluseu.ca|Dindigul|India|CAA843B7-F7C8-AD16-3F05-89DADB6EB288|1001.8761512414|245488|2016-06-17 21:19:22|59127.2684732433
-Galena D. Farmer|91-951-409-02162|sem@dictum.co.uk|Shillong|India|A46AF538-4BEF-D24F-870F-DD4CD20F4DC6|1022.2222771884|245487|2016-06-17 21:19:22|36667.9833050979
-Mollie E. Snow|91-816-823-87138|lectus@pharetra.org|Tiruvannamalai|India|A66AFB86-5297-61AC-2A2A-87C23F0E96E1|966.8626148916|245490|2016-06-17 21:19:23|28859.7500576673
-Abra E. Mccray|91-100-997-13066|Morbi.quis@risusDuis.net|Delhi|India|E49C0B30-586F-ECAA-FCBD-8F2898AA7771|1043.6824086302|245489|2016-06-17 21:19:23|79521.2870470628
-Andrew I. Lowe|91-859-767-29836|sed.turpis@Sed.co.uk|Aizwal|India|A928DE5C-6C18-9B9B-C4F0-3165F2E53C19|955.4198061291|245491|2016-06-17 21:19:23|34946.6776395577
-Zachery B. Marshall|91-103-978-47466|nonummy.Fusce@dui.co.uk|Vijayawada|India|E0D79F5F-2CB6-B9FC-B917-2D627141423B|969.8127419963|245492|2016-06-17 21:19:23|74671.8685208157
-Miranda M. Madden|91-310-815-30233|id@pharetranibh.co.uk|Brahmapur|India|55E04315-DE60-1C1D-78D6-878C9488BA06|963.5417067954|245493|2016-06-17 21:19:24|42089.8687934832
-Ian N. Johnson|91-651-594-79593|elit@pellentesquemassa.ca|Agartala|India|EDBCCD62-4871-4938-60C5-0CD3023CF333|1012.7053923589|245494|2016-06-17 21:19:24|97376.7240252239
-Macey V. Hinton|91-358-832-49086|eget.odio@Nunclectuspede.net|Sambalpur|India|49E4842E-0EB8-D48A-C1EC-A3F0FBAC92B4|982.9530728621|245496|2016-06-17 21:19:24|61850.2465402803
-Talon M. Calhoun|91-280-397-90948|lacinia.Sed@tinciduntvehicularisus.net|Pallavaram|India|E743767A-B1FF-C71C-B0D4-4C106D65AE3B|1040.9448702398|245495|2016-06-17 21:19:24|49928.6243188531
-Dillon A. Reese|91-900-545-84875|Mauris.vestibulum@sagittis.edu|Itanagar|India|9429B72A-36AA-8C78-2988-7FF48315094A|1029.5299833946|245498|2016-06-17 21:19:25|86990.7236411972
-Boris E. Boyd|91-480-944-74996|luctus.Curabitur.egestas@utmolestie.co.uk|Panjim|India|37962E2E-4B3D-27DD-1213-4FF1B5551CB4|969.407095963|245497|2016-06-17 21:19:25|32987.5284224486
-Inez U. Higgins|91-258-514-16019|Aliquam@utlacus.co.uk|Itanagar|India|D48C171C-A009-3813-5951-50A526BE6B58|1005.0207448325|245500|2016-06-17 21:19:25|68069.3550977988
-Leroy N. Snyder|91-682-436-52541|nisl.Maecenas@quam.com|Itanagar|India|BE155410-1F6D-AC60-C0B9-290E9426D5CF|993.1732235668|245499|2016-06-17 21:19:25|64376.3818134466
-George M. Case|91-760-901-60977|at.iaculis.quis@auctorvelit.com|Pondicherry|India|B924E744-29E3-8B6E-3EE9-A48A3BC9072B|987.476080645|245501|2016-06-17 21:19:26|97859.0229750841
-Kane E. Simon|91-461-207-64947|adipiscing.lobortis@tellus.ca|Haridwar|India|13C1AD7F-48D5-C6E5-892B-0345E7E070DB|983.5254219297|245502|2016-06-17 21:19:26|21912.3442110069
-Wanda U. Talley|91-398-803-35370|ipsum.Curabitur@eleifendnecmalesuada.com|Gangtok|India|5E2ED32D-621C-0ACE-71B6-F2F9746DD0C9|994.6344833977|245503|2016-06-17 21:19:26|91796.2772584239
-Asher U. Mccullough|91-149-955-32468|eget.massa.Suspendisse@Pellentesqueultricies.edu|Itanagar|India|3FA3514B-6974-06E3-EF7B-4841487A40D2|985.0811909206|245504|2016-06-17 21:19:26|71603.8572439688
-Malik E. Keller|91-391-507-50089|nec@Quisqueporttitor.net|Daman|India|05CDC866-65A3-E55C-EEE8-C73217F2071A|986.9798931399|245505|2016-06-17 21:19:27|65100.1941324091
-Yasir V. Howell|91-901-430-36468|nec.cursus.a@Cumsociisnatoque.ca|Mandya|India|F90D3C87-4C53-A5E9-D3F8-692C068CBCED|997.1935636906|245506|2016-06-17 21:19:27|35511.2898055096
[09/19] apex-malhar git commit: An application that demonstrates
throttling of the input operator output rate when the downsteam operators are
slower Fixed throttler to send requests once per state change instead of
repeatedly
Posted by th...@apache.org.
An application that demonstrates throttling of the input operator output rate when the downsteam operators are slower
Fixed throttler to send requests once per state change instead of repeatedly
Project: http://git-wip-us.apache.org/repos/asf/apex-malhar/repo
Commit: http://git-wip-us.apache.org/repos/asf/apex-malhar/commit/172b20a2
Tree: http://git-wip-us.apache.org/repos/asf/apex-malhar/tree/172b20a2
Diff: http://git-wip-us.apache.org/repos/asf/apex-malhar/diff/172b20a2
Branch: refs/heads/master
Commit: 172b20a2c0bb6ccbd584b34aff41779da9d7acfb
Parents: dd509f2
Author: Pramod Immaneni <pr...@datatorrent.com>
Authored: Sun Nov 6 01:33:05 2016 -0700
Committer: Lakshmi Prasanna Velineni <la...@datatorrent.com>
Committed: Sun Mar 26 11:43:48 2017 -0700
----------------------------------------------------------------------
examples/throttle/README.md | 8 +
.../throttle/XmlJavadocCommentsExtractor.xsl | 44 +++
examples/throttle/pom.xml | 274 +++++++++++++++++++
examples/throttle/src/assemble/appPackage.xml | 43 +++
.../examples/throttle/Application.java | 51 ++++
.../examples/throttle/PassThroughOperator.java | 20 ++
.../throttle/RandomNumberGenerator.java | 64 +++++
.../examples/throttle/SlowDevNullOperator.java | 35 +++
.../throttle/ThrottlingStatsListener.java | 150 ++++++++++
.../src/main/resources/META-INF/properties.xml | 20 ++
.../examples/throttle/ApplicationTest.java | 37 +++
.../src/test/resources/log4j.properties | 22 ++
12 files changed, 768 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/172b20a2/examples/throttle/README.md
----------------------------------------------------------------------
diff --git a/examples/throttle/README.md b/examples/throttle/README.md
new file mode 100644
index 0000000..7ed4670
--- /dev/null
+++ b/examples/throttle/README.md
@@ -0,0 +1,8 @@
+Application shows you how to throttle input operators in the application when the downstream
+operators are slower.
+
+It uses a combination of stats listener and operator request to achieve this. The throttler is
+a stats listener that is registered with the operators and when it notices that the window gap
+between operators is widening and crosses a configured threshold, it sends a request to the
+input operator to slow down and coversely when the gap falls below the threshold it requests
+the input operator to go back to its normal speed.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/172b20a2/examples/throttle/XmlJavadocCommentsExtractor.xsl
----------------------------------------------------------------------
diff --git a/examples/throttle/XmlJavadocCommentsExtractor.xsl b/examples/throttle/XmlJavadocCommentsExtractor.xsl
new file mode 100644
index 0000000..08075a9
--- /dev/null
+++ b/examples/throttle/XmlJavadocCommentsExtractor.xsl
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+
+<!--
+ Document : XmlJavadocCommentsExtractor.xsl
+ Created on : September 16, 2014, 11:30 AM
+ Description:
+ The transformation strips off all information except for comments and tags from xml javadoc generated by xml-doclet.
+-->
+
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+ <xsl:output method="xml" standalone="yes"/>
+
+ <!-- copy xml by selecting only the following nodes, attributes and text -->
+ <xsl:template match="node()|text()|@*">
+ <xsl:copy>
+ <xsl:apply-templates select="root|package|class|interface|method|field|type|comment|tag|text()|@name|@qualified|@text"/>
+ </xsl:copy>
+ </xsl:template>
+
+ <!-- Strip off the following paths from the selected xml -->
+ <xsl:template match="//root/package/interface/interface
+ |//root/package/interface/method/@qualified
+ |//root/package/class/interface
+ |//root/package/class/class
+ |//root/package/class/method/@qualified
+ |//root/package/class/field/@qualified" />
+
+ <xsl:strip-space elements="*"/>
+</xsl:stylesheet>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/172b20a2/examples/throttle/pom.xml
----------------------------------------------------------------------
diff --git a/examples/throttle/pom.xml b/examples/throttle/pom.xml
new file mode 100644
index 0000000..02fe22e
--- /dev/null
+++ b/examples/throttle/pom.xml
@@ -0,0 +1,274 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>com.datatorrent.examples</groupId>
+ <version>1.0-SNAPSHOT</version>
+ <artifactId>throttle</artifactId>
+ <packaging>jar</packaging>
+
+ <!-- change these to the appropriate values -->
+ <name>Throttle Application</name>
+ <description>Application demonstrating throttling input when downstream is slower</description>
+
+ <properties>
+ <!-- change this if you desire to use a different version of Apex Core -->
+ <apex.version>3.5.0</apex.version>
+ <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-eclipse-plugin</artifactId>
+ <version>2.9</version>
+ <configuration>
+ <downloadSources>true</downloadSources>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.3</version>
+ <configuration>
+ <encoding>UTF-8</encoding>
+ <source>1.7</source>
+ <target>1.7</target>
+ <debug>true</debug>
+ <optimize>false</optimize>
+ <showDeprecation>true</showDeprecation>
+ <showWarnings>true</showWarnings>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.8</version>
+ <executions>
+ <execution>
+ <id>copy-dependencies</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>target/deps</outputDirectory>
+ <includeScope>runtime</includeScope>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>app-package-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <finalName>${project.artifactId}-${project.version}-apexapp</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <descriptors>
+ <descriptor>src/assemble/appPackage.xml</descriptor>
+ </descriptors>
+ <archiverConfig>
+ <defaultDirectoryMode>0755</defaultDirectoryMode>
+ </archiverConfig>
+ <archive>
+ <manifestEntries>
+ <Class-Path>${apex.apppackage.classpath}</Class-Path>
+ <DT-Engine-Version>${apex.version}</DT-Engine-Version>
+ <DT-App-Package-Group-Id>${project.groupId}</DT-App-Package-Group-Id>
+ <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name>
+ <DT-App-Package-Version>${project.version}</DT-App-Package-Version>
+ <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name>
+ <DT-App-Package-Description>${project.description}</DT-App-Package-Description>
+ </manifestEntries>
+ </archive>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <configuration>
+ <target>
+ <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar"
+ tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa" />
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ <execution>
+ <!-- create resource directory for xml javadoc-->
+ <id>createJavadocDirectory</id>
+ <phase>generate-resources</phase>
+ <configuration>
+ <tasks>
+ <delete dir="${project.build.directory}/generated-resources/xml-javadoc"/>
+ <mkdir dir="${project.build.directory}/generated-resources/xml-javadoc"/>
+ </tasks>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.9.1</version>
+ <executions>
+ <execution>
+ <id>attach-artifacts</id>
+ <phase>package</phase>
+ <goals>
+ <goal>attach-artifact</goal>
+ </goals>
+ <configuration>
+ <artifacts>
+ <artifact>
+ <file>target/${project.artifactId}-${project.version}.apa</file>
+ <type>apa</type>
+ </artifact>
+ </artifacts>
+ <skipAttach>false</skipAttach>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- generate javdoc -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <executions>
+ <!-- generate xml javadoc -->
+ <execution>
+ <id>xml-doclet</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>javadoc</goal>
+ </goals>
+ <configuration>
+ <doclet>com.github.markusbernhardt.xmldoclet.XmlDoclet</doclet>
+ <additionalparam>-d ${project.build.directory}/generated-resources/xml-javadoc -filename ${project.artifactId}-${project.version}-javadoc.xml</additionalparam>
+ <useStandardDocletOptions>false</useStandardDocletOptions>
+ <docletArtifact>
+ <groupId>com.github.markusbernhardt</groupId>
+ <artifactId>xml-doclet</artifactId>
+ <version>1.0.4</version>
+ </docletArtifact>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <!-- Transform xml javadoc to stripped down version containing only class/interface comments and tags-->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>xml-maven-plugin</artifactId>
+ <version>1.0</version>
+ <executions>
+ <execution>
+ <id>transform-xmljavadoc</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>transform</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <transformationSets>
+ <transformationSet>
+ <dir>${project.build.directory}/generated-resources/xml-javadoc</dir>
+ <includes>
+ <include>${project.artifactId}-${project.version}-javadoc.xml</include>
+ </includes>
+ <stylesheet>XmlJavadocCommentsExtractor.xsl</stylesheet>
+ <outputDir>${project.build.directory}/generated-resources/xml-javadoc</outputDir>
+ </transformationSet>
+ </transformationSets>
+ </configuration>
+ </plugin>
+ <!-- copy xml javadoc to class jar -->
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>2.6</version>
+ <executions>
+ <execution>
+ <id>copy-resources</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${basedir}/target/classes</outputDirectory>
+ <resources>
+ <resource>
+ <directory>${project.build.directory}/generated-resources/xml-javadoc</directory>
+ <includes>
+ <include>${project.artifactId}-${project.version}-javadoc.xml</include>
+ </includes>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+
+ </build>
+
+ <dependencies>
+ <!-- add your dependencies here -->
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-library</artifactId>
+ <version>3.6.0</version>
+ <!--
+ If you know that your application does not need transitive dependencies pulled in by malhar-library,
+ uncomment the following to reduce the size of your app package.
+ -->
+ <!--
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ -->
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>apex-common</artifactId>
+ <version>${apex.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.10</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>apex-engine</artifactId>
+ <version>${apex.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/172b20a2/examples/throttle/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/throttle/src/assemble/appPackage.xml b/examples/throttle/src/assemble/appPackage.xml
new file mode 100644
index 0000000..7ad071c
--- /dev/null
+++ b/examples/throttle/src/assemble/appPackage.xml
@@ -0,0 +1,43 @@
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>appPackage</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/target/</directory>
+ <outputDirectory>/app</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/target/deps</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/site/conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/META-INF</directory>
+ <outputDirectory>/META-INF</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/app</directory>
+ <outputDirectory>/app</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/resources</directory>
+ <outputDirectory>/resources</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/172b20a2/examples/throttle/src/main/java/com/datatorrent/examples/throttle/Application.java
----------------------------------------------------------------------
diff --git a/examples/throttle/src/main/java/com/datatorrent/examples/throttle/Application.java b/examples/throttle/src/main/java/com/datatorrent/examples/throttle/Application.java
new file mode 100644
index 0000000..d789dbe
--- /dev/null
+++ b/examples/throttle/src/main/java/com/datatorrent/examples/throttle/Application.java
@@ -0,0 +1,51 @@
+/**
+ * Put your copyright and license info here.
+ */
+package com.datatorrent.examples.throttle;
+
+import java.util.Collection;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.collect.Lists;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StatsListener;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+
+@ApplicationAnnotation(name="ThrottleApplication")
+public class Application implements StreamingApplication
+{
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ // Creating an example application with three operators
+ // The last operator is slowing down the DAG
+ // With the use of the stats listener the input operator is slowed when the window difference crosses a threshold
+
+ RandomNumberGenerator randomGenerator = dag.addOperator("RandomGenerator", RandomNumberGenerator.class);
+ PassThroughOperator<Double> passThrough = dag.addOperator("PassThrough", PassThroughOperator.class);
+ SlowDevNullOperator<Double> devNull = dag.addOperator("SlowNull", SlowDevNullOperator.class);
+
+ // Important to use the same stats listener object for all operators so that we can centrally collect stats and make
+ // the decision
+ StatsListener statsListener = new ThrottlingStatsListener();
+ Collection<StatsListener> statsListeners = Lists.newArrayList(statsListener);
+ dag.setAttribute(randomGenerator, Context.OperatorContext.STATS_LISTENERS, statsListeners);
+ dag.setAttribute(passThrough, Context.OperatorContext.STATS_LISTENERS, statsListeners);
+ dag.setAttribute(devNull, Context.OperatorContext.STATS_LISTENERS, statsListeners);
+
+ // Increase timeout for the slow operator, this specifies the maximum timeout for an operator to process a window
+ // It is specified in number of windows, since 1 window is 500ms, 30 minutes is 30 * 60 * 2 = 3600 windows
+ dag.setAttribute(devNull, Context.OperatorContext.TIMEOUT_WINDOW_COUNT, 3600);
+
+ // If there are unifiers that are slow then set timeout for them
+ // dag.setUnifierAttribute(passThrough.output, Context.OperatorContext.TIMEOUT_WINDOW_COUNT, 3600);
+
+ dag.addStream("randomData", randomGenerator.out, passThrough.input);
+ dag.addStream("passData", passThrough.output, devNull.input);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/172b20a2/examples/throttle/src/main/java/com/datatorrent/examples/throttle/PassThroughOperator.java
----------------------------------------------------------------------
diff --git a/examples/throttle/src/main/java/com/datatorrent/examples/throttle/PassThroughOperator.java b/examples/throttle/src/main/java/com/datatorrent/examples/throttle/PassThroughOperator.java
new file mode 100644
index 0000000..b4630df
--- /dev/null
+++ b/examples/throttle/src/main/java/com/datatorrent/examples/throttle/PassThroughOperator.java
@@ -0,0 +1,20 @@
+package com.datatorrent.examples.throttle;
+
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.common.util.BaseOperator;
+
+/**
+ * Created by pramod on 9/27/16.
+ */
+public class PassThroughOperator<T> extends BaseOperator {
+
+ public transient final DefaultInputPort<T> input = new DefaultInputPort<T>() {
+ @Override
+ public void process(T t) {
+ output.emit(t);
+ }
+ };
+
+ public transient final DefaultOutputPort<T> output = new DefaultOutputPort<>();
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/172b20a2/examples/throttle/src/main/java/com/datatorrent/examples/throttle/RandomNumberGenerator.java
----------------------------------------------------------------------
diff --git a/examples/throttle/src/main/java/com/datatorrent/examples/throttle/RandomNumberGenerator.java b/examples/throttle/src/main/java/com/datatorrent/examples/throttle/RandomNumberGenerator.java
new file mode 100644
index 0000000..ea57b6d
--- /dev/null
+++ b/examples/throttle/src/main/java/com/datatorrent/examples/throttle/RandomNumberGenerator.java
@@ -0,0 +1,64 @@
+/**
+ * Put your copyright and license info here.
+ */
+package com.datatorrent.examples.throttle;
+
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.common.util.BaseOperator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This is a simple operator that emits random number.
+ */
+public class RandomNumberGenerator extends BaseOperator implements InputOperator
+{
+ private int numTuples = 1000;
+ private int origNumTuples = numTuples;
+ private transient int count = 0;
+
+ private static final Logger logger = LoggerFactory.getLogger(RandomNumberGenerator.class);
+
+ public final transient DefaultOutputPort<Double> out = new DefaultOutputPort<Double>();
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ count = 0;
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ if (count++ < numTuples) {
+ out.emit(Math.random());
+ }
+ }
+
+ // Simple suspend and
+ public void suspend() {
+ logger.debug("Slowing down");
+ numTuples = 0;
+ }
+
+ public void normal() {
+ logger.debug("Normal");
+ numTuples = origNumTuples;
+ }
+
+ public int getNumTuples()
+ {
+ return numTuples;
+ }
+
+ /**
+ * Sets the number of tuples to be emitted every window.
+ * @param numTuples number of tuples
+ */
+ public void setNumTuples(int numTuples)
+ {
+ this.numTuples = numTuples;
+ this.origNumTuples = numTuples;
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/172b20a2/examples/throttle/src/main/java/com/datatorrent/examples/throttle/SlowDevNullOperator.java
----------------------------------------------------------------------
diff --git a/examples/throttle/src/main/java/com/datatorrent/examples/throttle/SlowDevNullOperator.java b/examples/throttle/src/main/java/com/datatorrent/examples/throttle/SlowDevNullOperator.java
new file mode 100644
index 0000000..7d1451d
--- /dev/null
+++ b/examples/throttle/src/main/java/com/datatorrent/examples/throttle/SlowDevNullOperator.java
@@ -0,0 +1,35 @@
+package com.datatorrent.examples.throttle;
+
+import com.google.common.base.Throwables;
+
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.common.util.BaseOperator;
+
+/**
+ * Created by pramod on 9/27/16.
+ */
+public class SlowDevNullOperator<T> extends BaseOperator {
+
+ // Modify sleep time dynamically while app is running to increase and decrease sleep time
+ long sleepTime = 1;
+
+ public transient final DefaultInputPort<T> input = new DefaultInputPort<T>() {
+ @Override
+ public void process(T t) {
+ // Introduce an artificial delay for every tuple
+ try {
+ Thread.sleep(sleepTime);
+ } catch (InterruptedException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+ };
+
+ public long getSleepTime() {
+ return sleepTime;
+ }
+
+ public void setSleepTime(long sleepTime) {
+ this.sleepTime = sleepTime;
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/172b20a2/examples/throttle/src/main/java/com/datatorrent/examples/throttle/ThrottlingStatsListener.java
----------------------------------------------------------------------
diff --git a/examples/throttle/src/main/java/com/datatorrent/examples/throttle/ThrottlingStatsListener.java b/examples/throttle/src/main/java/com/datatorrent/examples/throttle/ThrottlingStatsListener.java
new file mode 100644
index 0000000..46e2e0e
--- /dev/null
+++ b/examples/throttle/src/main/java/com/datatorrent/examples/throttle/ThrottlingStatsListener.java
@@ -0,0 +1,150 @@
+package com.datatorrent.examples.throttle;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Maps;
+
+import com.datatorrent.api.Operator;
+import com.datatorrent.api.StatsListener;
+
+/**
+ * Created by pramod on 9/27/16.
+ */
+public class ThrottlingStatsListener implements StatsListener, Serializable {
+
+ private static final Logger logger = LoggerFactory.getLogger(ThrottlingStatsListener.class);
+
+ // Slowdown input if the window difference between operators increases beyond this value
+ long maxThreshold = 100;
+ // restore input operator to normal speed if the window difference falls below this threshold
+ long minThreshold = 100;
+
+ Map<Integer, ThrottleState> throttleStates = Maps.newHashMap();
+
+ static class ThrottleState {
+ // The current state of the operator, normal or throttled
+ boolean normal = true;
+ //The latest window id for which stats were received for the operator
+ long currentWindowId;
+ }
+
+ // This method runs on the app master side and is called whenever new stats are received from the operators
+ @Override
+ public Response processStats(BatchedOperatorStats batchedOperatorStats)
+ {
+ Response response = new Response();
+ int operatorId = batchedOperatorStats.getOperatorId();
+
+ ThrottleState throttleState = throttleStates.get(operatorId);
+ if (throttleState == null) {
+ throttleState = new ThrottleState();
+ throttleStates.put(operatorId, throttleState);
+ }
+
+ long windowId = batchedOperatorStats.getCurrentWindowId();
+ throttleState.currentWindowId = windowId;
+
+ // Find min and max window to compute difference
+ long minWindow = Long.MAX_VALUE;
+ long maxWindow = Long.MIN_VALUE;
+ for (ThrottleState state : throttleStates.values()) {
+ if (state.currentWindowId < minWindow) minWindow = state.currentWindowId;
+ if (state.currentWindowId > maxWindow) maxWindow = state.currentWindowId;
+ }
+ logger.debug("Operator {} min window {} max window {}", operatorId, minWindow, maxWindow);
+
+ if (throttleState.normal && ((maxWindow - minWindow) > maxThreshold)) {
+ // Send request to operator to slow down
+ logger.info("Sending suspend request");
+ List<OperatorRequest> operatorRequests = new ArrayList<OperatorRequest>();
+ operatorRequests.add(new InputSlowdownRequest());
+ response.operatorRequests = operatorRequests;
+ //logger.info("Setting suspend");
+ throttleState.normal = false;
+ } else if (!throttleState.normal && ((maxWindow - minWindow) <= minThreshold)) {
+ // Send request to operator to get back to normal
+ logger.info("Sending normal request");
+ List<OperatorRequest> operatorRequests = new ArrayList<OperatorRequest>();
+ operatorRequests.add(new InputNormalRequest());
+ response.operatorRequests = operatorRequests;
+ //logger.info("Setting normal");
+ throttleState.normal = true;
+ }
+
+ return response;
+ }
+
+ // This runs on the operator side
+ public static class InputSlowdownRequest implements OperatorRequest, Serializable
+ {
+ private static final Logger logger = LoggerFactory.getLogger(InputSlowdownRequest.class);
+
+ @Override
+ public OperatorResponse execute(Operator operator, int operatorId, long windowId) throws IOException
+ {
+ logger.debug("Received slowdown operator {} operatorId {} windowId {}", operator, operatorId, windowId);
+ if (operator instanceof RandomNumberGenerator) {
+ RandomNumberGenerator generator = (RandomNumberGenerator)operator;
+ generator.suspend();
+ }
+ return new InputOperatorResponse();
+ }
+ }
+
+ public static class InputNormalRequest implements OperatorRequest, Serializable
+ {
+ private static final Logger logger = LoggerFactory.getLogger(InputNormalRequest.class);
+
+ @Override
+ public OperatorResponse execute(Operator operator, int operatorId, long windowId) throws IOException
+ {
+ logger.debug("Received normal operator {} operatorId {} windowId {}", operator, operatorId, windowId);
+ if (operator instanceof RandomNumberGenerator) {
+ RandomNumberGenerator generator = (RandomNumberGenerator)operator;
+ generator.normal();
+ }
+ return new InputOperatorResponse();
+ }
+ }
+
+ public static class InputOperatorResponse implements OperatorResponse, Serializable
+ {
+
+ @Override
+ public Object getResponseId() {
+ return 1;
+ }
+
+ @Override
+ public Object getResponse() {
+ return "";
+ }
+ }
+
+ public long getMaxThreshold()
+ {
+ return maxThreshold;
+ }
+
+ public void setMaxThreshold(long maxThreshold)
+ {
+ this.maxThreshold = maxThreshold;
+ }
+
+ public long getMinThreshold()
+ {
+ return minThreshold;
+ }
+
+ public void setMinThreshold(long minThreshold)
+ {
+ this.minThreshold = minThreshold;
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/172b20a2/examples/throttle/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/throttle/src/main/resources/META-INF/properties.xml b/examples/throttle/src/main/resources/META-INF/properties.xml
new file mode 100644
index 0000000..a6ddc4c
--- /dev/null
+++ b/examples/throttle/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0"?>
+<configuration>
+ <!--
+ <property>
+ <name>dt.application.{appName}.operator.{opName}.prop.{propName}</name>
+ <value>some-default-value (if value is not specified, it is required from the user or custom config when launching)</value>
+ </property>
+ -->
+ <!-- memory assigned to app master
+ <property>
+ <name>dt.attr.MASTER_MEMORY_MB</name>
+ <value>1024</value>
+ </property>
+ -->
+ <property>
+ <name>dt.application.MyFirstApplication.operator.randomGenerator.prop.numTuples</name>
+ <value>1000</value>
+ </property>
+</configuration>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/172b20a2/examples/throttle/src/test/java/com/datatorrent/examples/throttle/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/throttle/src/test/java/com/datatorrent/examples/throttle/ApplicationTest.java b/examples/throttle/src/test/java/com/datatorrent/examples/throttle/ApplicationTest.java
new file mode 100644
index 0000000..02a96ba
--- /dev/null
+++ b/examples/throttle/src/test/java/com/datatorrent/examples/throttle/ApplicationTest.java
@@ -0,0 +1,37 @@
+/**
+ * Put your copyright and license info here.
+ */
+package com.datatorrent.examples.throttle;
+
+import java.io.IOException;
+
+import javax.validation.ConstraintViolationException;
+
+import org.junit.Assert;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+import com.datatorrent.api.LocalMode;
+import com.datatorrent.examples.throttle.Application;
+
+/**
+ * Test the DAG declaration in local mode.
+ */
+public class ApplicationTest {
+
+ @Test
+ public void testApplication() throws IOException, Exception {
+ try {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ lma.prepareDAG(new Application(), conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.run(10000); // runs for 10 seconds and quits
+ } catch (ConstraintViolationException e) {
+ Assert.fail("constraint violations: " + e.getConstraintViolations());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/172b20a2/examples/throttle/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/throttle/src/test/resources/log4j.properties b/examples/throttle/src/test/resources/log4j.properties
new file mode 100644
index 0000000..98544e8
--- /dev/null
+++ b/examples/throttle/src/test/resources/log4j.properties
@@ -0,0 +1,22 @@
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=debug
+log4j.logger.org.apache.apex=debug
+log4j.logger.org=info
[11/19] apex-malhar git commit: Add example of custom partitioner and
stream codec Update index and revise some README files
Posted by th...@apache.org.
Add example of custom partitioner and stream codec
Update index and revise some README files
Project: http://git-wip-us.apache.org/repos/asf/apex-malhar/repo
Commit: http://git-wip-us.apache.org/repos/asf/apex-malhar/commit/f7c7b7cf
Tree: http://git-wip-us.apache.org/repos/asf/apex-malhar/tree/f7c7b7cf
Diff: http://git-wip-us.apache.org/repos/asf/apex-malhar/diff/f7c7b7cf
Branch: refs/heads/master
Commit: f7c7b7cf476e0fe7ee82ffd85a2ed14a82cbcaed
Parents: 041af06
Author: Munagala V. Ramanath <ra...@datatorrent.com>
Authored: Sun Mar 13 17:37:45 2016 -0700
Committer: Lakshmi Prasanna Velineni <la...@datatorrent.com>
Committed: Sun Mar 26 11:43:48 2017 -0700
----------------------------------------------------------------------
examples/partition/README.md | 12 +
.../partition/XmlJavadocCommentsExtractor.xsl | 44 +++
examples/partition/pom.xml | 274 +++++++++++++++++++
examples/partition/src/assemble/appPackage.xml | 43 +++
.../java/com/example/myapexapp/Application.java | 27 ++
.../main/java/com/example/myapexapp/Codec3.java | 13 +
.../myapexapp/RandomNumberGenerator.java | 83 ++++++
.../com/example/myapexapp/TestPartition.java | 164 +++++++++++
.../src/main/resources/META-INF/properties.xml | 33 +++
.../src/main/resources/my-log4j.properties | 16 ++
.../com/example/myapexapp/ApplicationTest.java | 37 +++
.../src/test/resources/log4j.properties | 21 ++
12 files changed, 767 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/f7c7b7cf/examples/partition/README.md
----------------------------------------------------------------------
diff --git a/examples/partition/README.md b/examples/partition/README.md
new file mode 100644
index 0000000..6a46599
--- /dev/null
+++ b/examples/partition/README.md
@@ -0,0 +1,12 @@
+This example shows how to define custom partitions and a custom `StreamCodec` to customize
+the set of tuples that reach each partition.
+
+There are two operators: `RandomNumberGenerator` (generates random integers) and
+`TestPartition` (logs input tuples).
+
+The application also uses a StreamCodec called `Codec3` to tag each tuple with a
+partition tag based on whether the number is divisible by 2 or 4.
+
+`TestPartition` has code to create 3 partitions: one gets odd numbers, one gets multiples
+of 4 and the last gets the rest. The `PartitionKeys` associated with each partition use
+the partition tag to select the set of tuples to be handled by that partition.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/f7c7b7cf/examples/partition/XmlJavadocCommentsExtractor.xsl
----------------------------------------------------------------------
diff --git a/examples/partition/XmlJavadocCommentsExtractor.xsl b/examples/partition/XmlJavadocCommentsExtractor.xsl
new file mode 100644
index 0000000..08075a9
--- /dev/null
+++ b/examples/partition/XmlJavadocCommentsExtractor.xsl
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+
+<!--
+ Document : XmlJavadocCommentsExtractor.xsl
+ Created on : September 16, 2014, 11:30 AM
+ Description:
+ The transformation strips off all information except for comments and tags from xml javadoc generated by xml-doclet.
+-->
+
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+ <xsl:output method="xml" standalone="yes"/>
+
+ <!-- copy xml by selecting only the following nodes, attributes and text -->
+ <xsl:template match="node()|text()|@*">
+ <xsl:copy>
+ <xsl:apply-templates select="root|package|class|interface|method|field|type|comment|tag|text()|@name|@qualified|@text"/>
+ </xsl:copy>
+ </xsl:template>
+
+ <!-- Strip off the following paths from the selected xml -->
+ <xsl:template match="//root/package/interface/interface
+ |//root/package/interface/method/@qualified
+ |//root/package/class/interface
+ |//root/package/class/class
+ |//root/package/class/method/@qualified
+ |//root/package/class/field/@qualified" />
+
+ <xsl:strip-space elements="*"/>
+</xsl:stylesheet>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/f7c7b7cf/examples/partition/pom.xml
----------------------------------------------------------------------
diff --git a/examples/partition/pom.xml b/examples/partition/pom.xml
new file mode 100644
index 0000000..ac15981
--- /dev/null
+++ b/examples/partition/pom.xml
@@ -0,0 +1,274 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>com.example</groupId>
+ <version>1.0-SNAPSHOT</version>
+ <artifactId>Test-Ram</artifactId>
+ <packaging>jar</packaging>
+
+ <!-- change these to the appropriate values -->
+ <name>Test_ram</name>
+ <description>Test_ram</description>
+
+ <properties>
+ <!-- change this if you desire to use a different version of Apex core -->
+ <apex.version>3.5.0</apex.version>
+ <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-eclipse-plugin</artifactId>
+ <version>2.9</version>
+ <configuration>
+ <downloadSources>true</downloadSources>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.3</version>
+ <configuration>
+ <encoding>UTF-8</encoding>
+ <source>1.7</source>
+ <target>1.7</target>
+ <debug>true</debug>
+ <optimize>false</optimize>
+ <showDeprecation>true</showDeprecation>
+ <showWarnings>true</showWarnings>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.8</version>
+ <executions>
+ <execution>
+ <id>copy-dependencies</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>target/deps</outputDirectory>
+ <includeScope>runtime</includeScope>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>app-package-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <finalName>${project.artifactId}-${project.version}-apexapp</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <descriptors>
+ <descriptor>src/assemble/appPackage.xml</descriptor>
+ </descriptors>
+ <archiverConfig>
+ <defaultDirectoryMode>0755</defaultDirectoryMode>
+ </archiverConfig>
+ <archive>
+ <manifestEntries>
+ <Class-Path>${apex.apppackage.classpath}</Class-Path>
+ <DT-Engine-Version>${apex.version}</DT-Engine-Version>
+ <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name>
+ <DT-App-Package-Version>${project.version}</DT-App-Package-Version>
+ <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name>
+ <DT-App-Package-Description>${project.description}</DT-App-Package-Description>
+ </manifestEntries>
+ </archive>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <configuration>
+ <target>
+ <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar"
+ tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa" />
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ <execution>
+ <!-- create resource directory for xml javadoc-->
+ <id>createJavadocDirectory</id>
+ <phase>generate-resources</phase>
+ <configuration>
+ <tasks>
+ <delete dir="${project.build.directory}/generated-resources/xml-javadoc"/>
+ <mkdir dir="${project.build.directory}/generated-resources/xml-javadoc"/>
+ </tasks>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.9.1</version>
+ <executions>
+ <execution>
+ <id>attach-artifacts</id>
+ <phase>package</phase>
+ <goals>
+ <goal>attach-artifact</goal>
+ </goals>
+ <configuration>
+ <artifacts>
+ <artifact>
+ <file>target/${project.artifactId}-${project.version}.apa</file>
+ <type>apa</type>
+ </artifact>
+ </artifacts>
+ <skipAttach>false</skipAttach>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- generate javdoc -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <executions>
+ <!-- generate xml javadoc -->
+ <execution>
+ <id>xml-doclet</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>javadoc</goal>
+ </goals>
+ <configuration>
+ <doclet>com.github.markusbernhardt.xmldoclet.XmlDoclet</doclet>
+ <additionalparam>-d ${project.build.directory}/generated-resources/xml-javadoc -filename ${project.artifactId}-${project.version}-javadoc.xml</additionalparam>
+ <useStandardDocletOptions>false</useStandardDocletOptions>
+ <docletArtifact>
+ <groupId>com.github.markusbernhardt</groupId>
+ <artifactId>xml-doclet</artifactId>
+ <version>1.0.4</version>
+ </docletArtifact>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <!-- Transform xml javadoc to stripped down version containing only class/interface comments and tags-->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>xml-maven-plugin</artifactId>
+ <version>1.0</version>
+ <executions>
+ <execution>
+ <id>transform-xmljavadoc</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>transform</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <transformationSets>
+ <transformationSet>
+ <dir>${project.build.directory}/generated-resources/xml-javadoc</dir>
+ <includes>
+ <include>${project.artifactId}-${project.version}-javadoc.xml</include>
+ </includes>
+ <stylesheet>XmlJavadocCommentsExtractor.xsl</stylesheet>
+ <outputDir>${project.build.directory}/generated-resources/xml-javadoc</outputDir>
+ </transformationSet>
+ </transformationSets>
+ </configuration>
+ </plugin>
+ <!-- copy xml javadoc to class jar -->
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>2.6</version>
+ <executions>
+ <execution>
+ <id>copy-resources</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${basedir}/target/classes</outputDirectory>
+ <resources>
+ <resource>
+ <directory>${project.build.directory}/generated-resources/xml-javadoc</directory>
+ <includes>
+ <include>${project.artifactId}-${project.version}-javadoc.xml</include>
+ </includes>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+
+ </build>
+
+ <dependencies>
+ <!-- add your dependencies here -->
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-library</artifactId>
+ <version>3.6.0</version>
+
+ <!--
+ If you know that your application does not need transitive dependencies pulled in by malhar-library,
+ uncomment the following to reduce the size of your app package.
+ -->
+ <!--
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ -->
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>apex-common</artifactId>
+ <version>${apex.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.10</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>apex-engine</artifactId>
+ <version>${apex.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/f7c7b7cf/examples/partition/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/partition/src/assemble/appPackage.xml b/examples/partition/src/assemble/appPackage.xml
new file mode 100644
index 0000000..7ad071c
--- /dev/null
+++ b/examples/partition/src/assemble/appPackage.xml
@@ -0,0 +1,43 @@
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>appPackage</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/target/</directory>
+ <outputDirectory>/app</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/target/deps</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/site/conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/META-INF</directory>
+ <outputDirectory>/META-INF</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/app</directory>
+ <outputDirectory>/app</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/resources</directory>
+ <outputDirectory>/resources</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/f7c7b7cf/examples/partition/src/main/java/com/example/myapexapp/Application.java
----------------------------------------------------------------------
diff --git a/examples/partition/src/main/java/com/example/myapexapp/Application.java b/examples/partition/src/main/java/com/example/myapexapp/Application.java
new file mode 100644
index 0000000..e1ca2ff
--- /dev/null
+++ b/examples/partition/src/main/java/com/example/myapexapp/Application.java
@@ -0,0 +1,27 @@
+package com.example.myapexapp;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.Context.PortContext;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DAG.Locality;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+
+@ApplicationAnnotation(name="TestStuff")
+public class Application implements StreamingApplication
+{
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ RandomNumberGenerator random = dag.addOperator("randomInt", RandomNumberGenerator.class);
+ TestPartition testPartition = dag.addOperator("testPartition", TestPartition.class);
+ Codec3 codec = new Codec3();
+ dag.setInputPortAttribute(testPartition.in, PortContext.STREAM_CODEC, codec);
+
+ //Add locality if needed, e.g.: .setLocality(Locality.CONTAINER_LOCAL);
+ dag.addStream("randomData", random.out, testPartition.in);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/f7c7b7cf/examples/partition/src/main/java/com/example/myapexapp/Codec3.java
----------------------------------------------------------------------
diff --git a/examples/partition/src/main/java/com/example/myapexapp/Codec3.java b/examples/partition/src/main/java/com/example/myapexapp/Codec3.java
new file mode 100644
index 0000000..2754e9b
--- /dev/null
+++ b/examples/partition/src/main/java/com/example/myapexapp/Codec3.java
@@ -0,0 +1,13 @@
+package com.example.myapexapp;
+
+import com.datatorrent.lib.codec.KryoSerializableStreamCodec;
+
+public class Codec3 extends KryoSerializableStreamCodec<Integer> {
+ @Override
+ public int getPartition(Integer tuple) {
+ final int v = tuple;
+ return (1 == (v & 1)) ? 0 // odd
+ : (0 == (v & 3)) ? 1 // divisible by 4
+ : 2; // divisible by 2 but not 4
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/f7c7b7cf/examples/partition/src/main/java/com/example/myapexapp/RandomNumberGenerator.java
----------------------------------------------------------------------
diff --git a/examples/partition/src/main/java/com/example/myapexapp/RandomNumberGenerator.java b/examples/partition/src/main/java/com/example/myapexapp/RandomNumberGenerator.java
new file mode 100644
index 0000000..de2797b
--- /dev/null
+++ b/examples/partition/src/main/java/com/example/myapexapp/RandomNumberGenerator.java
@@ -0,0 +1,83 @@
+package com.example.myapexapp;
+
+import java.util.Random;
+
+import javax.validation.constraints.Min;
+import javax.validation.constraints.Size;
+import javax.validation.ConstraintViolation;
+import javax.validation.ValidatorFactory;
+import javax.validation.Validator;
+import javax.validation.Validation;
+
+import com.datatorrent.api.Attribute;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.common.util.BaseOperator;
+import com.datatorrent.api.Context;
+import com.datatorrent.api.Context.OperatorContext;
+
+/**
+ * This is a simple operator that emits random integer.
+ */
+public class RandomNumberGenerator extends BaseOperator implements InputOperator
+{
+ private static final Logger LOG = LoggerFactory.getLogger(RandomNumberGenerator.class);
+
+ @Min(1)
+ private int numTuples = 20;
+ private transient int count = 0;
+
+ private int sleepTime;
+ private transient long curWindowId;
+ private transient Random random = new Random();
+
+ public final transient DefaultOutputPort<Integer> out = new DefaultOutputPort<Integer>();
+
+ @Override
+ public void setup(Context.OperatorContext context)
+ {
+ super.setup(context);
+
+ long appWindowId = context.getValue(context.ACTIVATION_WINDOW_ID);
+ sleepTime = context.getValue(context.SPIN_MILLIS);
+ LOG.debug("Started setup, appWindowId = {}, sleepTime = {}", appWindowId, sleepTime);
+ }
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ count = 0;
+ LOG.debug("beginWindow: windowId = {}", windowId);
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ if (count++ < numTuples) {
+ out.emit(random.nextInt());
+ } else {
+ LOG.debug("count = {}, time = {}", count, System.currentTimeMillis());
+
+ try {
+ // avoid repeated calls to this function
+ Thread.sleep(sleepTime);
+ } catch (InterruptedException e) {
+ LOG.info("Sleep interrupted");
+ }
+ }
+ }
+
+ public int getNumTuples()
+ {
+ return numTuples;
+ }
+
+ public void setNumTuples(int numTuples)
+ {
+ this.numTuples = numTuples;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/f7c7b7cf/examples/partition/src/main/java/com/example/myapexapp/TestPartition.java
----------------------------------------------------------------------
diff --git a/examples/partition/src/main/java/com/example/myapexapp/TestPartition.java b/examples/partition/src/main/java/com/example/myapexapp/TestPartition.java
new file mode 100644
index 0000000..1f77e72
--- /dev/null
+++ b/examples/partition/src/main/java/com/example/myapexapp/TestPartition.java
@@ -0,0 +1,164 @@
+package com.example.myapexapp;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Random;
+
+import javax.validation.constraints.Max;
+import javax.validation.constraints.Min;
+import javax.validation.constraints.Size;
+import javax.validation.ConstraintViolation;
+import javax.validation.ValidatorFactory;
+import javax.validation.Validator;
+import javax.validation.Validation;
+
+import com.datatorrent.api.Attribute;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.DefaultPartition;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.api.Partitioner;
+import com.datatorrent.api.Partitioner.Partition;
+import com.datatorrent.api.Partitioner.PartitionKeys;
+import com.datatorrent.api.Partitioner.PartitioningContext;
+
+import com.datatorrent.common.util.BaseOperator;
+
+/**
+ * Simple operator to test partitioning
+ */
+public class TestPartition extends BaseOperator implements Partitioner<TestPartition>
+{
+ private static final Logger LOG = LoggerFactory.getLogger(TestPartition.class);
+
+ private transient int id; // operator/partition id
+ private transient long curWindowId; // current window id
+ private transient long cnt; // per-window tuple count
+
+ @Min(1) @Max(20)
+ private int nPartitions = 3;
+
+ public final transient DefaultInputPort<Integer> in = new DefaultInputPort<Integer>() {
+ @Override
+ public void process(Integer tuple)
+ {
+ LOG.debug("{}: tuple = {}, operator id = {}", cnt, tuple, id);
+ ++cnt;
+ }
+ };
+
+ //public final transient DefaultOutputPort<Integer> out = new DefaultOutputPort<Integer>();
+
+ @Override
+ public void setup(Context.OperatorContext context)
+ {
+ super.setup(context);
+
+ long appWindowId = context.getValue(context.ACTIVATION_WINDOW_ID);
+ id = context.getId();
+ LOG.debug("Started setup, appWindowId = {}, operator id = {}", appWindowId, id);
+ }
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ cnt = 0;
+ curWindowId = windowId;
+ LOG.debug("window id = {}, operator id = {}", curWindowId, id);
+ }
+
+ @Override
+ public void endWindow()
+ {
+ LOG.debug("window id = {}, operator id = {}, cnt = {}", curWindowId, id, cnt);
+ }
+
+ @Override
+ public void partitioned(Map<Integer, Partition<TestPartition>> partitions)
+ {
+ //Do nothing
+ }
+
+ @Override
+ public Collection<Partition<TestPartition>> definePartitions(
+ Collection<Partition<TestPartition>> partitions,
+ PartitioningContext context)
+ {
+ int oldSize = partitions.size();
+ LOG.debug("partitionCount: current = {} requested = {}", oldSize, nPartitions);
+
+ // each partition i in 0...nPartitions receives tuples divisible by i but not by any other
+ // j in that range; all other tuples ignored
+ //
+ if (3 != nPartitions) return getPartitions(partitions, context);
+
+ // special case of 3 partitions: All odd numbers to partition 0; even numbers divisible
+ // by 4 to partition 1, those divisible by 2 but not 4 to partition 2.
+
+ // mask used to extract discriminant from tuple hashcode
+ int mask = 0x03;
+
+ Partition<TestPartition>[] newPartitions = new Partition[] {
+ new DefaultPartition<TestPartition>(new TestPartition()),
+ new DefaultPartition<TestPartition>(new TestPartition()),
+ new DefaultPartition<TestPartition>(new TestPartition()) };
+
+ HashSet<Integer>[] set
+ = new HashSet[] {new HashSet<>(), new HashSet<>(), new HashSet<>()};
+ set[0].add(0);
+ set[1].add(1);
+ set[2].add(2);
+
+ PartitionKeys[] keys = {
+ new PartitionKeys(mask, set[0]),
+ new PartitionKeys(mask, set[1]),
+ new PartitionKeys(mask, set[2]) };
+
+ for (int i = 0; i < 3; ++i ) {
+ Partition<TestPartition> partition = newPartitions[i];
+ partition.getPartitionKeys().put(in, keys[i]);
+ }
+
+ return new ArrayList<Partition<TestPartition>>(Arrays.asList(newPartitions));
+ } // definePartitions
+
+ private Collection<Partition<TestPartition>> getPartitions(
+ Collection<Partition<TestPartition>> partitions,
+ PartitioningContext context)
+ {
+ // create array of partitions to return
+ Collection<Partition<TestPartition>> result
+ = new ArrayList<Partition<TestPartition>>(nPartitions);
+
+ int mask = getMask(nPartitions);
+ for (int i = 0; i < nPartitions; ++i) {
+ HashSet<Integer> set = new HashSet<>();
+ set.add(i);
+ PartitionKeys keys = new PartitionKeys(mask, set);
+ Partition partition = new DefaultPartition<TestPartition>(new TestPartition());
+ partition.getPartitionKeys().put(in, keys);
+ }
+
+ return result;
+ } // getPartitions
+
+ // return mask with bits 0..N set where N is the highest set bit of argument
+ private int getMask(final int n) {
+ return -1 >>> Integer.numberOfLeadingZeros(n);
+ } // getMask
+
+ // accessors
+ public int getNPartitions() { return nPartitions; }
+ public void setNPartitions(int v) { nPartitions = v; }
+}
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/f7c7b7cf/examples/partition/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/partition/src/main/resources/META-INF/properties.xml b/examples/partition/src/main/resources/META-INF/properties.xml
new file mode 100644
index 0000000..bf30603
--- /dev/null
+++ b/examples/partition/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0"?>
+<configuration>
+ <!--
+ <property>
+ <name>dt.application.{appName}.operator.{opName}.prop.{propName}</name>
+ <value>some-default-value (if value is not specified, it is required from the user or custom config when launching)</value>
+ </property>
+ -->
+ <!-- memory assigned to app master
+ <property>
+ <name>dt.attr.MASTER_MEMORY_MB</name>
+ <value>1024</value>
+ </property>
+ -->
+
+ <!-- log4j configuration for all the operators -->
+ <property>
+ <name>dt.operator.*.attr.JVM_OPTIONS</name>
+ <value> -Dlog4j.configuration=my-log4j.properties</value>
+ </property>
+
+ <property>
+ <name>dt.application.*.operator.randomInt.prop.numTuples</name>
+ <value>10</value>
+ </property>
+ <!--
+ <property>
+ <name>dt.application.TestStuff.operator.testPartition.attr.PARTITIONER</name>
+ <value>com.datatorrent.common.partitioner.StatelessPartitioner:2</value>
+ </property>
+ -->
+</configuration>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/f7c7b7cf/examples/partition/src/main/resources/my-log4j.properties
----------------------------------------------------------------------
diff --git a/examples/partition/src/main/resources/my-log4j.properties b/examples/partition/src/main/resources/my-log4j.properties
new file mode 100644
index 0000000..21ead89
--- /dev/null
+++ b/examples/partition/src/main/resources/my-log4j.properties
@@ -0,0 +1,16 @@
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} %M - %m%n
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+log4j.logger.org=INFO
+log4j.logger.org.apache=INFO
+log4j.logger.com.datatorrent=INFO
+
+#log4j.logger.com.example.myapexapp.TestPartition=DEBUG, CONSOLE
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/f7c7b7cf/examples/partition/src/test/java/com/example/myapexapp/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/partition/src/test/java/com/example/myapexapp/ApplicationTest.java b/examples/partition/src/test/java/com/example/myapexapp/ApplicationTest.java
new file mode 100644
index 0000000..5f490d8
--- /dev/null
+++ b/examples/partition/src/test/java/com/example/myapexapp/ApplicationTest.java
@@ -0,0 +1,37 @@
+/**
+ * Put your copyright and license info here.
+ */
+package com.example.myapexapp;
+
+import java.io.IOException;
+
+import javax.validation.ConstraintViolationException;
+
+import org.junit.Assert;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+import com.datatorrent.api.LocalMode;
+import com.example.myapexapp.Application;
+
+/**
+ * Test the DAG declaration in local mode.
+ */
+public class ApplicationTest {
+
+ @Test
+ public void testApplication() throws IOException, Exception {
+ try {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ lma.prepareDAG(new Application(), conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.run(5000); // runs for 5 seconds and quits
+ } catch (ConstraintViolationException e) {
+ Assert.fail("constraint violations: " + e.getConstraintViolations());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/f7c7b7cf/examples/partition/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/partition/src/test/resources/log4j.properties b/examples/partition/src/test/resources/log4j.properties
new file mode 100644
index 0000000..3bfcdc5
--- /dev/null
+++ b/examples/partition/src/test/resources/log4j.properties
@@ -0,0 +1,21 @@
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+log4j.logger.org=info
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=debug
[19/19] apex-malhar git commit: SPOI-8561 Record reader example app
Posted by th...@apache.org.
SPOI-8561 Record reader example app
2. Added entry for recordReader in README
3. Verification for number of records reaching to the output.
4. Using runAsync instead of run(10s)
5. Updated README
6. Incorporating review comments
Project: http://git-wip-us.apache.org/repos/asf/apex-malhar/repo
Commit: http://git-wip-us.apache.org/repos/asf/apex-malhar/commit/11f37671
Tree: http://git-wip-us.apache.org/repos/asf/apex-malhar/tree/11f37671
Diff: http://git-wip-us.apache.org/repos/asf/apex-malhar/diff/11f37671
Branch: refs/heads/master
Commit: 11f376711e35c456f2c8d2d846b1b6271495b82f
Parents: 0590c25
Author: yogidevendra <de...@datatorrent.com>
Authored: Mon Jul 4 17:17:05 2016 +0530
Committer: Lakshmi Prasanna Velineni <la...@datatorrent.com>
Committed: Sun Mar 26 11:43:48 2017 -0700
----------------------------------------------------------------------
examples/recordReader/README.md | 8 +
.../XmlJavadocCommentsExtractor.xsl | 44 +++
examples/recordReader/pom.xml | 297 +++++++++++++++++++
.../recordReader/src/assemble/appPackage.xml | 43 +++
.../com/example/recordReader/Application.java | 32 ++
.../recordReader/TransactionsSchema.java | 168 +++++++++++
.../src/main/resources/META-INF/CustomerTxnData | 100 +++++++
.../src/main/resources/META-INF/properties.xml | 159 ++++++++++
.../example/recordReader/ApplicationTest.java | 91 ++++++
.../src/test/resources/log4j.properties | 21 ++
10 files changed, 963 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/11f37671/examples/recordReader/README.md
----------------------------------------------------------------------
diff --git a/examples/recordReader/README.md b/examples/recordReader/README.md
new file mode 100644
index 0000000..ad715fc
--- /dev/null
+++ b/examples/recordReader/README.md
@@ -0,0 +1,8 @@
+This example shows how to use `FSRecordReaderModule` to read newline delimited records from a file convert them to plain old java objects (POJO) using `CSVParser`. These POJOs are converted to String using `CsvFormatter` and then written to output file using `StringFileOutputOperator`.
+
+The properties file `META-INF/properties.xml` shows how to configure the respective operators.
+
+The application can be run on an actual cluster or in local mode within your IDE by
+simply running the method `ApplicationTest.testApplication()`.
+
+One may tweak this example to add operator of their choice in between `CSVParser` and `CsvFormatter` to achieve functionality to suit your need.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/11f37671/examples/recordReader/XmlJavadocCommentsExtractor.xsl
----------------------------------------------------------------------
diff --git a/examples/recordReader/XmlJavadocCommentsExtractor.xsl b/examples/recordReader/XmlJavadocCommentsExtractor.xsl
new file mode 100644
index 0000000..08075a9
--- /dev/null
+++ b/examples/recordReader/XmlJavadocCommentsExtractor.xsl
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+
+<!--
+ Document : XmlJavadocCommentsExtractor.xsl
+ Created on : September 16, 2014, 11:30 AM
+ Description:
+ The transformation strips off all information except for comments and tags from xml javadoc generated by xml-doclet.
+-->
+
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+ <xsl:output method="xml" standalone="yes"/>
+
+ <!-- copy xml by selecting only the following nodes, attributes and text -->
+ <xsl:template match="node()|text()|@*">
+ <xsl:copy>
+ <xsl:apply-templates select="root|package|class|interface|method|field|type|comment|tag|text()|@name|@qualified|@text"/>
+ </xsl:copy>
+ </xsl:template>
+
+ <!-- Strip off the following paths from the selected xml -->
+ <xsl:template match="//root/package/interface/interface
+ |//root/package/interface/method/@qualified
+ |//root/package/class/interface
+ |//root/package/class/class
+ |//root/package/class/method/@qualified
+ |//root/package/class/field/@qualified" />
+
+ <xsl:strip-space elements="*"/>
+</xsl:stylesheet>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/11f37671/examples/recordReader/pom.xml
----------------------------------------------------------------------
diff --git a/examples/recordReader/pom.xml b/examples/recordReader/pom.xml
new file mode 100644
index 0000000..007eaef
--- /dev/null
+++ b/examples/recordReader/pom.xml
@@ -0,0 +1,297 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>com.example</groupId>
+ <version>1.0-SNAPSHOT</version>
+ <artifactId>recordReader</artifactId>
+ <packaging>jar</packaging>
+
+ <name>File Record Reader</name>
+ <description>Simple application illustrating use of record reader operator</description>
+
+ <properties>
+ <!-- change this if you desire to use a different version of Apex Core -->
+ <apex.version>3.5.0</apex.version>
+ <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath>
+ <malhar.version>3.6.0</malhar.version>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-eclipse-plugin</artifactId>
+ <version>2.9</version>
+ <configuration>
+ <downloadSources>true</downloadSources>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.3</version>
+ <configuration>
+ <encoding>UTF-8</encoding>
+ <source>1.7</source>
+ <target>1.7</target>
+ <debug>true</debug>
+ <optimize>false</optimize>
+ <showDeprecation>true</showDeprecation>
+ <showWarnings>true</showWarnings>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.8</version>
+ <executions>
+ <execution>
+ <id>copy-dependencies</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>target/deps</outputDirectory>
+ <includeScope>runtime</includeScope>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>app-package-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <finalName>${project.artifactId}-${project.version}-apexapp</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <descriptors>
+ <descriptor>src/assemble/appPackage.xml</descriptor>
+ </descriptors>
+ <archiverConfig>
+ <defaultDirectoryMode>0755</defaultDirectoryMode>
+ </archiverConfig>
+ <archive>
+ <manifestEntries>
+ <Class-Path>${apex.apppackage.classpath}</Class-Path>
+ <DT-Engine-Version>${apex.version}</DT-Engine-Version>
+ <DT-App-Package-Group-Id>${project.groupId}</DT-App-Package-Group-Id>
+ <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name>
+ <DT-App-Package-Version>${project.version}</DT-App-Package-Version>
+ <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name>
+ <DT-App-Package-Description>${project.description}</DT-App-Package-Description>
+ </manifestEntries>
+ </archive>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <configuration>
+ <target>
+ <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar"
+ tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa" />
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ <execution>
+ <!-- create resource directory for xml javadoc-->
+ <id>createJavadocDirectory</id>
+ <phase>generate-resources</phase>
+ <configuration>
+ <tasks>
+ <delete dir="${project.build.directory}/generated-resources/xml-javadoc"/>
+ <mkdir dir="${project.build.directory}/generated-resources/xml-javadoc"/>
+ </tasks>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.9.1</version>
+ <executions>
+ <execution>
+ <id>attach-artifacts</id>
+ <phase>package</phase>
+ <goals>
+ <goal>attach-artifact</goal>
+ </goals>
+ <configuration>
+ <artifacts>
+ <artifact>
+ <file>target/${project.artifactId}-${project.version}.apa</file>
+ <type>apa</type>
+ </artifact>
+ </artifacts>
+ <skipAttach>false</skipAttach>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- generate javdoc -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <executions>
+ <!-- generate xml javadoc -->
+ <execution>
+ <id>xml-doclet</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>javadoc</goal>
+ </goals>
+ <configuration>
+ <doclet>com.github.markusbernhardt.xmldoclet.XmlDoclet</doclet>
+ <additionalparam>-d ${project.build.directory}/generated-resources/xml-javadoc -filename ${project.artifactId}-${project.version}-javadoc.xml</additionalparam>
+ <useStandardDocletOptions>false</useStandardDocletOptions>
+ <docletArtifact>
+ <groupId>com.github.markusbernhardt</groupId>
+ <artifactId>xml-doclet</artifactId>
+ <version>1.0.4</version>
+ </docletArtifact>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <!-- Transform xml javadoc to stripped down version containing only class/interface comments and tags-->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>xml-maven-plugin</artifactId>
+ <version>1.0</version>
+ <executions>
+ <execution>
+ <id>transform-xmljavadoc</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>transform</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <transformationSets>
+ <transformationSet>
+ <dir>${project.build.directory}/generated-resources/xml-javadoc</dir>
+ <includes>
+ <include>${project.artifactId}-${project.version}-javadoc.xml</include>
+ </includes>
+ <stylesheet>XmlJavadocCommentsExtractor.xsl</stylesheet>
+ <outputDir>${project.build.directory}/generated-resources/xml-javadoc</outputDir>
+ </transformationSet>
+ </transformationSets>
+ </configuration>
+ </plugin>
+ <!-- copy xml javadoc to class jar -->
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>2.6</version>
+ <executions>
+ <execution>
+ <id>copy-resources</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${basedir}/target/classes</outputDirectory>
+ <resources>
+ <resource>
+ <directory>${project.build.directory}/generated-resources/xml-javadoc</directory>
+ <includes>
+ <include>${project.artifactId}-${project.version}-javadoc.xml</include>
+ </includes>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+
+ </build>
+
+ <dependencies>
+ <!-- add your dependencies here -->
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-library</artifactId>
+ <version>${malhar.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-contrib</artifactId>
+ <version>${malhar.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-contrib</artifactId>
+ <version>${malhar.version}</version>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>net.sf.supercsv</groupId>
+ <artifactId>super-csv</artifactId>
+ <version>2.4.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>apex-common</artifactId>
+ <version>${apex.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.10</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>apex-engine</artifactId>
+ <version>${apex.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/11f37671/examples/recordReader/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/recordReader/src/assemble/appPackage.xml b/examples/recordReader/src/assemble/appPackage.xml
new file mode 100644
index 0000000..7ad071c
--- /dev/null
+++ b/examples/recordReader/src/assemble/appPackage.xml
@@ -0,0 +1,43 @@
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>appPackage</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/target/</directory>
+ <outputDirectory>/app</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/target/deps</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/site/conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/META-INF</directory>
+ <outputDirectory>/META-INF</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/app</directory>
+ <outputDirectory>/app</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/resources</directory>
+ <outputDirectory>/resources</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/11f37671/examples/recordReader/src/main/java/com/example/recordReader/Application.java
----------------------------------------------------------------------
diff --git a/examples/recordReader/src/main/java/com/example/recordReader/Application.java b/examples/recordReader/src/main/java/com/example/recordReader/Application.java
new file mode 100644
index 0000000..8b0dd75
--- /dev/null
+++ b/examples/recordReader/src/main/java/com/example/recordReader/Application.java
@@ -0,0 +1,32 @@
+/**
+ * Put your copyright and license info here.
+ */
+package com.example.recordReader;
+
+import org.apache.apex.malhar.lib.fs.FSRecordReaderModule;
+import org.apache.apex.malhar.lib.fs.GenericFileOutputOperator.StringFileOutputOperator;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.contrib.formatter.CsvFormatter;
+import com.datatorrent.contrib.parser.CsvParser;
+
+@ApplicationAnnotation(name="RecordReaderExample")
+public class Application implements StreamingApplication
+{
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ FSRecordReaderModule recordReader = dag.addModule("recordReader", FSRecordReaderModule.class);
+ CsvParser csvParser = dag.addOperator("csvParser", CsvParser.class);
+ CsvFormatter formatter = dag.addOperator("formatter", new CsvFormatter());
+ StringFileOutputOperator fileOutput = dag.addOperator("fileOutput", new StringFileOutputOperator());
+
+ dag.addStream("record", recordReader.records, csvParser.in);
+ dag.addStream("pojo", csvParser.out, formatter.in);
+ dag.addStream("string", formatter.out, fileOutput.input);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/11f37671/examples/recordReader/src/main/java/com/example/recordReader/TransactionsSchema.java
----------------------------------------------------------------------
diff --git a/examples/recordReader/src/main/java/com/example/recordReader/TransactionsSchema.java b/examples/recordReader/src/main/java/com/example/recordReader/TransactionsSchema.java
new file mode 100644
index 0000000..de20516
--- /dev/null
+++ b/examples/recordReader/src/main/java/com/example/recordReader/TransactionsSchema.java
@@ -0,0 +1,168 @@
+/**
+ * Copyright (c) 2016 DataTorrent, Inc.
+ * All rights reserved.
+ */
+
+package com.example.recordReader;
+
+import java.util.Date;
+
+public class TransactionsSchema
+{
+ private String customerName;
+ private String customerPhone;
+ private String customerEmail;
+ private String city;
+ private String country;
+ private String uid;
+ private String accountNumber;
+ private long txId;
+ private Date txDate;
+ private double txAmount;
+
+ public String getCustomerName()
+ {
+ return customerName;
+ }
+
+ public void setCustomerName(String customerName)
+ {
+ this.customerName = customerName;
+ }
+
+ public String getCustomerPhone()
+ {
+ return customerPhone;
+ }
+
+ public void setCustomerPhone(String customerPhone)
+ {
+ this.customerPhone = customerPhone;
+ }
+
+ public String getCustomerEmail()
+ {
+ return customerEmail;
+ }
+
+ public void setCustomerEmail(String customerEmail)
+ {
+ this.customerEmail = customerEmail;
+ }
+
+ public String getCity()
+ {
+ return city;
+ }
+
+ public void setCity(String city)
+ {
+ this.city = city;
+ }
+
+ public String getCountry()
+ {
+ return country;
+ }
+
+ public void setCountry(String country)
+ {
+ this.country = country;
+ }
+
+ public String getUid()
+ {
+ return uid;
+ }
+
+ public void setUid(String uid)
+ {
+ this.uid = uid;
+ }
+
+ public String getAccountNumber()
+ {
+ return accountNumber;
+ }
+
+ public void setAccountNumber(String accountNumber)
+ {
+ this.accountNumber = accountNumber;
+ }
+
+ public long getTxId()
+ {
+ return txId;
+ }
+
+ public void setTxId(long txId)
+ {
+ this.txId = txId;
+ }
+
+ public Date getTxDate()
+ {
+ return txDate;
+ }
+
+ public void setTxDate(Date txDate)
+ {
+ this.txDate = txDate;
+ }
+
+ public double getTxAmount()
+ {
+ return txAmount;
+ }
+
+ public void setTxAmount(double txAmount)
+ {
+ this.txAmount = txAmount;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "TransactionsSchema [customerName=" + customerName + ", customerPhone=" + customerPhone + ", customerEmail="
+ + customerEmail + ", city=" + city + ", country=" + country + ", uid=" + uid + ", accountNumber="
+ + accountNumber + ", txId=" + txId + ", txDate=" + txDate + ", txAmount=" + txAmount
+ + "]\n";
+ }
+
+ @Override
+ public int hashCode()
+ {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((txDate == null) ? 0 : txDate.hashCode());
+ result = prime * result + (int)(txId ^ (txId >>> 32));
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj)
+ {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ TransactionsSchema other = (TransactionsSchema)obj;
+ if (txDate == null) {
+ if (other.txDate != null) {
+ return false;
+ }
+ } else if (!txDate.equals(other.txDate)) {
+ return false;
+ }
+ if (txId != other.txId) {
+ return false;
+ }
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/11f37671/examples/recordReader/src/main/resources/META-INF/CustomerTxnData
----------------------------------------------------------------------
diff --git a/examples/recordReader/src/main/resources/META-INF/CustomerTxnData b/examples/recordReader/src/main/resources/META-INF/CustomerTxnData
new file mode 100644
index 0000000..88322ab
--- /dev/null
+++ b/examples/recordReader/src/main/resources/META-INF/CustomerTxnData
@@ -0,0 +1,100 @@
+Desirae I. Hart|91-756-943-24728|rutrum.magna@dictummagnaUt.net|Delhi|India|7F2BAD18-D1D1-8DB6-A519-70DA658EF6DE|1048.2446157601|245407|2016-06-17 21:19:02|56095.4691099477
+Fay B. Page|91-952-299-03140|lobortis.risus@eutelluseu.com|Durg|India|5E8E12D3-34CC-53E1-7622-BC533542B930|939.9385330598|245408|2016-06-17 21:19:02|93013.7443360696
+Herrod F. Mcknight|91-609-146-21345|nec@egetmagna.co.uk|Haridwar|India|F589AF28-C10C-3EF1-E9DA-7CAAAD8E453D|1009.5714354316|245409|2016-06-17 21:19:03|57130.4933466919
+Suki K. Hutchinson|91-235-565-54483|justo.Praesent@nonquam.ca|Panjim|India|A263A773-E228-0C5B-D9CC-A5AE37A9F3F5|990.5143009573|245410|2016-06-17 21:19:03|88348.7049292828
+Tucker H. Strickland|91-353-281-78811|Suspendisse@felisNulla.edu|Aizwal|India|748AAD2B-8CB4-6010-2C80-F3B0FE107B4E|1027.2343235232|245411|2016-06-17 21:19:03|63602.53265577
+Carly L. Alvarado|91-364-259-91891|blandit.mattis.Cras@Nullam.co.uk|Mandya|India|292345E9-45E1-AE6C-9051-79681BE10E0A|964.9602866876|245412|2016-06-17 21:19:03|63777.4412787906
+Signe H. Carlson|91-736-760-21206|Curabitur@ac.edu|Basirhat|India|ABFCB28C-98B0-93E4-C5E9-D572648FE467|967.6980193409|245413|2016-06-17 21:19:04|81425.8673128708
+Timothy U. Cantrell|91-266-620-64253|ac.mattis@congueIn.com|Dibrugarh|India|46ED3B8F-5D19-D3FD-2833-7E6D956090D6|990.0113497158|245414|2016-06-17 21:19:04|20765.2379785903
+Todd M. Hunter|91-131-862-91105|vel@per.org|Pondicherry|India|73DDE1E7-7B86-53A0-8296-8DDCC0A8022A|999.2001773265|245416|2016-06-17 21:19:04|28079.5354148245
+Martin J. Joyce|91-312-843-82341|magnis.dis.parturient@penatibusetmagnis.co.uk|Gandhidham|India|46247FA6-883D-2DAA-78E4-626F41937905|974.8521165154|245415|2016-06-17 21:19:04|98219.3968546873
+Cruz D. Slater|91-399-580-01355|ante.dictum@tempor.co.uk|Kalyan|India|7A0AE886-CDBF-98FF-1808-1A977B06F939|989.5735205715|245418|2016-06-17 21:19:05|69472.6529645815
+Keefe M. Hodge|91-123-766-63270|molestie.arcu.Sed@eratnonummy.ca|Bokaro Steel City|India|5488D1DE-E840-827F-2D45-347A58ECF72C|1059.6676733575|245417|2016-06-17 21:19:05|60538.9043471269
+Minerva R. Velez|91-550-563-14205|dapibus.quam.quis@sitamet.co.uk|Bhusawal|India|C2EA3D6F-EEBE-7E7D-4DFD-DA90FAAAC04F|965.3100959637|245419|2016-06-17 21:19:05|22378.4665981816
+Gillian P. Bates|91-530-835-01656|In.nec@sedduiFusce.org|Kohima|India|8882E4F0-0AB1-6098-EEF8-0230BBB931B2|978.3566576884|245420|2016-06-17 21:19:05|57058.528246018
+Jael I. Humphrey|91-815-502-83243|dignissim@quisturpis.co.uk|Chandigarh|India|0D8ADC96-B858-60A9-4F65-3BACADC808A8|1047.1902156238|245421|2016-06-17 21:19:06|71348.2114533201
+Eleanor D. Hart|91-351-706-26576|nonummy@sitametdapibus.ca|Imphal|India|6DE39307-4D0B-77B0-8DAF-36417261DA2B|1021.3673266291|245422|2016-06-17 21:19:06|60474.8184384689
+Kaitlin U. Higgins|91-738-770-10870|sodales.at.velit@Inscelerisque.org|Kohima|India|C383AF31-636B-7897-6CF3-6B261BCDE018|1036.0644769672|245424|2016-06-17 21:19:06|48378.6503012718
+Giacomo A. Puckett|91-859-732-83951|sit.amet@atpedeCras.com|Silchar|India|5E41A5A4-A4B3-17FE-F83D-6F6BF36B0F2D|979.2148681363|245423|2016-06-17 21:19:06|33445.0439158336
+Jamalia Z. Hensley|91-395-592-56855|interdum@cursus.co.uk|Munger|India|7B98AA41-BC35-497E-6F2E-BBA815F84B28|1033.5638665941|245425|2016-06-17 21:19:07|63406.9037231582
+Raja L. Barrett|91-213-236-37863|tristique.senectus@orciUtsemper.com|Agartala|India|730F660A-4A6E-65B3-D64D-6A0AD4BD16F2|1051.51196186|245426|2016-06-17 21:19:07|16656.8948714763
+Audra D. Haley|91-909-551-98035|nostra@velitjusto.net|Shimla|India|5F6BD395-CFB1-86A3-232B-3B461C11B032|985.3099956934|245427|2016-06-17 21:19:07|42576.4326009901
+Burke G. Velez|91-480-634-47735|dictum@eulacusQuisque.com|Shimla|India|C0D51785-79E5-86C1-2F60-6E2B4BC368E9|1014.3374004511|245428|2016-06-17 21:19:07|68099.3435062983
+Helen O. Watson|91-388-550-33596|accumsan.neque.et@sagittis.ca|Rewa|India|267EA362-E266-01FC-2371-065D4B150D6B|995.395922684|245429|2016-06-17 21:19:08|80594.4138667748
+Dillon E. Manning|91-412-483-18804|vulputate.eu@diam.net|Kohima|India|08BD8752-4C34-1C2F-7EC5-063BE9C0025F|988.0527516287|245430|2016-06-17 21:19:08|34997.7855937537
+Darryl H. Preston|91-900-557-62018|consequat@blanditatnisi.ca|Srinagar|India|BAE0CB80-33BC-23AD-4F23-6EED717A1FA9|1005.2310251378|245432|2016-06-17 21:19:08|90995.8552344935
+Teagan L. Merritt|91-755-577-05229|nec.luctus.felis@ornarelibero.ca|Daman|India|A65E89EC-BD39-F826-BF3A-2E4DFD1FFE96|1031.6923612578|245431|2016-06-17 21:19:08|76129.6754380442
+Blythe V. Hester|91-143-693-74169|faucibus@convallisconvallis.net|Nagarcoil|India|457DA7DF-8473-FDA2-DFA8-9CA6FE70AF53|980.0187576543|245433|2016-06-17 21:19:09|1792.7823874316
+Felicia Z. Maynard|91-177-925-06452|facilisis@nequetellus.edu|Jammu|India|7511020D-252A-60E3-0259-B55174DB2BE0|962.3470002516|245434|2016-06-17 21:19:09|27442.1888130468
+Sopoline B. Swanson|91-512-190-23174|mauris.aliquam@vitae.edu|Agartala|India|92A84BDE-A231-715C-5B0A-A2C52FE4C726|1000.4777986036|245436|2016-06-17 21:19:09|32101.1072345695
+Christen T. Cabrera|91-956-276-72371|a.dui@sit.org|Tiruvannamalai|India|1B06945D-2595-5445-2694-12341158C45B|997.3852528283|245435|2016-06-17 21:19:09|86664.8353687926
+Chancellor C. Villarreal|91-882-782-21856|Nullam.ut.nisi@Nam.edu|Pondicherry|India|33EBFDA7-8EEF-48A5-B323-AF7A7A57C4F9|984.5578096612|245437|2016-06-17 21:19:10|46584.2856419789
+Adara A. Price|91-334-227-26172|elementum@aliquameuaccumsan.ca|Shahjahanpur|India|82F22680-CD3B-F1E7-45F9-8E2ADAA5ED42|988.0619905918|245438|2016-06-17 21:19:10|21626.2511858243
+Dana B. Lucas|91-781-212-83146|scelerisque.mollis.Phasellus@Sedeunibh.co.uk|Gangtok|India|88223C00-DCFA-F40B-AC4C-97FA79062789|976.7241552536|245440|2016-06-17 21:19:10|76481.528985468
+Alma X. Ingram|91-944-103-76290|senectus.et.netus@orci.net|Jammu|India|4C0C1266-DE7B-6FE7-6FFF-E0263987C15B|975.6210776138|245439|2016-06-17 21:19:10|64898.0024641107
+Cullen P. Glenn|91-388-717-53901|magna.Praesent@loremlorem.co.uk|Kohima|India|BA6BF2F4-8F94-C4C2-740E-2A3B587C4B3A|1031.9092818916|245442|2016-06-17 21:19:11|9437.7993039509
+Darius S. Allen|91-978-355-75025|mi.lacinia.mattis@enimSuspendissealiquet.co.uk|Trivandrum|India|D59B0CBB-A08D-DE3B-DE5F-426C0C96CC59|1032.6395532022|245441|2016-06-17 21:19:11|12454.7012410075
+Jordan J. Frye|91-399-558-07454|ridiculus.mus@idlibero.ca|Chandigarh|India|EBCC5349-6EC4-356A-F897-20BFBF220087|1019.8537249814|245444|2016-06-17 21:19:11|32040.2035007451
+August N. Chase|91-447-831-17890|libero.at.auctor@Vivamuseuismodurna.org|Agartala|India|8871CD66-C471-B709-61C4-DA8E11395864|1016.3890473484|245443|2016-06-17 21:19:11|45318.9872586436
+Teegan Z. House|91-834-850-62568|nunc.est.mollis@Nunc.edu|Aizwal|India|13B85BBD-EA23-70B2-1723-9F1D6046B42E|959.962281601|245445|2016-06-17 21:19:12|88784.9082302192
+Aquila B. Moran|91-851-475-57283|sapien.Cras.dolor@Suspendissetristiqueneque.com|Thalassery|India|823590C6-3381-1E11-8303-C3DA2A0E1821|993.9503426924|245446|2016-06-17 21:19:12|85906.741529358
+Dominic T. Dalton|91-504-884-93067|Ut@congue.co.uk|Rohtak|India|4EC579B1-4095-421D-27D5-95BD94551B81|988.6711045096|245448|2016-06-17 21:19:12|96034.4758503845
+Francis C. Gay|91-155-131-31144|elementum@CuraeDonec.co.uk|Panjim|India|BEA0E8B8-2153-55C3-9FAC-B645F75DC11F|1041.9520701363|245447|2016-06-17 21:19:12|28285.6913337028
+Florence K. Cox|91-509-817-87497|adipiscing.enim@ornarelectusjusto.edu|Bhilai|India|C613A6CF-0DDF-457B-1611-048AF476CD9F|984.7873386645|245449|2016-06-17 21:19:13|8045.8599610703
+Desirae F. Mooney|91-146-543-19767|ultricies@scelerisquenequeNullam.edu|Gangtok|India|58A86FB7-1F20-2C33-2682-9B20165B5CF9|965.4688425994|245450|2016-06-17 21:19:13|25829.5686136197
+Virginia O. Welch|91-338-235-55295|ultrices.posuere.cubilia@massanonante.co.uk|Srinagar|India|350C6D1C-8C2D-BE4A-EAB5-8098C2BC8054|981.8861112513|245451|2016-06-17 21:19:13|55435.8774902126
+Simon P. Estes|91-101-148-34019|a.aliquet@tortordictum.edu|Kavaratti|India|D42E0FB2-6093-FD6F-EA2E-DC470647340F|961.8009052379|245452|2016-06-17 21:19:13|26959.0061609105
+Garth M. Fuentes|91-401-871-57104|gravida@semperegestasurna.net|Vejalpur|India|1450A844-53D1-6315-4415-5C75B08FC228|974.9367040591|245454|2016-06-17 21:19:14|93043.0249827518
+Jaquelyn D. Rodgers|91-913-783-42186|lobortis.nisi.nibh@Donec.edu|Nagpur|India|D6CE7D75-0659-454A-40DF-3AF296DD4C58|976.7352615741|245453|2016-06-17 21:19:14|59609.6850497366
+Kasper F. Hatfield|91-632-379-95960|posuere@vulputatenisisem.ca|Korba|India|D59B8BE9-DA20-7B57-9942-2C15BCF28603|1000.4924219911|245456|2016-06-17 21:19:14|3275.3243581371
+Isaiah A. Warner|91-626-859-53011|diam.Sed@facilisisfacilisismagna.edu|Daman|India|20B5FE0D-0965-1E2D-681B-5E688DA1CA65|1001.1159994583|245455|2016-06-17 21:19:14|76836.6239820194
+Keaton Y. Solis|91-301-918-77318|sit.amet.nulla@elitelitfermentum.co.uk|Rajkot|India|16CFA137-3FD3-8D3D-7607-54577A6A18AB|987.5936149956|245458|2016-06-17 21:19:15|89465.3972332774
+Regan D. Underwood|91-869-304-48740|ornare@Curabituregestas.org|Daman|India|241FD5E5-01D8-7BEA-BE3A-3435B1D6577F|967.2325392222|245457|2016-06-17 21:19:15|81430.0742569319
+Laurel K. Hardy|91-535-589-44540|rutrum@penatibuset.edu|Shimla|India|46689123-0B18-3A05-2E9D-1995EABE674F|947.0982666227|245459|2016-06-17 21:19:15|14451.1023722487
+Vielka J. Roberson|91-545-218-53793|scelerisque.lorem.ipsum@Sed.org|Srinagar|India|E6384D6E-2AC4-3AF7-8C1C-A37DD4B013C6|933.9068440201|245460|2016-06-17 21:19:15|75600.2030709746
+Brynn S. Morrison|91-977-938-06240|natoque@enimgravidasit.edu|Daman|India|859E9B35-3A21-F7FD-A9A5-167F7FE6A7F5|1002.7243083243|245461|2016-06-17 21:19:16|19882.1569818146
+Graiden T. Jones|91-868-733-90988|lobortis.ultrices.Vivamus@faucibus.org|Nellore|India|C133E546-1E36-A6EF-9980-BB635F247290|973.7286142544|245462|2016-06-17 21:19:16|4393.9931397479
+Tanya X. Dominguez|91-553-110-11147|odio.Nam@antedictum.ca|Anantapur|India|DEF2CE91-CAF2-88BA-C50B-AB7EA1D73C63|959.1678617895|245464|2016-06-17 21:19:16|42536.8767788127
+Anne D. Pace|91-188-230-70605|pede.et.risus@Ut.co.uk|Agartala|India|32FC1F14-21F7-E940-7C9D-B80F44EE4063|1021.5754177485|245463|2016-06-17 21:19:16|32755.1536052658
+Hanna I. Sellers|91-322-901-42448|tincidunt.dui.augue@augue.org|Barddhaman|India|3D4F2FBC-FFCE-EE5C-5D2C-C60856D33B7D|993.2866819089|245466|2016-06-17 21:19:17|75662.1622559809
+Christine R. Gross|91-380-558-08928|convallis.convallis@convalliserateget.org|Shillong|India|DD81CFAC-DE87-E8A1-2850-E2637912543C|951.970651195|245465|2016-06-17 21:19:17|80146.9990770126
+Kirby A. Morse|91-299-538-45635|elit.Nulla.facilisi@acmattisvelit.net|Itanagar|India|CFF08336-B5C9-EECC-1F2D-07ADE42587FE|1009.5054026816|245467|2016-06-17 21:19:17|72602.1251286646
+Ryder B. Vega|91-320-455-77145|Aliquam.gravida@elit.co.uk|Gondiya|India|15F3478F-4D7E-562C-E189-945E655641E9|1006.8962892921|245468|2016-06-17 21:19:17|8089.8058610084
+Demetrius Q. Pope|91-179-308-28350|tempus@commodotinciduntnibh.ca|Imphal|India|CBF9C119-3E02-13F0-842A-BAF0AB46382D|1042.9498091927|245470|2016-06-17 21:19:18|83959.394047181
+Bryar M. Hendricks|91-617-948-40647|dui.Fusce.aliquam@loremvitaeodio.net|Kanchrapara|India|64188EFF-B7B1-AF75-EA3D-013CCA945F81|968.5880217334|245469|2016-06-17 21:19:18|85996.1671440447
+Ann P. Mann|91-395-300-37963|Cum@gravidanon.edu|Imphal|India|12818782-A86C-63A2-AB08-A5F281DA954B|937.3545836376|245472|2016-06-17 21:19:18|83236.1732288256
+Tyrone N. Bender|91-515-367-84014|at@fringillaporttitorvulputate.com|Chandigarh|India|D5109D0E-7DB6-A084-F0F9-E2BEF2B10E4A|1013.1764242663|245471|2016-06-17 21:19:18|36038.3036732134
+Miranda W. Sykes|91-774-614-92485|amet@accumsan.com|Gangtok|India|6EC7240D-5331-5F34-D1B8-38E2024A1925|992.518070982|245473|2016-06-17 21:19:19|97197.8457445721
+Ralph F. Walker|91-815-285-72212|molestie.in@vestibulumlorem.ca|Farrukhabad-cum-Fatehgarh|India|B7982CE7-B815-2794-504B-A8C3ECF0B592|992.3386856766|245474|2016-06-17 21:19:19|60698.1579345159
+Kelsie J. Mack|91-112-975-50329|pede.ac.urna@ligula.com|Hazaribag|India|5CACD398-932B-449D-80AC-5302AE35E027|1027.0394067971|245475|2016-06-17 21:19:19|48589.914064424
+Zenia F. Bond|91-445-745-77345|pede.Cum.sociis@Pellentesquetincidunt.com|Port Blair|India|68EDFE31-8E62-FF3B-3FBA-49B436D7142C|996.5071564618|245476|2016-06-17 21:19:19|40633.7414556007
+Todd V. Barton|91-312-429-52379|mauris.a@tristique.org|Dehradun|India|7D22103A-1DA0-2E4E-BFD7-8117133A983A|977.4969081079|245477|2016-06-17 21:19:20|49497.7649558266
+Kerry O. Franco|91-802-106-72427|Ut.sagittis.lobortis@dictumeueleifend.ca|Kohima|India|FC6049B3-9FD4-9A53-1A7E-C50394D4F4CD|1004.0387939017|245478|2016-06-17 21:19:20|43754.1503070853
+April I. Clarke|91-785-400-08200|sapien@mattisCras.edu|Daman|India|363E468F-BD44-6607-CDD6-226A6FB48B89|990.0985191984|245479|2016-06-17 21:19:20|90752.7220658782
+Randall B. Workman|91-722-286-44955|ut@Integervitae.net|Gujrat|India|1A402674-FD62-8F1B-7AE1-CFAA7E85006A|1012.6352839308|245480|2016-06-17 21:19:20|1815.8797213967
+Blaine J. Martinez|91-987-901-22060|libero.mauris.aliquam@egetlaoreet.co.uk|Kohima|India|7FDD906D-AB41-89EC-D546-B1E04D31FE49|1020.339672989|245481|2016-06-17 21:19:21|51409.0038738681
+May B. Schroeder|91-123-334-23814|aliquet@leo.edu|Orai|India|2E83862F-C3A4-4A4B-4D9C-C60AC4435A04|982.3424542846|245482|2016-06-17 21:19:21|89159.6292602544
+Whitney N. Tran|91-923-716-39943|Aliquam@leoMorbi.edu|Purulia|India|FEE99492-4B48-533F-EB36-5EA3528A121C|1007.9029941609|245483|2016-06-17 21:19:21|24692.473001632
+Ingrid A. Head|91-596-380-43300|ac@montes.ca|Port Blair|India|735007BF-1F13-95E9-7C3E-536C0452714D|995.4883512745|245484|2016-06-17 21:19:21|54890.8030672057
+Berk E. Gould|91-110-540-75246|ipsum@Nulla.org|Alwar|India|FB0297F7-8BC1-86F0-8CCD-C01538B7E5BE|984.2152917929|245486|2016-06-17 21:19:22|70311.7302535036
+Meredith G. Parks|91-154-792-20142|et.eros@auctor.com|Chandigarh|India|CD2691BE-ADB4-87DF-CA3B-88512330E745|960.8832140455|245485|2016-06-17 21:19:22|95293.2215958629
+Cora K. Carrillo|91-786-268-62522|elementum.lorem.ut@eutelluseu.ca|Dindigul|India|CAA843B7-F7C8-AD16-3F05-89DADB6EB288|1001.8761512414|245488|2016-06-17 21:19:22|59127.2684732433
+Galena D. Farmer|91-951-409-02162|sem@dictum.co.uk|Shillong|India|A46AF538-4BEF-D24F-870F-DD4CD20F4DC6|1022.2222771884|245487|2016-06-17 21:19:22|36667.9833050979
+Mollie E. Snow|91-816-823-87138|lectus@pharetra.org|Tiruvannamalai|India|A66AFB86-5297-61AC-2A2A-87C23F0E96E1|966.8626148916|245490|2016-06-17 21:19:23|28859.7500576673
+Abra E. Mccray|91-100-997-13066|Morbi.quis@risusDuis.net|Delhi|India|E49C0B30-586F-ECAA-FCBD-8F2898AA7771|1043.6824086302|245489|2016-06-17 21:19:23|79521.2870470628
+Andrew I. Lowe|91-859-767-29836|sed.turpis@Sed.co.uk|Aizwal|India|A928DE5C-6C18-9B9B-C4F0-3165F2E53C19|955.4198061291|245491|2016-06-17 21:19:23|34946.6776395577
+Zachery B. Marshall|91-103-978-47466|nonummy.Fusce@dui.co.uk|Vijayawada|India|E0D79F5F-2CB6-B9FC-B917-2D627141423B|969.8127419963|245492|2016-06-17 21:19:23|74671.8685208157
+Miranda M. Madden|91-310-815-30233|id@pharetranibh.co.uk|Brahmapur|India|55E04315-DE60-1C1D-78D6-878C9488BA06|963.5417067954|245493|2016-06-17 21:19:24|42089.8687934832
+Ian N. Johnson|91-651-594-79593|elit@pellentesquemassa.ca|Agartala|India|EDBCCD62-4871-4938-60C5-0CD3023CF333|1012.7053923589|245494|2016-06-17 21:19:24|97376.7240252239
+Macey V. Hinton|91-358-832-49086|eget.odio@Nunclectuspede.net|Sambalpur|India|49E4842E-0EB8-D48A-C1EC-A3F0FBAC92B4|982.9530728621|245496|2016-06-17 21:19:24|61850.2465402803
+Talon M. Calhoun|91-280-397-90948|lacinia.Sed@tinciduntvehicularisus.net|Pallavaram|India|E743767A-B1FF-C71C-B0D4-4C106D65AE3B|1040.9448702398|245495|2016-06-17 21:19:24|49928.6243188531
+Dillon A. Reese|91-900-545-84875|Mauris.vestibulum@sagittis.edu|Itanagar|India|9429B72A-36AA-8C78-2988-7FF48315094A|1029.5299833946|245498|2016-06-17 21:19:25|86990.7236411972
+Boris E. Boyd|91-480-944-74996|luctus.Curabitur.egestas@utmolestie.co.uk|Panjim|India|37962E2E-4B3D-27DD-1213-4FF1B5551CB4|969.407095963|245497|2016-06-17 21:19:25|32987.5284224486
+Inez U. Higgins|91-258-514-16019|Aliquam@utlacus.co.uk|Itanagar|India|D48C171C-A009-3813-5951-50A526BE6B58|1005.0207448325|245500|2016-06-17 21:19:25|68069.3550977988
+Leroy N. Snyder|91-682-436-52541|nisl.Maecenas@quam.com|Itanagar|India|BE155410-1F6D-AC60-C0B9-290E9426D5CF|993.1732235668|245499|2016-06-17 21:19:25|64376.3818134466
+George M. Case|91-760-901-60977|at.iaculis.quis@auctorvelit.com|Pondicherry|India|B924E744-29E3-8B6E-3EE9-A48A3BC9072B|987.476080645|245501|2016-06-17 21:19:26|97859.0229750841
+Kane E. Simon|91-461-207-64947|adipiscing.lobortis@tellus.ca|Haridwar|India|13C1AD7F-48D5-C6E5-892B-0345E7E070DB|983.5254219297|245502|2016-06-17 21:19:26|21912.3442110069
+Wanda U. Talley|91-398-803-35370|ipsum.Curabitur@eleifendnecmalesuada.com|Gangtok|India|5E2ED32D-621C-0ACE-71B6-F2F9746DD0C9|994.6344833977|245503|2016-06-17 21:19:26|91796.2772584239
+Asher U. Mccullough|91-149-955-32468|eget.massa.Suspendisse@Pellentesqueultricies.edu|Itanagar|India|3FA3514B-6974-06E3-EF7B-4841487A40D2|985.0811909206|245504|2016-06-17 21:19:26|71603.8572439688
+Malik E. Keller|91-391-507-50089|nec@Quisqueporttitor.net|Daman|India|05CDC866-65A3-E55C-EEE8-C73217F2071A|986.9798931399|245505|2016-06-17 21:19:27|65100.1941324091
+Yasir V. Howell|91-901-430-36468|nec.cursus.a@Cumsociisnatoque.ca|Mandya|India|F90D3C87-4C53-A5E9-D3F8-692C068CBCED|997.1935636906|245506|2016-06-17 21:19:27|35511.2898055096
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/11f37671/examples/recordReader/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/recordReader/src/main/resources/META-INF/properties.xml b/examples/recordReader/src/main/resources/META-INF/properties.xml
new file mode 100644
index 0000000..06961f1
--- /dev/null
+++ b/examples/recordReader/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,159 @@
+<?xml version="1.0"?>
+<configuration>
+ <!--
+ <property>
+ <name>dt.application.{appName}.operator.{opName}.prop.{propName}</name>
+ <value>some-default-value (if value is not specified, it is required from the user or custom config when launching)</value>
+ </property>
+ -->
+ <!-- memory assigned to app master
+ <property>
+ <name>dt.attr.MASTER_MEMORY_MB</name>
+ <value>1024</value>
+ </property>
+ -->
+ <property>
+ <name>dt.application.RecordReaderExample.operator.recordReader.prop.files</name>
+ <value>src/main/resources/META-INF/CustomerTxnData</value>
+ </property>
+ <property>
+ <name>dt.application.RecordReaderExample.attr.CHECKPOINT_WINDOW_COUNT</name>
+ <value>10</value>
+ </property>
+ <property>
+ <name>dt.application.RecordReaderExample.operator.csvParser.prop.schema</name>
+ <value>{
+ "separator": "|",
+ "quoteChar": "\"",
+ "fields": [
+ {
+ "name": "customerName",
+ "type": "String"
+ },
+ {
+ "name": "customerPhone",
+ "type": "String"
+ },
+ {
+ "name": "customerEmail",
+ "type": "String"
+ },
+ {
+ "name": "city",
+ "type": "String"
+ },
+ {
+ "name": "country",
+ "type": "String"
+ },
+ {
+ "name": "uid",
+ "type": "String"
+ },
+ {
+ "name": "accountNumber",
+ "type": "String"
+ },
+ {
+ "name": "txId",
+ "type": "long"
+ },
+ {
+ "name": "txDate",
+ "type": "Date",
+ "constraints" : {
+ "format" : "yyyy-MM-dd HH:mm:ss"
+ }
+ },
+ {
+ "name": "txAmount",
+ "type": "double"
+ }
+ ]
+ }
+ </value>
+ </property>
+ <property>
+ <name>dt.application.RecordReaderExample.operator.formatter.prop.schema</name>
+ <value>{
+ "separator": "|",
+ "quoteChar": "\"",
+ "lineDelimiter": "",
+ "fields": [
+ {
+ "name": "customerName",
+ "type": "String"
+ },
+ {
+ "name": "customerPhone",
+ "type": "String"
+ },
+ {
+ "name": "customerEmail",
+ "type": "String"
+ },
+ {
+ "name": "city",
+ "type": "String"
+ },
+ {
+ "name": "country",
+ "type": "String"
+ },
+ {
+ "name": "uid",
+ "type": "String"
+ },
+ {
+ "name": "accountNumber",
+ "type": "String"
+ },
+ {
+ "name": "txId",
+ "type": "long"
+ },
+ {
+ "name": "txDate",
+ "type": "Date",
+ "constraints" : {
+ "format" : "yyyy-MM-dd HH:mm:ss"
+ }
+ },
+ {
+ "name": "txAmount",
+ "type": "double"
+ }
+ ]
+ }
+ </value>
+ </property>
+ <property>
+ <name>dt.application.RecordReaderExample.operator.csvParser.port.out.attr.TUPLE_CLASS</name>
+ <value>com.example.recordReader.TransactionsSchema</value>
+ </property>
+ <property>
+ <name>dt.application.RecordReaderExample.operator.formatter.port.in.attr.TUPLE_CLASS</name>
+ <value>com.example.recordReader.TransactionsSchema</value>
+ </property>
+ <property>
+ <name>dt.application.RecordReaderExample.operator.fileOutput.prop.filePath</name>
+ <value>/tmp</value>
+ </property>
+ <property>
+ <name>dt.application.RecordReaderExample.operator.fileOutput.prop.outputFileName</name>
+ <value>output.txt</value>
+ </property>
+ <property>
+ <name>dt.application.RecordReaderExample.operator.fileOutput.prop.tupleSeparator</name>
+ <value></value>
+ </property>
+ <property>
+ <name>dt.application.RecordReaderExample.operator.fileOutput.prop.maxIdleWindows</name>
+ <value>2</value>
+ </property>
+ <property>
+ <name>dt.loggers.level</name>
+ <value>com.datatorrent.*:DEBUG,org.apache.apex.malhar.*:DEBUG</value>
+ </property>
+</configuration>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/11f37671/examples/recordReader/src/test/java/com/example/recordReader/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/recordReader/src/test/java/com/example/recordReader/ApplicationTest.java b/examples/recordReader/src/test/java/com/example/recordReader/ApplicationTest.java
new file mode 100644
index 0000000..222ad06
--- /dev/null
+++ b/examples/recordReader/src/test/java/com/example/recordReader/ApplicationTest.java
@@ -0,0 +1,91 @@
+package com.example.recordReader;
+
+import java.io.File;
+import java.io.IOException;
+
+import javax.validation.ConstraintViolationException;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.LocalMode;
+
+/**
+ * Test application in local mode.
+ */
+public class ApplicationTest
+{
+ private String outputDir;
+
+ public static class TestMeta extends TestWatcher
+ {
+ public String baseDirectory;
+
+ @Override
+ protected void starting(org.junit.runner.Description description)
+ {
+ this.baseDirectory = "target/" + description.getClassName() + "/" + description.getMethodName();
+ }
+
+ @Override
+ protected void finished(Description description)
+ {
+ super.finished(description);
+ try {
+ FileUtils.forceDelete(new File(baseDirectory));
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ }
+
+ @Rule
+ public TestMeta testMeta = new TestMeta();
+
+ @Before
+ public void setup() throws Exception
+ {
+ outputDir = testMeta.baseDirectory + File.separator + "output";
+ }
+
+ @Test
+ public void testApplication() throws IOException, Exception
+ {
+ try {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ conf.set("dt.application.RecordReaderExample.operator.fileOutput.prop.filePath", outputDir);
+ File outputfile = FileUtils.getFile(outputDir, "output.txt_5.0");
+
+ lma.prepareDAG(new Application(), conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.runAsync();
+
+ // wait for tuples to show up
+ while (!outputfile.exists()) {
+ System.out.println("Sleeping ....");
+ Thread.sleep(1000);
+ }
+
+ lc.shutdown();
+ Assert.assertTrue(
+ FileUtils.contentEquals(
+ FileUtils.getFile(
+ conf.get("dt.application.RecordReaderExample.operator.recordReader.prop.files")
+ ),outputfile));
+
+ } catch (ConstraintViolationException e) {
+ Assert.fail("constraint violations: " + e.getConstraintViolations());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/11f37671/examples/recordReader/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/recordReader/src/test/resources/log4j.properties b/examples/recordReader/src/test/resources/log4j.properties
new file mode 100644
index 0000000..3bfcdc5
--- /dev/null
+++ b/examples/recordReader/src/test/resources/log4j.properties
@@ -0,0 +1,21 @@
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+log4j.logger.org=info
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=debug
[04/19] apex-malhar git commit: SPOI-8728 Json Parser
Posted by th...@apache.org.
SPOI-8728 Json Parser
Project: http://git-wip-us.apache.org/repos/asf/apex-malhar/repo
Commit: http://git-wip-us.apache.org/repos/asf/apex-malhar/commit/8e200974
Tree: http://git-wip-us.apache.org/repos/asf/apex-malhar/tree/8e200974
Diff: http://git-wip-us.apache.org/repos/asf/apex-malhar/diff/8e200974
Branch: refs/heads/master
Commit: 8e200974435c30c498760ab669f39a1dbca8b270
Parents: 11f3767
Author: shubham <sh...@github.com>
Authored: Tue Jul 19 16:27:31 2016 +0530
Committer: Lakshmi Prasanna Velineni <la...@datatorrent.com>
Committed: Sun Mar 26 11:43:48 2017 -0700
----------------------------------------------------------------------
examples/parser/README.md | 37 +++
examples/parser/XmlJavadocCommentsExtractor.xsl | 44 +++
examples/parser/pom.xml | 309 +++++++++++++++++++
examples/parser/src/assemble/appPackage.xml | 43 +++
.../tutorial/jsonparser/Application.java | 35 +++
.../tutorial/jsonparser/Campaign.java | 74 +++++
.../tutorial/jsonparser/JsonGenerator.java | 83 +++++
.../src/main/resources/META-INF/properties.xml | 40 +++
.../tutorial/jsonparser/ApplicationTest.java | 36 +++
.../parser/src/test/resources/log4j.properties | 22 ++
10 files changed, 723 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/8e200974/examples/parser/README.md
----------------------------------------------------------------------
diff --git a/examples/parser/README.md b/examples/parser/README.md
new file mode 100644
index 0000000..da230e3
--- /dev/null
+++ b/examples/parser/README.md
@@ -0,0 +1,37 @@
+This project contains applications showcasing different **Parsers** and **Formatters** present in the Malhar Library. For all the apps, parameters are configurable in META_INF/properties.xml.
+
+* **Json Parser App**
+This app showcases **Json Parser**. Data generator sends Json data to the Json Parser which emits each record as POJO on the *output* port. The parser also has *parsedOutput* port that outputs each record as JSONObject and *error* port that emits a error records as key value pair.
+
+* **Csv Parser App**
+
+This application showcases how to use [CsvParser](https://datatorrent.com/docs/apidocs/com/datatorrent/contrib/parser/CsvParser.html) from [Apex Malhar](https://github.com/apache/apex-malhar) library. The CsvParser converts your delimited data to a key-value pair map or concrete java class also know as [POJO](https://en.wikipedia.org/wiki/Plain_Old_Java_Object). The parser emits key-value pair map on *parsedOutput* port. It emits POJO on *out* and error records on *err* port.
+
+Follow these steps to run this application:
+
+**Step 1**: Build the code:
+
+ shell> mvn clean install
+
+**Step 2**: Upload the `target/parser-1.0-SNAPSHOT.apa` to the UI console if available or launch it from
+the commandline using `apex` cli script.
+
+**Step 3**: During launch use `src/main/resources/META-INF/properties-csvParseApplication.xml` as a custom configuration file; then verify
+that the output by checking hdfs file path configured in properties-csvParseApplication.xml
+
+* **Xml Parser App**
+
+This application showcases how to use [XmlParser](https://datatorrent.com/docs/apidocs/com/datatorrent/lib/parser/XmlParser.html)
+from [Apex Malhar](https://github.com/apache/apex-malhar) library. The XmlParser Operator converts XML string to POJO.
+The parser emits dom based Document on *parsedOutput* port. It emits POJO on *out* and error records on *err* port.
+Follow these steps to run this application:
+
+**Step 1**: Build the code:
+
+ shell> mvn clean install
+
+**Step 2**: Upload the `target/parser-1.0-SNAPSHOT.apa` to the UI console if available or launch it from
+the commandline using `apex` cli script.
+
+**Step 3**: During launch use `src/main/resources/META-INF/properties-xmlParseApplication.xml` as a custom configuration file; then verify
+that the output by checking hdfs file path configured in properties-xmlParseApplication.xml
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/8e200974/examples/parser/XmlJavadocCommentsExtractor.xsl
----------------------------------------------------------------------
diff --git a/examples/parser/XmlJavadocCommentsExtractor.xsl b/examples/parser/XmlJavadocCommentsExtractor.xsl
new file mode 100644
index 0000000..08075a9
--- /dev/null
+++ b/examples/parser/XmlJavadocCommentsExtractor.xsl
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+
+<!--
+ Document : XmlJavadocCommentsExtractor.xsl
+ Created on : September 16, 2014, 11:30 AM
+ Description:
+ The transformation strips off all information except for comments and tags from xml javadoc generated by xml-doclet.
+-->
+
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+ <xsl:output method="xml" standalone="yes"/>
+
+ <!-- copy xml by selecting only the following nodes, attributes and text -->
+ <xsl:template match="node()|text()|@*">
+ <xsl:copy>
+ <xsl:apply-templates select="root|package|class|interface|method|field|type|comment|tag|text()|@name|@qualified|@text"/>
+ </xsl:copy>
+ </xsl:template>
+
+ <!-- Strip off the following paths from the selected xml -->
+ <xsl:template match="//root/package/interface/interface
+ |//root/package/interface/method/@qualified
+ |//root/package/class/interface
+ |//root/package/class/class
+ |//root/package/class/method/@qualified
+ |//root/package/class/field/@qualified" />
+
+ <xsl:strip-space elements="*"/>
+</xsl:stylesheet>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/8e200974/examples/parser/pom.xml
----------------------------------------------------------------------
diff --git a/examples/parser/pom.xml b/examples/parser/pom.xml
new file mode 100644
index 0000000..4eefe75
--- /dev/null
+++ b/examples/parser/pom.xml
@@ -0,0 +1,309 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>com.example</groupId>
+ <version>1.0-SNAPSHOT</version>
+ <artifactId>parser</artifactId>
+ <packaging>jar</packaging>
+
+ <!-- change these to the appropriate values -->
+ <name>Parser Apps</name>
+ <description>Applications to showcase different parsers</description>
+
+ <properties>
+ <!-- change this if you desire to use a different version of Apex Core -->
+ <apex.version>3.5.0</apex.version>
+ <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath>
+ <malhar.version>3.7.0-SNAPSHOT</malhar.version>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-eclipse-plugin</artifactId>
+ <version>2.9</version>
+ <configuration>
+ <downloadSources>true</downloadSources>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.3</version>
+ <configuration>
+ <encoding>UTF-8</encoding>
+ <source>1.7</source>
+ <target>1.7</target>
+ <debug>true</debug>
+ <optimize>false</optimize>
+ <showDeprecation>true</showDeprecation>
+ <showWarnings>true</showWarnings>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.8</version>
+ <executions>
+ <execution>
+ <id>copy-dependencies</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>target/deps</outputDirectory>
+ <includeScope>runtime</includeScope>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>app-package-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <finalName>${project.artifactId}-${project.version}-apexapp</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <descriptors>
+ <descriptor>src/assemble/appPackage.xml</descriptor>
+ </descriptors>
+ <archiverConfig>
+ <defaultDirectoryMode>0755</defaultDirectoryMode>
+ </archiverConfig>
+ <archive>
+ <manifestEntries>
+ <Class-Path>${apex.apppackage.classpath}</Class-Path>
+ <DT-Engine-Version>${apex.version}</DT-Engine-Version>
+ <DT-App-Package-Group-Id>${project.groupId}</DT-App-Package-Group-Id>
+ <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name>
+ <DT-App-Package-Version>${project.version}</DT-App-Package-Version>
+ <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name>
+ <DT-App-Package-Description>${project.description}</DT-App-Package-Description>
+ </manifestEntries>
+ </archive>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <configuration>
+ <target>
+ <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar"
+ tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa" />
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ <execution>
+ <!-- create resource directory for xml javadoc-->
+ <id>createJavadocDirectory</id>
+ <phase>generate-resources</phase>
+ <configuration>
+ <tasks>
+ <delete dir="${project.build.directory}/generated-resources/xml-javadoc"/>
+ <mkdir dir="${project.build.directory}/generated-resources/xml-javadoc"/>
+ </tasks>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.9.1</version>
+ <executions>
+ <execution>
+ <id>attach-artifacts</id>
+ <phase>package</phase>
+ <goals>
+ <goal>attach-artifact</goal>
+ </goals>
+ <configuration>
+ <artifacts>
+ <artifact>
+ <file>target/${project.artifactId}-${project.version}.apa</file>
+ <type>apa</type>
+ </artifact>
+ </artifacts>
+ <skipAttach>false</skipAttach>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- generate javdoc -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <executions>
+ <!-- generate xml javadoc -->
+ <execution>
+ <id>xml-doclet</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>javadoc</goal>
+ </goals>
+ <configuration>
+ <doclet>com.github.markusbernhardt.xmldoclet.XmlDoclet</doclet>
+ <additionalparam>-d ${project.build.directory}/generated-resources/xml-javadoc -filename ${project.artifactId}-${project.version}-javadoc.xml</additionalparam>
+ <useStandardDocletOptions>false</useStandardDocletOptions>
+ <docletArtifact>
+ <groupId>com.github.markusbernhardt</groupId>
+ <artifactId>xml-doclet</artifactId>
+ <version>1.0.4</version>
+ </docletArtifact>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <!-- Transform xml javadoc to stripped down version containing only class/interface comments and tags-->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>xml-maven-plugin</artifactId>
+ <version>1.0</version>
+ <executions>
+ <execution>
+ <id>transform-xmljavadoc</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>transform</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <transformationSets>
+ <transformationSet>
+ <dir>${project.build.directory}/generated-resources/xml-javadoc</dir>
+ <includes>
+ <include>${pmalhar.versioroject.artifactId}-${project.version}-javadoc.xml</include>
+ </includes>
+ <stylesheet>XmlJavadocCommentsExtractor.xsl</stylesheet>
+ <outputDir>${project.build.directory}/generated-resources/xml-javadoc</outputDir>
+ </transformationSet>
+ </transformationSets>
+ </configuration>
+ </plugin>
+ <!-- copy xml javadoc to class jar -->
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>2.6</version>
+ <executions>
+ <execution>
+ <id>copy-resources</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${basedir}/target/classes</outputDirectory>
+ <resources>
+ <resource>
+ <directory>${project.build.directory}/generated-resources/xml-javadoc</directory>
+ <includes>
+ <include>${project.artifactId}-${project.version}-javadoc.xml</include>
+ </includes>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+
+ </build>
+
+ <dependencies>
+ <!-- add your dependencies here -->
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-library</artifactId>
+ <version>${malhar.version}</version>
+ <!--
+ If you know that your application does not need transitive dependencies pulled in by malhar-library,
+ uncomment the following to reduce the size of your app package.
+ -->
+ <!--
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ -->
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-contrib</artifactId>
+ <version>${malhar.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>2.5.4</version>
+ </dependency>
+ <dependency>
+ <groupId>com.github.fge</groupId>
+ <artifactId>json-schema-validator</artifactId>
+ <version>2.0.1</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>net.sf.supercsv</groupId>
+ <artifactId>super-csv</artifactId>
+ <version>2.4.0</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>com.univocity</groupId>
+ <artifactId>univocity-parsers</artifactId>
+ <version>2.0.0</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.janino</groupId>
+ <artifactId>janino</artifactId>
+ <version>2.7.8</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>apex-common</artifactId>
+ <version>${apex.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.10</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>apex-engine</artifactId>
+ <version>${apex.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/8e200974/examples/parser/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/parser/src/assemble/appPackage.xml b/examples/parser/src/assemble/appPackage.xml
new file mode 100644
index 0000000..7ad071c
--- /dev/null
+++ b/examples/parser/src/assemble/appPackage.xml
@@ -0,0 +1,43 @@
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>appPackage</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/target/</directory>
+ <outputDirectory>/app</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/target/deps</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/site/conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/META-INF</directory>
+ <outputDirectory>/META-INF</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/app</directory>
+ <outputDirectory>/app</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/resources</directory>
+ <outputDirectory>/resources</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/8e200974/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/Application.java
----------------------------------------------------------------------
diff --git a/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/Application.java b/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/Application.java
new file mode 100644
index 0000000..e75541d
--- /dev/null
+++ b/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/Application.java
@@ -0,0 +1,35 @@
+package com.datatorrent.tutorial.jsonparser;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.contrib.parser.JsonParser;
+import com.datatorrent.lib.formatter.JsonFormatter;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+
+@ApplicationAnnotation(name = "JsonProcessor")
+public class Application implements StreamingApplication
+{
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+
+ JsonGenerator generator = dag.addOperator("JsonGenerator", JsonGenerator.class);
+ JsonParser parser = dag.addOperator("JsonParser", JsonParser.class);
+ JsonFormatter formatter = dag.addOperator("JsonFormatter", JsonFormatter.class);
+
+ ConsoleOutputOperator jsonString = dag.addOperator("JsonString", ConsoleOutputOperator.class);
+ ConsoleOutputOperator jsonObject = dag.addOperator("JsonObject", ConsoleOutputOperator.class);
+ ConsoleOutputOperator error = dag.addOperator("Error", ConsoleOutputOperator.class);
+
+ dag.addStream("json", generator.out, parser.in);
+ dag.addStream("pojo", parser.out, formatter.in);
+ dag.addStream("jsonString", formatter.out, jsonString.input);
+ dag.addStream("jsonObject", parser.parsedOutput, jsonObject.input);
+ dag.addStream("error", parser.err, error.input);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/8e200974/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/Campaign.java
----------------------------------------------------------------------
diff --git a/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/Campaign.java b/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/Campaign.java
new file mode 100644
index 0000000..f0245eb
--- /dev/null
+++ b/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/Campaign.java
@@ -0,0 +1,74 @@
+package com.datatorrent.tutorial.jsonparser;
+
+import java.util.Date;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class Campaign
+{
+ private int adId;
+ private String campaignName;
+ @JsonProperty("budget")
+ private double campaignBudget;
+ private boolean weatherTargeting;
+ @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy/MM/dd")
+ private Date startDate;
+
+ public int getAdId()
+ {
+ return adId;
+ }
+
+ public void setAdId(int adId)
+ {
+ this.adId = adId;
+ }
+
+ public String getCampaignName()
+ {
+ return campaignName;
+ }
+
+ public void setCampaignName(String campaignName)
+ {
+ this.campaignName = campaignName;
+ }
+
+ public double getCampaignBudget()
+ {
+ return campaignBudget;
+ }
+
+ public void setCampaignBudget(double campaignBudget)
+ {
+ this.campaignBudget = campaignBudget;
+ }
+
+ public boolean isWeatherTargeting()
+ {
+ return weatherTargeting;
+ }
+
+ public void setWeatherTargeting(boolean weatherTargeting)
+ {
+ this.weatherTargeting = weatherTargeting;
+ }
+
+ public Date getStartDate()
+ {
+ return startDate;
+ }
+
+ public void setStartDate(Date startDate)
+ {
+ this.startDate = startDate;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "Campaign [adId=" + adId + ", campaignName=" + campaignName + ", campaignBudget=" + campaignBudget
+ + ", weatherTargeting=" + weatherTargeting + ", startDate=" + startDate + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/8e200974/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/JsonGenerator.java
----------------------------------------------------------------------
diff --git a/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/JsonGenerator.java b/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/JsonGenerator.java
new file mode 100644
index 0000000..90beb13
--- /dev/null
+++ b/examples/parser/src/main/java/com/datatorrent/tutorial/jsonparser/JsonGenerator.java
@@ -0,0 +1,83 @@
+package com.datatorrent.tutorial.jsonparser;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Random;
+
+import javax.validation.constraints.Min;
+
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.common.util.BaseOperator;
+
+public class JsonGenerator extends BaseOperator implements InputOperator
+{
+
+ private static final Logger LOG = LoggerFactory.getLogger(JsonGenerator.class);
+
+ @Min(1)
+ private int numTuples = 20;
+ private transient int count = 0;
+
+ public static Random rand = new Random();
+ public static SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd");
+ public static int[] adId = { 1, 2, 3, 4, 5 };
+ public static String[] campaignName = { "cmp1", "cmp2", "cmp3", "cmp4" };
+ public static double[] campaignBudget = { 10000.0, 20000.0, 300000.0 };
+ public static boolean[] weatherTargeting = { true, false };
+ private int sleepTime;
+
+ public final transient DefaultOutputPort<byte[]> out = new DefaultOutputPort<byte[]>();
+
+ private static String getNext(int num)
+ {
+
+ JSONObject obj = new JSONObject();
+ try {
+ obj.put("adId", adId[num % adId.length]);
+ obj.put("campaignName", campaignName[num % campaignName.length]);
+ obj.put("campaignBudget", campaignBudget[num % campaignBudget.length]);
+ obj.put("weatherTargeting", weatherTargeting[num % weatherTargeting.length]);
+ obj.put("startDate", sdf.format(new Date()));
+ } catch (JSONException e) {
+ return null;
+ }
+ return obj.toString();
+ }
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ count = 0;
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ if (count++ < numTuples) {
+ out.emit(getNext(rand.nextInt(numTuples) + 1).getBytes());
+ } else {
+ try {
+ Thread.sleep(sleepTime);
+ } catch (InterruptedException e) {
+ LOG.info("Sleep interrupted");
+ }
+ }
+ }
+
+ public int getNumTuples()
+ {
+ return numTuples;
+ }
+
+ public void setNumTuples(int numTuples)
+ {
+ this.numTuples = numTuples;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/8e200974/examples/parser/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/parser/src/main/resources/META-INF/properties.xml b/examples/parser/src/main/resources/META-INF/properties.xml
new file mode 100644
index 0000000..d8ac178
--- /dev/null
+++ b/examples/parser/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0"?>
+<configuration>
+ <!-- <property> <name>dt.application.{appName}.operator.{opName}.prop.{propName}</name>
+ <value>some-default-value (if value is not specified, it is required from
+ the user or custom config when launching)</value> </property> -->
+ <!-- memory assigned to app master <property> <name>dt.attr.MASTER_MEMORY_MB</name>
+ <value>1024</value> </property> -->
+
+ <!-- these values override those set in populateDAG -->
+ <property>
+ <name>dt.application.JsonProcessor.operator.JsonParser.prop.sleepTime
+ </name>
+ <value>100</value>
+ </property>
+ <property>
+ <name>dt.application.JsonProcessor.operator.JsonParser.port.out.attr.TUPLE_CLASS
+ </name>
+ <value>com.datatorrent.tutorial.jsonparser.Campaign</value>
+ </property>
+ <property>
+ <name>dt.application.JsonProcessor.operator.JsonFormatter.port.in.attr.TUPLE_CLASS
+ </name>
+ <value>com.datatorrent.tutorial.jsonparser.Campaign</value>
+ </property>
+ <property>
+ <name>dt.application.JsonProcessor.operator.JsonString.prop.stringFormat
+ </name>
+ <value>JsonString:%s</value>
+ </property>
+ <property>
+ <name>dt.application.JsonProcessor.operator.JsonObject.prop.stringFormat
+ </name>
+ <value>JsonObject:%s</value>
+ </property>
+ <property>
+ <name>dt.application.JsonProcessor.operator.Error.prop.stringFormat
+ </name>
+ <value>Error:%s</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/8e200974/examples/parser/src/test/java/com/datatorrent/tutorial/jsonparser/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/parser/src/test/java/com/datatorrent/tutorial/jsonparser/ApplicationTest.java b/examples/parser/src/test/java/com/datatorrent/tutorial/jsonparser/ApplicationTest.java
new file mode 100644
index 0000000..390d4cb
--- /dev/null
+++ b/examples/parser/src/test/java/com/datatorrent/tutorial/jsonparser/ApplicationTest.java
@@ -0,0 +1,36 @@
+package com.datatorrent.tutorial.jsonparser;
+
+import java.io.IOException;
+
+import javax.validation.ConstraintViolationException;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.LocalMode;
+import com.datatorrent.tutorial.jsonparser.Application;
+
+/**
+ * Test the DAG declaration in local mode.
+ */
+public class ApplicationTest
+{
+
+ @Test
+ public void testApplication() throws IOException, Exception
+ {
+
+ try {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ lma.prepareDAG(new Application(), conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.run(10 * 1000); // runs for 30 seconds and quits
+ } catch (ConstraintViolationException e) {
+ Assert.fail("constraint violations: " + e.getConstraintViolations());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/8e200974/examples/parser/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/parser/src/test/resources/log4j.properties b/examples/parser/src/test/resources/log4j.properties
new file mode 100644
index 0000000..98544e8
--- /dev/null
+++ b/examples/parser/src/test/resources/log4j.properties
@@ -0,0 +1,22 @@
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=debug
+log4j.logger.org.apache.apex=debug
+log4j.logger.org=info
[17/19] apex-malhar git commit: APEXMALHAR-2233 Updated the examples
to follow the structure of apex-malhar examples. Specified dependencies in
pom.xmls of individual examples correctly.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/dynamic-partition/src/main/java/org/apache/apex/examples/dynamic/App.java
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/src/main/java/org/apache/apex/examples/dynamic/App.java b/examples/dynamic-partition/src/main/java/org/apache/apex/examples/dynamic/App.java
new file mode 100644
index 0000000..1a40cb5
--- /dev/null
+++ b/examples/dynamic-partition/src/main/java/org/apache/apex/examples/dynamic/App.java
@@ -0,0 +1,23 @@
+package org.apache.apex.examples.dynamic;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+
+import com.datatorrent.lib.stream.DevNull;
+
+@ApplicationAnnotation(name="Dyn")
+public class App implements StreamingApplication
+{
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ Gen gen = dag.addOperator("gen", Gen.class);
+ DevNull devNull = dag.addOperator("devNull", DevNull.class);
+
+ dag.addStream("data", gen.out, devNull.data);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/dynamic-partition/src/main/java/org/apache/apex/examples/dynamic/Gen.java
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/src/main/java/org/apache/apex/examples/dynamic/Gen.java b/examples/dynamic-partition/src/main/java/org/apache/apex/examples/dynamic/Gen.java
new file mode 100644
index 0000000..29f79ac
--- /dev/null
+++ b/examples/dynamic-partition/src/main/java/org/apache/apex/examples/dynamic/Gen.java
@@ -0,0 +1,171 @@
+package org.apache.apex.examples.dynamic;
+
+import java.io.ByteArrayOutputStream;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import javax.validation.constraints.NotNull;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.io.IOUtils;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+import com.google.common.collect.Lists;
+
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.DefaultPartition;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.api.Partitioner;
+import com.datatorrent.api.StatsListener;
+import com.datatorrent.common.util.BaseOperator;
+
+/**
+ * Operator that dynamically partitions itself after 500 tuples have been emitted
+ */
+public class Gen extends BaseOperator implements InputOperator, Partitioner<Gen>, StatsListener
+{
+ private static final Logger LOG = LoggerFactory.getLogger(Gen.class);
+
+ private static final int MAX_PARTITIONS = 4; // maximum number of partitions
+
+ private int partitions = 2; // initial number of partitions
+
+ @NotNull
+ private int numTuples; // number of tuples to emit per window
+
+ private transient int count = 0;
+
+ public final transient DefaultOutputPort<Double> out = new DefaultOutputPort<Double>();
+
+ @Override
+ public void partitioned(Map<Integer, Partition<Gen>> map)
+ {
+ if (partitions != map.size()) {
+ String msg = String.format("partitions = %d, map.size = %d%n", partitions, map.size());
+ throw new RuntimeException(msg);
+ }
+ }
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ count = 0;
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ if (count < numTuples) {
+ ++count;
+ out.emit(Math.random());
+ }
+ }
+
+ public int getNumTuples()
+ {
+ return numTuples;
+ }
+
+ /**
+ * Sets the number of tuples to be emitted every window.
+ * @param numTuples number of tuples
+ */
+ public void setNumTuples(int numTuples)
+ {
+ this.numTuples = numTuples;
+ }
+
+ @Override
+ public Response processStats(BatchedOperatorStats batchedOperatorStats) {
+
+ final long emittedCount = batchedOperatorStats.getTuplesEmittedPSMA();
+
+ // we only perform a single dynamic repartition
+ Response res = new Response();
+ res.repartitionRequired = false;
+ if (emittedCount > 500 && partitions < MAX_PARTITIONS) {
+ LOG.info("processStats: trying repartition of input operator current {} required {}",
+ partitions, MAX_PARTITIONS);
+ LOG.info("**** operator id = {}, window id = {}, tuplesProcessedPSMA = {}, tuplesEmittedPSMA = {}",
+ batchedOperatorStats.getOperatorId(),
+ batchedOperatorStats.getCurrentWindowId(),
+ batchedOperatorStats.getTuplesProcessedPSMA(),
+ emittedCount);
+ partitions = MAX_PARTITIONS;
+ res.repartitionRequired = true;
+ }
+
+ return res;
+ } // processStats
+
+ /**
+ * Clone object by serializing and deserializing using Kryo.
+ * Note this is different from using {@link Kryo#copy(Object)}, which will attempt to also clone transient fields.
+ *
+ * @param kryo kryo object used to clone objects
+ * @param src src object that copy from
+ * @return cloned object
+ */
+ @SuppressWarnings("unchecked")
+ private static <SRC> SRC cloneObject(Kryo kryo, SRC src)
+ {
+ kryo.setClassLoader(src.getClass().getClassLoader());
+ ByteArrayOutputStream bos = null;
+ Output output;
+ Input input = null;
+ try {
+ bos = new ByteArrayOutputStream();
+ output = new Output(bos);
+ kryo.writeObject(output, src);
+ output.close();
+ input = new Input(bos.toByteArray());
+ return (SRC)kryo.readObject(input, src.getClass());
+ } finally {
+ IOUtils.closeQuietly(input);
+ IOUtils.closeQuietly(bos);
+ }
+ }
+
+ @Override
+ public Collection<Partition<Gen>> definePartitions(
+ Collection<Partition<Gen>> list, PartitioningContext context)
+ {
+ if (partitions < 0) { // error
+ String msg = String.format("Error: Bad value: partitions = %d%n", partitions);
+ LOG.error(msg);
+ throw new RuntimeException(msg);
+ }
+
+ final int prevCount = list.size();
+ if (1 == prevCount) { // initial call
+ LOG.info("definePartitions: First call, prevCount = {}, partitions = {}",
+ prevCount, partitions);
+ }
+
+ if (prevCount == partitions) {
+ LOG.info("definePartitions: Nothing to do in definePartitions");
+ return list; // nothing to do
+ }
+
+ LOG.debug("definePartitions: Repartitioning from {} to {}", prevCount, partitions);
+
+ Kryo kryo = new Kryo();
+
+ // return value: new list of partitions (includes old list)
+ List<Partition<Gen>> newPartitions = Lists.newArrayListWithExpectedSize(partitions);
+
+ for (int i = 0; i < partitions; i++) {
+ Gen oper = cloneObject(kryo, this);
+ newPartitions.add(new DefaultPartition<>(oper));
+ }
+
+ LOG.info("definePartition: returning {} partitions", newPartitions.size());
+ return newPartitions;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/dynamic-partition/src/test/java/com/example/dynamic/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/src/test/java/com/example/dynamic/ApplicationTest.java b/examples/dynamic-partition/src/test/java/com/example/dynamic/ApplicationTest.java
deleted file mode 100644
index 788b9d3..0000000
--- a/examples/dynamic-partition/src/test/java/com/example/dynamic/ApplicationTest.java
+++ /dev/null
@@ -1,34 +0,0 @@
-package com.example.dynamic;
-
-import java.io.IOException;
-
-import javax.validation.ConstraintViolationException;
-
-import org.junit.Assert;
-
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Test;
-
-import com.datatorrent.api.LocalMode;
-//import com.example.myapexapp.Application;
-
-/**
- * Test the DAG declaration in local mode.
- */
-public class ApplicationTest {
-
- @Test
- public void testApplication() throws IOException, Exception {
- try {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
- lma.prepareDAG(new App(), conf);
- LocalMode.Controller lc = lma.getController();
- lc.run(10000); // runs for 10 seconds and quits
- } catch (ConstraintViolationException e) {
- Assert.fail("constraint violations: " + e.getConstraintViolations());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/dynamic-partition/src/test/java/org/apache/apex/examples/dynamic/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/src/test/java/org/apache/apex/examples/dynamic/ApplicationTest.java b/examples/dynamic-partition/src/test/java/org/apache/apex/examples/dynamic/ApplicationTest.java
new file mode 100644
index 0000000..a4c8076
--- /dev/null
+++ b/examples/dynamic-partition/src/test/java/org/apache/apex/examples/dynamic/ApplicationTest.java
@@ -0,0 +1,33 @@
+package org.apache.apex.examples.dynamic;
+
+import java.io.IOException;
+
+import javax.validation.ConstraintViolationException;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.LocalMode;
+
+/**
+ * Test the DAG declaration in local mode.
+ */
+public class ApplicationTest {
+
+ @Test
+ public void testApplication() throws IOException, Exception {
+ try {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ lma.prepareDAG(new App(), conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.run(10000); // runs for 10 seconds and quits
+ } catch (ConstraintViolationException e) {
+ Assert.fail("constraint violations: " + e.getConstraintViolations());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/enricher/pom.xml
----------------------------------------------------------------------
diff --git a/examples/enricher/pom.xml b/examples/enricher/pom.xml
index a93bcf5..7a55d32 100644
--- a/examples/enricher/pom.xml
+++ b/examples/enricher/pom.xml
@@ -1,263 +1,25 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
-
- <groupId>com.example</groupId>
- <version>1.0-SNAPSHOT</version>
- <artifactId>enricher</artifactId>
+
+ <parent>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-examples</artifactId>
+ <version>3.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>malhar-examples-enricher</artifactId>
<packaging>jar</packaging>
<!-- change these to the appropriate values -->
<name>Enricher</name>
<description>Example Use of POJO Enricher</description>
- <properties>
- <!-- change this if you desire to use a different version of Apex Core -->
- <apex.version>3.5.0</apex.version>
- <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath>
- <malhar.version>3.6.0</malhar.version>
- </properties>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-eclipse-plugin</artifactId>
- <version>2.9</version>
- <configuration>
- <downloadSources>true</downloadSources>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>3.3</version>
- <configuration>
- <encoding>UTF-8</encoding>
- <source>1.7</source>
- <target>1.7</target>
- <debug>true</debug>
- <optimize>false</optimize>
- <showDeprecation>true</showDeprecation>
- <showWarnings>true</showWarnings>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <version>2.8</version>
- <executions>
- <execution>
- <id>copy-dependencies</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>copy-dependencies</goal>
- </goals>
- <configuration>
- <outputDirectory>target/deps</outputDirectory>
- <includeScope>runtime</includeScope>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <executions>
- <execution>
- <id>app-package-assembly</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <finalName>${project.artifactId}-${project.version}-apexapp</finalName>
- <appendAssemblyId>false</appendAssemblyId>
- <descriptors>
- <descriptor>src/assemble/appPackage.xml</descriptor>
- </descriptors>
- <archiverConfig>
- <defaultDirectoryMode>0755</defaultDirectoryMode>
- </archiverConfig>
- <archive>
- <manifestEntries>
- <Class-Path>${apex.apppackage.classpath}</Class-Path>
- <DT-Engine-Version>${apex.version}</DT-Engine-Version>
- <DT-App-Package-Group-Id>${project.groupId}</DT-App-Package-Group-Id>
- <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name>
- <DT-App-Package-Version>${project.version}</DT-App-Package-Version>
- <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name>
- <DT-App-Package-Description>${project.description}</DT-App-Package-Description>
- </manifestEntries>
- </archive>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <artifactId>maven-antrun-plugin</artifactId>
- <version>1.7</version>
- <executions>
- <execution>
- <phase>package</phase>
- <configuration>
- <target>
- <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar"
- tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa" />
- </target>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- <execution>
- <!-- create resource directory for xml javadoc-->
- <id>createJavadocDirectory</id>
- <phase>generate-resources</phase>
- <configuration>
- <tasks>
- <delete dir="${project.build.directory}/generated-resources/xml-javadoc"/>
- <mkdir dir="${project.build.directory}/generated-resources/xml-javadoc"/>
- </tasks>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>1.9.1</version>
- <executions>
- <execution>
- <id>attach-artifacts</id>
- <phase>package</phase>
- <goals>
- <goal>attach-artifact</goal>
- </goals>
- <configuration>
- <artifacts>
- <artifact>
- <file>target/${project.artifactId}-${project.version}.apa</file>
- <type>apa</type>
- </artifact>
- </artifacts>
- <skipAttach>false</skipAttach>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <!-- generate javdoc -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <executions>
- <!-- generate xml javadoc -->
- <execution>
- <id>xml-doclet</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>javadoc</goal>
- </goals>
- <configuration>
- <doclet>com.github.markusbernhardt.xmldoclet.XmlDoclet</doclet>
- <additionalparam>-d ${project.build.directory}/generated-resources/xml-javadoc -filename ${project.artifactId}-${project.version}-javadoc.xml</additionalparam>
- <useStandardDocletOptions>false</useStandardDocletOptions>
- <docletArtifact>
- <groupId>com.github.markusbernhardt</groupId>
- <artifactId>xml-doclet</artifactId>
- <version>1.0.4</version>
- </docletArtifact>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <!-- Transform xml javadoc to stripped down version containing only class/interface comments and tags-->
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>xml-maven-plugin</artifactId>
- <version>1.0</version>
- <executions>
- <execution>
- <id>transform-xmljavadoc</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>transform</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <transformationSets>
- <transformationSet>
- <dir>${project.build.directory}/generated-resources/xml-javadoc</dir>
- <includes>
- <include>${project.artifactId}-${project.version}-javadoc.xml</include>
- </includes>
- <stylesheet>XmlJavadocCommentsExtractor.xsl</stylesheet>
- <outputDir>${project.build.directory}/generated-resources/xml-javadoc</outputDir>
- </transformationSet>
- </transformationSets>
- </configuration>
- </plugin>
- <!-- copy xml javadoc to class jar -->
- <plugin>
- <artifactId>maven-resources-plugin</artifactId>
- <version>2.6</version>
- <executions>
- <execution>
- <id>copy-resources</id>
- <phase>process-resources</phase>
- <goals>
- <goal>copy-resources</goal>
- </goals>
- <configuration>
- <outputDirectory>${basedir}/target/classes</outputDirectory>
- <resources>
- <resource>
- <directory>${project.build.directory}/generated-resources/xml-javadoc</directory>
- <includes>
- <include>${project.artifactId}-${project.version}-javadoc.xml</include>
- </includes>
- <filtering>true</filtering>
- </resource>
- </resources>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
-
- </build>
-
<dependencies>
- <!-- add your dependencies here -->
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-library</artifactId>
- <version>${malhar.version}</version>
- <!--
- If you know that your application does not need transitive dependencies pulled in by malhar-library,
- uncomment the following to reduce the size of your app package.
- -->
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<dependency>
<groupId>org.apache.apex</groupId>
<artifactId>malhar-contrib</artifactId>
- <version>${malhar.version}</version>
- <!--
- If you know that your application does not need transitive dependencies pulled in by malhar-library,
- uncomment the following to reduce the size of your app package.
- -->
+ <version>${project.version}</version>
<exclusions>
<exclusion>
<groupId>*</groupId>
@@ -265,27 +27,6 @@
</exclusion>
</exclusions>
</dependency>
-
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>apex-common</artifactId>
- <version>${apex.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.10</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>apex-engine</artifactId>
- <version>${apex.version}</version>
- <scope>test</scope>
- </dependency>
-
<dependency>
<groupId>com.github.fge</groupId>
<artifactId>json-schema-validator</artifactId>
@@ -303,6 +44,25 @@
<artifactId>janino</artifactId>
<version>2.7.8</version>
</dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ <version>1.9.13</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <version>0.13.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jettison</groupId>
+ <artifactId>jettison</artifactId>
+ <version>1.1</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>2.7.0</version>
+ </dependency>
</dependencies>
-
</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/enricher/src/main/java/com/example/myapexapp/DataGenerator.java
----------------------------------------------------------------------
diff --git a/examples/enricher/src/main/java/com/example/myapexapp/DataGenerator.java b/examples/enricher/src/main/java/com/example/myapexapp/DataGenerator.java
deleted file mode 100644
index 3afbb87..0000000
--- a/examples/enricher/src/main/java/com/example/myapexapp/DataGenerator.java
+++ /dev/null
@@ -1,94 +0,0 @@
-package com.example.myapexapp;
-
-import java.util.Random;
-
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.InputOperator;
-import com.datatorrent.common.util.BaseOperator;
-
-/**
- * Generates Subscriber Data:
- * A Party Phone
- * A Party IMEI
- * A Party IMSI
- * Circle Id
- */
-public class DataGenerator extends BaseOperator implements InputOperator
-{
- public static int NUM_CIRCLES = 10;
-
- private Random r;
- private int count = 0;
- private int limit = 1000;
-
- public final transient DefaultOutputPort<byte[]> output = new DefaultOutputPort<>();
-
- @Override
- public void setup(OperatorContext context)
- {
- r = new Random(System.currentTimeMillis());
- }
-
- @Override
- public void beginWindow(long windowId) {
- super.beginWindow(windowId);
- count = 0;
- }
-
- @Override
- public void emitTuples()
- {
- if(count++ < limit) {
- output.emit(getRecord());
- }
- }
-
- private byte[] getRecord()
- {
- String phone = getRandomNumber(10);
- String imsi = getHashInRange(phone, 15);
- String imei = getHashInRange(imsi, 15);
- String circleId = Math.abs(phone.hashCode()) % NUM_CIRCLES + "";
-// String record = MessageFormat.format(baseDataTemplate, phone, imsi, imei, circleId);
- String record = "{" +
- "\"phone\":\"" + phone + "\"," +
- "\"imei\":\"" + imei+ "\"," +
- "\"imsi\":\"" + imsi+ "\"," +
- "\"circleId\":" + circleId +
- "}";
- return record.getBytes();
- }
-
- private String getRandomNumber(int numDigits)
- {
- String retVal = (r.nextInt((9 - 1) + 1) + 1) + "";
-
- for (int i = 0; i < numDigits - 1; i++) {
- retVal += (r.nextInt((9 - 0) + 1) + 0);
- }
- return retVal;
- }
-
- private String getHashInRange(String s, int n)
- {
- StringBuilder retVal = new StringBuilder();
- for (int i = 0, j = 0; i < n && j < s.length(); i++, j++) {
- retVal.append(Math.abs(s.charAt(j) + "".hashCode()) % 10);
- if (j == s.length() - 1) {
- j = -1;
- }
- }
- return retVal.toString();
- }
-
- public int getLimit()
- {
- return limit;
- }
-
- public void setLimit(int limit)
- {
- this.limit = limit;
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/enricher/src/main/java/com/example/myapexapp/EnricherAppWithJSONFile.java
----------------------------------------------------------------------
diff --git a/examples/enricher/src/main/java/com/example/myapexapp/EnricherAppWithJSONFile.java b/examples/enricher/src/main/java/com/example/myapexapp/EnricherAppWithJSONFile.java
deleted file mode 100644
index a0dab64..0000000
--- a/examples/enricher/src/main/java/com/example/myapexapp/EnricherAppWithJSONFile.java
+++ /dev/null
@@ -1,47 +0,0 @@
-package com.example.myapexapp;
-
-import java.util.ArrayList;
-
-import com.datatorrent.contrib.enrich.JsonFSLoader;
-import com.datatorrent.contrib.enrich.POJOEnricher;
-import com.datatorrent.contrib.parser.JsonParser;
-import com.datatorrent.lib.io.ConsoleOutputOperator;
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-
-@ApplicationAnnotation(name="EnricherAppWithJSONFile")
-public class EnricherAppWithJSONFile implements StreamingApplication
-{
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- DataGenerator dataGenerator = dag.addOperator("DataGenerator", DataGenerator.class);
- JsonParser parser = dag.addOperator("Parser", JsonParser.class);
-
- /**
- * FSLoader is used to configure Enricher backend. Property of FSLoader file which is fileName is set in
- * properties.xml file.
- * The format that is used to read the file is present as an example in resources/circleMapping.txt file.
- */
- JsonFSLoader fsLoader = new JsonFSLoader();
- POJOEnricher enrich = dag.addOperator("Enrich", POJOEnricher.class);
- enrich.setStore(fsLoader);
-
- ArrayList includeFields = new ArrayList();
- includeFields.add("circleName");
- ArrayList lookupFields = new ArrayList();
- lookupFields.add("circleId");
-
- enrich.setIncludeFields(includeFields);
- enrich.setLookupFields(lookupFields);
-
- ConsoleOutputOperator console = dag.addOperator("Console", ConsoleOutputOperator.class);
-
- dag.addStream("Parse", dataGenerator.output, parser.in);
- dag.addStream("Enrich", parser.out, enrich.input);
- dag.addStream("Console", enrich.output, console.input);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/enricher/src/main/java/com/example/myapexapp/LineOutputOperator.java
----------------------------------------------------------------------
diff --git a/examples/enricher/src/main/java/com/example/myapexapp/LineOutputOperator.java b/examples/enricher/src/main/java/com/example/myapexapp/LineOutputOperator.java
deleted file mode 100644
index 3b7a298..0000000
--- a/examples/enricher/src/main/java/com/example/myapexapp/LineOutputOperator.java
+++ /dev/null
@@ -1,34 +0,0 @@
-package com.example.myapexapp;
-
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-
-import javax.validation.constraints.NotNull;
-
-import com.datatorrent.lib.io.fs.AbstractFileOutputOperator;
-
-/**
- * Converts each tuple to a string and writes it as a new line to the output file
- */
-public class LineOutputOperator extends AbstractFileOutputOperator<Object>
-{
- private static final String NL = System.lineSeparator();
- private static final Charset CS = StandardCharsets.UTF_8;
-
- @NotNull
- private String baseName;
-
- @Override
- public byte[] getBytesForTuple(Object t) {
- String result = new String(t.toString().getBytes(), CS) + NL;
- return result.getBytes(CS);
- }
-
- @Override
- protected String getFileName(Object tuple) {
- return baseName;
- }
-
- public String getBaseName() { return baseName; }
- public void setBaseName(String v) { baseName = v; }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/enricher/src/main/java/com/example/myapexapp/POJO.java
----------------------------------------------------------------------
diff --git a/examples/enricher/src/main/java/com/example/myapexapp/POJO.java b/examples/enricher/src/main/java/com/example/myapexapp/POJO.java
deleted file mode 100644
index 32845e8..0000000
--- a/examples/enricher/src/main/java/com/example/myapexapp/POJO.java
+++ /dev/null
@@ -1,49 +0,0 @@
-package com.example.myapexapp;
-
-public class POJO
-{
- private String phone;
- private String imei;
- private String imsi;
- private int circleId;
-
- public String getPhone()
- {
- return phone;
- }
-
- public void setPhone(String phone)
- {
- this.phone = phone;
- }
-
- public String getImei()
- {
- return imei;
- }
-
- public void setImei(String imei)
- {
- this.imei = imei;
- }
-
- public String getImsi()
- {
- return imsi;
- }
-
- public void setImsi(String imsi)
- {
- this.imsi = imsi;
- }
-
- public int getCircleId()
- {
- return circleId;
- }
-
- public void setCircleId(int circleId)
- {
- this.circleId = circleId;
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/enricher/src/main/java/com/example/myapexapp/POJOEnriched.java
----------------------------------------------------------------------
diff --git a/examples/enricher/src/main/java/com/example/myapexapp/POJOEnriched.java b/examples/enricher/src/main/java/com/example/myapexapp/POJOEnriched.java
deleted file mode 100644
index bed2cfb..0000000
--- a/examples/enricher/src/main/java/com/example/myapexapp/POJOEnriched.java
+++ /dev/null
@@ -1,71 +0,0 @@
-package com.example.myapexapp;
-
-public class POJOEnriched
-{
- private String phone;
- private String imei;
- private String imsi;
- private int circleId;
- private String circleName;
-
- public String getPhone()
- {
- return phone;
- }
-
- public void setPhone(String phone)
- {
- this.phone = phone;
- }
-
- public String getImei()
- {
- return imei;
- }
-
- public void setImei(String imei)
- {
- this.imei = imei;
- }
-
- public String getImsi()
- {
- return imsi;
- }
-
- public void setImsi(String imsi)
- {
- this.imsi = imsi;
- }
-
- public int getCircleId()
- {
- return circleId;
- }
-
- public void setCircleId(int circleId)
- {
- this.circleId = circleId;
- }
-
- public String getCircleName()
- {
- return circleName;
- }
-
- public void setCircleName(String circleName)
- {
- this.circleName = circleName;
- }
-
- @Override public String toString()
- {
- return "POJOEnriched{" +
- "phone='" + phone + '\'' +
- ", imei='" + imei + '\'' +
- ", imsi='" + imsi + '\'' +
- ", circleId=" + circleId +
- ", circleName='" + circleName + '\'' +
- '}';
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/enricher/src/main/java/org/apache/apex/examples/enricher/DataGenerator.java
----------------------------------------------------------------------
diff --git a/examples/enricher/src/main/java/org/apache/apex/examples/enricher/DataGenerator.java b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/DataGenerator.java
new file mode 100644
index 0000000..2ba5567
--- /dev/null
+++ b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/DataGenerator.java
@@ -0,0 +1,94 @@
+package org.apache.apex.examples.enricher;
+
+import java.util.Random;
+
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.common.util.BaseOperator;
+
+/**
+ * Generates Subscriber Data:
+ * A Party Phone
+ * A Party IMEI
+ * A Party IMSI
+ * Circle Id
+ */
+public class DataGenerator extends BaseOperator implements InputOperator
+{
+ public static int NUM_CIRCLES = 10;
+
+ private Random r;
+ private int count = 0;
+ private int limit = 1000;
+
+ public final transient DefaultOutputPort<byte[]> output = new DefaultOutputPort<>();
+
+ @Override
+ public void setup(OperatorContext context)
+ {
+ r = new Random(System.currentTimeMillis());
+ }
+
+ @Override
+ public void beginWindow(long windowId) {
+ super.beginWindow(windowId);
+ count = 0;
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ if(count++ < limit) {
+ output.emit(getRecord());
+ }
+ }
+
+ private byte[] getRecord()
+ {
+ String phone = getRandomNumber(10);
+ String imsi = getHashInRange(phone, 15);
+ String imei = getHashInRange(imsi, 15);
+ String circleId = Math.abs(phone.hashCode()) % NUM_CIRCLES + "";
+// String record = MessageFormat.format(baseDataTemplate, phone, imsi, imei, circleId);
+ String record = "{" +
+ "\"phone\":\"" + phone + "\"," +
+ "\"imei\":\"" + imei+ "\"," +
+ "\"imsi\":\"" + imsi+ "\"," +
+ "\"circleId\":" + circleId +
+ "}";
+ return record.getBytes();
+ }
+
+ private String getRandomNumber(int numDigits)
+ {
+ String retVal = (r.nextInt((9 - 1) + 1) + 1) + "";
+
+ for (int i = 0; i < numDigits - 1; i++) {
+ retVal += (r.nextInt((9 - 0) + 1) + 0);
+ }
+ return retVal;
+ }
+
+ private String getHashInRange(String s, int n)
+ {
+ StringBuilder retVal = new StringBuilder();
+ for (int i = 0, j = 0; i < n && j < s.length(); i++, j++) {
+ retVal.append(Math.abs(s.charAt(j) + "".hashCode()) % 10);
+ if (j == s.length() - 1) {
+ j = -1;
+ }
+ }
+ return retVal.toString();
+ }
+
+ public int getLimit()
+ {
+ return limit;
+ }
+
+ public void setLimit(int limit)
+ {
+ this.limit = limit;
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/enricher/src/main/java/org/apache/apex/examples/enricher/EnricherAppWithJSONFile.java
----------------------------------------------------------------------
diff --git a/examples/enricher/src/main/java/org/apache/apex/examples/enricher/EnricherAppWithJSONFile.java b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/EnricherAppWithJSONFile.java
new file mode 100644
index 0000000..1a420c4
--- /dev/null
+++ b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/EnricherAppWithJSONFile.java
@@ -0,0 +1,47 @@
+package org.apache.apex.examples.enricher;
+
+import java.util.ArrayList;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.contrib.enrich.JsonFSLoader;
+import com.datatorrent.contrib.enrich.POJOEnricher;
+import com.datatorrent.contrib.parser.JsonParser;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+
+@ApplicationAnnotation(name="EnricherAppWithJSONFile")
+public class EnricherAppWithJSONFile implements StreamingApplication
+{
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ DataGenerator dataGenerator = dag.addOperator("DataGenerator", DataGenerator.class);
+ JsonParser parser = dag.addOperator("Parser", JsonParser.class);
+
+ /**
+ * FSLoader is used to configure Enricher backend. Property of FSLoader file which is fileName is set in
+ * properties.xml file.
+ * The format that is used to read the file is present as an example in resources/circleMapping.txt file.
+ */
+ JsonFSLoader fsLoader = new JsonFSLoader();
+ POJOEnricher enrich = dag.addOperator("Enrich", POJOEnricher.class);
+ enrich.setStore(fsLoader);
+
+ ArrayList includeFields = new ArrayList();
+ includeFields.add("circleName");
+ ArrayList lookupFields = new ArrayList();
+ lookupFields.add("circleId");
+
+ enrich.setIncludeFields(includeFields);
+ enrich.setLookupFields(lookupFields);
+
+ ConsoleOutputOperator console = dag.addOperator("Console", ConsoleOutputOperator.class);
+
+ dag.addStream("Parse", dataGenerator.output, parser.in);
+ dag.addStream("Enrich", parser.out, enrich.input);
+ dag.addStream("Console", enrich.output, console.input);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/enricher/src/main/java/org/apache/apex/examples/enricher/LineOutputOperator.java
----------------------------------------------------------------------
diff --git a/examples/enricher/src/main/java/org/apache/apex/examples/enricher/LineOutputOperator.java b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/LineOutputOperator.java
new file mode 100644
index 0000000..aca7df6
--- /dev/null
+++ b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/LineOutputOperator.java
@@ -0,0 +1,34 @@
+package org.apache.apex.examples.enricher;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+
+import javax.validation.constraints.NotNull;
+
+import com.datatorrent.lib.io.fs.AbstractFileOutputOperator;
+
+/**
+ * Converts each tuple to a string and writes it as a new line to the output file
+ */
+public class LineOutputOperator extends AbstractFileOutputOperator<Object>
+{
+ private static final String NL = System.lineSeparator();
+ private static final Charset CS = StandardCharsets.UTF_8;
+
+ @NotNull
+ private String baseName;
+
+ @Override
+ public byte[] getBytesForTuple(Object t) {
+ String result = new String(t.toString().getBytes(), CS) + NL;
+ return result.getBytes(CS);
+ }
+
+ @Override
+ protected String getFileName(Object tuple) {
+ return baseName;
+ }
+
+ public String getBaseName() { return baseName; }
+ public void setBaseName(String v) { baseName = v; }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/enricher/src/main/java/org/apache/apex/examples/enricher/POJO.java
----------------------------------------------------------------------
diff --git a/examples/enricher/src/main/java/org/apache/apex/examples/enricher/POJO.java b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/POJO.java
new file mode 100644
index 0000000..d48bd1a
--- /dev/null
+++ b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/POJO.java
@@ -0,0 +1,49 @@
+package org.apache.apex.examples.enricher;
+
+public class POJO
+{
+ private String phone;
+ private String imei;
+ private String imsi;
+ private int circleId;
+
+ public String getPhone()
+ {
+ return phone;
+ }
+
+ public void setPhone(String phone)
+ {
+ this.phone = phone;
+ }
+
+ public String getImei()
+ {
+ return imei;
+ }
+
+ public void setImei(String imei)
+ {
+ this.imei = imei;
+ }
+
+ public String getImsi()
+ {
+ return imsi;
+ }
+
+ public void setImsi(String imsi)
+ {
+ this.imsi = imsi;
+ }
+
+ public int getCircleId()
+ {
+ return circleId;
+ }
+
+ public void setCircleId(int circleId)
+ {
+ this.circleId = circleId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/enricher/src/main/java/org/apache/apex/examples/enricher/POJOEnriched.java
----------------------------------------------------------------------
diff --git a/examples/enricher/src/main/java/org/apache/apex/examples/enricher/POJOEnriched.java b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/POJOEnriched.java
new file mode 100644
index 0000000..df08c7f
--- /dev/null
+++ b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/POJOEnriched.java
@@ -0,0 +1,71 @@
+package org.apache.apex.examples.enricher;
+
+public class POJOEnriched
+{
+ private String phone;
+ private String imei;
+ private String imsi;
+ private int circleId;
+ private String circleName;
+
+ public String getPhone()
+ {
+ return phone;
+ }
+
+ public void setPhone(String phone)
+ {
+ this.phone = phone;
+ }
+
+ public String getImei()
+ {
+ return imei;
+ }
+
+ public void setImei(String imei)
+ {
+ this.imei = imei;
+ }
+
+ public String getImsi()
+ {
+ return imsi;
+ }
+
+ public void setImsi(String imsi)
+ {
+ this.imsi = imsi;
+ }
+
+ public int getCircleId()
+ {
+ return circleId;
+ }
+
+ public void setCircleId(int circleId)
+ {
+ this.circleId = circleId;
+ }
+
+ public String getCircleName()
+ {
+ return circleName;
+ }
+
+ public void setCircleName(String circleName)
+ {
+ this.circleName = circleName;
+ }
+
+ @Override public String toString()
+ {
+ return "POJOEnriched{" +
+ "phone='" + phone + '\'' +
+ ", imei='" + imei + '\'' +
+ ", imsi='" + imsi + '\'' +
+ ", circleId=" + circleId +
+ ", circleName='" + circleName + '\'' +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/enricher/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/enricher/src/main/resources/META-INF/properties.xml b/examples/enricher/src/main/resources/META-INF/properties.xml
index 9ecf899..543b99a 100644
--- a/examples/enricher/src/main/resources/META-INF/properties.xml
+++ b/examples/enricher/src/main/resources/META-INF/properties.xml
@@ -3,17 +3,17 @@
<!-- Parser -->
<property>
<name>dt.application.EnricherAppWithJSONFile.operator.Parser.port.out.attr.TUPLE_CLASS</name>
- <value>com.example.myapexapp.POJO</value>
+ <value>org.apache.apex.examples.enricher.POJO</value>
</property>
<!-- Enrich -->
<property>
<name>dt.application.EnricherAppWithJSONFile.operator.Enrich.port.input.attr.TUPLE_CLASS</name>
- <value>com.example.myapexapp.POJO</value>
+ <value>org.apache.apex.examples.enricher.POJO</value>
</property>
<property>
<name>dt.application.EnricherAppWithJSONFile.operator.Enrich.port.output.attr.TUPLE_CLASS</name>
- <value>com.example.myapexapp.POJOEnriched</value>
+ <value>org.apache.apex.examples.enricher.POJOEnriched</value>
</property>
<property>
<name>dt.application.EnricherAppWithJSONFile.operator.Enrich.prop.store.fileName</name>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/enricher/src/test/java/com/example/myapexapp/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/enricher/src/test/java/com/example/myapexapp/ApplicationTest.java b/examples/enricher/src/test/java/com/example/myapexapp/ApplicationTest.java
deleted file mode 100644
index 4b04603..0000000
--- a/examples/enricher/src/test/java/com/example/myapexapp/ApplicationTest.java
+++ /dev/null
@@ -1,31 +0,0 @@
-package com.example.myapexapp;
-
-import javax.validation.ConstraintViolationException;
-
-import org.junit.Assert;
-
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Test;
-
-import com.datatorrent.api.LocalMode;
-
-/**
- * Test the DAG declaration in local mode.
- */
-public class ApplicationTest {
-
- @Test
- public void testApplication() throws Exception {
- try {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
- lma.prepareDAG(new EnricherAppWithJSONFile(), conf);
- LocalMode.Controller lc = lma.getController();
- lc.run(10000); // runs for 10 seconds and quits
- } catch (ConstraintViolationException e) {
- Assert.fail("constraint violations: " + e.getConstraintViolations());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/enricher/src/test/java/org/apache/apex/examples/enricher/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/enricher/src/test/java/org/apache/apex/examples/enricher/ApplicationTest.java b/examples/enricher/src/test/java/org/apache/apex/examples/enricher/ApplicationTest.java
new file mode 100644
index 0000000..6b6698e
--- /dev/null
+++ b/examples/enricher/src/test/java/org/apache/apex/examples/enricher/ApplicationTest.java
@@ -0,0 +1,31 @@
+package org.apache.apex.examples.enricher;
+
+import javax.validation.ConstraintViolationException;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.LocalMode;
+
+/**
+ * Test the DAG declaration in local mode.
+ */
+public class ApplicationTest {
+
+ @Test
+ public void testApplication() throws Exception {
+ try {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ lma.prepareDAG(new EnricherAppWithJSONFile(), conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.run(10000); // runs for 10 seconds and quits
+ } catch (ConstraintViolationException e) {
+ Assert.fail("constraint violations: " + e.getConstraintViolations());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/filter/pom.xml
----------------------------------------------------------------------
diff --git a/examples/filter/pom.xml b/examples/filter/pom.xml
index 9407818..7ef038e 100644
--- a/examples/filter/pom.xml
+++ b/examples/filter/pom.xml
@@ -1,266 +1,26 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
-
- <groupId>com.datatorrent.tutorial</groupId>
- <version>1.0-SNAPSHOT</version>
- <artifactId>filter</artifactId>
+
+ <parent>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-examples</artifactId>
+ <version>3.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>malhar-examples-filter</artifactId>
<packaging>jar</packaging>
<!-- change these to the appropriate values -->
<name>Filter Operator</name>
<description>Apex application demonstrating filter operator</description>
- <properties>
- <!-- change this if you desire to use a different version of Apex Core -->
- <apex.version>3.5.0</apex.version>
- <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath>
- <malhar.version>3.6.0</malhar.version>
- </properties>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-eclipse-plugin</artifactId>
- <version>2.9</version>
- <configuration>
- <downloadSources>true</downloadSources>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>3.3</version>
- <configuration>
- <encoding>UTF-8</encoding>
- <source>1.7</source>
- <target>1.7</target>
- <debug>true</debug>
- <optimize>false</optimize>
- <showDeprecation>true</showDeprecation>
- <showWarnings>true</showWarnings>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <version>2.8</version>
- <executions>
- <execution>
- <id>copy-dependencies</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>copy-dependencies</goal>
- </goals>
- <configuration>
- <outputDirectory>target/deps</outputDirectory>
- <includeScope>runtime</includeScope>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <executions>
- <execution>
- <id>app-package-assembly</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <finalName>${project.artifactId}-${project.version}-apexapp</finalName>
- <appendAssemblyId>false</appendAssemblyId>
- <descriptors>
- <descriptor>src/assemble/appPackage.xml</descriptor>
- </descriptors>
- <archiverConfig>
- <defaultDirectoryMode>0755</defaultDirectoryMode>
- </archiverConfig>
- <archive>
- <manifestEntries>
- <Class-Path>${apex.apppackage.classpath}</Class-Path>
- <DT-Engine-Version>${apex.version}</DT-Engine-Version>
- <DT-App-Package-Group-Id>${project.groupId}</DT-App-Package-Group-Id>
- <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name>
- <DT-App-Package-Version>${project.version}</DT-App-Package-Version>
- <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name>
- <DT-App-Package-Description>${project.description}</DT-App-Package-Description>
- </manifestEntries>
- </archive>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <artifactId>maven-antrun-plugin</artifactId>
- <version>1.7</version>
- <executions>
- <execution>
- <phase>package</phase>
- <configuration>
- <target>
- <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar"
- tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa" />
- </target>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- <execution>
- <!-- create resource directory for xml javadoc-->
- <id>createJavadocDirectory</id>
- <phase>generate-resources</phase>
- <configuration>
- <tasks>
- <delete dir="${project.build.directory}/generated-resources/xml-javadoc"/>
- <mkdir dir="${project.build.directory}/generated-resources/xml-javadoc"/>
- </tasks>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>1.9.1</version>
- <executions>
- <execution>
- <id>attach-artifacts</id>
- <phase>package</phase>
- <goals>
- <goal>attach-artifact</goal>
- </goals>
- <configuration>
- <artifacts>
- <artifact>
- <file>target/${project.artifactId}-${project.version}.apa</file>
- <type>apa</type>
- </artifact>
- </artifacts>
- <skipAttach>false</skipAttach>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <!-- generate javdoc -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <executions>
- <!-- generate xml javadoc -->
- <execution>
- <id>xml-doclet</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>javadoc</goal>
- </goals>
- <configuration>
- <doclet>com.github.markusbernhardt.xmldoclet.XmlDoclet</doclet>
- <additionalparam>-d ${project.build.directory}/generated-resources/xml-javadoc -filename ${project.artifactId}-${project.version}-javadoc.xml</additionalparam>
- <useStandardDocletOptions>false</useStandardDocletOptions>
- <docletArtifact>
- <groupId>com.github.markusbernhardt</groupId>
- <artifactId>xml-doclet</artifactId>
- <version>1.0.4</version>
- </docletArtifact>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <!-- Transform xml javadoc to stripped down version containing only class/interface comments and tags-->
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>xml-maven-plugin</artifactId>
- <version>1.0</version>
- <executions>
- <execution>
- <id>transform-xmljavadoc</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>transform</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <transformationSets>
- <transformationSet>
- <dir>${project.build.directory}/generated-resources/xml-javadoc</dir>
- <includes>
- <include>${project.artifactId}-${project.version}-javadoc.xml</include>
- </includes>
- <stylesheet>XmlJavadocCommentsExtractor.xsl</stylesheet>
- <outputDir>${project.build.directory}/generated-resources/xml-javadoc</outputDir>
- </transformationSet>
- </transformationSets>
- </configuration>
- </plugin>
- <!-- copy xml javadoc to class jar -->
- <plugin>
- <artifactId>maven-resources-plugin</artifactId>
- <version>2.6</version>
- <executions>
- <execution>
- <id>copy-resources</id>
- <phase>process-resources</phase>
- <goals>
- <goal>copy-resources</goal>
- </goals>
- <configuration>
- <outputDirectory>${basedir}/target/classes</outputDirectory>
- <resources>
- <resource>
- <directory>${project.build.directory}/generated-resources/xml-javadoc</directory>
- <includes>
- <include>${project.artifactId}-${project.version}-javadoc.xml</include>
- </includes>
- <filtering>true</filtering>
- </resource>
- </resources>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
-
- </build>
-
<dependencies>
- <!-- add your dependencies here -->
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-library</artifactId>
- <version>${malhar.version}</version>
- <!--
- If you know that your application does not need transitive dependencies pulled in by malhar-library,
- uncomment the following to reduce the size of your app package.
- -->
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- <version>3.1</version>
- <type>jar</type>
- </dependency>
<dependency>
<groupId>org.apache.apex</groupId>
<artifactId>malhar-contrib</artifactId>
- <version>${malhar.version}</version>
- <!--
+ <version>${project.version}</version>
+ <!--
If you know that your application does not need transitive dependencies pulled in by malhar-library,
uncomment the following to reduce the size of your app package.
-->
@@ -277,23 +37,10 @@
<version>2.4.0</version>
<optional>true</optional>
</dependency>
-
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>apex-common</artifactId>
- <version>${apex.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.10</version>
- <scope>test</scope>
- </dependency>
<dependency>
<groupId>org.apache.apex</groupId>
<artifactId>apex-engine</artifactId>
- <version>${apex.version}</version>
+ <version>${apex.core.version}</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -308,5 +55,4 @@
<version>2.7.8</version>
</dependency>
</dependencies>
-
</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/filter/src/main/java/com/datatorrent/tutorial/filter/Application.java
----------------------------------------------------------------------
diff --git a/examples/filter/src/main/java/com/datatorrent/tutorial/filter/Application.java b/examples/filter/src/main/java/com/datatorrent/tutorial/filter/Application.java
deleted file mode 100644
index 4ebb153..0000000
--- a/examples/filter/src/main/java/com/datatorrent/tutorial/filter/Application.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Copyright (c) 2016 DataTorrent, Inc.
- * All rights reserved.
- */
-
-package com.datatorrent.tutorial.filter;
-
-import org.apache.apex.malhar.lib.fs.FSRecordReaderModule;
-import org.apache.apex.malhar.lib.fs.GenericFileOutputOperator.StringFileOutputOperator;
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.contrib.formatter.CsvFormatter;
-import com.datatorrent.contrib.parser.CsvParser;
-import com.datatorrent.lib.filter.FilterOperator;
-
-/**
- * Simple application illustrating filter operator
- */
-@ApplicationAnnotation(name="FilterExample")
-public class Application implements StreamingApplication
-{
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
-
- FSRecordReaderModule recordReader = dag.addModule("recordReader", FSRecordReaderModule.class);
- CsvParser csvParser = dag.addOperator("csvParser", CsvParser.class);
- FilterOperator filterOperator = dag.addOperator("filterOperator", new FilterOperator());
-
- CsvFormatter selectedFormatter = dag.addOperator("selectedFormatter", new CsvFormatter());
- CsvFormatter rejectedFormatter = dag.addOperator("rejectedFormatter", new CsvFormatter());
-
- StringFileOutputOperator selectedOutput = dag.addOperator("selectedOutput", new StringFileOutputOperator());
- StringFileOutputOperator rejectedOutput = dag.addOperator("rejectedOutput", new StringFileOutputOperator());
-
- dag.addStream("record", recordReader.records, csvParser.in);
- dag.addStream("pojo", csvParser.out, filterOperator.input);
-
- dag.addStream("pojoSelected", filterOperator.truePort, selectedFormatter.in);
- dag.addStream("pojoRejected", filterOperator.falsePort, rejectedFormatter.in);
-
- dag.addStream("csvSelected", selectedFormatter.out, selectedOutput.input);
- dag.addStream("csvRejected", rejectedFormatter.out, rejectedOutput.input);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/filter/src/main/java/com/datatorrent/tutorial/filter/TransactionPOJO.java
----------------------------------------------------------------------
diff --git a/examples/filter/src/main/java/com/datatorrent/tutorial/filter/TransactionPOJO.java b/examples/filter/src/main/java/com/datatorrent/tutorial/filter/TransactionPOJO.java
deleted file mode 100644
index c0d8817..0000000
--- a/examples/filter/src/main/java/com/datatorrent/tutorial/filter/TransactionPOJO.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Copyright (c) 2016 DataTorrent, Inc.
- * All rights reserved.
- */
-
-package com.datatorrent.tutorial.filter;
-
-public class TransactionPOJO
-{
-
- private long trasactionId;
- private double amount;
- private long accountNumber;
- private String type;
-
- public long getTrasactionId()
- {
- return trasactionId;
- }
-
- public void setTrasactionId(long trasactionId)
- {
- this.trasactionId = trasactionId;
- }
-
- public double getAmount()
- {
- return amount;
- }
-
- public void setAmount(double amount)
- {
- this.amount = amount;
- }
-
- public long getAccountNumber()
- {
- return accountNumber;
- }
-
- public void setAccountNumber(long accountNumber)
- {
- this.accountNumber = accountNumber;
- }
-
- public String getType()
- {
- return type;
- }
-
- public void setType(String type)
- {
- this.type = type;
- }
-
- @Override
- public String toString()
- {
- return "TransactionPOJO [trasactionId=" + trasactionId + ", amount=" + amount + ", accountNumber=" + accountNumber
- + ", type=" + type + "]";
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/filter/src/main/java/org/apache/apex/examples/filter/Application.java
----------------------------------------------------------------------
diff --git a/examples/filter/src/main/java/org/apache/apex/examples/filter/Application.java b/examples/filter/src/main/java/org/apache/apex/examples/filter/Application.java
new file mode 100644
index 0000000..5bbf159
--- /dev/null
+++ b/examples/filter/src/main/java/org/apache/apex/examples/filter/Application.java
@@ -0,0 +1,49 @@
+/**
+ * Copyright (c) 2016 DataTorrent, Inc.
+ * All rights reserved.
+ */
+
+package org.apache.apex.examples.filter;
+
+import org.apache.apex.malhar.lib.fs.FSRecordReaderModule;
+import org.apache.apex.malhar.lib.fs.GenericFileOutputOperator.StringFileOutputOperator;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.contrib.formatter.CsvFormatter;
+import com.datatorrent.contrib.parser.CsvParser;
+import com.datatorrent.lib.filter.FilterOperator;
+
+/**
+ * Simple application illustrating filter operator
+ */
+@ApplicationAnnotation(name="FilterExample")
+public class Application implements StreamingApplication
+{
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+
+ FSRecordReaderModule recordReader = dag.addModule("recordReader", FSRecordReaderModule.class);
+ CsvParser csvParser = dag.addOperator("csvParser", CsvParser.class);
+ FilterOperator filterOperator = dag.addOperator("filterOperator", new FilterOperator());
+
+ CsvFormatter selectedFormatter = dag.addOperator("selectedFormatter", new CsvFormatter());
+ CsvFormatter rejectedFormatter = dag.addOperator("rejectedFormatter", new CsvFormatter());
+
+ StringFileOutputOperator selectedOutput = dag.addOperator("selectedOutput", new StringFileOutputOperator());
+ StringFileOutputOperator rejectedOutput = dag.addOperator("rejectedOutput", new StringFileOutputOperator());
+
+ dag.addStream("record", recordReader.records, csvParser.in);
+ dag.addStream("pojo", csvParser.out, filterOperator.input);
+
+ dag.addStream("pojoSelected", filterOperator.truePort, selectedFormatter.in);
+ dag.addStream("pojoRejected", filterOperator.falsePort, rejectedFormatter.in);
+
+ dag.addStream("csvSelected", selectedFormatter.out, selectedOutput.input);
+ dag.addStream("csvRejected", rejectedFormatter.out, rejectedOutput.input);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/filter/src/main/java/org/apache/apex/examples/filter/TransactionPOJO.java
----------------------------------------------------------------------
diff --git a/examples/filter/src/main/java/org/apache/apex/examples/filter/TransactionPOJO.java b/examples/filter/src/main/java/org/apache/apex/examples/filter/TransactionPOJO.java
new file mode 100644
index 0000000..4b0d7a9
--- /dev/null
+++ b/examples/filter/src/main/java/org/apache/apex/examples/filter/TransactionPOJO.java
@@ -0,0 +1,62 @@
+/**
+ * Copyright (c) 2016 DataTorrent, Inc.
+ * All rights reserved.
+ */
+
+package org.apache.apex.examples.filter;
+
+public class TransactionPOJO
+{
+
+ private long trasactionId;
+ private double amount;
+ private long accountNumber;
+ private String type;
+
+ public long getTrasactionId()
+ {
+ return trasactionId;
+ }
+
+ public void setTrasactionId(long trasactionId)
+ {
+ this.trasactionId = trasactionId;
+ }
+
+ public double getAmount()
+ {
+ return amount;
+ }
+
+ public void setAmount(double amount)
+ {
+ this.amount = amount;
+ }
+
+ public long getAccountNumber()
+ {
+ return accountNumber;
+ }
+
+ public void setAccountNumber(long accountNumber)
+ {
+ this.accountNumber = accountNumber;
+ }
+
+ public String getType()
+ {
+ return type;
+ }
+
+ public void setType(String type)
+ {
+ this.type = type;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "TransactionPOJO [trasactionId=" + trasactionId + ", amount=" + amount + ", accountNumber=" + accountNumber
+ + ", type=" + type + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/filter/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/filter/src/main/resources/META-INF/properties.xml b/examples/filter/src/main/resources/META-INF/properties.xml
index 079fb3f..8c1a4a3 100644
--- a/examples/filter/src/main/resources/META-INF/properties.xml
+++ b/examples/filter/src/main/resources/META-INF/properties.xml
@@ -33,7 +33,7 @@
</property>
<property>
<name>dt.application.FilterExample.operator.csvParser.port.out.attr.TUPLE_CLASS</name>
- <value>com.datatorrent.tutorial.filter.TransactionPOJO</value>
+ <value>org.apache.apex.examples.filter.TransactionPOJO</value>
</property>
<property>
<name>dt.application.FilterExample.operator.selectedOutput.prop.maxIdleWindows</name>
@@ -45,15 +45,15 @@
</property>
<property>
<name>dt.application.FilterExample.operator.filterOperator.port.input.attr.TUPLE_CLASS</name>
- <value>com.datatorrent.tutorial.filter.TransactionPOJO</value>
+ <value>org.apache.apex.examples.filter.TransactionPOJO</value>
</property>
<property>
<name>dt.application.FilterExample.operator.selectedFormatter.port.in.attr.TUPLE_CLASS</name>
- <value>com.datatorrent.tutorial.filter.TransactionPOJO</value>
+ <value>org.apache.apex.examples.filter.TransactionPOJO</value>
</property>
<property>
<name>dt.application.FilterExample.operator.rejectedFormatter.port.in.attr.TUPLE_CLASS</name>
- <value>com.datatorrent.tutorial.filter.TransactionPOJO</value>
+ <value>org.apache.apex.examples.filter.TransactionPOJO</value>
</property>
<property>
<name>dt.application.FilterExample.operator.filterOperator.prop.condition</name>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/filter/src/test/java/com/datatorrent/tutorial/filter/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/filter/src/test/java/com/datatorrent/tutorial/filter/ApplicationTest.java b/examples/filter/src/test/java/com/datatorrent/tutorial/filter/ApplicationTest.java
deleted file mode 100644
index a90e822..0000000
--- a/examples/filter/src/test/java/com/datatorrent/tutorial/filter/ApplicationTest.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/**
- * Copyright (c) 2016 DataTorrent, Inc.
- * All rights reserved.
- */
-
-package com.datatorrent.tutorial.filter;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.concurrent.Callable;
-
-import javax.validation.ConstraintViolationException;
-
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestWatcher;
-import org.junit.runner.Description;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.LocalMode;
-import com.datatorrent.stram.StramLocalCluster;
-
-/**
- * Test the DAG declaration in local mode.
- */
-public class ApplicationTest
-{
- private String outputDir;
-
- public static class TestMeta extends TestWatcher
- {
- public String baseDirectory;
-
- @Override
- protected void starting(org.junit.runner.Description description)
- {
- this.baseDirectory = "target/" + description.getClassName() + "/" + description.getMethodName();
- }
-
- @Override
- protected void finished(Description description)
- {
- super.finished(description);
- try {
- FileUtils.forceDelete(new File(baseDirectory));
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- }
-
- @Rule
- public TestMeta testMeta = new TestMeta();
-
- @Before
- public void setup() throws Exception
- {
- outputDir = testMeta.baseDirectory + File.separator + "output";
- }
-
- @Test
- public void testApplication() throws IOException, Exception
- {
-
- try {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
- conf.set("dt.application.FilterExample.operator.selectedOutput.prop.filePath", outputDir);
- conf.set("dt.application.FilterExample.operator.rejectedOutput.prop.filePath", outputDir);
- final File selectedfile = FileUtils.getFile(outputDir, "selected.txt_8.0");
- final File rejectedfile = FileUtils.getFile(outputDir, "rejected.txt_6.0");
-
- lma.prepareDAG(new Application(), conf);
- LocalMode.Controller lc = lma.getController();
-
- ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>()
- {
- @Override
- public Boolean call() throws Exception
- {
- if (selectedfile.exists() && rejectedfile.exists()) {
- return true;
- }
- return false;
- }
- });
-
- lc.run(40000);
- Assert.assertTrue(
- FileUtils.contentEquals(
- FileUtils.getFile(
- "src/main/resources/META-INF/selected_output.txt"
- ),selectedfile));
-
- Assert.assertTrue(
- FileUtils.contentEquals(
- FileUtils.getFile(
- "src/main/resources/META-INF/rejected_output.txt"
- ),rejectedfile));
-
- } catch (ConstraintViolationException e) {
- Assert.fail("constraint violations: " + e.getConstraintViolations());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/filter/src/test/java/org/apache/apex/examples/filter/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/filter/src/test/java/org/apache/apex/examples/filter/ApplicationTest.java b/examples/filter/src/test/java/org/apache/apex/examples/filter/ApplicationTest.java
new file mode 100644
index 0000000..2170753
--- /dev/null
+++ b/examples/filter/src/test/java/org/apache/apex/examples/filter/ApplicationTest.java
@@ -0,0 +1,96 @@
+/**
+ * Copyright (c) 2016 DataTorrent, Inc.
+ * All rights reserved.
+ */
+
+package org.apache.apex.examples.filter;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.concurrent.Callable;
+
+import javax.validation.ConstraintViolationException;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.LocalMode;
+import com.datatorrent.stram.StramLocalCluster;
+
+/**
+ * Test the DAG declaration in local mode.
+ */
+public class ApplicationTest
+{
+ private String outputDir;
+
+ public static class TestMeta extends TestWatcher
+ {
+ public String baseDirectory;
+
+ @Override
+ protected void starting(org.junit.runner.Description description)
+ {
+ this.baseDirectory = "target/" + description.getClassName() + "/" + description.getMethodName();
+ }
+
+ @Override
+ protected void finished(Description description)
+ {
+ super.finished(description);
+ try {
+ FileUtils.forceDelete(new File(baseDirectory));
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ @Rule
+ public TestMeta testMeta = new TestMeta();
+
+ @Before
+ public void setup() throws Exception
+ {
+ outputDir = testMeta.baseDirectory + File.separator + "output";
+ }
+
+ @Test
+ public void testApplication() throws IOException, Exception
+ {
+ try {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ conf.set("dt.application.FilterExample.operator.selectedOutput.prop.filePath", outputDir);
+ conf.set("dt.application.FilterExample.operator.rejectedOutput.prop.filePath", outputDir);
+ final File selectedfile = FileUtils.getFile(outputDir, "selected.txt_8.0");
+ final File rejectedfile = FileUtils.getFile(outputDir, "rejected.txt_6.0");
+
+ lma.prepareDAG(new Application(), conf);
+ LocalMode.Controller lc = lma.getController();
+
+ ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>()
+ {
+ @Override
+ public Boolean call() throws Exception
+ {
+ if (selectedfile.exists() && rejectedfile.exists()) {
+ return true;
+ }
+ return false;
+ }
+ });
+
+ } catch (ConstraintViolationException e) {
+ Assert.fail("constraint violations: " + e.getConstraintViolations());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/innerjoin/pom.xml
----------------------------------------------------------------------
diff --git a/examples/innerjoin/pom.xml b/examples/innerjoin/pom.xml
index 29c1b90..78193ca 100644
--- a/examples/innerjoin/pom.xml
+++ b/examples/innerjoin/pom.xml
@@ -2,266 +2,49 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>com.example</groupId>
- <version>1.0-SNAPSHOT</version>
- <artifactId>innerjoin</artifactId>
+ <parent>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-examples</artifactId>
+ <version>3.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>malhar-examples-innerjoin</artifactId>
<packaging>jar</packaging>
<name>Inner Join Application</name>
<description>Sample Application for Inner Join</description>
- <properties>
- <!-- change this if you desire to use a different version of Apex Core -->
- <apex.version>3.5.0</apex.version>
- <malhar.version>3.6.0</malhar.version>
- <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath>
- </properties>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-eclipse-plugin</artifactId>
- <version>2.9</version>
- <configuration>
- <downloadSources>true</downloadSources>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>3.3</version>
- <configuration>
- <encoding>UTF-8</encoding>
- <source>1.7</source>
- <target>1.7</target>
- <debug>true</debug>
- <optimize>false</optimize>
- <showDeprecation>true</showDeprecation>
- <showWarnings>true</showWarnings>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <version>2.8</version>
- <executions>
- <execution>
- <id>copy-dependencies</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>copy-dependencies</goal>
- </goals>
- <configuration>
- <outputDirectory>target/deps</outputDirectory>
- <includeScope>runtime</includeScope>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <executions>
- <execution>
- <id>app-package-assembly</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <finalName>${project.artifactId}-${project.version}-apexapp</finalName>
- <appendAssemblyId>false</appendAssemblyId>
- <descriptors>
- <descriptor>src/assemble/appPackage.xml</descriptor>
- </descriptors>
- <archiverConfig>
- <defaultDirectoryMode>0755</defaultDirectoryMode>
- </archiverConfig>
- <archive>
- <manifestEntries>
- <Class-Path>${apex.apppackage.classpath}</Class-Path>
- <DT-Engine-Version>${apex.version}</DT-Engine-Version>
- <DT-App-Package-Group-Id>${project.groupId}</DT-App-Package-Group-Id>
- <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name>
- <DT-App-Package-Version>${project.version}</DT-App-Package-Version>
- <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name>
- <DT-App-Package-Description>${project.description}</DT-App-Package-Description>
- </manifestEntries>
- </archive>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <artifactId>maven-antrun-plugin</artifactId>
- <version>1.7</version>
- <executions>
- <execution>
- <phase>package</phase>
- <configuration>
- <target>
- <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar"
- tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa" />
- </target>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- <execution>
- <!-- create resource directory for xml javadoc-->
- <id>createJavadocDirectory</id>
- <phase>generate-resources</phase>
- <configuration>
- <tasks>
- <delete dir="${project.build.directory}/generated-resources/xml-javadoc"/>
- <mkdir dir="${project.build.directory}/generated-resources/xml-javadoc"/>
- </tasks>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>1.9.1</version>
- <executions>
- <execution>
- <id>attach-artifacts</id>
- <phase>package</phase>
- <goals>
- <goal>attach-artifact</goal>
- </goals>
- <configuration>
- <artifacts>
- <artifact>
- <file>target/${project.artifactId}-${project.version}.apa</file>
- <type>apa</type>
- </artifact>
- </artifacts>
- <skipAttach>false</skipAttach>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <!-- generate javdoc -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <executions>
- <!-- generate xml javadoc -->
- <execution>
- <id>xml-doclet</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>javadoc</goal>
- </goals>
- <configuration>
- <doclet>com.github.markusbernhardt.xmldoclet.XmlDoclet</doclet>
- <additionalparam>-d ${project.build.directory}/generated-resources/xml-javadoc -filename ${project.artifactId}-${project.version}-javadoc.xml</additionalparam>
- <useStandardDocletOptions>false</useStandardDocletOptions>
- <docletArtifact>
- <groupId>com.github.markusbernhardt</groupId>
- <artifactId>xml-doclet</artifactId>
- <version>1.0.4</version>
- </docletArtifact>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <!-- Transform xml javadoc to stripped down version containing only class/interface comments and tags-->
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>xml-maven-plugin</artifactId>
- <version>1.0</version>
- <executions>
- <execution>
- <id>transform-xmljavadoc</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>transform</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <transformationSets>
- <transformationSet>
- <dir>${project.build.directory}/generated-resources/xml-javadoc</dir>
- <includes>
- <include>${project.artifactId}-${project.version}-javadoc.xml</include>
- </includes>
- <stylesheet>XmlJavadocCommentsExtractor.xsl</stylesheet>
- <outputDir>${project.build.directory}/generated-resources/xml-javadoc</outputDir>
- </transformationSet>
- </transformationSets>
- </configuration>
- </plugin>
- <!-- copy xml javadoc to class jar -->
- <plugin>
- <artifactId>maven-resources-plugin</artifactId>
- <version>2.6</version>
- <executions>
- <execution>
- <id>copy-resources</id>
- <phase>process-resources</phase>
- <goals>
- <goal>copy-resources</goal>
- </goals>
- <configuration>
- <outputDirectory>${basedir}/target/classes</outputDirectory>
- <resources>
- <resource>
- <directory>${project.build.directory}/generated-resources/xml-javadoc</directory>
- <includes>
- <include>${project.artifactId}-${project.version}-javadoc.xml</include>
- </includes>
- <filtering>true</filtering>
- </resource>
- </resources>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
-
- </build>
-
<dependencies>
- <!-- add your dependencies here -->
<dependency>
<groupId>org.apache.apex</groupId>
<artifactId>malhar-library</artifactId>
- <version>${malhar.version}</version>
+ <version>${project.version}</version>
+ <!--
+ If you know that your application does not need transitive dependencies pulled in by malhar-library,
+ uncomment the following to reduce the size of your app package.
+ -->
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>apex-common</artifactId>
- <version>${apex.version}</version>
- <scope>provided</scope>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.14.1</version>
</dependency>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.10</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>apex-engine</artifactId>
- <version>${apex.version}</version>
- <scope>test</scope>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ <version>2.9.1</version>
</dependency>
<dependency>
<groupId>org.codehaus.janino</groupId>
- <artifactId>janino</artifactId>
+ <artifactId>commons-compiler</artifactId>
<version>2.7.8</version>
+ <type>jar</type>
</dependency>
</dependencies>
-
</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/innerjoin/src/main/java/com/example/join/InnerJoinApplication.java
----------------------------------------------------------------------
diff --git a/examples/innerjoin/src/main/java/com/example/join/InnerJoinApplication.java b/examples/innerjoin/src/main/java/com/example/join/InnerJoinApplication.java
deleted file mode 100644
index 0b2f663..0000000
--- a/examples/innerjoin/src/main/java/com/example/join/InnerJoinApplication.java
+++ /dev/null
@@ -1,39 +0,0 @@
-package com.example.join;
-
-import org.apache.apex.malhar.lib.join.POJOInnerJoinOperator;
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.common.partitioner.StatelessPartitioner;
-import com.datatorrent.lib.io.ConsoleOutputOperator;
-
-@ApplicationAnnotation(name="InnerJoinExample")
-public class InnerJoinApplication implements StreamingApplication
-{
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- // SalesEvent Generator
- POJOGenerator salesGenerator = dag.addOperator("Input1", new POJOGenerator());
- // ProductEvent Generator
- POJOGenerator productGenerator = dag.addOperator("Input2", new POJOGenerator());
- productGenerator.setSalesEvent(false);
-
- // Inner join Operator
- POJOInnerJoinOperator join = dag.addOperator("Join", new POJOInnerJoinOperator());
- ConsoleOutputOperator output = dag.addOperator("Output", new ConsoleOutputOperator());
-
- // Streams
- dag.addStream("SalesToJoin", salesGenerator.output, join.input1);
- dag.addStream("ProductToJoin", productGenerator.output, join.input2);
- dag.addStream("JoinToConsole", join.outputPort, output.input);
-
- // Setting tuple class properties to the ports of join operator
- dag.setInputPortAttribute(join.input1, Context.PortContext.TUPLE_CLASS, POJOGenerator.SalesEvent.class);
- dag.setInputPortAttribute(join.input2, Context.PortContext.TUPLE_CLASS, POJOGenerator.ProductEvent.class);
- dag.setOutputPortAttribute(join.outputPort,Context.PortContext.TUPLE_CLASS, POJOGenerator.SalesEvent.class);
- }
-}
[15/19] apex-malhar git commit: APEXMALHAR-2233 Updated the examples
to follow the structure of apex-malhar examples. Specified dependencies in
pom.xmls of individual examples correctly.
Posted by th...@apache.org.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/recordReader/pom.xml
----------------------------------------------------------------------
diff --git a/examples/recordReader/pom.xml b/examples/recordReader/pom.xml
index 007eaef..25ac191 100644
--- a/examples/recordReader/pom.xml
+++ b/examples/recordReader/pom.xml
@@ -1,267 +1,28 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
-
- <groupId>com.example</groupId>
- <version>1.0-SNAPSHOT</version>
- <artifactId>recordReader</artifactId>
+
+ <parent>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-examples</artifactId>
+ <version>3.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>malhar-examples-recordReader</artifactId>
<packaging>jar</packaging>
<name>File Record Reader</name>
<description>Simple application illustrating use of record reader operator</description>
- <properties>
- <!-- change this if you desire to use a different version of Apex Core -->
- <apex.version>3.5.0</apex.version>
- <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath>
- <malhar.version>3.6.0</malhar.version>
- </properties>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-eclipse-plugin</artifactId>
- <version>2.9</version>
- <configuration>
- <downloadSources>true</downloadSources>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>3.3</version>
- <configuration>
- <encoding>UTF-8</encoding>
- <source>1.7</source>
- <target>1.7</target>
- <debug>true</debug>
- <optimize>false</optimize>
- <showDeprecation>true</showDeprecation>
- <showWarnings>true</showWarnings>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <version>2.8</version>
- <executions>
- <execution>
- <id>copy-dependencies</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>copy-dependencies</goal>
- </goals>
- <configuration>
- <outputDirectory>target/deps</outputDirectory>
- <includeScope>runtime</includeScope>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <executions>
- <execution>
- <id>app-package-assembly</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <finalName>${project.artifactId}-${project.version}-apexapp</finalName>
- <appendAssemblyId>false</appendAssemblyId>
- <descriptors>
- <descriptor>src/assemble/appPackage.xml</descriptor>
- </descriptors>
- <archiverConfig>
- <defaultDirectoryMode>0755</defaultDirectoryMode>
- </archiverConfig>
- <archive>
- <manifestEntries>
- <Class-Path>${apex.apppackage.classpath}</Class-Path>
- <DT-Engine-Version>${apex.version}</DT-Engine-Version>
- <DT-App-Package-Group-Id>${project.groupId}</DT-App-Package-Group-Id>
- <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name>
- <DT-App-Package-Version>${project.version}</DT-App-Package-Version>
- <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name>
- <DT-App-Package-Description>${project.description}</DT-App-Package-Description>
- </manifestEntries>
- </archive>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <artifactId>maven-antrun-plugin</artifactId>
- <version>1.7</version>
- <executions>
- <execution>
- <phase>package</phase>
- <configuration>
- <target>
- <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar"
- tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa" />
- </target>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- <execution>
- <!-- create resource directory for xml javadoc-->
- <id>createJavadocDirectory</id>
- <phase>generate-resources</phase>
- <configuration>
- <tasks>
- <delete dir="${project.build.directory}/generated-resources/xml-javadoc"/>
- <mkdir dir="${project.build.directory}/generated-resources/xml-javadoc"/>
- </tasks>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>1.9.1</version>
- <executions>
- <execution>
- <id>attach-artifacts</id>
- <phase>package</phase>
- <goals>
- <goal>attach-artifact</goal>
- </goals>
- <configuration>
- <artifacts>
- <artifact>
- <file>target/${project.artifactId}-${project.version}.apa</file>
- <type>apa</type>
- </artifact>
- </artifacts>
- <skipAttach>false</skipAttach>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <!-- generate javdoc -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <executions>
- <!-- generate xml javadoc -->
- <execution>
- <id>xml-doclet</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>javadoc</goal>
- </goals>
- <configuration>
- <doclet>com.github.markusbernhardt.xmldoclet.XmlDoclet</doclet>
- <additionalparam>-d ${project.build.directory}/generated-resources/xml-javadoc -filename ${project.artifactId}-${project.version}-javadoc.xml</additionalparam>
- <useStandardDocletOptions>false</useStandardDocletOptions>
- <docletArtifact>
- <groupId>com.github.markusbernhardt</groupId>
- <artifactId>xml-doclet</artifactId>
- <version>1.0.4</version>
- </docletArtifact>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <!-- Transform xml javadoc to stripped down version containing only class/interface comments and tags-->
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>xml-maven-plugin</artifactId>
- <version>1.0</version>
- <executions>
- <execution>
- <id>transform-xmljavadoc</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>transform</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <transformationSets>
- <transformationSet>
- <dir>${project.build.directory}/generated-resources/xml-javadoc</dir>
- <includes>
- <include>${project.artifactId}-${project.version}-javadoc.xml</include>
- </includes>
- <stylesheet>XmlJavadocCommentsExtractor.xsl</stylesheet>
- <outputDir>${project.build.directory}/generated-resources/xml-javadoc</outputDir>
- </transformationSet>
- </transformationSets>
- </configuration>
- </plugin>
- <!-- copy xml javadoc to class jar -->
- <plugin>
- <artifactId>maven-resources-plugin</artifactId>
- <version>2.6</version>
- <executions>
- <execution>
- <id>copy-resources</id>
- <phase>process-resources</phase>
- <goals>
- <goal>copy-resources</goal>
- </goals>
- <configuration>
- <outputDirectory>${basedir}/target/classes</outputDirectory>
- <resources>
- <resource>
- <directory>${project.build.directory}/generated-resources/xml-javadoc</directory>
- <includes>
- <include>${project.artifactId}-${project.version}-javadoc.xml</include>
- </includes>
- <filtering>true</filtering>
- </resource>
- </resources>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
-
- </build>
-
<dependencies>
- <!-- add your dependencies here -->
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-library</artifactId>
- <version>${malhar.version}</version>
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<dependency>
<groupId>org.apache.apex</groupId>
<artifactId>malhar-contrib</artifactId>
- <version>${malhar.version}</version>
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-contrib</artifactId>
- <version>${malhar.version}</version>
- <classifier>tests</classifier>
- <scope>test</scope>
+ <version>${project.version}</version>
+ <!--
+ If you know that your application does not need transitive dependencies pulled in by malhar-library,
+ uncomment the following to reduce the size of your app package.
+ -->
<exclusions>
<exclusion>
<groupId>*</groupId>
@@ -274,24 +35,5 @@
<artifactId>super-csv</artifactId>
<version>2.4.0</version>
</dependency>
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>apex-common</artifactId>
- <version>${apex.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.10</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>apex-engine</artifactId>
- <version>${apex.version}</version>
- <scope>test</scope>
- </dependency>
</dependencies>
-
</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/recordReader/src/main/java/com/example/recordReader/Application.java
----------------------------------------------------------------------
diff --git a/examples/recordReader/src/main/java/com/example/recordReader/Application.java b/examples/recordReader/src/main/java/com/example/recordReader/Application.java
deleted file mode 100644
index 8b0dd75..0000000
--- a/examples/recordReader/src/main/java/com/example/recordReader/Application.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Put your copyright and license info here.
- */
-package com.example.recordReader;
-
-import org.apache.apex.malhar.lib.fs.FSRecordReaderModule;
-import org.apache.apex.malhar.lib.fs.GenericFileOutputOperator.StringFileOutputOperator;
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.contrib.formatter.CsvFormatter;
-import com.datatorrent.contrib.parser.CsvParser;
-
-@ApplicationAnnotation(name="RecordReaderExample")
-public class Application implements StreamingApplication
-{
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- FSRecordReaderModule recordReader = dag.addModule("recordReader", FSRecordReaderModule.class);
- CsvParser csvParser = dag.addOperator("csvParser", CsvParser.class);
- CsvFormatter formatter = dag.addOperator("formatter", new CsvFormatter());
- StringFileOutputOperator fileOutput = dag.addOperator("fileOutput", new StringFileOutputOperator());
-
- dag.addStream("record", recordReader.records, csvParser.in);
- dag.addStream("pojo", csvParser.out, formatter.in);
- dag.addStream("string", formatter.out, fileOutput.input);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/recordReader/src/main/java/com/example/recordReader/TransactionsSchema.java
----------------------------------------------------------------------
diff --git a/examples/recordReader/src/main/java/com/example/recordReader/TransactionsSchema.java b/examples/recordReader/src/main/java/com/example/recordReader/TransactionsSchema.java
deleted file mode 100644
index de20516..0000000
--- a/examples/recordReader/src/main/java/com/example/recordReader/TransactionsSchema.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/**
- * Copyright (c) 2016 DataTorrent, Inc.
- * All rights reserved.
- */
-
-package com.example.recordReader;
-
-import java.util.Date;
-
-public class TransactionsSchema
-{
- private String customerName;
- private String customerPhone;
- private String customerEmail;
- private String city;
- private String country;
- private String uid;
- private String accountNumber;
- private long txId;
- private Date txDate;
- private double txAmount;
-
- public String getCustomerName()
- {
- return customerName;
- }
-
- public void setCustomerName(String customerName)
- {
- this.customerName = customerName;
- }
-
- public String getCustomerPhone()
- {
- return customerPhone;
- }
-
- public void setCustomerPhone(String customerPhone)
- {
- this.customerPhone = customerPhone;
- }
-
- public String getCustomerEmail()
- {
- return customerEmail;
- }
-
- public void setCustomerEmail(String customerEmail)
- {
- this.customerEmail = customerEmail;
- }
-
- public String getCity()
- {
- return city;
- }
-
- public void setCity(String city)
- {
- this.city = city;
- }
-
- public String getCountry()
- {
- return country;
- }
-
- public void setCountry(String country)
- {
- this.country = country;
- }
-
- public String getUid()
- {
- return uid;
- }
-
- public void setUid(String uid)
- {
- this.uid = uid;
- }
-
- public String getAccountNumber()
- {
- return accountNumber;
- }
-
- public void setAccountNumber(String accountNumber)
- {
- this.accountNumber = accountNumber;
- }
-
- public long getTxId()
- {
- return txId;
- }
-
- public void setTxId(long txId)
- {
- this.txId = txId;
- }
-
- public Date getTxDate()
- {
- return txDate;
- }
-
- public void setTxDate(Date txDate)
- {
- this.txDate = txDate;
- }
-
- public double getTxAmount()
- {
- return txAmount;
- }
-
- public void setTxAmount(double txAmount)
- {
- this.txAmount = txAmount;
- }
-
- @Override
- public String toString()
- {
- return "TransactionsSchema [customerName=" + customerName + ", customerPhone=" + customerPhone + ", customerEmail="
- + customerEmail + ", city=" + city + ", country=" + country + ", uid=" + uid + ", accountNumber="
- + accountNumber + ", txId=" + txId + ", txDate=" + txDate + ", txAmount=" + txAmount
- + "]\n";
- }
-
- @Override
- public int hashCode()
- {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((txDate == null) ? 0 : txDate.hashCode());
- result = prime * result + (int)(txId ^ (txId >>> 32));
- return result;
- }
-
- @Override
- public boolean equals(Object obj)
- {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- TransactionsSchema other = (TransactionsSchema)obj;
- if (txDate == null) {
- if (other.txDate != null) {
- return false;
- }
- } else if (!txDate.equals(other.txDate)) {
- return false;
- }
- if (txId != other.txId) {
- return false;
- }
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/recordReader/src/main/java/org/apache/apex/examples/recordReader/Application.java
----------------------------------------------------------------------
diff --git a/examples/recordReader/src/main/java/org/apache/apex/examples/recordReader/Application.java b/examples/recordReader/src/main/java/org/apache/apex/examples/recordReader/Application.java
new file mode 100644
index 0000000..916b061
--- /dev/null
+++ b/examples/recordReader/src/main/java/org/apache/apex/examples/recordReader/Application.java
@@ -0,0 +1,32 @@
+/**
+ * Put your copyright and license info here.
+ */
+package org.apache.apex.examples.recordReader;
+
+import org.apache.apex.malhar.lib.fs.FSRecordReaderModule;
+import org.apache.apex.malhar.lib.fs.GenericFileOutputOperator.StringFileOutputOperator;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.contrib.formatter.CsvFormatter;
+import com.datatorrent.contrib.parser.CsvParser;
+
+@ApplicationAnnotation(name="RecordReaderExample")
+public class Application implements StreamingApplication
+{
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ FSRecordReaderModule recordReader = dag.addModule("recordReader", FSRecordReaderModule.class);
+ CsvParser csvParser = dag.addOperator("csvParser", CsvParser.class);
+ CsvFormatter formatter = dag.addOperator("formatter", new CsvFormatter());
+ StringFileOutputOperator fileOutput = dag.addOperator("fileOutput", new StringFileOutputOperator());
+
+ dag.addStream("record", recordReader.records, csvParser.in);
+ dag.addStream("pojo", csvParser.out, formatter.in);
+ dag.addStream("string", formatter.out, fileOutput.input);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/recordReader/src/main/java/org/apache/apex/examples/recordReader/TransactionsSchema.java
----------------------------------------------------------------------
diff --git a/examples/recordReader/src/main/java/org/apache/apex/examples/recordReader/TransactionsSchema.java b/examples/recordReader/src/main/java/org/apache/apex/examples/recordReader/TransactionsSchema.java
new file mode 100644
index 0000000..28c90f2
--- /dev/null
+++ b/examples/recordReader/src/main/java/org/apache/apex/examples/recordReader/TransactionsSchema.java
@@ -0,0 +1,168 @@
+/**
+ * Copyright (c) 2016 DataTorrent, Inc.
+ * All rights reserved.
+ */
+
+package org.apache.apex.examples.recordReader;
+
+import java.util.Date;
+
+public class TransactionsSchema
+{
+ private String customerName;
+ private String customerPhone;
+ private String customerEmail;
+ private String city;
+ private String country;
+ private String uid;
+ private String accountNumber;
+ private long txId;
+ private Date txDate;
+ private double txAmount;
+
+ public String getCustomerName()
+ {
+ return customerName;
+ }
+
+ public void setCustomerName(String customerName)
+ {
+ this.customerName = customerName;
+ }
+
+ public String getCustomerPhone()
+ {
+ return customerPhone;
+ }
+
+ public void setCustomerPhone(String customerPhone)
+ {
+ this.customerPhone = customerPhone;
+ }
+
+ public String getCustomerEmail()
+ {
+ return customerEmail;
+ }
+
+ public void setCustomerEmail(String customerEmail)
+ {
+ this.customerEmail = customerEmail;
+ }
+
+ public String getCity()
+ {
+ return city;
+ }
+
+ public void setCity(String city)
+ {
+ this.city = city;
+ }
+
+ public String getCountry()
+ {
+ return country;
+ }
+
+ public void setCountry(String country)
+ {
+ this.country = country;
+ }
+
+ public String getUid()
+ {
+ return uid;
+ }
+
+ public void setUid(String uid)
+ {
+ this.uid = uid;
+ }
+
+ public String getAccountNumber()
+ {
+ return accountNumber;
+ }
+
+ public void setAccountNumber(String accountNumber)
+ {
+ this.accountNumber = accountNumber;
+ }
+
+ public long getTxId()
+ {
+ return txId;
+ }
+
+ public void setTxId(long txId)
+ {
+ this.txId = txId;
+ }
+
+ public Date getTxDate()
+ {
+ return txDate;
+ }
+
+ public void setTxDate(Date txDate)
+ {
+ this.txDate = txDate;
+ }
+
+ public double getTxAmount()
+ {
+ return txAmount;
+ }
+
+ public void setTxAmount(double txAmount)
+ {
+ this.txAmount = txAmount;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "TransactionsSchema [customerName=" + customerName + ", customerPhone=" + customerPhone + ", customerEmail="
+ + customerEmail + ", city=" + city + ", country=" + country + ", uid=" + uid + ", accountNumber="
+ + accountNumber + ", txId=" + txId + ", txDate=" + txDate + ", txAmount=" + txAmount
+ + "]\n";
+ }
+
+ @Override
+ public int hashCode()
+ {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((txDate == null) ? 0 : txDate.hashCode());
+ result = prime * result + (int)(txId ^ (txId >>> 32));
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj)
+ {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ TransactionsSchema other = (TransactionsSchema)obj;
+ if (txDate == null) {
+ if (other.txDate != null) {
+ return false;
+ }
+ } else if (!txDate.equals(other.txDate)) {
+ return false;
+ }
+ if (txId != other.txId) {
+ return false;
+ }
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/recordReader/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/recordReader/src/main/resources/META-INF/properties.xml b/examples/recordReader/src/main/resources/META-INF/properties.xml
index 06961f1..612e3fa 100644
--- a/examples/recordReader/src/main/resources/META-INF/properties.xml
+++ b/examples/recordReader/src/main/resources/META-INF/properties.xml
@@ -129,11 +129,11 @@
</property>
<property>
<name>dt.application.RecordReaderExample.operator.csvParser.port.out.attr.TUPLE_CLASS</name>
- <value>com.example.recordReader.TransactionsSchema</value>
+ <value>org.apache.apex.examples.recordReader.TransactionsSchema</value>
</property>
<property>
<name>dt.application.RecordReaderExample.operator.formatter.port.in.attr.TUPLE_CLASS</name>
- <value>com.example.recordReader.TransactionsSchema</value>
+ <value>org.apache.apex.examples.recordReader.TransactionsSchema</value>
</property>
<property>
<name>dt.application.RecordReaderExample.operator.fileOutput.prop.filePath</name>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/recordReader/src/test/java/com/example/recordReader/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/recordReader/src/test/java/com/example/recordReader/ApplicationTest.java b/examples/recordReader/src/test/java/com/example/recordReader/ApplicationTest.java
deleted file mode 100644
index 222ad06..0000000
--- a/examples/recordReader/src/test/java/com/example/recordReader/ApplicationTest.java
+++ /dev/null
@@ -1,91 +0,0 @@
-package com.example.recordReader;
-
-import java.io.File;
-import java.io.IOException;
-
-import javax.validation.ConstraintViolationException;
-
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestWatcher;
-import org.junit.runner.Description;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.LocalMode;
-
-/**
- * Test application in local mode.
- */
-public class ApplicationTest
-{
- private String outputDir;
-
- public static class TestMeta extends TestWatcher
- {
- public String baseDirectory;
-
- @Override
- protected void starting(org.junit.runner.Description description)
- {
- this.baseDirectory = "target/" + description.getClassName() + "/" + description.getMethodName();
- }
-
- @Override
- protected void finished(Description description)
- {
- super.finished(description);
- try {
- FileUtils.forceDelete(new File(baseDirectory));
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- }
-
- @Rule
- public TestMeta testMeta = new TestMeta();
-
- @Before
- public void setup() throws Exception
- {
- outputDir = testMeta.baseDirectory + File.separator + "output";
- }
-
- @Test
- public void testApplication() throws IOException, Exception
- {
- try {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
- conf.set("dt.application.RecordReaderExample.operator.fileOutput.prop.filePath", outputDir);
- File outputfile = FileUtils.getFile(outputDir, "output.txt_5.0");
-
- lma.prepareDAG(new Application(), conf);
- LocalMode.Controller lc = lma.getController();
- lc.runAsync();
-
- // wait for tuples to show up
- while (!outputfile.exists()) {
- System.out.println("Sleeping ....");
- Thread.sleep(1000);
- }
-
- lc.shutdown();
- Assert.assertTrue(
- FileUtils.contentEquals(
- FileUtils.getFile(
- conf.get("dt.application.RecordReaderExample.operator.recordReader.prop.files")
- ),outputfile));
-
- } catch (ConstraintViolationException e) {
- Assert.fail("constraint violations: " + e.getConstraintViolations());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/recordReader/src/test/java/org/apache/apex/examples/recordReader/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/recordReader/src/test/java/org/apache/apex/examples/recordReader/ApplicationTest.java b/examples/recordReader/src/test/java/org/apache/apex/examples/recordReader/ApplicationTest.java
new file mode 100644
index 0000000..7e133ff
--- /dev/null
+++ b/examples/recordReader/src/test/java/org/apache/apex/examples/recordReader/ApplicationTest.java
@@ -0,0 +1,91 @@
+package org.apache.apex.examples.recordReader;
+
+import java.io.File;
+import java.io.IOException;
+
+import javax.validation.ConstraintViolationException;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.LocalMode;
+
+/**
+ * Test application in local mode.
+ */
+public class ApplicationTest
+{
+ private String outputDir;
+
+ public static class TestMeta extends TestWatcher
+ {
+ public String baseDirectory;
+
+ @Override
+ protected void starting(org.junit.runner.Description description)
+ {
+ this.baseDirectory = "target/" + description.getClassName() + "/" + description.getMethodName();
+ }
+
+ @Override
+ protected void finished(Description description)
+ {
+ super.finished(description);
+ try {
+ FileUtils.forceDelete(new File(baseDirectory));
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ }
+
+ @Rule
+ public TestMeta testMeta = new TestMeta();
+
+ @Before
+ public void setup() throws Exception
+ {
+ outputDir = testMeta.baseDirectory + File.separator + "output";
+ }
+
+ @Test
+ public void testApplication() throws IOException, Exception
+ {
+ try {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ conf.set("dt.application.RecordReaderExample.operator.fileOutput.prop.filePath", outputDir);
+ File outputfile = FileUtils.getFile(outputDir, "output.txt_5.0");
+
+ lma.prepareDAG(new Application(), conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.runAsync();
+
+ // wait for tuples to show up
+ while (!outputfile.exists()) {
+ System.out.println("Sleeping ....");
+ Thread.sleep(1000);
+ }
+
+ lc.shutdown();
+ Assert.assertTrue(
+ FileUtils.contentEquals(
+ FileUtils.getFile(
+ conf.get("dt.application.RecordReaderExample.operator.recordReader.prop.files")
+ ),outputfile));
+
+ } catch (ConstraintViolationException e) {
+ Assert.fail("constraint violations: " + e.getConstraintViolations());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/throttle/pom.xml
----------------------------------------------------------------------
diff --git a/examples/throttle/pom.xml b/examples/throttle/pom.xml
index 02fe22e..d312268 100644
--- a/examples/throttle/pom.xml
+++ b/examples/throttle/pom.xml
@@ -1,263 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-examples</artifactId>
+ <version>3.7.0-SNAPSHOT</version>
+ </parent>
- <groupId>com.datatorrent.examples</groupId>
- <version>1.0-SNAPSHOT</version>
- <artifactId>throttle</artifactId>
+ <artifactId>malhar-examples-throttle</artifactId>
<packaging>jar</packaging>
<!-- change these to the appropriate values -->
<name>Throttle Application</name>
<description>Application demonstrating throttling input when downstream is slower</description>
- <properties>
- <!-- change this if you desire to use a different version of Apex Core -->
- <apex.version>3.5.0</apex.version>
- <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath>
- </properties>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-eclipse-plugin</artifactId>
- <version>2.9</version>
- <configuration>
- <downloadSources>true</downloadSources>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>3.3</version>
- <configuration>
- <encoding>UTF-8</encoding>
- <source>1.7</source>
- <target>1.7</target>
- <debug>true</debug>
- <optimize>false</optimize>
- <showDeprecation>true</showDeprecation>
- <showWarnings>true</showWarnings>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <version>2.8</version>
- <executions>
- <execution>
- <id>copy-dependencies</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>copy-dependencies</goal>
- </goals>
- <configuration>
- <outputDirectory>target/deps</outputDirectory>
- <includeScope>runtime</includeScope>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <executions>
- <execution>
- <id>app-package-assembly</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <finalName>${project.artifactId}-${project.version}-apexapp</finalName>
- <appendAssemblyId>false</appendAssemblyId>
- <descriptors>
- <descriptor>src/assemble/appPackage.xml</descriptor>
- </descriptors>
- <archiverConfig>
- <defaultDirectoryMode>0755</defaultDirectoryMode>
- </archiverConfig>
- <archive>
- <manifestEntries>
- <Class-Path>${apex.apppackage.classpath}</Class-Path>
- <DT-Engine-Version>${apex.version}</DT-Engine-Version>
- <DT-App-Package-Group-Id>${project.groupId}</DT-App-Package-Group-Id>
- <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name>
- <DT-App-Package-Version>${project.version}</DT-App-Package-Version>
- <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name>
- <DT-App-Package-Description>${project.description}</DT-App-Package-Description>
- </manifestEntries>
- </archive>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <artifactId>maven-antrun-plugin</artifactId>
- <version>1.7</version>
- <executions>
- <execution>
- <phase>package</phase>
- <configuration>
- <target>
- <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar"
- tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa" />
- </target>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- <execution>
- <!-- create resource directory for xml javadoc-->
- <id>createJavadocDirectory</id>
- <phase>generate-resources</phase>
- <configuration>
- <tasks>
- <delete dir="${project.build.directory}/generated-resources/xml-javadoc"/>
- <mkdir dir="${project.build.directory}/generated-resources/xml-javadoc"/>
- </tasks>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>1.9.1</version>
- <executions>
- <execution>
- <id>attach-artifacts</id>
- <phase>package</phase>
- <goals>
- <goal>attach-artifact</goal>
- </goals>
- <configuration>
- <artifacts>
- <artifact>
- <file>target/${project.artifactId}-${project.version}.apa</file>
- <type>apa</type>
- </artifact>
- </artifacts>
- <skipAttach>false</skipAttach>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <!-- generate javdoc -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <executions>
- <!-- generate xml javadoc -->
- <execution>
- <id>xml-doclet</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>javadoc</goal>
- </goals>
- <configuration>
- <doclet>com.github.markusbernhardt.xmldoclet.XmlDoclet</doclet>
- <additionalparam>-d ${project.build.directory}/generated-resources/xml-javadoc -filename ${project.artifactId}-${project.version}-javadoc.xml</additionalparam>
- <useStandardDocletOptions>false</useStandardDocletOptions>
- <docletArtifact>
- <groupId>com.github.markusbernhardt</groupId>
- <artifactId>xml-doclet</artifactId>
- <version>1.0.4</version>
- </docletArtifact>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <!-- Transform xml javadoc to stripped down version containing only class/interface comments and tags-->
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>xml-maven-plugin</artifactId>
- <version>1.0</version>
- <executions>
- <execution>
- <id>transform-xmljavadoc</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>transform</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <transformationSets>
- <transformationSet>
- <dir>${project.build.directory}/generated-resources/xml-javadoc</dir>
- <includes>
- <include>${project.artifactId}-${project.version}-javadoc.xml</include>
- </includes>
- <stylesheet>XmlJavadocCommentsExtractor.xsl</stylesheet>
- <outputDir>${project.build.directory}/generated-resources/xml-javadoc</outputDir>
- </transformationSet>
- </transformationSets>
- </configuration>
- </plugin>
- <!-- copy xml javadoc to class jar -->
- <plugin>
- <artifactId>maven-resources-plugin</artifactId>
- <version>2.6</version>
- <executions>
- <execution>
- <id>copy-resources</id>
- <phase>process-resources</phase>
- <goals>
- <goal>copy-resources</goal>
- </goals>
- <configuration>
- <outputDirectory>${basedir}/target/classes</outputDirectory>
- <resources>
- <resource>
- <directory>${project.build.directory}/generated-resources/xml-javadoc</directory>
- <includes>
- <include>${project.artifactId}-${project.version}-javadoc.xml</include>
- </includes>
- <filtering>true</filtering>
- </resource>
- </resources>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
-
- </build>
-
<dependencies>
<!-- add your dependencies here -->
<dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-library</artifactId>
- <version>3.6.0</version>
- <!--
- If you know that your application does not need transitive dependencies pulled in by malhar-library,
- uncomment the following to reduce the size of your app package.
- -->
- <!--
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- -->
- </dependency>
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>apex-common</artifactId>
- <version>${apex.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.10</version>
@@ -266,7 +26,7 @@
<dependency>
<groupId>org.apache.apex</groupId>
<artifactId>apex-engine</artifactId>
- <version>${apex.version}</version>
+ <version>${apex.core.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/throttle/src/main/java/com/datatorrent/examples/throttle/Application.java
----------------------------------------------------------------------
diff --git a/examples/throttle/src/main/java/com/datatorrent/examples/throttle/Application.java b/examples/throttle/src/main/java/com/datatorrent/examples/throttle/Application.java
deleted file mode 100644
index d789dbe..0000000
--- a/examples/throttle/src/main/java/com/datatorrent/examples/throttle/Application.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Put your copyright and license info here.
- */
-package com.datatorrent.examples.throttle;
-
-import java.util.Collection;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.google.common.collect.Lists;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.StatsListener;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-
-@ApplicationAnnotation(name="ThrottleApplication")
-public class Application implements StreamingApplication
-{
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- // Creating an example application with three operators
- // The last operator is slowing down the DAG
- // With the use of the stats listener the input operator is slowed when the window difference crosses a threshold
-
- RandomNumberGenerator randomGenerator = dag.addOperator("RandomGenerator", RandomNumberGenerator.class);
- PassThroughOperator<Double> passThrough = dag.addOperator("PassThrough", PassThroughOperator.class);
- SlowDevNullOperator<Double> devNull = dag.addOperator("SlowNull", SlowDevNullOperator.class);
-
- // Important to use the same stats listener object for all operators so that we can centrally collect stats and make
- // the decision
- StatsListener statsListener = new ThrottlingStatsListener();
- Collection<StatsListener> statsListeners = Lists.newArrayList(statsListener);
- dag.setAttribute(randomGenerator, Context.OperatorContext.STATS_LISTENERS, statsListeners);
- dag.setAttribute(passThrough, Context.OperatorContext.STATS_LISTENERS, statsListeners);
- dag.setAttribute(devNull, Context.OperatorContext.STATS_LISTENERS, statsListeners);
-
- // Increase timeout for the slow operator, this specifies the maximum timeout for an operator to process a window
- // It is specified in number of windows, since 1 window is 500ms, 30 minutes is 30 * 60 * 2 = 3600 windows
- dag.setAttribute(devNull, Context.OperatorContext.TIMEOUT_WINDOW_COUNT, 3600);
-
- // If there are unifiers that are slow then set timeout for them
- // dag.setUnifierAttribute(passThrough.output, Context.OperatorContext.TIMEOUT_WINDOW_COUNT, 3600);
-
- dag.addStream("randomData", randomGenerator.out, passThrough.input);
- dag.addStream("passData", passThrough.output, devNull.input);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/throttle/src/main/java/com/datatorrent/examples/throttle/PassThroughOperator.java
----------------------------------------------------------------------
diff --git a/examples/throttle/src/main/java/com/datatorrent/examples/throttle/PassThroughOperator.java b/examples/throttle/src/main/java/com/datatorrent/examples/throttle/PassThroughOperator.java
deleted file mode 100644
index b4630df..0000000
--- a/examples/throttle/src/main/java/com/datatorrent/examples/throttle/PassThroughOperator.java
+++ /dev/null
@@ -1,20 +0,0 @@
-package com.datatorrent.examples.throttle;
-
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.common.util.BaseOperator;
-
-/**
- * Created by pramod on 9/27/16.
- */
-public class PassThroughOperator<T> extends BaseOperator {
-
- public transient final DefaultInputPort<T> input = new DefaultInputPort<T>() {
- @Override
- public void process(T t) {
- output.emit(t);
- }
- };
-
- public transient final DefaultOutputPort<T> output = new DefaultOutputPort<>();
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/throttle/src/main/java/com/datatorrent/examples/throttle/RandomNumberGenerator.java
----------------------------------------------------------------------
diff --git a/examples/throttle/src/main/java/com/datatorrent/examples/throttle/RandomNumberGenerator.java b/examples/throttle/src/main/java/com/datatorrent/examples/throttle/RandomNumberGenerator.java
deleted file mode 100644
index ea57b6d..0000000
--- a/examples/throttle/src/main/java/com/datatorrent/examples/throttle/RandomNumberGenerator.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Put your copyright and license info here.
- */
-package com.datatorrent.examples.throttle;
-
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.InputOperator;
-import com.datatorrent.common.util.BaseOperator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This is a simple operator that emits random number.
- */
-public class RandomNumberGenerator extends BaseOperator implements InputOperator
-{
- private int numTuples = 1000;
- private int origNumTuples = numTuples;
- private transient int count = 0;
-
- private static final Logger logger = LoggerFactory.getLogger(RandomNumberGenerator.class);
-
- public final transient DefaultOutputPort<Double> out = new DefaultOutputPort<Double>();
-
- @Override
- public void beginWindow(long windowId)
- {
- count = 0;
- }
-
- @Override
- public void emitTuples()
- {
- if (count++ < numTuples) {
- out.emit(Math.random());
- }
- }
-
- // Simple suspend and
- public void suspend() {
- logger.debug("Slowing down");
- numTuples = 0;
- }
-
- public void normal() {
- logger.debug("Normal");
- numTuples = origNumTuples;
- }
-
- public int getNumTuples()
- {
- return numTuples;
- }
-
- /**
- * Sets the number of tuples to be emitted every window.
- * @param numTuples number of tuples
- */
- public void setNumTuples(int numTuples)
- {
- this.numTuples = numTuples;
- this.origNumTuples = numTuples;
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/throttle/src/main/java/com/datatorrent/examples/throttle/SlowDevNullOperator.java
----------------------------------------------------------------------
diff --git a/examples/throttle/src/main/java/com/datatorrent/examples/throttle/SlowDevNullOperator.java b/examples/throttle/src/main/java/com/datatorrent/examples/throttle/SlowDevNullOperator.java
deleted file mode 100644
index 7d1451d..0000000
--- a/examples/throttle/src/main/java/com/datatorrent/examples/throttle/SlowDevNullOperator.java
+++ /dev/null
@@ -1,35 +0,0 @@
-package com.datatorrent.examples.throttle;
-
-import com.google.common.base.Throwables;
-
-import com.datatorrent.api.DefaultInputPort;
-import com.datatorrent.common.util.BaseOperator;
-
-/**
- * Created by pramod on 9/27/16.
- */
-public class SlowDevNullOperator<T> extends BaseOperator {
-
- // Modify sleep time dynamically while app is running to increase and decrease sleep time
- long sleepTime = 1;
-
- public transient final DefaultInputPort<T> input = new DefaultInputPort<T>() {
- @Override
- public void process(T t) {
- // Introduce an artificial delay for every tuple
- try {
- Thread.sleep(sleepTime);
- } catch (InterruptedException e) {
- throw Throwables.propagate(e);
- }
- }
- };
-
- public long getSleepTime() {
- return sleepTime;
- }
-
- public void setSleepTime(long sleepTime) {
- this.sleepTime = sleepTime;
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/throttle/src/main/java/com/datatorrent/examples/throttle/ThrottlingStatsListener.java
----------------------------------------------------------------------
diff --git a/examples/throttle/src/main/java/com/datatorrent/examples/throttle/ThrottlingStatsListener.java b/examples/throttle/src/main/java/com/datatorrent/examples/throttle/ThrottlingStatsListener.java
deleted file mode 100644
index 46e2e0e..0000000
--- a/examples/throttle/src/main/java/com/datatorrent/examples/throttle/ThrottlingStatsListener.java
+++ /dev/null
@@ -1,150 +0,0 @@
-package com.datatorrent.examples.throttle;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Maps;
-
-import com.datatorrent.api.Operator;
-import com.datatorrent.api.StatsListener;
-
-/**
- * Created by pramod on 9/27/16.
- */
-public class ThrottlingStatsListener implements StatsListener, Serializable {
-
- private static final Logger logger = LoggerFactory.getLogger(ThrottlingStatsListener.class);
-
- // Slowdown input if the window difference between operators increases beyond this value
- long maxThreshold = 100;
- // restore input operator to normal speed if the window difference falls below this threshold
- long minThreshold = 100;
-
- Map<Integer, ThrottleState> throttleStates = Maps.newHashMap();
-
- static class ThrottleState {
- // The current state of the operator, normal or throttled
- boolean normal = true;
- //The latest window id for which stats were received for the operator
- long currentWindowId;
- }
-
- // This method runs on the app master side and is called whenever new stats are received from the operators
- @Override
- public Response processStats(BatchedOperatorStats batchedOperatorStats)
- {
- Response response = new Response();
- int operatorId = batchedOperatorStats.getOperatorId();
-
- ThrottleState throttleState = throttleStates.get(operatorId);
- if (throttleState == null) {
- throttleState = new ThrottleState();
- throttleStates.put(operatorId, throttleState);
- }
-
- long windowId = batchedOperatorStats.getCurrentWindowId();
- throttleState.currentWindowId = windowId;
-
- // Find min and max window to compute difference
- long minWindow = Long.MAX_VALUE;
- long maxWindow = Long.MIN_VALUE;
- for (ThrottleState state : throttleStates.values()) {
- if (state.currentWindowId < minWindow) minWindow = state.currentWindowId;
- if (state.currentWindowId > maxWindow) maxWindow = state.currentWindowId;
- }
- logger.debug("Operator {} min window {} max window {}", operatorId, minWindow, maxWindow);
-
- if (throttleState.normal && ((maxWindow - minWindow) > maxThreshold)) {
- // Send request to operator to slow down
- logger.info("Sending suspend request");
- List<OperatorRequest> operatorRequests = new ArrayList<OperatorRequest>();
- operatorRequests.add(new InputSlowdownRequest());
- response.operatorRequests = operatorRequests;
- //logger.info("Setting suspend");
- throttleState.normal = false;
- } else if (!throttleState.normal && ((maxWindow - minWindow) <= minThreshold)) {
- // Send request to operator to get back to normal
- logger.info("Sending normal request");
- List<OperatorRequest> operatorRequests = new ArrayList<OperatorRequest>();
- operatorRequests.add(new InputNormalRequest());
- response.operatorRequests = operatorRequests;
- //logger.info("Setting normal");
- throttleState.normal = true;
- }
-
- return response;
- }
-
- // This runs on the operator side
- public static class InputSlowdownRequest implements OperatorRequest, Serializable
- {
- private static final Logger logger = LoggerFactory.getLogger(InputSlowdownRequest.class);
-
- @Override
- public OperatorResponse execute(Operator operator, int operatorId, long windowId) throws IOException
- {
- logger.debug("Received slowdown operator {} operatorId {} windowId {}", operator, operatorId, windowId);
- if (operator instanceof RandomNumberGenerator) {
- RandomNumberGenerator generator = (RandomNumberGenerator)operator;
- generator.suspend();
- }
- return new InputOperatorResponse();
- }
- }
-
- public static class InputNormalRequest implements OperatorRequest, Serializable
- {
- private static final Logger logger = LoggerFactory.getLogger(InputNormalRequest.class);
-
- @Override
- public OperatorResponse execute(Operator operator, int operatorId, long windowId) throws IOException
- {
- logger.debug("Received normal operator {} operatorId {} windowId {}", operator, operatorId, windowId);
- if (operator instanceof RandomNumberGenerator) {
- RandomNumberGenerator generator = (RandomNumberGenerator)operator;
- generator.normal();
- }
- return new InputOperatorResponse();
- }
- }
-
- public static class InputOperatorResponse implements OperatorResponse, Serializable
- {
-
- @Override
- public Object getResponseId() {
- return 1;
- }
-
- @Override
- public Object getResponse() {
- return "";
- }
- }
-
- public long getMaxThreshold()
- {
- return maxThreshold;
- }
-
- public void setMaxThreshold(long maxThreshold)
- {
- this.maxThreshold = maxThreshold;
- }
-
- public long getMinThreshold()
- {
- return minThreshold;
- }
-
- public void setMinThreshold(long minThreshold)
- {
- this.minThreshold = minThreshold;
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/throttle/src/main/java/org/apache/apex/examples/throttle/Application.java
----------------------------------------------------------------------
diff --git a/examples/throttle/src/main/java/org/apache/apex/examples/throttle/Application.java b/examples/throttle/src/main/java/org/apache/apex/examples/throttle/Application.java
new file mode 100644
index 0000000..9adad32
--- /dev/null
+++ b/examples/throttle/src/main/java/org/apache/apex/examples/throttle/Application.java
@@ -0,0 +1,51 @@
+/**
+ * Put your copyright and license info here.
+ */
+package org.apache.apex.examples.throttle;
+
+import java.util.Collection;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.collect.Lists;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StatsListener;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+
+@ApplicationAnnotation(name="ThrottleApplication")
+public class Application implements StreamingApplication
+{
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ // Creating an example application with three operators
+ // The last operator is slowing down the DAG
+ // With the use of the stats listener the input operator is slowed when the window difference crosses a threshold
+
+ RandomNumberGenerator randomGenerator = dag.addOperator("RandomGenerator", RandomNumberGenerator.class);
+ PassThroughOperator<Double> passThrough = dag.addOperator("PassThrough", PassThroughOperator.class);
+ SlowDevNullOperator<Double> devNull = dag.addOperator("SlowNull", SlowDevNullOperator.class);
+
+ // Important to use the same stats listener object for all operators so that we can centrally collect stats and make
+ // the decision
+ StatsListener statsListener = new ThrottlingStatsListener();
+ Collection<StatsListener> statsListeners = Lists.newArrayList(statsListener);
+ dag.setAttribute(randomGenerator, Context.OperatorContext.STATS_LISTENERS, statsListeners);
+ dag.setAttribute(passThrough, Context.OperatorContext.STATS_LISTENERS, statsListeners);
+ dag.setAttribute(devNull, Context.OperatorContext.STATS_LISTENERS, statsListeners);
+
+ // Increase timeout for the slow operator, this specifies the maximum timeout for an operator to process a window
+ // It is specified in number of windows, since 1 window is 500ms, 30 minutes is 30 * 60 * 2 = 3600 windows
+ dag.setAttribute(devNull, Context.OperatorContext.TIMEOUT_WINDOW_COUNT, 3600);
+
+ // If there are unifiers that are slow then set timeout for them
+ // dag.setUnifierAttribute(passThrough.output, Context.OperatorContext.TIMEOUT_WINDOW_COUNT, 3600);
+
+ dag.addStream("randomData", randomGenerator.out, passThrough.input);
+ dag.addStream("passData", passThrough.output, devNull.input);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/throttle/src/main/java/org/apache/apex/examples/throttle/PassThroughOperator.java
----------------------------------------------------------------------
diff --git a/examples/throttle/src/main/java/org/apache/apex/examples/throttle/PassThroughOperator.java b/examples/throttle/src/main/java/org/apache/apex/examples/throttle/PassThroughOperator.java
new file mode 100644
index 0000000..bf0fbce
--- /dev/null
+++ b/examples/throttle/src/main/java/org/apache/apex/examples/throttle/PassThroughOperator.java
@@ -0,0 +1,20 @@
+package org.apache.apex.examples.throttle;
+
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.common.util.BaseOperator;
+
+/**
+ * Created by pramod on 9/27/16.
+ */
+public class PassThroughOperator<T> extends BaseOperator {
+
+ public transient final DefaultInputPort<T> input = new DefaultInputPort<T>() {
+ @Override
+ public void process(T t) {
+ output.emit(t);
+ }
+ };
+
+ public transient final DefaultOutputPort<T> output = new DefaultOutputPort<>();
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/throttle/src/main/java/org/apache/apex/examples/throttle/RandomNumberGenerator.java
----------------------------------------------------------------------
diff --git a/examples/throttle/src/main/java/org/apache/apex/examples/throttle/RandomNumberGenerator.java b/examples/throttle/src/main/java/org/apache/apex/examples/throttle/RandomNumberGenerator.java
new file mode 100644
index 0000000..6825340
--- /dev/null
+++ b/examples/throttle/src/main/java/org/apache/apex/examples/throttle/RandomNumberGenerator.java
@@ -0,0 +1,64 @@
+/**
+ * Put your copyright and license info here.
+ */
+package org.apache.apex.examples.throttle;
+
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.common.util.BaseOperator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This is a simple operator that emits random number.
+ */
+public class RandomNumberGenerator extends BaseOperator implements InputOperator
+{
+ private int numTuples = 1000;
+ private int origNumTuples = numTuples;
+ private transient int count = 0;
+
+ private static final Logger logger = LoggerFactory.getLogger(RandomNumberGenerator.class);
+
+ public final transient DefaultOutputPort<Double> out = new DefaultOutputPort<Double>();
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ count = 0;
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ if (count++ < numTuples) {
+ out.emit(Math.random());
+ }
+ }
+
+ // Simple suspend and
+ public void suspend() {
+ logger.debug("Slowing down");
+ numTuples = 0;
+ }
+
+ public void normal() {
+ logger.debug("Normal");
+ numTuples = origNumTuples;
+ }
+
+ public int getNumTuples()
+ {
+ return numTuples;
+ }
+
+ /**
+ * Sets the number of tuples to be emitted every window.
+ * @param numTuples number of tuples
+ */
+ public void setNumTuples(int numTuples)
+ {
+ this.numTuples = numTuples;
+ this.origNumTuples = numTuples;
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/throttle/src/main/java/org/apache/apex/examples/throttle/SlowDevNullOperator.java
----------------------------------------------------------------------
diff --git a/examples/throttle/src/main/java/org/apache/apex/examples/throttle/SlowDevNullOperator.java b/examples/throttle/src/main/java/org/apache/apex/examples/throttle/SlowDevNullOperator.java
new file mode 100644
index 0000000..b49b3c8
--- /dev/null
+++ b/examples/throttle/src/main/java/org/apache/apex/examples/throttle/SlowDevNullOperator.java
@@ -0,0 +1,35 @@
+package org.apache.apex.examples.throttle;
+
+import com.google.common.base.Throwables;
+
+import com.datatorrent.api.DefaultInputPort;
+import com.datatorrent.common.util.BaseOperator;
+
+/**
+ * Created by pramod on 9/27/16.
+ */
+public class SlowDevNullOperator<T> extends BaseOperator {
+
+ // Modify sleep time dynamically while app is running to increase and decrease sleep time
+ long sleepTime = 1;
+
+ public transient final DefaultInputPort<T> input = new DefaultInputPort<T>() {
+ @Override
+ public void process(T t) {
+ // Introduce an artificial delay for every tuple
+ try {
+ Thread.sleep(sleepTime);
+ } catch (InterruptedException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+ };
+
+ public long getSleepTime() {
+ return sleepTime;
+ }
+
+ public void setSleepTime(long sleepTime) {
+ this.sleepTime = sleepTime;
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/throttle/src/main/java/org/apache/apex/examples/throttle/ThrottlingStatsListener.java
----------------------------------------------------------------------
diff --git a/examples/throttle/src/main/java/org/apache/apex/examples/throttle/ThrottlingStatsListener.java b/examples/throttle/src/main/java/org/apache/apex/examples/throttle/ThrottlingStatsListener.java
new file mode 100644
index 0000000..7e9e70f
--- /dev/null
+++ b/examples/throttle/src/main/java/org/apache/apex/examples/throttle/ThrottlingStatsListener.java
@@ -0,0 +1,150 @@
+package org.apache.apex.examples.throttle;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Maps;
+
+import com.datatorrent.api.Operator;
+import com.datatorrent.api.StatsListener;
+
+/**
+ * Created by pramod on 9/27/16.
+ */
+public class ThrottlingStatsListener implements StatsListener, Serializable {
+
+ private static final Logger logger = LoggerFactory.getLogger(ThrottlingStatsListener.class);
+
+ // Slowdown input if the window difference between operators increases beyond this value
+ long maxThreshold = 100;
+ // restore input operator to normal speed if the window difference falls below this threshold
+ long minThreshold = 100;
+
+ Map<Integer, ThrottleState> throttleStates = Maps.newHashMap();
+
+ static class ThrottleState {
+ // The current state of the operator, normal or throttled
+ boolean normal = true;
+ //The latest window id for which stats were received for the operator
+ long currentWindowId;
+ }
+
+ // This method runs on the app master side and is called whenever new stats are received from the operators
+ @Override
+ public Response processStats(BatchedOperatorStats batchedOperatorStats)
+ {
+ Response response = new Response();
+ int operatorId = batchedOperatorStats.getOperatorId();
+
+ ThrottleState throttleState = throttleStates.get(operatorId);
+ if (throttleState == null) {
+ throttleState = new ThrottleState();
+ throttleStates.put(operatorId, throttleState);
+ }
+
+ long windowId = batchedOperatorStats.getCurrentWindowId();
+ throttleState.currentWindowId = windowId;
+
+ // Find min and max window to compute difference
+ long minWindow = Long.MAX_VALUE;
+ long maxWindow = Long.MIN_VALUE;
+ for (ThrottleState state : throttleStates.values()) {
+ if (state.currentWindowId < minWindow) minWindow = state.currentWindowId;
+ if (state.currentWindowId > maxWindow) maxWindow = state.currentWindowId;
+ }
+ logger.debug("Operator {} min window {} max window {}", operatorId, minWindow, maxWindow);
+
+ if (throttleState.normal && ((maxWindow - minWindow) > maxThreshold)) {
+ // Send request to operator to slow down
+ logger.info("Sending suspend request");
+ List<OperatorRequest> operatorRequests = new ArrayList<OperatorRequest>();
+ operatorRequests.add(new InputSlowdownRequest());
+ response.operatorRequests = operatorRequests;
+ //logger.info("Setting suspend");
+ throttleState.normal = false;
+ } else if (!throttleState.normal && ((maxWindow - minWindow) <= minThreshold)) {
+ // Send request to operator to get back to normal
+ logger.info("Sending normal request");
+ List<OperatorRequest> operatorRequests = new ArrayList<OperatorRequest>();
+ operatorRequests.add(new InputNormalRequest());
+ response.operatorRequests = operatorRequests;
+ //logger.info("Setting normal");
+ throttleState.normal = true;
+ }
+
+ return response;
+ }
+
+ // This runs on the operator side
+ public static class InputSlowdownRequest implements OperatorRequest, Serializable
+ {
+ private static final Logger logger = LoggerFactory.getLogger(InputSlowdownRequest.class);
+
+ @Override
+ public OperatorResponse execute(Operator operator, int operatorId, long windowId) throws IOException
+ {
+ logger.debug("Received slowdown operator {} operatorId {} windowId {}", operator, operatorId, windowId);
+ if (operator instanceof RandomNumberGenerator) {
+ RandomNumberGenerator generator = (RandomNumberGenerator)operator;
+ generator.suspend();
+ }
+ return new InputOperatorResponse();
+ }
+ }
+
+ public static class InputNormalRequest implements OperatorRequest, Serializable
+ {
+ private static final Logger logger = LoggerFactory.getLogger(InputNormalRequest.class);
+
+ @Override
+ public OperatorResponse execute(Operator operator, int operatorId, long windowId) throws IOException
+ {
+ logger.debug("Received normal operator {} operatorId {} windowId {}", operator, operatorId, windowId);
+ if (operator instanceof RandomNumberGenerator) {
+ RandomNumberGenerator generator = (RandomNumberGenerator)operator;
+ generator.normal();
+ }
+ return new InputOperatorResponse();
+ }
+ }
+
+ public static class InputOperatorResponse implements OperatorResponse, Serializable
+ {
+
+ @Override
+ public Object getResponseId() {
+ return 1;
+ }
+
+ @Override
+ public Object getResponse() {
+ return "";
+ }
+ }
+
+ public long getMaxThreshold()
+ {
+ return maxThreshold;
+ }
+
+ public void setMaxThreshold(long maxThreshold)
+ {
+ this.maxThreshold = maxThreshold;
+ }
+
+ public long getMinThreshold()
+ {
+ return minThreshold;
+ }
+
+ public void setMinThreshold(long minThreshold)
+ {
+ this.minThreshold = minThreshold;
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/throttle/src/test/java/com/datatorrent/examples/throttle/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/throttle/src/test/java/com/datatorrent/examples/throttle/ApplicationTest.java b/examples/throttle/src/test/java/com/datatorrent/examples/throttle/ApplicationTest.java
deleted file mode 100644
index 02a96ba..0000000
--- a/examples/throttle/src/test/java/com/datatorrent/examples/throttle/ApplicationTest.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Put your copyright and license info here.
- */
-package com.datatorrent.examples.throttle;
-
-import java.io.IOException;
-
-import javax.validation.ConstraintViolationException;
-
-import org.junit.Assert;
-
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Test;
-
-import com.datatorrent.api.LocalMode;
-import com.datatorrent.examples.throttle.Application;
-
-/**
- * Test the DAG declaration in local mode.
- */
-public class ApplicationTest {
-
- @Test
- public void testApplication() throws IOException, Exception {
- try {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
- lma.prepareDAG(new Application(), conf);
- LocalMode.Controller lc = lma.getController();
- lc.run(10000); // runs for 10 seconds and quits
- } catch (ConstraintViolationException e) {
- Assert.fail("constraint violations: " + e.getConstraintViolations());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/throttle/src/test/java/org/apache/apex/examples/throttle/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/throttle/src/test/java/org/apache/apex/examples/throttle/ApplicationTest.java b/examples/throttle/src/test/java/org/apache/apex/examples/throttle/ApplicationTest.java
new file mode 100644
index 0000000..e85c5c5
--- /dev/null
+++ b/examples/throttle/src/test/java/org/apache/apex/examples/throttle/ApplicationTest.java
@@ -0,0 +1,36 @@
+/**
+ * Put your copyright and license info here.
+ */
+package org.apache.apex.examples.throttle;
+
+import java.io.IOException;
+
+import javax.validation.ConstraintViolationException;
+
+import org.junit.Assert;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+import com.datatorrent.api.LocalMode;
+
+/**
+ * Test the DAG declaration in local mode.
+ */
+public class ApplicationTest {
+
+ @Test
+ public void testApplication() throws IOException, Exception {
+ try {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ lma.prepareDAG(new Application(), conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.run(10000); // runs for 10 seconds and quits
+ } catch (ConstraintViolationException e) {
+ Assert.fail("constraint violations: " + e.getConstraintViolations());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/transform/pom.xml
----------------------------------------------------------------------
diff --git a/examples/transform/pom.xml b/examples/transform/pom.xml
index e8846d3..93dbad1 100644
--- a/examples/transform/pom.xml
+++ b/examples/transform/pom.xml
@@ -2,250 +2,21 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>com.example</groupId>
- <version>1.0-SNAPSHOT</version>
- <artifactId>transform</artifactId>
+ <parent>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-examples</artifactId>
+ <version>3.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>malhar-examples-transform</artifactId>
<packaging>jar</packaging>
<name>Transform Application</name>
<description>Sample application for transform operator</description>
- <properties>
- <!-- change this if you desire to use a different version of Apex Core -->
- <apex.version>3.5.0</apex.version>
- <malhar.version>3.6.0</malhar.version>
- <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath>
- </properties>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-eclipse-plugin</artifactId>
- <version>2.9</version>
- <configuration>
- <downloadSources>true</downloadSources>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>3.3</version>
- <configuration>
- <encoding>UTF-8</encoding>
- <source>1.7</source>
- <target>1.7</target>
- <debug>true</debug>
- <optimize>false</optimize>
- <showDeprecation>true</showDeprecation>
- <showWarnings>true</showWarnings>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <version>2.8</version>
- <executions>
- <execution>
- <id>copy-dependencies</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>copy-dependencies</goal>
- </goals>
- <configuration>
- <outputDirectory>target/deps</outputDirectory>
- <includeScope>runtime</includeScope>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <executions>
- <execution>
- <id>app-package-assembly</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <finalName>${project.artifactId}-${project.version}-apexapp</finalName>
- <appendAssemblyId>false</appendAssemblyId>
- <descriptors>
- <descriptor>src/assemble/appPackage.xml</descriptor>
- </descriptors>
- <archiverConfig>
- <defaultDirectoryMode>0755</defaultDirectoryMode>
- </archiverConfig>
- <archive>
- <manifestEntries>
- <Class-Path>${apex.apppackage.classpath}</Class-Path>
- <DT-Engine-Version>${apex.version}</DT-Engine-Version>
- <DT-App-Package-Group-Id>${project.groupId}</DT-App-Package-Group-Id>
- <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name>
- <DT-App-Package-Version>${project.version}</DT-App-Package-Version>
- <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name>
- <DT-App-Package-Description>${project.description}</DT-App-Package-Description>
- </manifestEntries>
- </archive>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <artifactId>maven-antrun-plugin</artifactId>
- <version>1.7</version>
- <executions>
- <execution>
- <phase>package</phase>
- <configuration>
- <target>
- <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar"
- tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa" />
- </target>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- <execution>
- <!-- create resource directory for xml javadoc-->
- <id>createJavadocDirectory</id>
- <phase>generate-resources</phase>
- <configuration>
- <tasks>
- <delete dir="${project.build.directory}/generated-resources/xml-javadoc"/>
- <mkdir dir="${project.build.directory}/generated-resources/xml-javadoc"/>
- </tasks>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>1.9.1</version>
- <executions>
- <execution>
- <id>attach-artifacts</id>
- <phase>package</phase>
- <goals>
- <goal>attach-artifact</goal>
- </goals>
- <configuration>
- <artifacts>
- <artifact>
- <file>target/${project.artifactId}-${project.version}.apa</file>
- <type>apa</type>
- </artifact>
- </artifacts>
- <skipAttach>false</skipAttach>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <!-- generate javdoc -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <executions>
- <!-- generate xml javadoc -->
- <execution>
- <id>xml-doclet</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>javadoc</goal>
- </goals>
- <configuration>
- <doclet>com.github.markusbernhardt.xmldoclet.XmlDoclet</doclet>
- <additionalparam>-d ${project.build.directory}/generated-resources/xml-javadoc -filename ${project.artifactId}-${project.version}-javadoc.xml</additionalparam>
- <useStandardDocletOptions>false</useStandardDocletOptions>
- <docletArtifact>
- <groupId>com.github.markusbernhardt</groupId>
- <artifactId>xml-doclet</artifactId>
- <version>1.0.4</version>
- </docletArtifact>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <!-- Transform xml javadoc to stripped down version containing only class/interface comments and tags-->
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>xml-maven-plugin</artifactId>
- <version>1.0</version>
- <executions>
- <execution>
- <id>transform-xmljavadoc</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>transform</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <transformationSets>
- <transformationSet>
- <dir>${project.build.directory}/generated-resources/xml-javadoc</dir>
- <includes>
- <include>${project.artifactId}-${project.version}-javadoc.xml</include>
- </includes>
- <stylesheet>XmlJavadocCommentsExtractor.xsl</stylesheet>
- <outputDir>${project.build.directory}/generated-resources/xml-javadoc</outputDir>
- </transformationSet>
- </transformationSets>
- </configuration>
- </plugin>
- <!-- copy xml javadoc to class jar -->
- <plugin>
- <artifactId>maven-resources-plugin</artifactId>
- <version>2.6</version>
- <executions>
- <execution>
- <id>copy-resources</id>
- <phase>process-resources</phase>
- <goals>
- <goal>copy-resources</goal>
- </goals>
- <configuration>
- <outputDirectory>${basedir}/target/classes</outputDirectory>
- <resources>
- <resource>
- <directory>${project.build.directory}/generated-resources/xml-javadoc</directory>
- <includes>
- <include>${project.artifactId}-${project.version}-javadoc.xml</include>
- </includes>
- <filtering>true</filtering>
- </resource>
- </resources>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
-
- </build>
-
<dependencies>
<!-- add your dependencies here -->
<dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-library</artifactId>
- <version>${malhar.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>apex-common</artifactId>
- <version>${apex.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.10</version>
@@ -254,7 +25,7 @@
<dependency>
<groupId>org.apache.apex</groupId>
<artifactId>apex-engine</artifactId>
- <version>${apex.version}</version>
+ <version>${apex.core.version}</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -262,6 +33,11 @@
<artifactId>janino</artifactId>
<version>2.7.8</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>3.1</version>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/transform/src/main/java/com/example/transform/Application.java
----------------------------------------------------------------------
diff --git a/examples/transform/src/main/java/com/example/transform/Application.java b/examples/transform/src/main/java/com/example/transform/Application.java
deleted file mode 100644
index d73b47d..0000000
--- a/examples/transform/src/main/java/com/example/transform/Application.java
+++ /dev/null
@@ -1,39 +0,0 @@
-package com.example.transform;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.common.partitioner.StatelessPartitioner;
-import com.datatorrent.lib.io.ConsoleOutputOperator;
-import com.datatorrent.lib.transform.TransformOperator;
-
-@ApplicationAnnotation(name="TransformExample")
-public class Application implements StreamingApplication
-{
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- POJOGenerator input = dag.addOperator("Input", new POJOGenerator());
- TransformOperator transform = dag.addOperator("Process", new TransformOperator());
- // Set expression map
- Map<String, String> expMap = new HashMap<>();
- expMap.put("name", "{$.firstName}.concat(\" \").concat({$.lastName})");
- expMap.put("age", "(new java.util.Date()).getYear() - {$.dateOfBirth}.getYear()");
- expMap.put("address", "{$.address}.toLowerCase()");
- transform.setExpressionMap(expMap);
- ConsoleOutputOperator output = dag.addOperator("Output", new ConsoleOutputOperator());
-
- dag.addStream("InputToTransform", input.output, transform.input);
- dag.addStream("TransformToOutput", transform.output, output.input);
-
- dag.setInputPortAttribute(transform.input, Context.PortContext.TUPLE_CLASS, CustomerEvent.class);
- dag.setOutputPortAttribute(transform.output, Context.PortContext.TUPLE_CLASS, CustomerInfo.class);
- dag.setAttribute(transform, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<TransformOperator>(2));
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/transform/src/main/java/com/example/transform/CustomerEvent.java
----------------------------------------------------------------------
diff --git a/examples/transform/src/main/java/com/example/transform/CustomerEvent.java b/examples/transform/src/main/java/com/example/transform/CustomerEvent.java
deleted file mode 100644
index 14a9c82..0000000
--- a/examples/transform/src/main/java/com/example/transform/CustomerEvent.java
+++ /dev/null
@@ -1,74 +0,0 @@
-package com.example.transform;
-
-import java.util.Date;
-
-public class CustomerEvent
-{
- private int customerId;
- private String firstName;
- private String lastName;
- private Date dateOfBirth;
- private String address;
-
- public int getCustomerId()
- {
- return customerId;
- }
-
- public void setCustomerId(int customerId)
- {
- this.customerId = customerId;
- }
-
- public String getFirstName()
- {
- return firstName;
- }
-
- public void setFirstName(String firstName)
- {
- this.firstName = firstName;
- }
-
- public String getLastName()
- {
- return lastName;
- }
-
- public void setLastName(String lastName)
- {
- this.lastName = lastName;
- }
-
- public Date getDateOfBirth()
- {
- return dateOfBirth;
- }
-
- public void setDateOfBirth(Date dateOfBirth)
- {
- this.dateOfBirth = dateOfBirth;
- }
-
- public String getAddress()
- {
- return address;
- }
-
- public void setAddress(String address)
- {
- this.address = address;
- }
-
- @Override
- public String toString()
- {
- return "CustomerEvent{" +
- "customerId=" + customerId +
- ", firstName='" + firstName + '\'' +
- ", lastName='" + lastName + '\'' +
- ", dateOfBirth=" + dateOfBirth +
- ", address='" + address + '\'' +
- '}';
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/transform/src/main/java/com/example/transform/CustomerInfo.java
----------------------------------------------------------------------
diff --git a/examples/transform/src/main/java/com/example/transform/CustomerInfo.java b/examples/transform/src/main/java/com/example/transform/CustomerInfo.java
deleted file mode 100644
index 28a9ccb..0000000
--- a/examples/transform/src/main/java/com/example/transform/CustomerInfo.java
+++ /dev/null
@@ -1,60 +0,0 @@
-package com.example.transform;
-
-public class CustomerInfo
-{
- private int customerId;
- private String name;
- private int age;
- private String address;
-
- public int getCustomerId()
- {
- return customerId;
- }
-
- public void setCustomerId(int customerId)
- {
- this.customerId = customerId;
- }
-
- public String getName()
- {
- return name;
- }
-
- public void setName(String name)
- {
- this.name = name;
- }
-
- public int getAge()
- {
- return age;
- }
-
- public void setAge(int age)
- {
- this.age = age;
- }
-
- public String getAddress()
- {
- return address;
- }
-
- public void setAddress(String address)
- {
- this.address = address;
- }
-
- @Override
- public String toString()
- {
- return "CustomerInfo{" +
- "customerId=" + customerId +
- ", name='" + name + '\'' +
- ", age=" + age +
- ", address='" + address + '\'' +
- '}';
- }
-}
[10/19] apex-malhar git commit: SPOI-9032 Added sample application
for Inner Join
Posted by th...@apache.org.
SPOI-9032 Added sample application for Inner Join
Project: http://git-wip-us.apache.org/repos/asf/apex-malhar/repo
Commit: http://git-wip-us.apache.org/repos/asf/apex-malhar/commit/ca12bca0
Tree: http://git-wip-us.apache.org/repos/asf/apex-malhar/tree/ca12bca0
Diff: http://git-wip-us.apache.org/repos/asf/apex-malhar/diff/ca12bca0
Branch: refs/heads/master
Commit: ca12bca03b0ac0525fe779895cb40340f6666147
Parents: 172b20a
Author: chaitanya <ch...@apache.org>
Authored: Tue Sep 13 12:07:40 2016 +0530
Committer: Lakshmi Prasanna Velineni <la...@datatorrent.com>
Committed: Sun Mar 26 11:43:48 2017 -0700
----------------------------------------------------------------------
examples/innerjoin/README.md | 6 +
.../innerjoin/XmlJavadocCommentsExtractor.xsl | 44 +++
examples/innerjoin/pom.xml | 267 +++++++++++++++++++
examples/innerjoin/src/assemble/appPackage.xml | 43 +++
.../com/example/join/InnerJoinApplication.java | 39 +++
.../java/com/example/join/POJOGenerator.java | 260 ++++++++++++++++++
.../src/main/resources/META-INF/properties.xml | 36 +++
.../example/join/InnerJoinApplicationTest.java | 21 ++
.../src/test/resources/log4j.properties | 22 ++
9 files changed, 738 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/ca12bca0/examples/innerjoin/README.md
----------------------------------------------------------------------
diff --git a/examples/innerjoin/README.md b/examples/innerjoin/README.md
new file mode 100644
index 0000000..7cc1d7d
--- /dev/null
+++ b/examples/innerjoin/README.md
@@ -0,0 +1,6 @@
+Sample application to show how to use the POJOInnerJoinOperator to join two data streams.
+
+Operators in sample application are as follows:
+1) POJOGenerator which emits SalesEvent/ProductEvent POJO's
+2) POJOInnerJoinOperator which joins two POJO streams and emits POJO tuple.
+3) ConsoleOutputOperator which write joined POJO tuples to stdout.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/ca12bca0/examples/innerjoin/XmlJavadocCommentsExtractor.xsl
----------------------------------------------------------------------
diff --git a/examples/innerjoin/XmlJavadocCommentsExtractor.xsl b/examples/innerjoin/XmlJavadocCommentsExtractor.xsl
new file mode 100644
index 0000000..08075a9
--- /dev/null
+++ b/examples/innerjoin/XmlJavadocCommentsExtractor.xsl
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+
+<!--
+ Document : XmlJavadocCommentsExtractor.xsl
+ Created on : September 16, 2014, 11:30 AM
+ Description:
+ The transformation strips off all information except for comments and tags from xml javadoc generated by xml-doclet.
+-->
+
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+ <xsl:output method="xml" standalone="yes"/>
+
+ <!-- copy xml by selecting only the following nodes, attributes and text -->
+ <xsl:template match="node()|text()|@*">
+ <xsl:copy>
+ <xsl:apply-templates select="root|package|class|interface|method|field|type|comment|tag|text()|@name|@qualified|@text"/>
+ </xsl:copy>
+ </xsl:template>
+
+ <!-- Strip off the following paths from the selected xml -->
+ <xsl:template match="//root/package/interface/interface
+ |//root/package/interface/method/@qualified
+ |//root/package/class/interface
+ |//root/package/class/class
+ |//root/package/class/method/@qualified
+ |//root/package/class/field/@qualified" />
+
+ <xsl:strip-space elements="*"/>
+</xsl:stylesheet>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/ca12bca0/examples/innerjoin/pom.xml
----------------------------------------------------------------------
diff --git a/examples/innerjoin/pom.xml b/examples/innerjoin/pom.xml
new file mode 100644
index 0000000..29c1b90
--- /dev/null
+++ b/examples/innerjoin/pom.xml
@@ -0,0 +1,267 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>com.example</groupId>
+ <version>1.0-SNAPSHOT</version>
+ <artifactId>innerjoin</artifactId>
+ <packaging>jar</packaging>
+
+ <name>Inner Join Application</name>
+ <description>Sample Application for Inner Join</description>
+
+ <properties>
+ <!-- change this if you desire to use a different version of Apex Core -->
+ <apex.version>3.5.0</apex.version>
+ <malhar.version>3.6.0</malhar.version>
+ <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-eclipse-plugin</artifactId>
+ <version>2.9</version>
+ <configuration>
+ <downloadSources>true</downloadSources>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.3</version>
+ <configuration>
+ <encoding>UTF-8</encoding>
+ <source>1.7</source>
+ <target>1.7</target>
+ <debug>true</debug>
+ <optimize>false</optimize>
+ <showDeprecation>true</showDeprecation>
+ <showWarnings>true</showWarnings>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.8</version>
+ <executions>
+ <execution>
+ <id>copy-dependencies</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>target/deps</outputDirectory>
+ <includeScope>runtime</includeScope>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>app-package-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <finalName>${project.artifactId}-${project.version}-apexapp</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <descriptors>
+ <descriptor>src/assemble/appPackage.xml</descriptor>
+ </descriptors>
+ <archiverConfig>
+ <defaultDirectoryMode>0755</defaultDirectoryMode>
+ </archiverConfig>
+ <archive>
+ <manifestEntries>
+ <Class-Path>${apex.apppackage.classpath}</Class-Path>
+ <DT-Engine-Version>${apex.version}</DT-Engine-Version>
+ <DT-App-Package-Group-Id>${project.groupId}</DT-App-Package-Group-Id>
+ <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name>
+ <DT-App-Package-Version>${project.version}</DT-App-Package-Version>
+ <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name>
+ <DT-App-Package-Description>${project.description}</DT-App-Package-Description>
+ </manifestEntries>
+ </archive>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <configuration>
+ <target>
+ <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar"
+ tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa" />
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ <execution>
+ <!-- create resource directory for xml javadoc-->
+ <id>createJavadocDirectory</id>
+ <phase>generate-resources</phase>
+ <configuration>
+ <tasks>
+ <delete dir="${project.build.directory}/generated-resources/xml-javadoc"/>
+ <mkdir dir="${project.build.directory}/generated-resources/xml-javadoc"/>
+ </tasks>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.9.1</version>
+ <executions>
+ <execution>
+ <id>attach-artifacts</id>
+ <phase>package</phase>
+ <goals>
+ <goal>attach-artifact</goal>
+ </goals>
+ <configuration>
+ <artifacts>
+ <artifact>
+ <file>target/${project.artifactId}-${project.version}.apa</file>
+ <type>apa</type>
+ </artifact>
+ </artifacts>
+ <skipAttach>false</skipAttach>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- generate javdoc -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <executions>
+ <!-- generate xml javadoc -->
+ <execution>
+ <id>xml-doclet</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>javadoc</goal>
+ </goals>
+ <configuration>
+ <doclet>com.github.markusbernhardt.xmldoclet.XmlDoclet</doclet>
+ <additionalparam>-d ${project.build.directory}/generated-resources/xml-javadoc -filename ${project.artifactId}-${project.version}-javadoc.xml</additionalparam>
+ <useStandardDocletOptions>false</useStandardDocletOptions>
+ <docletArtifact>
+ <groupId>com.github.markusbernhardt</groupId>
+ <artifactId>xml-doclet</artifactId>
+ <version>1.0.4</version>
+ </docletArtifact>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <!-- Transform xml javadoc to stripped down version containing only class/interface comments and tags-->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>xml-maven-plugin</artifactId>
+ <version>1.0</version>
+ <executions>
+ <execution>
+ <id>transform-xmljavadoc</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>transform</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <transformationSets>
+ <transformationSet>
+ <dir>${project.build.directory}/generated-resources/xml-javadoc</dir>
+ <includes>
+ <include>${project.artifactId}-${project.version}-javadoc.xml</include>
+ </includes>
+ <stylesheet>XmlJavadocCommentsExtractor.xsl</stylesheet>
+ <outputDir>${project.build.directory}/generated-resources/xml-javadoc</outputDir>
+ </transformationSet>
+ </transformationSets>
+ </configuration>
+ </plugin>
+ <!-- copy xml javadoc to class jar -->
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>2.6</version>
+ <executions>
+ <execution>
+ <id>copy-resources</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${basedir}/target/classes</outputDirectory>
+ <resources>
+ <resource>
+ <directory>${project.build.directory}/generated-resources/xml-javadoc</directory>
+ <includes>
+ <include>${project.artifactId}-${project.version}-javadoc.xml</include>
+ </includes>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+
+ </build>
+
+ <dependencies>
+ <!-- add your dependencies here -->
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-library</artifactId>
+ <version>${malhar.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>apex-common</artifactId>
+ <version>${apex.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.10</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>apex-engine</artifactId>
+ <version>${apex.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.janino</groupId>
+ <artifactId>janino</artifactId>
+ <version>2.7.8</version>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/ca12bca0/examples/innerjoin/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/innerjoin/src/assemble/appPackage.xml b/examples/innerjoin/src/assemble/appPackage.xml
new file mode 100644
index 0000000..7ad071c
--- /dev/null
+++ b/examples/innerjoin/src/assemble/appPackage.xml
@@ -0,0 +1,43 @@
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>appPackage</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/target/</directory>
+ <outputDirectory>/app</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/target/deps</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/site/conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/META-INF</directory>
+ <outputDirectory>/META-INF</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/app</directory>
+ <outputDirectory>/app</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/resources</directory>
+ <outputDirectory>/resources</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/ca12bca0/examples/innerjoin/src/main/java/com/example/join/InnerJoinApplication.java
----------------------------------------------------------------------
diff --git a/examples/innerjoin/src/main/java/com/example/join/InnerJoinApplication.java b/examples/innerjoin/src/main/java/com/example/join/InnerJoinApplication.java
new file mode 100644
index 0000000..0b2f663
--- /dev/null
+++ b/examples/innerjoin/src/main/java/com/example/join/InnerJoinApplication.java
@@ -0,0 +1,39 @@
+package com.example.join;
+
+import org.apache.apex.malhar.lib.join.POJOInnerJoinOperator;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.common.partitioner.StatelessPartitioner;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+
+@ApplicationAnnotation(name="InnerJoinExample")
+public class InnerJoinApplication implements StreamingApplication
+{
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ // SalesEvent Generator
+ POJOGenerator salesGenerator = dag.addOperator("Input1", new POJOGenerator());
+ // ProductEvent Generator
+ POJOGenerator productGenerator = dag.addOperator("Input2", new POJOGenerator());
+ productGenerator.setSalesEvent(false);
+
+ // Inner join Operator
+ POJOInnerJoinOperator join = dag.addOperator("Join", new POJOInnerJoinOperator());
+ ConsoleOutputOperator output = dag.addOperator("Output", new ConsoleOutputOperator());
+
+ // Streams
+ dag.addStream("SalesToJoin", salesGenerator.output, join.input1);
+ dag.addStream("ProductToJoin", productGenerator.output, join.input2);
+ dag.addStream("JoinToConsole", join.outputPort, output.input);
+
+ // Setting tuple class properties to the ports of join operator
+ dag.setInputPortAttribute(join.input1, Context.PortContext.TUPLE_CLASS, POJOGenerator.SalesEvent.class);
+ dag.setInputPortAttribute(join.input2, Context.PortContext.TUPLE_CLASS, POJOGenerator.ProductEvent.class);
+ dag.setOutputPortAttribute(join.outputPort,Context.PortContext.TUPLE_CLASS, POJOGenerator.SalesEvent.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/ca12bca0/examples/innerjoin/src/main/java/com/example/join/POJOGenerator.java
----------------------------------------------------------------------
diff --git a/examples/innerjoin/src/main/java/com/example/join/POJOGenerator.java b/examples/innerjoin/src/main/java/com/example/join/POJOGenerator.java
new file mode 100644
index 0000000..613c80e
--- /dev/null
+++ b/examples/innerjoin/src/main/java/com/example/join/POJOGenerator.java
@@ -0,0 +1,260 @@
+package com.example.join;
+
+import java.util.Random;
+
+import javax.validation.constraints.Min;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+
+/**
+ * Generates and emits the SalesEvent/ProductEvent based on isSalesEvent.
+ */
+public class POJOGenerator implements InputOperator
+{
+ @Min(1)
+ private int maxProductId = 100000;
+ @Min(1)
+ private int maxCustomerId = 100000;
+ @Min(1)
+ private int maxProductCategories = 100;
+ private double maxAmount = 100.0;
+ private long tuplesCounter;
+ private long time;
+ private long timeIncrement;
+ private boolean isSalesEvent = true;
+ // Limit number of emitted tuples per window
+ @Min(0)
+ private long maxTuplesPerWindow = 100;
+ private final Random random = new Random();
+ public final transient DefaultOutputPort<Object> output = new DefaultOutputPort<>();
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ tuplesCounter = 0;
+ }
+
+ @Override
+ public void endWindow()
+ {
+ time += timeIncrement;
+ }
+
+ @Override
+ public void setup(Context.OperatorContext context)
+ {
+ time = System.currentTimeMillis();
+ timeIncrement = context.getValue(Context.OperatorContext.APPLICATION_WINDOW_COUNT) *
+ context.getValue(Context.DAGContext.STREAMING_WINDOW_SIZE_MILLIS);
+ }
+
+ @Override
+ public void teardown()
+ {
+
+ }
+
+ SalesEvent generateSalesEvent() throws Exception {
+
+ SalesEvent salesEvent = new SalesEvent();
+ salesEvent.productId = randomId(maxProductId);
+ salesEvent.customerId = randomId(maxCustomerId);
+ salesEvent.amount = randomAmount();
+ salesEvent.timestamp = time;
+ return salesEvent;
+ }
+
+ ProductEvent generateProductEvent() throws Exception {
+ ProductEvent productEvent = new ProductEvent();
+ productEvent.productId = randomId(maxProductId);
+ productEvent.productCategory = randomId(maxProductCategories);
+ productEvent.timestamp = time;
+ return productEvent;
+ }
+
+ private int randomId(int max) {
+ if (max < 1) return 1;
+ return 1 + random.nextInt(max);
+ }
+
+ private double randomAmount() {
+ return maxAmount * random.nextDouble();
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ while (tuplesCounter++ < maxTuplesPerWindow) {
+ try {
+ if (isSalesEvent) {
+ SalesEvent event = generateSalesEvent();
+ this.output.emit(event);
+ } else {
+ ProductEvent event = generateProductEvent();
+ this.output.emit(event);
+ }
+
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+ }
+
+ public static class SalesEvent
+ {
+ public int customerId;
+ public int productId;
+ public int productCategory;
+ public double amount;
+ public long timestamp;
+
+ public int getCustomerId()
+ {
+ return customerId;
+ }
+
+ public void setCustomerId(int customerId)
+ {
+ this.customerId = customerId;
+ }
+
+ public int getProductId()
+ {
+ return productId;
+ }
+
+ public void setProductId(int productId)
+ {
+ this.productId = productId;
+ }
+
+ public int getProductCategory()
+ {
+ return productCategory;
+ }
+
+ public void setProductCategory(int productCategory)
+ {
+ this.productCategory = productCategory;
+ }
+
+ public double getAmount()
+ {
+ return amount;
+ }
+
+ public void setAmount(double amount)
+ {
+ this.amount = amount;
+ }
+
+ public long getTimestamp()
+ {
+ return timestamp;
+ }
+
+ public void setTimestamp(long timestamp)
+ {
+ this.timestamp = timestamp;
+ }
+ }
+
+ public static class ProductEvent
+ {
+ public int productId;
+ public int productCategory;
+ public long timestamp;
+
+ public int getProductId()
+ {
+ return productId;
+ }
+
+ public void setProductId(int productId)
+ {
+ this.productId = productId;
+ }
+
+ public int getProductCategory()
+ {
+ return productCategory;
+ }
+
+ public void setProductCategory(int productCategory)
+ {
+ this.productCategory = productCategory;
+ }
+
+ public long getTimestamp()
+ {
+ return timestamp;
+ }
+
+ public void setTimestamp(long timestamp)
+ {
+ this.timestamp = timestamp;
+ }
+ }
+
+ public int getMaxProductId()
+ {
+ return maxProductId;
+ }
+
+ public void setMaxProductId(int maxProductId)
+ {
+ this.maxProductId = maxProductId;
+ }
+
+ public int getMaxCustomerId()
+ {
+ return maxCustomerId;
+ }
+
+ public void setMaxCustomerId(int maxCustomerId)
+ {
+ this.maxCustomerId = maxCustomerId;
+ }
+
+ public int getMaxProductCategories()
+ {
+ return maxProductCategories;
+ }
+
+ public void setMaxProductCategories(int maxProductCategories)
+ {
+ this.maxProductCategories = maxProductCategories;
+ }
+
+ public double getMaxAmount()
+ {
+ return maxAmount;
+ }
+
+ public void setMaxAmount(double maxAmount)
+ {
+ this.maxAmount = maxAmount;
+ }
+
+ public boolean isSalesEvent()
+ {
+ return isSalesEvent;
+ }
+
+ public void setSalesEvent(boolean salesEvent)
+ {
+ isSalesEvent = salesEvent;
+ }
+
+ public long getMaxTuplesPerWindow()
+ {
+ return maxTuplesPerWindow;
+ }
+
+ public void setMaxTuplesPerWindow(long maxTuplesPerWindow)
+ {
+ this.maxTuplesPerWindow = maxTuplesPerWindow;
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/ca12bca0/examples/innerjoin/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/innerjoin/src/main/resources/META-INF/properties.xml b/examples/innerjoin/src/main/resources/META-INF/properties.xml
new file mode 100644
index 0000000..5c9cb75
--- /dev/null
+++ b/examples/innerjoin/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0"?>
+<configuration>
+ <property>
+ <name>dt.application.InnerJoinExample.operator.Join.attr.MEMORY_MB</name>
+ <value>1024</value>
+ </property>
+ <property>
+ <name>dt.application.InnerJoinExample.operator.Join.prop.leftKeyExpression</name>
+ <value>productId</value>
+ </property>
+ <property>
+ <name>dt.application.InnerJoinExample.operator.Join.prop.rightKeyExpression</name>
+ <value>productId</value>
+ </property>
+ <property>
+ <name>dt.application.InnerJoinExample.operator.Join.prop.timeFieldsStr</name>
+ <value>timestamp,timestamp</value>
+ </property>
+ <property>
+ <name>dt.application.InnerJoinExample.operator.Join.prop.expiryTime</name>
+ <value>3600000</value>
+ </property>
+ <property>
+ <name>dt.application.InnerJoinExample.operator.Join.prop.includeFieldStr</name>
+ <value>customerId,productId,amount;productCategory,timestamp</value>
+ </property>
+ <property>
+ <name>dt.application.InnerJoinExample.operator.Join.prop.noOfBuckets</name>
+ <value>2</value>
+ </property>
+ <property>
+ <name>dt.application.InnerJoinExample.operator.Join.attr.PARTITIONER</name>
+ <value>com.datatorrent.common.partitioner.StatelessPartitioner:3</value>
+ </property>
+</configuration>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/ca12bca0/examples/innerjoin/src/test/java/com/example/join/InnerJoinApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/innerjoin/src/test/java/com/example/join/InnerJoinApplicationTest.java b/examples/innerjoin/src/test/java/com/example/join/InnerJoinApplicationTest.java
new file mode 100644
index 0000000..6da3908
--- /dev/null
+++ b/examples/innerjoin/src/test/java/com/example/join/InnerJoinApplicationTest.java
@@ -0,0 +1,21 @@
+package com.example.join;
+
+import org.junit.Test;
+import org.apache.hadoop.conf.Configuration;
+import com.datatorrent.api.LocalMode;
+
+public class InnerJoinApplicationTest
+{
+ @Test
+ public void testApplication() throws Exception
+ {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ lma.prepareDAG(new InnerJoinApplication(), conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.runAsync();
+ Thread.sleep(10 * 1000);
+ lc.shutdown();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/ca12bca0/examples/innerjoin/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/innerjoin/src/test/resources/log4j.properties b/examples/innerjoin/src/test/resources/log4j.properties
new file mode 100644
index 0000000..98544e8
--- /dev/null
+++ b/examples/innerjoin/src/test/resources/log4j.properties
@@ -0,0 +1,22 @@
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=debug
+log4j.logger.org.apache.apex=debug
+log4j.logger.org=info
[18/19] apex-malhar git commit: APEXMALHAR-2233 Updated the examples
to follow the structure of apex-malhar examples. Specified dependencies in
pom.xmls of individual examples correctly.
Posted by th...@apache.org.
APEXMALHAR-2233 Updated the examples to follow the structure of apex-malhar examples. Specified dependencies in pom.xmls of individual examples correctly.
Project: http://git-wip-us.apache.org/repos/asf/apex-malhar/repo
Commit: http://git-wip-us.apache.org/repos/asf/apex-malhar/commit/9c154f20
Tree: http://git-wip-us.apache.org/repos/asf/apex-malhar/tree/9c154f20
Diff: http://git-wip-us.apache.org/repos/asf/apex-malhar/diff/9c154f20
Branch: refs/heads/master
Commit: 9c154f204042a9e1974c2466e8783505b2c6da03
Parents: 8e20097
Author: Lakshmi Prasanna Velineni <la...@datatorrent.com>
Authored: Sun Mar 19 22:40:04 2017 -0700
Committer: Lakshmi Prasanna Velineni <la...@datatorrent.com>
Committed: Sun Mar 26 11:43:48 2017 -0700
----------------------------------------------------------------------
examples/csvformatter/pom.xml | 299 +++----------------
.../java/com/demo/myapexapp/Application.java | 45 ---
.../com/demo/myapexapp/HDFSOutputOperator.java | 87 ------
.../java/com/demo/myapexapp/JsonGenerator.java | 78 -----
.../main/java/com/demo/myapexapp/PojoEvent.java | 141 ---------
.../apex/examples/csvformatter/Application.java | 37 +++
.../csvformatter/HDFSOutputOperator.java | 87 ++++++
.../examples/csvformatter/JsonGenerator.java | 76 +++++
.../apex/examples/csvformatter/PojoEvent.java | 141 +++++++++
.../src/main/resources/META-INF/properties.xml | 2 +-
.../com/demo/myapexapp/ApplicationTest.java | 67 -----
.../examples/csvformatter/ApplicationTest.java | 65 ++++
examples/dedup/pom.xml | 277 +----------------
.../java/com/example/dedup/Application.java | 123 --------
.../apache/apex/examples/dedup/Application.java | 122 ++++++++
.../java/com/example/dedup/ApplicationTest.java | 38 ---
.../apex/examples/dedup/ApplicationTest.java | 37 +++
examples/dynamic-partition/pom.xml | 270 +----------------
.../src/main/java/com/example/dynamic/App.java | 23 --
.../src/main/java/com/example/dynamic/Gen.java | 169 -----------
.../org/apache/apex/examples/dynamic/App.java | 23 ++
.../org/apache/apex/examples/dynamic/Gen.java | 171 +++++++++++
.../com/example/dynamic/ApplicationTest.java | 34 ---
.../apex/examples/dynamic/ApplicationTest.java | 33 ++
examples/enricher/pom.xml | 298 ++----------------
.../com/example/myapexapp/DataGenerator.java | 94 ------
.../myapexapp/EnricherAppWithJSONFile.java | 47 ---
.../example/myapexapp/LineOutputOperator.java | 34 ---
.../main/java/com/example/myapexapp/POJO.java | 49 ---
.../com/example/myapexapp/POJOEnriched.java | 71 -----
.../apex/examples/enricher/DataGenerator.java | 94 ++++++
.../enricher/EnricherAppWithJSONFile.java | 47 +++
.../examples/enricher/LineOutputOperator.java | 34 +++
.../org/apache/apex/examples/enricher/POJO.java | 49 +++
.../apex/examples/enricher/POJOEnriched.java | 71 +++++
.../src/main/resources/META-INF/properties.xml | 6 +-
.../com/example/myapexapp/ApplicationTest.java | 31 --
.../apex/examples/enricher/ApplicationTest.java | 31 ++
examples/filter/pom.xml | 276 +----------------
.../tutorial/filter/Application.java | 49 ---
.../tutorial/filter/TransactionPOJO.java | 64 ----
.../apex/examples/filter/Application.java | 49 +++
.../apex/examples/filter/TransactionPOJO.java | 62 ++++
.../src/main/resources/META-INF/properties.xml | 8 +-
.../tutorial/filter/ApplicationTest.java | 111 -------
.../apex/examples/filter/ApplicationTest.java | 96 ++++++
examples/innerjoin/pom.xml | 269 ++---------------
.../com/example/join/InnerJoinApplication.java | 39 ---
.../java/com/example/join/POJOGenerator.java | 260 ----------------
.../innerjoin/InnerJoinApplication.java | 38 +++
.../apex/examples/innerjoin/POJOGenerator.java | 260 ++++++++++++++++
.../example/join/InnerJoinApplicationTest.java | 21 --
.../innerjoin/InnerJoinApplicationTest.java | 21 ++
examples/parser/pom.xml | 268 ++---------------
.../tutorial/jsonparser/Application.java | 35 ---
.../tutorial/jsonparser/Campaign.java | 74 -----
.../tutorial/jsonparser/JsonGenerator.java | 83 -----
.../examples/parser/jsonparser/Application.java | 35 +++
.../examples/parser/jsonparser/Campaign.java | 74 +++++
.../parser/jsonparser/JsonGenerator.java | 83 +++++
.../src/main/resources/META-INF/properties.xml | 4 +-
.../tutorial/jsonparser/ApplicationTest.java | 36 ---
.../parser/jsonparser/ApplicationTest.java | 35 +++
examples/partition/pom.xml | 276 +----------------
.../java/com/example/myapexapp/Application.java | 27 --
.../main/java/com/example/myapexapp/Codec3.java | 13 -
.../myapexapp/RandomNumberGenerator.java | 83 -----
.../com/example/myapexapp/TestPartition.java | 164 ----------
.../apex/examples/partition/Application.java | 25 ++
.../apache/apex/examples/partition/Codec3.java | 13 +
.../partition/RandomNumberGenerator.java | 76 +++++
.../apex/examples/partition/TestPartition.java | 149 +++++++++
.../src/main/resources/my-log4j.properties | 2 +-
.../com/example/myapexapp/ApplicationTest.java | 37 ---
.../examples/partition/ApplicationTest.java | 36 +++
examples/pom.xml | 10 +
examples/recordReader/pom.xml | 284 +-----------------
.../com/example/recordReader/Application.java | 32 --
.../recordReader/TransactionsSchema.java | 168 -----------
.../apex/examples/recordReader/Application.java | 32 ++
.../recordReader/TransactionsSchema.java | 168 +++++++++++
.../src/main/resources/META-INF/properties.xml | 4 +-
.../example/recordReader/ApplicationTest.java | 91 ------
.../examples/recordReader/ApplicationTest.java | 91 ++++++
examples/throttle/pom.xml | 256 +---------------
.../examples/throttle/Application.java | 51 ----
.../examples/throttle/PassThroughOperator.java | 20 --
.../throttle/RandomNumberGenerator.java | 64 ----
.../examples/throttle/SlowDevNullOperator.java | 35 ---
.../throttle/ThrottlingStatsListener.java | 150 ----------
.../apex/examples/throttle/Application.java | 51 ++++
.../examples/throttle/PassThroughOperator.java | 20 ++
.../throttle/RandomNumberGenerator.java | 64 ++++
.../examples/throttle/SlowDevNullOperator.java | 35 +++
.../throttle/ThrottlingStatsListener.java | 150 ++++++++++
.../examples/throttle/ApplicationTest.java | 37 ---
.../apex/examples/throttle/ApplicationTest.java | 36 +++
examples/transform/pom.xml | 250 +---------------
.../java/com/example/transform/Application.java | 39 ---
.../com/example/transform/CustomerEvent.java | 74 -----
.../com/example/transform/CustomerInfo.java | 60 ----
.../transform/DynamicTransformApplication.java | 52 ----
.../com/example/transform/POJOGenerator.java | 125 --------
.../apex/examples/transform/Application.java | 39 +++
.../apex/examples/transform/CustomerEvent.java | 74 +++++
.../apex/examples/transform/CustomerInfo.java | 60 ++++
.../transform/DynamicTransformApplication.java | 51 ++++
.../apex/examples/transform/POJOGenerator.java | 125 ++++++++
.../com/example/transform/ApplicationTest.java | 21 --
.../examples/transform/ApplicationTest.java | 21 ++
110 files changed, 3404 insertions(+), 6088 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/csvformatter/pom.xml
----------------------------------------------------------------------
diff --git a/examples/csvformatter/pom.xml b/examples/csvformatter/pom.xml
index 9033db5..be3be7a 100644
--- a/examples/csvformatter/pom.xml
+++ b/examples/csvformatter/pom.xml
@@ -1,266 +1,61 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>com.example</groupId>
- <version>1.0-SNAPSHOT</version>
- <artifactId>formatter</artifactId>
+ <parent>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-examples</artifactId>
+ <version>3.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>malhar-examples-csvformatter</artifactId>
<packaging>jar</packaging>
<!-- change these to the appropriate values -->
<name>Formatter Apps</name>
<description>Applications to showcase different formatters</description>
- <properties>
- <!-- change this if you desire to use a different version of Apex Core -->
- <apex.version>3.5.0</apex.version>
- <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath>
- <malhar.version>3.6.0</malhar.version>
- </properties>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-eclipse-plugin</artifactId>
- <version>2.9</version>
- <configuration>
- <downloadSources>true</downloadSources>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>3.3</version>
- <configuration>
- <encoding>UTF-8</encoding>
- <source>1.7</source>
- <target>1.7</target>
- <debug>true</debug>
- <optimize>false</optimize>
- <showDeprecation>true</showDeprecation>
- <showWarnings>true</showWarnings>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <version>2.8</version>
- <executions>
- <execution>
- <id>copy-dependencies</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>copy-dependencies</goal>
- </goals>
- <configuration>
- <outputDirectory>target/deps</outputDirectory>
- <includeScope>runtime</includeScope>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <executions>
- <execution>
- <id>app-package-assembly</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <finalName>${project.artifactId}-${project.version}-apexapp</finalName>
- <appendAssemblyId>false</appendAssemblyId>
- <descriptors>
- <descriptor>src/assemble/appPackage.xml</descriptor>
- </descriptors>
- <archiverConfig>
- <defaultDirectoryMode>0755</defaultDirectoryMode>
- </archiverConfig>
- <archive>
- <manifestEntries>
- <Class-Path>${apex.apppackage.classpath}</Class-Path>
- <DT-Engine-Version>${apex.version}</DT-Engine-Version>
- <DT-App-Package-Group-Id>${project.groupId}</DT-App-Package-Group-Id>
- <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name>
- <DT-App-Package-Version>${project.version}</DT-App-Package-Version>
- <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name>
- <DT-App-Package-Description>${project.description}</DT-App-Package-Description>
- </manifestEntries>
- </archive>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <artifactId>maven-antrun-plugin</artifactId>
- <version>1.7</version>
- <executions>
- <execution>
- <phase>package</phase>
- <configuration>
- <target>
- <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar"
- tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa" />
- </target>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- <execution>
- <!-- create resource directory for xml javadoc-->
- <id>createJavadocDirectory</id>
- <phase>generate-resources</phase>
- <configuration>
- <tasks>
- <delete dir="${project.build.directory}/generated-resources/xml-javadoc"/>
- <mkdir dir="${project.build.directory}/generated-resources/xml-javadoc"/>
- </tasks>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>1.9.1</version>
- <executions>
- <execution>
- <id>attach-artifacts</id>
- <phase>package</phase>
- <goals>
- <goal>attach-artifact</goal>
- </goals>
- <configuration>
- <artifacts>
- <artifact>
- <file>target/${project.artifactId}-${project.version}.apa</file>
- <type>apa</type>
- </artifact>
- </artifacts>
- <skipAttach>false</skipAttach>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <!-- generate javdoc -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <executions>
- <!-- generate xml javadoc -->
- <execution>
- <id>xml-doclet</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>javadoc</goal>
- </goals>
- <configuration>
- <doclet>com.github.markusbernhardt.xmldoclet.XmlDoclet</doclet>
- <additionalparam>-d ${project.build.directory}/generated-resources/xml-javadoc -filename ${project.artifactId}-${project.version}-javadoc.xml</additionalparam>
- <useStandardDocletOptions>false</useStandardDocletOptions>
- <docletArtifact>
- <groupId>com.github.markusbernhardt</groupId>
- <artifactId>xml-doclet</artifactId>
- <version>1.0.4</version>
- </docletArtifact>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <!-- Transform xml javadoc to stripped down version containing only class/interface comments and tags-->
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>xml-maven-plugin</artifactId>
- <version>1.0</version>
- <executions>
- <execution>
- <id>transform-xmljavadoc</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>transform</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <transformationSets>
- <transformationSet>
- <dir>${project.build.directory}/generated-resources/xml-javadoc</dir>
- <includes>
- <include>${project.artifactId}-${project.version}-javadoc.xml</include>
- </includes>
- <stylesheet>XmlJavadocCommentsExtractor.xsl</stylesheet>
- <outputDir>${project.build.directory}/generated-resources/xml-javadoc</outputDir>
- </transformationSet>
- </transformationSets>
- </configuration>
- </plugin>
- <!-- copy xml javadoc to class jar -->
- <plugin>
- <artifactId>maven-resources-plugin</artifactId>
- <version>2.6</version>
- <executions>
- <execution>
- <id>copy-resources</id>
- <phase>process-resources</phase>
- <goals>
- <goal>copy-resources</goal>
- </goals>
- <configuration>
- <outputDirectory>${basedir}/target/classes</outputDirectory>
- <resources>
- <resource>
- <directory>${project.build.directory}/generated-resources/xml-javadoc</directory>
- <includes>
- <include>${project.artifactId}-${project.version}-javadoc.xml</include>
- </includes>
- <filtering>true</filtering>
- </resource>
- </resources>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
-
- </build>
-
<dependencies>
- <!-- add your dependencies here -->
<dependency>
<groupId>org.apache.apex</groupId>
- <artifactId>malhar-library</artifactId>
- <version>${malhar.version}</version>
- <!--
- If you know that your application does not need transitive dependencies pulled in by malhar-library,
- uncomment the following to reduce the size of your app package.
- -->
- <!--
+ <artifactId>malhar-contrib</artifactId>
+ <version>${project.version}</version>
<exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
</exclusions>
- -->
</dependency>
<dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-contrib</artifactId>
- <version>${malhar.version}</version>
+ <groupId>org.codehaus.jettison</groupId>
+ <artifactId>jettison</artifactId>
+ <version>1.1</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
- <version>2.5.4</version>
+ <version>2.7.0</version>
</dependency>
<dependency>
<groupId>com.github.fge</groupId>
@@ -269,29 +64,9 @@
<optional>true</optional>
</dependency>
<dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>apex-common</artifactId>
- <version>${apex.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.10</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>apex-engine</artifactId>
- <version>${apex.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
<groupId>net.sf.supercsv</groupId>
<artifactId>super-csv</artifactId>
<version>2.4.0</version>
</dependency>
</dependencies>
-
</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/csvformatter/src/main/java/com/demo/myapexapp/Application.java
----------------------------------------------------------------------
diff --git a/examples/csvformatter/src/main/java/com/demo/myapexapp/Application.java b/examples/csvformatter/src/main/java/com/demo/myapexapp/Application.java
deleted file mode 100644
index a4ff06f..0000000
--- a/examples/csvformatter/src/main/java/com/demo/myapexapp/Application.java
+++ /dev/null
@@ -1,45 +0,0 @@
-package com.demo.myapexapp;
-
-import java.util.Arrays;
-
-import com.datatorrent.contrib.parser.JsonParser;
-
-import org.apache.apex.malhar.contrib.parser.StreamingJsonParser;
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.Context;
-import com.datatorrent.api.Context.PortContext;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.StatsListener;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.contrib.formatter.CsvFormatter;
-import com.datatorrent.lib.appdata.schemas.SchemaUtils;
-import com.datatorrent.lib.io.fs.AbstractFileInputOperator.DirectoryScanner;
-import com.datatorrent.lib.partitioner.StatelessThroughputBasedPartitioner;
-
-@ApplicationAnnotation(name = "CustomOutputFormatter")
-public class Application implements StreamingApplication
-{
- //Set the delimiters and schema structure for the custom output in schema.json
- private static final String filename = "schema.json";
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- JsonGenerator generator = dag.addOperator("JsonGenerator", JsonGenerator.class);
- JsonParser jsonParser = dag.addOperator("jsonParser", JsonParser.class);
-
- CsvFormatter formatter = dag.addOperator("formatter", CsvFormatter.class);
- formatter.setSchema(SchemaUtils.jarResourceFileToString(filename));
- dag.setInputPortAttribute(formatter.in, PortContext.TUPLE_CLASS, PojoEvent.class);
-
- HDFSOutputOperator<String> hdfsOutput = dag.addOperator("HDFSOutputOperator", HDFSOutputOperator.class);
- hdfsOutput.setLineDelimiter("");
-
- dag.addStream("parserStream", generator.out, jsonParser.in);
- dag.addStream("formatterStream", jsonParser.out, formatter.in);
- dag.addStream("outputStream", formatter.out, hdfsOutput.input);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/csvformatter/src/main/java/com/demo/myapexapp/HDFSOutputOperator.java
----------------------------------------------------------------------
diff --git a/examples/csvformatter/src/main/java/com/demo/myapexapp/HDFSOutputOperator.java b/examples/csvformatter/src/main/java/com/demo/myapexapp/HDFSOutputOperator.java
deleted file mode 100644
index 5cb162c..0000000
--- a/examples/csvformatter/src/main/java/com/demo/myapexapp/HDFSOutputOperator.java
+++ /dev/null
@@ -1,87 +0,0 @@
-package com.demo.myapexapp;
-
-import javax.validation.constraints.NotNull;
-
-import com.datatorrent.api.Context.OperatorContext;
-import com.datatorrent.lib.io.fs.AbstractFileOutputOperator;
-
-/**
- * HDFSoutput operator with implementation to write Objects to HDFS
- *
- * @param <T>
- */
-public class HDFSOutputOperator<T> extends AbstractFileOutputOperator<T>
-{
-
- @NotNull
- String outFileName;
-
- //setting default value
- String lineDelimiter = "\n";
-
- //Switch to write the files to HDFS - set to false to diable writes
- private boolean writeFilesFlag = true;
-
- int id;
-
- @Override
- public void setup(OperatorContext context)
- {
- super.setup(context);
- id = context.getId();
- }
-
- public boolean isWriteFilesFlag()
- {
- return writeFilesFlag;
- }
-
- public void setWriteFilesFlag(boolean writeFilesFlag)
- {
- this.writeFilesFlag = writeFilesFlag;
- }
-
- public String getOutFileName()
- {
- return outFileName;
- }
-
- public void setOutFileName(String outFileName)
- {
- this.outFileName = outFileName;
- }
-
- @Override
- protected String getFileName(T tuple)
- {
- return getOutFileName() + id;
- }
-
- public String getLineDelimiter()
- {
- return lineDelimiter;
- }
-
- public void setLineDelimiter(String lineDelimiter)
- {
- this.lineDelimiter = lineDelimiter;
- }
-
- @Override
- protected byte[] getBytesForTuple(T tuple)
- {
- String temp = tuple.toString().concat(String.valueOf(lineDelimiter));
- byte[] theByteArray = temp.getBytes();
-
- return theByteArray;
- }
-
- @Override
- protected void processTuple(T tuple)
- {
- if (writeFilesFlag) {
- }
- super.processTuple(tuple);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/csvformatter/src/main/java/com/demo/myapexapp/JsonGenerator.java
----------------------------------------------------------------------
diff --git a/examples/csvformatter/src/main/java/com/demo/myapexapp/JsonGenerator.java b/examples/csvformatter/src/main/java/com/demo/myapexapp/JsonGenerator.java
deleted file mode 100644
index f50f300..0000000
--- a/examples/csvformatter/src/main/java/com/demo/myapexapp/JsonGenerator.java
+++ /dev/null
@@ -1,78 +0,0 @@
-package com.demo.myapexapp;
-
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.Random;
-
-import javax.validation.constraints.Min;
-
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.InputOperator;
-import com.datatorrent.common.util.BaseOperator;
-
-public class JsonGenerator extends BaseOperator implements InputOperator
-{
-
- private static final Logger LOG = LoggerFactory.getLogger(JsonGenerator.class);
-
- @Min(1)
- private int numTuples = 20;
- private transient int count = 0;
-
- public static Random rand = new Random();
- private int sleepTime=5;
-
- public final transient DefaultOutputPort<byte[]> out = new DefaultOutputPort<byte[]>();
-
- private static String getJson()
- {
-
- JSONObject obj = new JSONObject();
- try {
- obj.put("campaignId", 1234);
- obj.put("campaignName", "SimpleCsvFormatterExample");
- obj.put("campaignBudget", 10000.0);
- obj.put("weatherTargeting", "false");
- obj.put("securityCode", "APEX");
- } catch (JSONException e) {
- return null;
- }
- return obj.toString();
- }
-
- @Override
- public void beginWindow(long windowId)
- {
- count = 0;
- }
-
- @Override
- public void emitTuples()
- {
- if (count++ < numTuples) {
- out.emit(getJson().getBytes());
- } else {
- try {
- Thread.sleep(sleepTime);
- } catch (InterruptedException e) {
- LOG.info("Sleep interrupted");
- }
- }
- }
-
- public int getNumTuples()
- {
- return numTuples;
- }
-
- public void setNumTuples(int numTuples)
- {
- this.numTuples = numTuples;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/csvformatter/src/main/java/com/demo/myapexapp/PojoEvent.java
----------------------------------------------------------------------
diff --git a/examples/csvformatter/src/main/java/com/demo/myapexapp/PojoEvent.java b/examples/csvformatter/src/main/java/com/demo/myapexapp/PojoEvent.java
deleted file mode 100644
index 8514856..0000000
--- a/examples/csvformatter/src/main/java/com/demo/myapexapp/PojoEvent.java
+++ /dev/null
@@ -1,141 +0,0 @@
-package com.demo.myapexapp;
-
-import java.util.Date;
-
-public class PojoEvent
-{
-
- private int advId;
- private int campaignId;
- private String campaignName;
- private double campaignBudget;
- private Date startDate;
- private Date endDate;
- private String securityCode;
- private boolean weatherTargeting;
- private boolean optimized;
- private String parentCampaign;
- private Character weatherTargeted;
- private String valid;
-
- public int getAdvId()
- {
- return advId;
- }
-
- public void setAdvId(int AdId)
- {
- this.advId = advId;
- }
-
- public int getCampaignId()
- {
- return campaignId;
- }
-
- public void setCampaignId(int campaignId)
- {
- this.campaignId = campaignId;
- }
-
- public String getCampaignName()
- {
- return campaignName;
- }
-
- public void setCampaignName(String campaignName)
- {
- this.campaignName = campaignName;
- }
-
- public double getCampaignBudget()
- {
- return campaignBudget;
- }
-
- public void setCampaignBudget(double campaignBudget)
- {
- this.campaignBudget = campaignBudget;
- }
-
- public Date getStartDate()
- {
- return startDate;
- }
-
- public void setStartDate(Date startDate)
- {
- this.startDate = startDate;
- }
-
- public Date getEndDate()
- {
- return endDate;
- }
-
- public void setEndDate(Date endDate)
- {
- this.endDate = endDate;
- }
-
- public String getSecurityCode()
- {
- return securityCode;
- }
-
- public void setSecurityCode(String securityCode)
- {
- this.securityCode = securityCode;
- }
-
- public boolean isWeatherTargeting()
- {
- return weatherTargeting;
- }
-
- public void setWeatherTargeting(boolean weatherTargeting)
- {
- this.weatherTargeting = weatherTargeting;
- }
-
- public boolean isOptimized()
- {
- return optimized;
- }
-
- public void setOptimized(boolean optimized)
- {
- this.optimized = optimized;
- }
-
- public String getParentCampaign()
- {
- return parentCampaign;
- }
-
- public void setParentCampaign(String parentCampaign)
- {
- this.parentCampaign = parentCampaign;
- }
-
- public Character getWeatherTargeted()
- {
- return weatherTargeted;
- }
-
- public void setWeatherTargeted(Character weatherTargeted)
- {
- this.weatherTargeted = weatherTargeted;
- }
-
- public String getValid()
- {
- return valid;
- }
-
- public void setValid(String valid)
- {
- this.valid = valid;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/Application.java
----------------------------------------------------------------------
diff --git a/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/Application.java b/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/Application.java
new file mode 100644
index 0000000..cc9ee79
--- /dev/null
+++ b/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/Application.java
@@ -0,0 +1,37 @@
+package org.apache.apex.examples.csvformatter;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.Context.PortContext;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.contrib.formatter.CsvFormatter;
+import com.datatorrent.contrib.parser.JsonParser;
+import com.datatorrent.lib.appdata.schemas.SchemaUtils;
+
+@ApplicationAnnotation(name = "CustomOutputFormatter")
+public class Application implements StreamingApplication
+{
+ //Set the delimiters and schema structure for the custom output in schema.json
+ private static final String filename = "schema.json";
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ JsonGenerator generator = dag.addOperator("JsonGenerator", JsonGenerator.class);
+ JsonParser jsonParser = dag.addOperator("jsonParser", JsonParser.class);
+
+ CsvFormatter formatter = dag.addOperator("formatter", CsvFormatter.class);
+ formatter.setSchema(SchemaUtils.jarResourceFileToString(filename));
+ dag.setInputPortAttribute(formatter.in, PortContext.TUPLE_CLASS, PojoEvent.class);
+
+ HDFSOutputOperator<String> hdfsOutput = dag.addOperator("HDFSOutputOperator", HDFSOutputOperator.class);
+ hdfsOutput.setLineDelimiter("");
+
+ dag.addStream("parserStream", generator.out, jsonParser.in);
+ dag.addStream("formatterStream", jsonParser.out, formatter.in);
+ dag.addStream("outputStream", formatter.out, hdfsOutput.input);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/HDFSOutputOperator.java
----------------------------------------------------------------------
diff --git a/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/HDFSOutputOperator.java b/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/HDFSOutputOperator.java
new file mode 100644
index 0000000..7cdd8bb
--- /dev/null
+++ b/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/HDFSOutputOperator.java
@@ -0,0 +1,87 @@
+package org.apache.apex.examples.csvformatter;
+
+import javax.validation.constraints.NotNull;
+
+import com.datatorrent.api.Context.OperatorContext;
+import com.datatorrent.lib.io.fs.AbstractFileOutputOperator;
+
+/**
+ * HDFSoutput operator with implementation to write Objects to HDFS
+ *
+ * @param <T>
+ */
+public class HDFSOutputOperator<T> extends AbstractFileOutputOperator<T>
+{
+
+ @NotNull
+ String outFileName;
+
+ //setting default value
+ String lineDelimiter = "\n";
+
+ //Switch to write the files to HDFS - set to false to diable writes
+ private boolean writeFilesFlag = true;
+
+ int id;
+
+ @Override
+ public void setup(OperatorContext context)
+ {
+ super.setup(context);
+ id = context.getId();
+ }
+
+ public boolean isWriteFilesFlag()
+ {
+ return writeFilesFlag;
+ }
+
+ public void setWriteFilesFlag(boolean writeFilesFlag)
+ {
+ this.writeFilesFlag = writeFilesFlag;
+ }
+
+ public String getOutFileName()
+ {
+ return outFileName;
+ }
+
+ public void setOutFileName(String outFileName)
+ {
+ this.outFileName = outFileName;
+ }
+
+ @Override
+ protected String getFileName(T tuple)
+ {
+ return getOutFileName() + id;
+ }
+
+ public String getLineDelimiter()
+ {
+ return lineDelimiter;
+ }
+
+ public void setLineDelimiter(String lineDelimiter)
+ {
+ this.lineDelimiter = lineDelimiter;
+ }
+
+ @Override
+ protected byte[] getBytesForTuple(T tuple)
+ {
+ String temp = tuple.toString().concat(String.valueOf(lineDelimiter));
+ byte[] theByteArray = temp.getBytes();
+
+ return theByteArray;
+ }
+
+ @Override
+ protected void processTuple(T tuple)
+ {
+ if (writeFilesFlag) {
+ }
+ super.processTuple(tuple);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/JsonGenerator.java
----------------------------------------------------------------------
diff --git a/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/JsonGenerator.java b/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/JsonGenerator.java
new file mode 100644
index 0000000..9b7698c
--- /dev/null
+++ b/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/JsonGenerator.java
@@ -0,0 +1,76 @@
+package org.apache.apex.examples.csvformatter;
+
+import java.util.Random;
+
+import javax.validation.constraints.Min;
+
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.common.util.BaseOperator;
+
+public class JsonGenerator extends BaseOperator implements InputOperator
+{
+
+ private static final Logger LOG = LoggerFactory.getLogger(JsonGenerator.class);
+
+ @Min(1)
+ private int numTuples = 20;
+ private transient int count = 0;
+
+ public static Random rand = new Random();
+ private int sleepTime=5;
+
+ public final transient DefaultOutputPort<byte[]> out = new DefaultOutputPort<byte[]>();
+
+ private static String getJson()
+ {
+
+ JSONObject obj = new JSONObject();
+ try {
+ obj.put("campaignId", 1234);
+ obj.put("campaignName", "SimpleCsvFormatterExample");
+ obj.put("campaignBudget", 10000.0);
+ obj.put("weatherTargeting", "false");
+ obj.put("securityCode", "APEX");
+ } catch (JSONException e) {
+ return null;
+ }
+ return obj.toString();
+ }
+
+ @Override
+ public void beginWindow(long windowId)
+ {
+ count = 0;
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ if (count++ < numTuples) {
+ out.emit(getJson().getBytes());
+ } else {
+ try {
+ Thread.sleep(sleepTime);
+ } catch (InterruptedException e) {
+ LOG.info("Sleep interrupted");
+ }
+ }
+ }
+
+ public int getNumTuples()
+ {
+ return numTuples;
+ }
+
+ public void setNumTuples(int numTuples)
+ {
+ this.numTuples = numTuples;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/PojoEvent.java
----------------------------------------------------------------------
diff --git a/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/PojoEvent.java b/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/PojoEvent.java
new file mode 100644
index 0000000..03fda93
--- /dev/null
+++ b/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/PojoEvent.java
@@ -0,0 +1,141 @@
+package org.apache.apex.examples.csvformatter;
+
+import java.util.Date;
+
+public class PojoEvent
+{
+
+ private int advId;
+ private int campaignId;
+ private String campaignName;
+ private double campaignBudget;
+ private Date startDate;
+ private Date endDate;
+ private String securityCode;
+ private boolean weatherTargeting;
+ private boolean optimized;
+ private String parentCampaign;
+ private Character weatherTargeted;
+ private String valid;
+
+ public int getAdvId()
+ {
+ return advId;
+ }
+
+ public void setAdvId(int AdId)
+ {
+ this.advId = advId;
+ }
+
+ public int getCampaignId()
+ {
+ return campaignId;
+ }
+
+ public void setCampaignId(int campaignId)
+ {
+ this.campaignId = campaignId;
+ }
+
+ public String getCampaignName()
+ {
+ return campaignName;
+ }
+
+ public void setCampaignName(String campaignName)
+ {
+ this.campaignName = campaignName;
+ }
+
+ public double getCampaignBudget()
+ {
+ return campaignBudget;
+ }
+
+ public void setCampaignBudget(double campaignBudget)
+ {
+ this.campaignBudget = campaignBudget;
+ }
+
+ public Date getStartDate()
+ {
+ return startDate;
+ }
+
+ public void setStartDate(Date startDate)
+ {
+ this.startDate = startDate;
+ }
+
+ public Date getEndDate()
+ {
+ return endDate;
+ }
+
+ public void setEndDate(Date endDate)
+ {
+ this.endDate = endDate;
+ }
+
+ public String getSecurityCode()
+ {
+ return securityCode;
+ }
+
+ public void setSecurityCode(String securityCode)
+ {
+ this.securityCode = securityCode;
+ }
+
+ public boolean isWeatherTargeting()
+ {
+ return weatherTargeting;
+ }
+
+ public void setWeatherTargeting(boolean weatherTargeting)
+ {
+ this.weatherTargeting = weatherTargeting;
+ }
+
+ public boolean isOptimized()
+ {
+ return optimized;
+ }
+
+ public void setOptimized(boolean optimized)
+ {
+ this.optimized = optimized;
+ }
+
+ public String getParentCampaign()
+ {
+ return parentCampaign;
+ }
+
+ public void setParentCampaign(String parentCampaign)
+ {
+ this.parentCampaign = parentCampaign;
+ }
+
+ public Character getWeatherTargeted()
+ {
+ return weatherTargeted;
+ }
+
+ public void setWeatherTargeted(Character weatherTargeted)
+ {
+ this.weatherTargeted = weatherTargeted;
+ }
+
+ public String getValid()
+ {
+ return valid;
+ }
+
+ public void setValid(String valid)
+ {
+ this.valid = valid;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/csvformatter/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/csvformatter/src/main/resources/META-INF/properties.xml b/examples/csvformatter/src/main/resources/META-INF/properties.xml
index ed2b5ce..8d67c93 100644
--- a/examples/csvformatter/src/main/resources/META-INF/properties.xml
+++ b/examples/csvformatter/src/main/resources/META-INF/properties.xml
@@ -20,7 +20,7 @@
<property>
<name>dt.application.CustomOutputFormatter.operator.jsonParser.port.out.attr.TUPLE_CLASS
</name>
- <value>com.demo.myapexapp.PojoEvent</value>
+ <value>org.apache.apex.examples.csvformatter.PojoEvent</value>
</property>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/csvformatter/src/test/java/com/demo/myapexapp/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/csvformatter/src/test/java/com/demo/myapexapp/ApplicationTest.java b/examples/csvformatter/src/test/java/com/demo/myapexapp/ApplicationTest.java
deleted file mode 100644
index efe5946..0000000
--- a/examples/csvformatter/src/test/java/com/demo/myapexapp/ApplicationTest.java
+++ /dev/null
@@ -1,67 +0,0 @@
-package com.demo.myapexapp;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Collection;
-
-import javax.validation.ConstraintViolationException;
-
-import org.apache.commons.io.FileUtils;
-
-import org.junit.AfterClass;
-import org.junit.Assert;
-
-import org.apache.hadoop.conf.Configuration;
-
-import org.junit.Test;
-
-import com.datatorrent.api.LocalMode;
-
-/**
- * Test the DAG declaration in local mode.
- */
-public class ApplicationTest
-{
-
- private static final String FILE_NAME = "/tmp/formatterApp";
-
- @AfterClass
- public static void cleanup()
- {
- try {
- FileUtils.deleteDirectory(new File(FILE_NAME));
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Test
- public void testApplication() throws Exception
- {
- try {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
- lma.prepareDAG(new Application(), conf);
- LocalMode.Controller lc = lma.getController();
- lc.runAsync();
-
- // wait for output files to roll
- Thread.sleep(5000);
-
- String[] extensions = {"dat.0", "tmp"};
- Collection<File> list = FileUtils.listFiles(new File(FILE_NAME), extensions, false);
-
- for (File file : list) {
- for (String line : FileUtils.readLines(file)) {
- Assert.assertEquals("Delimiter in record", true, (line.equals(
- "1234|0|SimpleCsvFormatterExample|10000.0|||APEX|false|false||")));
- }
- }
-
- } catch (ConstraintViolationException e) {
- Assert.fail("constraint violations: " + e.getConstraintViolations());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/csvformatter/src/test/java/org/apache/apex/examples/csvformatter/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/csvformatter/src/test/java/org/apache/apex/examples/csvformatter/ApplicationTest.java b/examples/csvformatter/src/test/java/org/apache/apex/examples/csvformatter/ApplicationTest.java
new file mode 100644
index 0000000..67d5fd0
--- /dev/null
+++ b/examples/csvformatter/src/test/java/org/apache/apex/examples/csvformatter/ApplicationTest.java
@@ -0,0 +1,65 @@
+package org.apache.apex.examples.csvformatter;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+
+import javax.validation.ConstraintViolationException;
+
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.LocalMode;
+
+/**
+ * Test the DAG declaration in local mode.
+ */
+public class ApplicationTest
+{
+
+ private static final String FILE_NAME = "/tmp/formatterApp";
+
+ @AfterClass
+ public static void cleanup()
+ {
+ try {
+ FileUtils.deleteDirectory(new File(FILE_NAME));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Test
+ public void testApplication() throws Exception
+ {
+ try {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ lma.prepareDAG(new Application(), conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.runAsync();
+
+ // wait for output files to roll
+ Thread.sleep(5000);
+
+ String[] extensions = {"dat.0", "tmp"};
+ Collection<File> list = FileUtils.listFiles(new File(FILE_NAME), extensions, false);
+
+ for (File file : list) {
+ for (String line : FileUtils.readLines(file)) {
+ Assert.assertEquals("Delimiter in record", true, (line.equals(
+ "1234|0|SimpleCsvFormatterExample|10000.0|||APEX|false|false||")));
+ }
+ }
+
+ } catch (ConstraintViolationException e) {
+ Assert.fail("constraint violations: " + e.getConstraintViolations());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/dedup/pom.xml
----------------------------------------------------------------------
diff --git a/examples/dedup/pom.xml b/examples/dedup/pom.xml
index f777784..ba5a24d 100644
--- a/examples/dedup/pom.xml
+++ b/examples/dedup/pom.xml
@@ -2,279 +2,30 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>com.example</groupId>
- <version>1.0-SNAPSHOT</version>
- <artifactId>dedup</artifactId>
+ <parent>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-examples</artifactId>
+ <version>3.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>malhar-examples-dedup</artifactId>
<packaging>jar</packaging>
<!-- change these to the appropriate values -->
- <name>My Apex Application</name>
- <description>My Apex Application Description</description>
-
- <properties>
- <!-- change this if you desire to use a different version of Apex Core -->
- <apex.version>3.5.0</apex.version>
- <malhar.version>3.6.0</malhar.version>
- <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath>
- </properties>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-eclipse-plugin</artifactId>
- <version>2.9</version>
- <configuration>
- <downloadSources>true</downloadSources>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>3.3</version>
- <configuration>
- <encoding>UTF-8</encoding>
- <source>1.7</source>
- <target>1.7</target>
- <debug>true</debug>
- <optimize>false</optimize>
- <showDeprecation>true</showDeprecation>
- <showWarnings>true</showWarnings>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <version>2.8</version>
- <executions>
- <execution>
- <id>copy-dependencies</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>copy-dependencies</goal>
- </goals>
- <configuration>
- <outputDirectory>target/deps</outputDirectory>
- <includeScope>runtime</includeScope>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <executions>
- <execution>
- <id>app-package-assembly</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <finalName>${project.artifactId}-${project.version}-apexapp</finalName>
- <appendAssemblyId>false</appendAssemblyId>
- <descriptors>
- <descriptor>src/assemble/appPackage.xml</descriptor>
- </descriptors>
- <archiverConfig>
- <defaultDirectoryMode>0755</defaultDirectoryMode>
- </archiverConfig>
- <archive>
- <manifestEntries>
- <Class-Path>${apex.apppackage.classpath}</Class-Path>
- <DT-Engine-Version>${apex.version}</DT-Engine-Version>
- <DT-App-Package-Group-Id>${project.groupId}</DT-App-Package-Group-Id>
- <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name>
- <DT-App-Package-Version>${project.version}</DT-App-Package-Version>
- <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name>
- <DT-App-Package-Description>${project.description}</DT-App-Package-Description>
- </manifestEntries>
- </archive>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <artifactId>maven-antrun-plugin</artifactId>
- <version>1.7</version>
- <executions>
- <execution>
- <phase>package</phase>
- <configuration>
- <target>
- <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar"
- tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa" />
- </target>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- <execution>
- <!-- create resource directory for xml javadoc-->
- <id>createJavadocDirectory</id>
- <phase>generate-resources</phase>
- <configuration>
- <tasks>
- <delete dir="${project.build.directory}/generated-resources/xml-javadoc"/>
- <mkdir dir="${project.build.directory}/generated-resources/xml-javadoc"/>
- </tasks>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>1.9.1</version>
- <executions>
- <execution>
- <id>attach-artifacts</id>
- <phase>package</phase>
- <goals>
- <goal>attach-artifact</goal>
- </goals>
- <configuration>
- <artifacts>
- <artifact>
- <file>target/${project.artifactId}-${project.version}.apa</file>
- <type>apa</type>
- </artifact>
- </artifacts>
- <skipAttach>false</skipAttach>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <!-- generate javdoc -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <executions>
- <!-- generate xml javadoc -->
- <execution>
- <id>xml-doclet</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>javadoc</goal>
- </goals>
- <configuration>
- <doclet>com.github.markusbernhardt.xmldoclet.XmlDoclet</doclet>
- <additionalparam>-d ${project.build.directory}/generated-resources/xml-javadoc -filename ${project.artifactId}-${project.version}-javadoc.xml</additionalparam>
- <useStandardDocletOptions>false</useStandardDocletOptions>
- <docletArtifact>
- <groupId>com.github.markusbernhardt</groupId>
- <artifactId>xml-doclet</artifactId>
- <version>1.0.4</version>
- </docletArtifact>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <!-- Transform xml javadoc to stripped down version containing only class/interface comments and tags-->
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>xml-maven-plugin</artifactId>
- <version>1.0</version>
- <executions>
- <execution>
- <id>transform-xmljavadoc</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>transform</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <transformationSets>
- <transformationSet>
- <dir>${project.build.directory}/generated-resources/xml-javadoc</dir>
- <includes>
- <include>${project.artifactId}-${project.version}-javadoc.xml</include>
- </includes>
- <stylesheet>XmlJavadocCommentsExtractor.xsl</stylesheet>
- <outputDir>${project.build.directory}/generated-resources/xml-javadoc</outputDir>
- </transformationSet>
- </transformationSets>
- </configuration>
- </plugin>
- <!-- copy xml javadoc to class jar -->
- <plugin>
- <artifactId>maven-resources-plugin</artifactId>
- <version>2.6</version>
- <executions>
- <execution>
- <id>copy-resources</id>
- <phase>process-resources</phase>
- <goals>
- <goal>copy-resources</goal>
- </goals>
- <configuration>
- <outputDirectory>${basedir}/target/classes</outputDirectory>
- <resources>
- <resource>
- <directory>${project.build.directory}/generated-resources/xml-javadoc</directory>
- <includes>
- <include>${project.artifactId}-${project.version}-javadoc.xml</include>
- </includes>
- <filtering>true</filtering>
- </resource>
- </resources>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
-
- </build>
+ <name>Dedup Application</name>
+ <description>Dedup Application</description>
<dependencies>
- <!-- add your dependencies here -->
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-library</artifactId>
- <version>${malhar.version}</version>
- <!--
- If you know that your application does not need transitive dependencies pulled in by malhar-library,
- uncomment the following to reduce the size of your app package.
- -->
- <!--
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- -->
- </dependency>
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>apex-common</artifactId>
- <version>${apex.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.10</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>apex-engine</artifactId>
- <version>${apex.version}</version>
- <scope>test</scope>
- </dependency>
<dependency>
<groupId>org.codehaus.janino</groupId>
<artifactId>janino</artifactId>
<version>2.7.8</version>
</dependency>
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ <version>2.9.1</version>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/dedup/src/main/java/com/example/dedup/Application.java
----------------------------------------------------------------------
diff --git a/examples/dedup/src/main/java/com/example/dedup/Application.java b/examples/dedup/src/main/java/com/example/dedup/Application.java
deleted file mode 100644
index cabdce2..0000000
--- a/examples/dedup/src/main/java/com/example/dedup/Application.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/**
- * Put your copyright and license info here.
- */
-package com.example.dedup;
-
-import java.util.Date;
-import java.util.Random;
-
-import org.apache.apex.malhar.lib.dedup.TimeBasedDedupOperator;
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.common.partitioner.StatelessPartitioner;
-import com.datatorrent.common.util.BaseOperator;
-import com.datatorrent.lib.io.ConsoleOutputOperator;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.Context;
-import com.datatorrent.api.DAG;
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.InputOperator;
-
-@ApplicationAnnotation(name="DedupExample")
-public class Application implements StreamingApplication
-{
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- // Test Data Generator Operator
- RandomDataGeneratorOperator gen = dag.addOperator("RandomGenerator", new RandomDataGeneratorOperator());
-
- // Dedup Operator. Configuration through resources/META-INF/properties.xml
- TimeBasedDedupOperator dedup = dag.addOperator("Deduper", new TimeBasedDedupOperator());
-
- // Console output operator for unique tuples
- ConsoleOutputOperator consoleUnique = dag.addOperator("ConsoleUnique", new ConsoleOutputOperator());
-
- // Console output operator for duplicate tuples
- ConsoleOutputOperator consoleDuplicate = dag.addOperator("ConsoleDuplicate", new ConsoleOutputOperator());
-
- // Console output operator for duplicate tuples
- ConsoleOutputOperator consoleExpired = dag.addOperator("ConsoleExpired", new ConsoleOutputOperator());
-
- // Streams
- dag.addStream("Generator to Dedup", gen.output, dedup.input);
-
- // Connect Dedup unique to Console
- dag.addStream("Dedup Unique to Console", dedup.unique, consoleUnique.input);
- // Connect Dedup duplicate to Console
- dag.addStream("Dedup Duplicate to Console", dedup.duplicate, consoleDuplicate.input);
- // Connect Dedup expired to Console
- dag.addStream("Dedup Expired to Console", dedup.expired, consoleExpired.input);
-
- // Set Attribute TUPLE_CLASS for supplying schema information to the port
- dag.setInputPortAttribute(dedup.input, Context.PortContext.TUPLE_CLASS, TestEvent.class);
-
- // Uncomment the following line to create multiple partitions for Dedup operator. In this case: 2
- // dag.setAttribute(dedup, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<TimeBasedDedupOperator>(2));
- }
-
- public static class RandomDataGeneratorOperator extends BaseOperator implements InputOperator
- {
-
- public final transient DefaultOutputPort<TestEvent> output = new DefaultOutputPort<>();
- private final transient Random r = new Random();
- private int tuplesPerWindow = 100;
- private transient int count = 0;
-
- @Override
- public void beginWindow(long windowId) {
- count = 0;
- }
-
- @Override
- public void emitTuples()
- {
- if (count++ > tuplesPerWindow) {
- return;
- }
- TestEvent event = new TestEvent();
- event.id = r.nextInt(100);
- event.eventTime = new Date(System.currentTimeMillis() - (r.nextInt(60 * 1000)));
- output.emit(event);
- }
- }
-
- public static class TestEvent
- {
- private int id;
- private Date eventTime;
-
- public TestEvent()
- {
- }
-
- public int getId()
- {
- return id;
- }
-
- public void setId(int id)
- {
- this.id = id;
- }
-
- public Date getEventTime()
- {
- return eventTime;
- }
-
- public void setEventTime(Date eventTime)
- {
- this.eventTime = eventTime;
- }
-
- @Override
- public String toString() {
- return "TestEvent [id=" + id + ", eventTime=" + eventTime + "]";
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/dedup/src/main/java/org/apache/apex/examples/dedup/Application.java
----------------------------------------------------------------------
diff --git a/examples/dedup/src/main/java/org/apache/apex/examples/dedup/Application.java b/examples/dedup/src/main/java/org/apache/apex/examples/dedup/Application.java
new file mode 100644
index 0000000..2498d62
--- /dev/null
+++ b/examples/dedup/src/main/java/org/apache/apex/examples/dedup/Application.java
@@ -0,0 +1,122 @@
+/**
+ * Put your copyright and license info here.
+ */
+package org.apache.apex.examples.dedup;
+
+import java.util.Date;
+import java.util.Random;
+
+import org.apache.apex.malhar.lib.dedup.TimeBasedDedupOperator;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.common.util.BaseOperator;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+
+@ApplicationAnnotation(name="DedupExample")
+public class Application implements StreamingApplication
+{
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ // Test Data Generator Operator
+ RandomDataGeneratorOperator gen = dag.addOperator("RandomGenerator", new RandomDataGeneratorOperator());
+
+ // Dedup Operator. Configuration through resources/META-INF/properties.xml
+ TimeBasedDedupOperator dedup = dag.addOperator("Deduper", new TimeBasedDedupOperator());
+
+ // Console output operator for unique tuples
+ ConsoleOutputOperator consoleUnique = dag.addOperator("ConsoleUnique", new ConsoleOutputOperator());
+
+ // Console output operator for duplicate tuples
+ ConsoleOutputOperator consoleDuplicate = dag.addOperator("ConsoleDuplicate", new ConsoleOutputOperator());
+
+ // Console output operator for duplicate tuples
+ ConsoleOutputOperator consoleExpired = dag.addOperator("ConsoleExpired", new ConsoleOutputOperator());
+
+ // Streams
+ dag.addStream("Generator to Dedup", gen.output, dedup.input);
+
+ // Connect Dedup unique to Console
+ dag.addStream("Dedup Unique to Console", dedup.unique, consoleUnique.input);
+ // Connect Dedup duplicate to Console
+ dag.addStream("Dedup Duplicate to Console", dedup.duplicate, consoleDuplicate.input);
+ // Connect Dedup expired to Console
+ dag.addStream("Dedup Expired to Console", dedup.expired, consoleExpired.input);
+
+ // Set Attribute TUPLE_CLASS for supplying schema information to the port
+ dag.setInputPortAttribute(dedup.input, Context.PortContext.TUPLE_CLASS, TestEvent.class);
+
+ // Uncomment the following line to create multiple partitions for Dedup operator. In this case: 2
+ // dag.setAttribute(dedup, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<TimeBasedDedupOperator>(2));
+ }
+
+ public static class RandomDataGeneratorOperator extends BaseOperator implements InputOperator
+ {
+
+ public final transient DefaultOutputPort<TestEvent> output = new DefaultOutputPort<>();
+ private final transient Random r = new Random();
+ private int tuplesPerWindow = 100;
+ private transient int count = 0;
+
+ @Override
+ public void beginWindow(long windowId) {
+ count = 0;
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ if (count++ > tuplesPerWindow) {
+ return;
+ }
+ TestEvent event = new TestEvent();
+ event.id = r.nextInt(100);
+ event.eventTime = new Date(System.currentTimeMillis() - (r.nextInt(60 * 1000)));
+ output.emit(event);
+ }
+ }
+
+ public static class TestEvent
+ {
+ private int id;
+ private Date eventTime;
+
+ public TestEvent()
+ {
+ }
+
+ public int getId()
+ {
+ return id;
+ }
+
+ public void setId(int id)
+ {
+ this.id = id;
+ }
+
+ public Date getEventTime()
+ {
+ return eventTime;
+ }
+
+ public void setEventTime(Date eventTime)
+ {
+ this.eventTime = eventTime;
+ }
+
+ @Override
+ public String toString() {
+ return "TestEvent [id=" + id + ", eventTime=" + eventTime + "]";
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/dedup/src/test/java/com/example/dedup/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/dedup/src/test/java/com/example/dedup/ApplicationTest.java b/examples/dedup/src/test/java/com/example/dedup/ApplicationTest.java
deleted file mode 100644
index 9c9f17c..0000000
--- a/examples/dedup/src/test/java/com/example/dedup/ApplicationTest.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Put your copyright and license info here.
- */
-package com.example.dedup;
-
-import java.io.IOException;
-
-import javax.validation.ConstraintViolationException;
-
-import org.junit.Assert;
-
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Test;
-
-import com.datatorrent.api.LocalMode;
-import com.example.dedup.Application;
-
-/**
- * Test the DAG declaration in local mode.
- */
-public class ApplicationTest {
-
- @Test
- public void testApplication() throws IOException, Exception {
- try {
- LocalMode lma = LocalMode.newInstance();
- Configuration conf = new Configuration(false);
- conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
- lma.prepareDAG(new Application(), conf);
- LocalMode.Controller lc = lma.getController();
- lc.runAsync();
- Thread.sleep(10 * 1000);
- lc.shutdown();
- } catch (ConstraintViolationException e) {
- Assert.fail("constraint violations: " + e.getConstraintViolations());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/dedup/src/test/java/org/apache/apex/examples/dedup/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/dedup/src/test/java/org/apache/apex/examples/dedup/ApplicationTest.java b/examples/dedup/src/test/java/org/apache/apex/examples/dedup/ApplicationTest.java
new file mode 100644
index 0000000..3304a04
--- /dev/null
+++ b/examples/dedup/src/test/java/org/apache/apex/examples/dedup/ApplicationTest.java
@@ -0,0 +1,37 @@
+/**
+ * Put your copyright and license info here.
+ */
+package org.apache.apex.examples.dedup;
+
+import java.io.IOException;
+
+import javax.validation.ConstraintViolationException;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.LocalMode;
+
+/**
+ * Test the DAG declaration in local mode.
+ */
+public class ApplicationTest {
+
+ @Test
+ public void testApplication() throws IOException, Exception {
+ try {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ lma.prepareDAG(new Application(), conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.runAsync();
+ Thread.sleep(10 * 1000);
+ lc.shutdown();
+ } catch (ConstraintViolationException e) {
+ Assert.fail("constraint violations: " + e.getConstraintViolations());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/dynamic-partition/pom.xml
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/pom.xml b/examples/dynamic-partition/pom.xml
index 34e91ee..21b1c30 100644
--- a/examples/dynamic-partition/pom.xml
+++ b/examples/dynamic-partition/pom.xml
@@ -1,273 +1,25 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
-
- <groupId>com.example</groupId>
- <version>1.0-SNAPSHOT</version>
- <artifactId>dynamic-partition</artifactId>
+
+ <parent>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-examples</artifactId>
+ <version>3.7.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>malhar-examples-dynamic-partition</artifactId>
<packaging>jar</packaging>
<!-- change these to the appropriate values -->
<name>Dynamic Partitioning</name>
<description>Example showing dynamic partitioning</description>
- <properties>
- <!-- change this if you desire to use a different version of Apex Core -->
- <apex.version>3.5.0</apex.version>
- <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath>
- </properties>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-eclipse-plugin</artifactId>
- <version>2.9</version>
- <configuration>
- <downloadSources>true</downloadSources>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>3.3</version>
- <configuration>
- <encoding>UTF-8</encoding>
- <source>1.7</source>
- <target>1.7</target>
- <debug>true</debug>
- <optimize>false</optimize>
- <showDeprecation>true</showDeprecation>
- <showWarnings>true</showWarnings>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <version>2.8</version>
- <executions>
- <execution>
- <id>copy-dependencies</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>copy-dependencies</goal>
- </goals>
- <configuration>
- <outputDirectory>target/deps</outputDirectory>
- <includeScope>runtime</includeScope>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <executions>
- <execution>
- <id>app-package-assembly</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <finalName>${project.artifactId}-${project.version}-apexapp</finalName>
- <appendAssemblyId>false</appendAssemblyId>
- <descriptors>
- <descriptor>src/assemble/appPackage.xml</descriptor>
- </descriptors>
- <archiverConfig>
- <defaultDirectoryMode>0755</defaultDirectoryMode>
- </archiverConfig>
- <archive>
- <manifestEntries>
- <Class-Path>${apex.apppackage.classpath}</Class-Path>
- <DT-Engine-Version>${apex.version}</DT-Engine-Version>
- <DT-App-Package-Group-Id>${project.groupId}</DT-App-Package-Group-Id>
- <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name>
- <DT-App-Package-Version>${project.version}</DT-App-Package-Version>
- <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name>
- <DT-App-Package-Description>${project.description}</DT-App-Package-Description>
- </manifestEntries>
- </archive>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <artifactId>maven-antrun-plugin</artifactId>
- <version>1.7</version>
- <executions>
- <execution>
- <phase>package</phase>
- <configuration>
- <target>
- <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar"
- tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa" />
- </target>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- <execution>
- <!-- create resource directory for xml javadoc-->
- <id>createJavadocDirectory</id>
- <phase>generate-resources</phase>
- <configuration>
- <tasks>
- <delete dir="${project.build.directory}/generated-resources/xml-javadoc"/>
- <mkdir dir="${project.build.directory}/generated-resources/xml-javadoc"/>
- </tasks>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <version>1.9.1</version>
- <executions>
- <execution>
- <id>attach-artifacts</id>
- <phase>package</phase>
- <goals>
- <goal>attach-artifact</goal>
- </goals>
- <configuration>
- <artifacts>
- <artifact>
- <file>target/${project.artifactId}-${project.version}.apa</file>
- <type>apa</type>
- </artifact>
- </artifacts>
- <skipAttach>false</skipAttach>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <!-- generate javdoc -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <executions>
- <!-- generate xml javadoc -->
- <execution>
- <id>xml-doclet</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>javadoc</goal>
- </goals>
- <configuration>
- <doclet>com.github.markusbernhardt.xmldoclet.XmlDoclet</doclet>
- <additionalparam>-d ${project.build.directory}/generated-resources/xml-javadoc -filename ${project.artifactId}-${project.version}-javadoc.xml</additionalparam>
- <useStandardDocletOptions>false</useStandardDocletOptions>
- <docletArtifact>
- <groupId>com.github.markusbernhardt</groupId>
- <artifactId>xml-doclet</artifactId>
- <version>1.0.4</version>
- </docletArtifact>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <!-- Transform xml javadoc to stripped down version containing only class/interface comments and tags-->
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>xml-maven-plugin</artifactId>
- <version>1.0</version>
- <executions>
- <execution>
- <id>transform-xmljavadoc</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>transform</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <transformationSets>
- <transformationSet>
- <dir>${project.build.directory}/generated-resources/xml-javadoc</dir>
- <includes>
- <include>${project.artifactId}-${project.version}-javadoc.xml</include>
- </includes>
- <stylesheet>XmlJavadocCommentsExtractor.xsl</stylesheet>
- <outputDir>${project.build.directory}/generated-resources/xml-javadoc</outputDir>
- </transformationSet>
- </transformationSets>
- </configuration>
- </plugin>
- <!-- copy xml javadoc to class jar -->
- <plugin>
- <artifactId>maven-resources-plugin</artifactId>
- <version>2.6</version>
- <executions>
- <execution>
- <id>copy-resources</id>
- <phase>process-resources</phase>
- <goals>
- <goal>copy-resources</goal>
- </goals>
- <configuration>
- <outputDirectory>${basedir}/target/classes</outputDirectory>
- <resources>
- <resource>
- <directory>${project.build.directory}/generated-resources/xml-javadoc</directory>
- <includes>
- <include>${project.artifactId}-${project.version}-javadoc.xml</include>
- </includes>
- <filtering>true</filtering>
- </resource>
- </resources>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
-
- </build>
-
<dependencies>
- <!-- add your dependencies here -->
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>malhar-library</artifactId>
- <version>3.6.0</version>
- <!--
- If you know that your application does not need transitive dependencies pulled in by malhar-library,
- uncomment the following to reduce the size of your app package.
- -->
- <!--
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- -->
- </dependency>
- <dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>apex-common</artifactId>
- <version>${apex.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.10</version>
- <scope>test</scope>
- </dependency>
<dependency>
- <groupId>org.apache.apex</groupId>
- <artifactId>apex-engine</artifactId>
- <version>${apex.version}</version>
- <scope>test</scope>
+ <groupId>com.esotericsoftware.kryo</groupId>
+ <artifactId>kryo</artifactId>
+ <version>2.24.0</version>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/dynamic-partition/src/main/java/com/example/dynamic/App.java
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/src/main/java/com/example/dynamic/App.java b/examples/dynamic-partition/src/main/java/com/example/dynamic/App.java
deleted file mode 100644
index 9eec263..0000000
--- a/examples/dynamic-partition/src/main/java/com/example/dynamic/App.java
+++ /dev/null
@@ -1,23 +0,0 @@
-package com.example.dynamic;
-
-import org.apache.hadoop.conf.Configuration;
-
-import com.datatorrent.api.annotation.ApplicationAnnotation;
-import com.datatorrent.api.StreamingApplication;
-import com.datatorrent.api.DAG;
-
-import com.datatorrent.lib.stream.DevNull;
-
-@ApplicationAnnotation(name="Dyn")
-public class App implements StreamingApplication
-{
-
- @Override
- public void populateDAG(DAG dag, Configuration conf)
- {
- Gen gen = dag.addOperator("gen", Gen.class);
- DevNull devNull = dag.addOperator("devNull", DevNull.class);
-
- dag.addStream("data", gen.out, devNull.data);
- }
-}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/9c154f20/examples/dynamic-partition/src/main/java/com/example/dynamic/Gen.java
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/src/main/java/com/example/dynamic/Gen.java b/examples/dynamic-partition/src/main/java/com/example/dynamic/Gen.java
deleted file mode 100644
index 4cccd23..0000000
--- a/examples/dynamic-partition/src/main/java/com/example/dynamic/Gen.java
+++ /dev/null
@@ -1,169 +0,0 @@
-package com.example.dynamic;
-
-import com.esotericsoftware.kryo.Kryo;
-import com.esotericsoftware.kryo.io.Input;
-import com.esotericsoftware.kryo.io.Output;
-import com.google.common.collect.Lists;
-import org.apache.commons.io.IOUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.validation.constraints.NotNull;
-import java.io.ByteArrayOutputStream;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-
-import com.datatorrent.api.DefaultPartition;
-import com.datatorrent.api.Partitioner;
-import com.datatorrent.api.StatsListener;
-
-import com.datatorrent.api.DefaultOutputPort;
-import com.datatorrent.api.InputOperator;
-import com.datatorrent.common.util.BaseOperator;
-
-/**
- * Operator that dynamically partitions itself after 500 tuples have been emitted
- */
-public class Gen extends BaseOperator implements InputOperator, Partitioner<Gen>, StatsListener
-{
- private static final Logger LOG = LoggerFactory.getLogger(Gen.class);
-
- private static final int MAX_PARTITIONS = 4; // maximum number of partitions
-
- private int partitions = 2; // initial number of partitions
-
- @NotNull
- private int numTuples; // number of tuples to emit per window
-
- private transient int count = 0;
-
- public final transient DefaultOutputPort<Double> out = new DefaultOutputPort<Double>();
-
- @Override
- public void partitioned(Map<Integer, Partition<Gen>> map)
- {
- if (partitions != map.size()) {
- String msg = String.format("partitions = %d, map.size = %d%n", partitions, map.size());
- throw new RuntimeException(msg);
- }
- }
-
- @Override
- public void beginWindow(long windowId)
- {
- count = 0;
- }
-
- @Override
- public void emitTuples()
- {
- if (count < numTuples) {
- ++count;
- out.emit(Math.random());
- }
- }
-
- public int getNumTuples()
- {
- return numTuples;
- }
-
- /**
- * Sets the number of tuples to be emitted every window.
- * @param numTuples number of tuples
- */
- public void setNumTuples(int numTuples)
- {
- this.numTuples = numTuples;
- }
-
- @Override
- public Response processStats(BatchedOperatorStats batchedOperatorStats) {
-
- final long emittedCount = batchedOperatorStats.getTuplesEmittedPSMA();
-
- // we only perform a single dynamic repartition
- Response res = new Response();
- res.repartitionRequired = false;
- if (emittedCount > 500 && partitions < MAX_PARTITIONS) {
- LOG.info("processStats: trying repartition of input operator current {} required {}",
- partitions, MAX_PARTITIONS);
- LOG.info("**** operator id = {}, window id = {}, tuplesProcessedPSMA = {}, tuplesEmittedPSMA = {}",
- batchedOperatorStats.getOperatorId(),
- batchedOperatorStats.getCurrentWindowId(),
- batchedOperatorStats.getTuplesProcessedPSMA(),
- emittedCount);
- partitions = MAX_PARTITIONS;
- res.repartitionRequired = true;
- }
-
- return res;
- } // processStats
-
- /**
- * Clone object by serializing and deserializing using Kryo.
- * Note this is different from using {@link Kryo#copy(Object)}, which will attempt to also clone transient fields.
- *
- * @param kryo kryo object used to clone objects
- * @param src src object that copy from
- * @return cloned object
- */
- @SuppressWarnings("unchecked")
- private static <SRC> SRC cloneObject(Kryo kryo, SRC src)
- {
- kryo.setClassLoader(src.getClass().getClassLoader());
- ByteArrayOutputStream bos = null;
- Output output;
- Input input = null;
- try {
- bos = new ByteArrayOutputStream();
- output = new Output(bos);
- kryo.writeObject(output, src);
- output.close();
- input = new Input(bos.toByteArray());
- return (SRC)kryo.readObject(input, src.getClass());
- } finally {
- IOUtils.closeQuietly(input);
- IOUtils.closeQuietly(bos);
- }
- }
-
- @Override
- public Collection<Partition<Gen>> definePartitions(
- Collection<Partition<Gen>> list, PartitioningContext context)
- {
- if (partitions < 0) { // error
- String msg = String.format("Error: Bad value: partitions = %d%n", partitions);
- LOG.error(msg);
- throw new RuntimeException(msg);
- }
-
- final int prevCount = list.size();
- if (1 == prevCount) { // initial call
- LOG.info("definePartitions: First call, prevCount = {}, partitions = {}",
- prevCount, partitions);
- }
-
- if (prevCount == partitions) {
- LOG.info("definePartitions: Nothing to do in definePartitions");
- return list; // nothing to do
- }
-
- LOG.debug("definePartitions: Repartitioning from {} to {}", prevCount, partitions);
-
- Kryo kryo = new Kryo();
-
- // return value: new list of partitions (includes old list)
- List<Partition<Gen>> newPartitions = Lists.newArrayListWithExpectedSize(partitions);
-
- for (int i = 0; i < partitions; i++) {
- Gen oper = cloneObject(kryo, this);
- newPartitions.add(new DefaultPartition<>(oper));
- }
-
- LOG.info("definePartition: returning {} partitions", newPartitions.size());
- return newPartitions;
- }
-
-}
[03/19] apex-malhar git commit: Fixed checkstyle and added license
headers.
Posted by th...@apache.org.
Fixed checkstyle and added license headers.
Project: http://git-wip-us.apache.org/repos/asf/apex-malhar/repo
Commit: http://git-wip-us.apache.org/repos/asf/apex-malhar/commit/b8a83a45
Tree: http://git-wip-us.apache.org/repos/asf/apex-malhar/tree/b8a83a45
Diff: http://git-wip-us.apache.org/repos/asf/apex-malhar/diff/b8a83a45
Branch: refs/heads/master
Commit: b8a83a451875d615417f67808bbdf47a9ea607e7
Parents: 9c154f2
Author: Apex Dev <de...@apex.apache.org>
Authored: Mon Mar 20 13:45:54 2017 -0700
Committer: Lakshmi Prasanna Velineni <la...@datatorrent.com>
Committed: Sun Mar 26 11:43:48 2017 -0700
----------------------------------------------------------------------
.../XmlJavadocCommentsExtractor.xsl | 44 ----
examples/csvformatter/pom.xml | 2 +-
.../csvformatter/src/assemble/appPackage.xml | 20 ++
.../apex/examples/csvformatter/Application.java | 19 ++
.../csvformatter/HDFSOutputOperator.java | 23 +-
.../examples/csvformatter/JsonGenerator.java | 21 +-
.../apex/examples/csvformatter/PojoEvent.java | 19 ++
.../src/main/resources/META-INF/properties.xml | 22 +-
.../examples/csvformatter/ApplicationTest.java | 22 +-
.../src/test/resources/log4j.properties | 19 ++
examples/dedup/XmlJavadocCommentsExtractor.xsl | 44 ----
examples/dedup/pom.xml | 22 +-
examples/dedup/src/assemble/appPackage.xml | 20 ++
.../apache/apex/examples/dedup/Application.java | 25 +-
.../src/main/resources/META-INF/properties.xml | 20 ++
.../apex/examples/dedup/ApplicationTest.java | 24 +-
.../dedup/src/test/resources/log4j.properties | 19 ++
.../XmlJavadocCommentsExtractor.xsl | 44 ----
examples/dynamic-partition/pom.xml | 22 +-
.../src/assemble/appPackage.xml | 20 ++
.../org/apache/apex/examples/dynamic/App.java | 21 +-
.../org/apache/apex/examples/dynamic/Gen.java | 33 ++-
.../src/main/resources/META-INF/properties.xml | 24 +-
.../src/main/resources/unused-log4j.properties | 19 ++
.../apex/examples/dynamic/ApplicationTest.java | 26 +-
.../src/test/resources/log4j.properties | 19 ++
.../enricher/XmlJavadocCommentsExtractor.xsl | 44 ----
examples/enricher/pom.xml | 22 +-
examples/enricher/src/assemble/appPackage.xml | 20 ++
.../apex/examples/enricher/DataGenerator.java | 34 ++-
.../enricher/EnricherAppWithJSONFile.java | 21 +-
.../examples/enricher/LineOutputOperator.java | 38 ++-
.../org/apache/apex/examples/enricher/POJO.java | 117 +++++----
.../apex/examples/enricher/POJOEnriched.java | 22 +-
.../src/main/resources/META-INF/properties.xml | 20 ++
.../apex/examples/enricher/ApplicationTest.java | 24 +-
.../src/test/resources/log4j.properties | 19 ++
examples/filter/XmlJavadocCommentsExtractor.xsl | 44 ----
examples/filter/pom.xml | 22 +-
examples/filter/src/assemble/appPackage.xml | 20 ++
.../apex/examples/filter/Application.java | 21 +-
.../apex/examples/filter/TransactionPOJO.java | 19 +-
.../src/main/resources/META-INF/properties.xml | 24 +-
.../apex/examples/filter/ApplicationTest.java | 19 +-
.../filter/src/test/resources/log4j.properties | 19 ++
.../innerjoin/XmlJavadocCommentsExtractor.xsl | 44 ----
examples/innerjoin/pom.xml | 22 +-
examples/innerjoin/src/assemble/appPackage.xml | 20 ++
.../innerjoin/InnerJoinApplication.java | 21 +-
.../apex/examples/innerjoin/POJOGenerator.java | 36 ++-
.../src/main/resources/META-INF/properties.xml | 20 ++
.../innerjoin/InnerJoinApplicationTest.java | 21 +-
.../src/test/resources/log4j.properties | 19 ++
examples/parser/README.md | 5 +-
examples/parser/XmlJavadocCommentsExtractor.xsl | 44 ----
examples/parser/pom.xml | 22 +-
examples/parser/src/assemble/appPackage.xml | 20 ++
.../examples/parser/jsonparser/Application.java | 20 ++
.../examples/parser/jsonparser/Campaign.java | 19 ++
.../parser/jsonparser/JsonGenerator.java | 27 +-
.../src/main/resources/META-INF/properties.xml | 26 +-
.../parser/jsonparser/ApplicationTest.java | 19 ++
.../parser/src/test/resources/log4j.properties | 19 ++
.../partition/XmlJavadocCommentsExtractor.xsl | 44 ----
examples/partition/pom.xml | 22 +-
examples/partition/src/assemble/appPackage.xml | 20 ++
.../apex/examples/partition/Application.java | 25 +-
.../apache/apex/examples/partition/Codec3.java | 33 ++-
.../partition/RandomNumberGenerator.java | 21 +-
.../apex/examples/partition/TestPartition.java | 54 ++--
.../src/main/resources/META-INF/properties.xml | 22 +-
.../src/main/resources/my-log4j.properties | 19 ++
.../examples/partition/ApplicationTest.java | 28 ++-
.../src/test/resources/log4j.properties | 19 ++
.../XmlJavadocCommentsExtractor.xsl | 44 ----
examples/recordReader/pom.xml | 22 +-
.../recordReader/src/assemble/appPackage.xml | 20 ++
.../apex/examples/recordReader/Application.java | 19 +-
.../recordReader/TransactionsSchema.java | 30 ++-
.../src/main/resources/META-INF/CustomerTxnData | 100 --------
.../main/resources/META-INF/CustomerTxnData.txt | 100 ++++++++
.../src/main/resources/META-INF/properties.xml | 24 +-
.../examples/recordReader/ApplicationTest.java | 43 +++-
.../src/test/resources/log4j.properties | 19 ++
.../throttle/XmlJavadocCommentsExtractor.xsl | 44 ----
examples/throttle/pom.xml | 24 +-
examples/throttle/src/assemble/appPackage.xml | 20 ++
.../apex/examples/throttle/Application.java | 20 +-
.../examples/throttle/PassThroughOperator.java | 39 ++-
.../throttle/RandomNumberGenerator.java | 29 ++-
.../examples/throttle/SlowDevNullOperator.java | 65 +++--
.../throttle/ThrottlingStatsListener.java | 250 ++++++++++---------
.../src/main/resources/META-INF/properties.xml | 20 ++
.../apex/examples/throttle/ApplicationTest.java | 25 +-
.../src/test/resources/log4j.properties | 19 ++
.../transform/XmlJavadocCommentsExtractor.xsl | 44 ----
examples/transform/pom.xml | 22 +-
examples/transform/src/assemble/appPackage.xml | 21 ++
.../apex/examples/transform/Application.java | 21 +-
.../apex/examples/transform/CustomerEvent.java | 19 ++
.../apex/examples/transform/CustomerInfo.java | 19 ++
.../transform/DynamicTransformApplication.java | 21 +-
.../apex/examples/transform/POJOGenerator.java | 30 ++-
.../src/main/resources/META-INF/properties.xml | 21 ++
.../examples/transform/ApplicationTest.java | 19 ++
.../src/test/resources/log4j.properties | 19 ++
106 files changed, 2216 insertions(+), 929 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/csvformatter/XmlJavadocCommentsExtractor.xsl
----------------------------------------------------------------------
diff --git a/examples/csvformatter/XmlJavadocCommentsExtractor.xsl b/examples/csvformatter/XmlJavadocCommentsExtractor.xsl
deleted file mode 100644
index 08075a9..0000000
--- a/examples/csvformatter/XmlJavadocCommentsExtractor.xsl
+++ /dev/null
@@ -1,44 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
--->
-
-<!--
- Document : XmlJavadocCommentsExtractor.xsl
- Created on : September 16, 2014, 11:30 AM
- Description:
- The transformation strips off all information except for comments and tags from xml javadoc generated by xml-doclet.
--->
-
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
- <xsl:output method="xml" standalone="yes"/>
-
- <!-- copy xml by selecting only the following nodes, attributes and text -->
- <xsl:template match="node()|text()|@*">
- <xsl:copy>
- <xsl:apply-templates select="root|package|class|interface|method|field|type|comment|tag|text()|@name|@qualified|@text"/>
- </xsl:copy>
- </xsl:template>
-
- <!-- Strip off the following paths from the selected xml -->
- <xsl:template match="//root/package/interface/interface
- |//root/package/interface/method/@qualified
- |//root/package/class/interface
- |//root/package/class/class
- |//root/package/class/method/@qualified
- |//root/package/class/field/@qualified" />
-
- <xsl:strip-space elements="*"/>
-</xsl:stylesheet>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/csvformatter/pom.xml
----------------------------------------------------------------------
diff --git a/examples/csvformatter/pom.xml b/examples/csvformatter/pom.xml
index be3be7a..7df58e4 100644
--- a/examples/csvformatter/pom.xml
+++ b/examples/csvformatter/pom.xml
@@ -32,7 +32,7 @@
<packaging>jar</packaging>
<!-- change these to the appropriate values -->
- <name>Formatter Apps</name>
+ <name>Apache Apex Malhar CSV Formatter Example</name>
<description>Applications to showcase different formatters</description>
<dependencies>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/csvformatter/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/csvformatter/src/assemble/appPackage.xml b/examples/csvformatter/src/assemble/appPackage.xml
index 7ad071c..a870807 100644
--- a/examples/csvformatter/src/assemble/appPackage.xml
+++ b/examples/csvformatter/src/assemble/appPackage.xml
@@ -1,3 +1,23 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/Application.java
----------------------------------------------------------------------
diff --git a/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/Application.java b/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/Application.java
index cc9ee79..6dbb977 100644
--- a/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/Application.java
+++ b/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/Application.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.csvformatter;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/HDFSOutputOperator.java
----------------------------------------------------------------------
diff --git a/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/HDFSOutputOperator.java b/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/HDFSOutputOperator.java
index 7cdd8bb..90fcc3d 100644
--- a/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/HDFSOutputOperator.java
+++ b/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/HDFSOutputOperator.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.csvformatter;
import javax.validation.constraints.NotNull;
@@ -79,8 +98,8 @@ public class HDFSOutputOperator<T> extends AbstractFileOutputOperator<T>
@Override
protected void processTuple(T tuple)
{
- if (writeFilesFlag) {
- }
+ //if (writeFilesFlag) {
+ //}
super.processTuple(tuple);
}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/JsonGenerator.java
----------------------------------------------------------------------
diff --git a/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/JsonGenerator.java b/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/JsonGenerator.java
index 9b7698c..628171b 100644
--- a/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/JsonGenerator.java
+++ b/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/JsonGenerator.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.csvformatter;
import java.util.Random;
@@ -23,7 +42,7 @@ public class JsonGenerator extends BaseOperator implements InputOperator
private transient int count = 0;
public static Random rand = new Random();
- private int sleepTime=5;
+ private int sleepTime = 5;
public final transient DefaultOutputPort<byte[]> out = new DefaultOutputPort<byte[]>();
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/PojoEvent.java
----------------------------------------------------------------------
diff --git a/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/PojoEvent.java b/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/PojoEvent.java
index 03fda93..8a71496 100644
--- a/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/PojoEvent.java
+++ b/examples/csvformatter/src/main/java/org/apache/apex/examples/csvformatter/PojoEvent.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.csvformatter;
import java.util.Date;
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/csvformatter/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/csvformatter/src/main/resources/META-INF/properties.xml b/examples/csvformatter/src/main/resources/META-INF/properties.xml
index 8d67c93..e46be31 100644
--- a/examples/csvformatter/src/main/resources/META-INF/properties.xml
+++ b/examples/csvformatter/src/main/resources/META-INF/properties.xml
@@ -1,6 +1,26 @@
<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<configuration>
- <!--
+ <!--
<property>
<name>dt.application.{appName}.operator.{opName}.prop.{propName}</name>
<value>some-default-value (if value is not specified, it is required from the user or custom config when launching)</value>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/csvformatter/src/test/java/org/apache/apex/examples/csvformatter/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/csvformatter/src/test/java/org/apache/apex/examples/csvformatter/ApplicationTest.java b/examples/csvformatter/src/test/java/org/apache/apex/examples/csvformatter/ApplicationTest.java
index 67d5fd0..beedbd2 100644
--- a/examples/csvformatter/src/test/java/org/apache/apex/examples/csvformatter/ApplicationTest.java
+++ b/examples/csvformatter/src/test/java/org/apache/apex/examples/csvformatter/ApplicationTest.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
package org.apache.apex.examples.csvformatter;
import java.io.File;
@@ -52,8 +70,8 @@ public class ApplicationTest
for (File file : list) {
for (String line : FileUtils.readLines(file)) {
- Assert.assertEquals("Delimiter in record", true, (line.equals(
- "1234|0|SimpleCsvFormatterExample|10000.0|||APEX|false|false||")));
+ Assert.assertEquals("Delimiter in record", true,
+ (line.equals("1234|0|SimpleCsvFormatterExample|10000.0|||APEX|false|false||")));
}
}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/csvformatter/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/csvformatter/src/test/resources/log4j.properties b/examples/csvformatter/src/test/resources/log4j.properties
index 98544e8..1c9776b 100644
--- a/examples/csvformatter/src/test/resources/log4j.properties
+++ b/examples/csvformatter/src/test/resources/log4j.properties
@@ -1,3 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
log4j.rootLogger=DEBUG,CONSOLE
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/dedup/XmlJavadocCommentsExtractor.xsl
----------------------------------------------------------------------
diff --git a/examples/dedup/XmlJavadocCommentsExtractor.xsl b/examples/dedup/XmlJavadocCommentsExtractor.xsl
deleted file mode 100644
index 08075a9..0000000
--- a/examples/dedup/XmlJavadocCommentsExtractor.xsl
+++ /dev/null
@@ -1,44 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
--->
-
-<!--
- Document : XmlJavadocCommentsExtractor.xsl
- Created on : September 16, 2014, 11:30 AM
- Description:
- The transformation strips off all information except for comments and tags from xml javadoc generated by xml-doclet.
--->
-
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
- <xsl:output method="xml" standalone="yes"/>
-
- <!-- copy xml by selecting only the following nodes, attributes and text -->
- <xsl:template match="node()|text()|@*">
- <xsl:copy>
- <xsl:apply-templates select="root|package|class|interface|method|field|type|comment|tag|text()|@name|@qualified|@text"/>
- </xsl:copy>
- </xsl:template>
-
- <!-- Strip off the following paths from the selected xml -->
- <xsl:template match="//root/package/interface/interface
- |//root/package/interface/method/@qualified
- |//root/package/class/interface
- |//root/package/class/class
- |//root/package/class/method/@qualified
- |//root/package/class/field/@qualified" />
-
- <xsl:strip-space elements="*"/>
-</xsl:stylesheet>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/dedup/pom.xml
----------------------------------------------------------------------
diff --git a/examples/dedup/pom.xml b/examples/dedup/pom.xml
index ba5a24d..acba0f6 100644
--- a/examples/dedup/pom.xml
+++ b/examples/dedup/pom.xml
@@ -1,4 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@@ -12,7 +32,7 @@
<packaging>jar</packaging>
<!-- change these to the appropriate values -->
- <name>Dedup Application</name>
+ <name>Apache Apex Malhar Dedup Example</name>
<description>Dedup Application</description>
<dependencies>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/dedup/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/dedup/src/assemble/appPackage.xml b/examples/dedup/src/assemble/appPackage.xml
index 7ad071c..a870807 100644
--- a/examples/dedup/src/assemble/appPackage.xml
+++ b/examples/dedup/src/assemble/appPackage.xml
@@ -1,3 +1,23 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/dedup/src/main/java/org/apache/apex/examples/dedup/Application.java
----------------------------------------------------------------------
diff --git a/examples/dedup/src/main/java/org/apache/apex/examples/dedup/Application.java b/examples/dedup/src/main/java/org/apache/apex/examples/dedup/Application.java
index 2498d62..077e52c 100644
--- a/examples/dedup/src/main/java/org/apache/apex/examples/dedup/Application.java
+++ b/examples/dedup/src/main/java/org/apache/apex/examples/dedup/Application.java
@@ -1,5 +1,20 @@
/**
- * Put your copyright and license info here.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
package org.apache.apex.examples.dedup;
@@ -18,7 +33,7 @@ import com.datatorrent.api.annotation.ApplicationAnnotation;
import com.datatorrent.common.util.BaseOperator;
import com.datatorrent.lib.io.ConsoleOutputOperator;
-@ApplicationAnnotation(name="DedupExample")
+@ApplicationAnnotation(name = "DedupExample")
public class Application implements StreamingApplication
{
@@ -66,7 +81,8 @@ public class Application implements StreamingApplication
private transient int count = 0;
@Override
- public void beginWindow(long windowId) {
+ public void beginWindow(long windowId)
+ {
count = 0;
}
@@ -113,7 +129,8 @@ public class Application implements StreamingApplication
}
@Override
- public String toString() {
+ public String toString()
+ {
return "TestEvent [id=" + id + ", eventTime=" + eventTime + "]";
}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/dedup/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/dedup/src/main/resources/META-INF/properties.xml b/examples/dedup/src/main/resources/META-INF/properties.xml
index ffe3350..ca8d96e 100644
--- a/examples/dedup/src/main/resources/META-INF/properties.xml
+++ b/examples/dedup/src/main/resources/META-INF/properties.xml
@@ -1,4 +1,24 @@
<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<configuration>
<property>
<name>dt.application.DedupExample.operator.RandomGenerator.prop.tuplesPerWindow</name>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/dedup/src/test/java/org/apache/apex/examples/dedup/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/dedup/src/test/java/org/apache/apex/examples/dedup/ApplicationTest.java b/examples/dedup/src/test/java/org/apache/apex/examples/dedup/ApplicationTest.java
index 3304a04..704515a 100644
--- a/examples/dedup/src/test/java/org/apache/apex/examples/dedup/ApplicationTest.java
+++ b/examples/dedup/src/test/java/org/apache/apex/examples/dedup/ApplicationTest.java
@@ -1,6 +1,22 @@
/**
- * Put your copyright and license info here.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
+
package org.apache.apex.examples.dedup;
import java.io.IOException;
@@ -17,10 +33,12 @@ import com.datatorrent.api.LocalMode;
/**
* Test the DAG declaration in local mode.
*/
-public class ApplicationTest {
+public class ApplicationTest
+{
@Test
- public void testApplication() throws IOException, Exception {
+ public void testApplication() throws IOException, Exception
+ {
try {
LocalMode lma = LocalMode.newInstance();
Configuration conf = new Configuration(false);
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/dedup/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/dedup/src/test/resources/log4j.properties b/examples/dedup/src/test/resources/log4j.properties
index 98544e8..1c9776b 100644
--- a/examples/dedup/src/test/resources/log4j.properties
+++ b/examples/dedup/src/test/resources/log4j.properties
@@ -1,3 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
log4j.rootLogger=DEBUG,CONSOLE
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/dynamic-partition/XmlJavadocCommentsExtractor.xsl
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/XmlJavadocCommentsExtractor.xsl b/examples/dynamic-partition/XmlJavadocCommentsExtractor.xsl
deleted file mode 100644
index 08075a9..0000000
--- a/examples/dynamic-partition/XmlJavadocCommentsExtractor.xsl
+++ /dev/null
@@ -1,44 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
--->
-
-<!--
- Document : XmlJavadocCommentsExtractor.xsl
- Created on : September 16, 2014, 11:30 AM
- Description:
- The transformation strips off all information except for comments and tags from xml javadoc generated by xml-doclet.
--->
-
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
- <xsl:output method="xml" standalone="yes"/>
-
- <!-- copy xml by selecting only the following nodes, attributes and text -->
- <xsl:template match="node()|text()|@*">
- <xsl:copy>
- <xsl:apply-templates select="root|package|class|interface|method|field|type|comment|tag|text()|@name|@qualified|@text"/>
- </xsl:copy>
- </xsl:template>
-
- <!-- Strip off the following paths from the selected xml -->
- <xsl:template match="//root/package/interface/interface
- |//root/package/interface/method/@qualified
- |//root/package/class/interface
- |//root/package/class/class
- |//root/package/class/method/@qualified
- |//root/package/class/field/@qualified" />
-
- <xsl:strip-space elements="*"/>
-</xsl:stylesheet>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/dynamic-partition/pom.xml
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/pom.xml b/examples/dynamic-partition/pom.xml
index 21b1c30..b12c09f 100644
--- a/examples/dynamic-partition/pom.xml
+++ b/examples/dynamic-partition/pom.xml
@@ -1,4 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@@ -12,7 +32,7 @@
<packaging>jar</packaging>
<!-- change these to the appropriate values -->
- <name>Dynamic Partitioning</name>
+ <name>Apache Apex Malhar Dynamic Partitioning Example</name>
<description>Example showing dynamic partitioning</description>
<dependencies>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/dynamic-partition/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/src/assemble/appPackage.xml b/examples/dynamic-partition/src/assemble/appPackage.xml
index 7ad071c..a870807 100644
--- a/examples/dynamic-partition/src/assemble/appPackage.xml
+++ b/examples/dynamic-partition/src/assemble/appPackage.xml
@@ -1,3 +1,23 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/dynamic-partition/src/main/java/org/apache/apex/examples/dynamic/App.java
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/src/main/java/org/apache/apex/examples/dynamic/App.java b/examples/dynamic-partition/src/main/java/org/apache/apex/examples/dynamic/App.java
index 1a40cb5..60862ac 100644
--- a/examples/dynamic-partition/src/main/java/org/apache/apex/examples/dynamic/App.java
+++ b/examples/dynamic-partition/src/main/java/org/apache/apex/examples/dynamic/App.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.dynamic;
import org.apache.hadoop.conf.Configuration;
@@ -8,7 +27,7 @@ import com.datatorrent.api.annotation.ApplicationAnnotation;
import com.datatorrent.lib.stream.DevNull;
-@ApplicationAnnotation(name="Dyn")
+@ApplicationAnnotation(name = "Dyn")
public class App implements StreamingApplication
{
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/dynamic-partition/src/main/java/org/apache/apex/examples/dynamic/Gen.java
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/src/main/java/org/apache/apex/examples/dynamic/Gen.java b/examples/dynamic-partition/src/main/java/org/apache/apex/examples/dynamic/Gen.java
index 29f79ac..bb3f202 100644
--- a/examples/dynamic-partition/src/main/java/org/apache/apex/examples/dynamic/Gen.java
+++ b/examples/dynamic-partition/src/main/java/org/apache/apex/examples/dynamic/Gen.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.dynamic;
import java.io.ByteArrayOutputStream;
@@ -81,8 +100,8 @@ public class Gen extends BaseOperator implements InputOperator, Partitioner<Gen>
}
@Override
- public Response processStats(BatchedOperatorStats batchedOperatorStats) {
-
+ public Response processStats(BatchedOperatorStats batchedOperatorStats)
+ {
final long emittedCount = batchedOperatorStats.getTuplesEmittedPSMA();
// we only perform a single dynamic repartition
@@ -90,12 +109,10 @@ public class Gen extends BaseOperator implements InputOperator, Partitioner<Gen>
res.repartitionRequired = false;
if (emittedCount > 500 && partitions < MAX_PARTITIONS) {
LOG.info("processStats: trying repartition of input operator current {} required {}",
- partitions, MAX_PARTITIONS);
+ partitions, MAX_PARTITIONS);
LOG.info("**** operator id = {}, window id = {}, tuplesProcessedPSMA = {}, tuplesEmittedPSMA = {}",
- batchedOperatorStats.getOperatorId(),
- batchedOperatorStats.getCurrentWindowId(),
- batchedOperatorStats.getTuplesProcessedPSMA(),
- emittedCount);
+ batchedOperatorStats.getOperatorId(), batchedOperatorStats.getCurrentWindowId(),
+ batchedOperatorStats.getTuplesProcessedPSMA(), emittedCount);
partitions = MAX_PARTITIONS;
res.repartitionRequired = true;
}
@@ -144,7 +161,7 @@ public class Gen extends BaseOperator implements InputOperator, Partitioner<Gen>
final int prevCount = list.size();
if (1 == prevCount) { // initial call
LOG.info("definePartitions: First call, prevCount = {}, partitions = {}",
- prevCount, partitions);
+ prevCount, partitions);
}
if (prevCount == partitions) {
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/dynamic-partition/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/src/main/resources/META-INF/properties.xml b/examples/dynamic-partition/src/main/resources/META-INF/properties.xml
index 5bbb4b0..253bd26 100644
--- a/examples/dynamic-partition/src/main/resources/META-INF/properties.xml
+++ b/examples/dynamic-partition/src/main/resources/META-INF/properties.xml
@@ -1,6 +1,26 @@
<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<configuration>
- <!--
+ <!--
<property>
<name>dt.application.{appName}.operator.{opName}.prop.{propName}</name>
<value>some-default-value (if value is not specified, it is required from the user or custom config when launching)</value>
@@ -20,6 +40,6 @@
<name>dt.application.Dyn.operator.gen.prop.numTuples</name>
<value>1000</value>
</property>
-
+
</configuration>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/dynamic-partition/src/main/resources/unused-log4j.properties
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/src/main/resources/unused-log4j.properties b/examples/dynamic-partition/src/main/resources/unused-log4j.properties
index a83a30b..625497b 100644
--- a/examples/dynamic-partition/src/main/resources/unused-log4j.properties
+++ b/examples/dynamic-partition/src/main/resources/unused-log4j.properties
@@ -1,3 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
log4j.rootLogger=DEBUG,CONSOLE
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/dynamic-partition/src/test/java/org/apache/apex/examples/dynamic/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/src/test/java/org/apache/apex/examples/dynamic/ApplicationTest.java b/examples/dynamic-partition/src/test/java/org/apache/apex/examples/dynamic/ApplicationTest.java
index a4c8076..8964a14 100644
--- a/examples/dynamic-partition/src/test/java/org/apache/apex/examples/dynamic/ApplicationTest.java
+++ b/examples/dynamic-partition/src/test/java/org/apache/apex/examples/dynamic/ApplicationTest.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.dynamic;
import java.io.IOException;
@@ -14,10 +33,11 @@ import com.datatorrent.api.LocalMode;
/**
* Test the DAG declaration in local mode.
*/
-public class ApplicationTest {
-
+public class ApplicationTest
+{
@Test
- public void testApplication() throws IOException, Exception {
+ public void testApplication() throws IOException, Exception
+ {
try {
LocalMode lma = LocalMode.newInstance();
Configuration conf = new Configuration(false);
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/dynamic-partition/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/dynamic-partition/src/test/resources/log4j.properties b/examples/dynamic-partition/src/test/resources/log4j.properties
index 58d6913..3b76ecc 100644
--- a/examples/dynamic-partition/src/test/resources/log4j.properties
+++ b/examples/dynamic-partition/src/test/resources/log4j.properties
@@ -1,3 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
log4j.rootLogger=DEBUG,CONSOLE
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/enricher/XmlJavadocCommentsExtractor.xsl
----------------------------------------------------------------------
diff --git a/examples/enricher/XmlJavadocCommentsExtractor.xsl b/examples/enricher/XmlJavadocCommentsExtractor.xsl
deleted file mode 100644
index 08075a9..0000000
--- a/examples/enricher/XmlJavadocCommentsExtractor.xsl
+++ /dev/null
@@ -1,44 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
--->
-
-<!--
- Document : XmlJavadocCommentsExtractor.xsl
- Created on : September 16, 2014, 11:30 AM
- Description:
- The transformation strips off all information except for comments and tags from xml javadoc generated by xml-doclet.
--->
-
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
- <xsl:output method="xml" standalone="yes"/>
-
- <!-- copy xml by selecting only the following nodes, attributes and text -->
- <xsl:template match="node()|text()|@*">
- <xsl:copy>
- <xsl:apply-templates select="root|package|class|interface|method|field|type|comment|tag|text()|@name|@qualified|@text"/>
- </xsl:copy>
- </xsl:template>
-
- <!-- Strip off the following paths from the selected xml -->
- <xsl:template match="//root/package/interface/interface
- |//root/package/interface/method/@qualified
- |//root/package/class/interface
- |//root/package/class/class
- |//root/package/class/method/@qualified
- |//root/package/class/field/@qualified" />
-
- <xsl:strip-space elements="*"/>
-</xsl:stylesheet>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/enricher/pom.xml
----------------------------------------------------------------------
diff --git a/examples/enricher/pom.xml b/examples/enricher/pom.xml
index 7a55d32..170efa6 100644
--- a/examples/enricher/pom.xml
+++ b/examples/enricher/pom.xml
@@ -1,4 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@@ -12,7 +32,7 @@
<packaging>jar</packaging>
<!-- change these to the appropriate values -->
- <name>Enricher</name>
+ <name>Apache Apex Malhar Enricher Example</name>
<description>Example Use of POJO Enricher</description>
<dependencies>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/enricher/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/enricher/src/assemble/appPackage.xml b/examples/enricher/src/assemble/appPackage.xml
index 7ad071c..a870807 100644
--- a/examples/enricher/src/assemble/appPackage.xml
+++ b/examples/enricher/src/assemble/appPackage.xml
@@ -1,3 +1,23 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/enricher/src/main/java/org/apache/apex/examples/enricher/DataGenerator.java
----------------------------------------------------------------------
diff --git a/examples/enricher/src/main/java/org/apache/apex/examples/enricher/DataGenerator.java b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/DataGenerator.java
index 2ba5567..50f5c26 100644
--- a/examples/enricher/src/main/java/org/apache/apex/examples/enricher/DataGenerator.java
+++ b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/DataGenerator.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.enricher;
import java.util.Random;
@@ -31,7 +50,8 @@ public class DataGenerator extends BaseOperator implements InputOperator
}
@Override
- public void beginWindow(long windowId) {
+ public void beginWindow(long windowId)
+ {
super.beginWindow(windowId);
count = 0;
}
@@ -39,7 +59,7 @@ public class DataGenerator extends BaseOperator implements InputOperator
@Override
public void emitTuples()
{
- if(count++ < limit) {
+ if (count++ < limit) {
output.emit(getRecord());
}
}
@@ -51,12 +71,10 @@ public class DataGenerator extends BaseOperator implements InputOperator
String imei = getHashInRange(imsi, 15);
String circleId = Math.abs(phone.hashCode()) % NUM_CIRCLES + "";
// String record = MessageFormat.format(baseDataTemplate, phone, imsi, imei, circleId);
- String record = "{" +
- "\"phone\":\"" + phone + "\"," +
- "\"imei\":\"" + imei+ "\"," +
- "\"imsi\":\"" + imsi+ "\"," +
- "\"circleId\":" + circleId +
- "}";
+ String record = "{" + "\"phone\":\"" + phone + "\","
+ + "\"imei\":\"" + imei + "\","
+ + "\"imsi\":\"" + imsi + "\","
+ + "\"circleId\":" + circleId + "}";
return record.getBytes();
}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/enricher/src/main/java/org/apache/apex/examples/enricher/EnricherAppWithJSONFile.java
----------------------------------------------------------------------
diff --git a/examples/enricher/src/main/java/org/apache/apex/examples/enricher/EnricherAppWithJSONFile.java b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/EnricherAppWithJSONFile.java
index 1a420c4..6894830 100644
--- a/examples/enricher/src/main/java/org/apache/apex/examples/enricher/EnricherAppWithJSONFile.java
+++ b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/EnricherAppWithJSONFile.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.enricher;
import java.util.ArrayList;
@@ -12,7 +31,7 @@ import com.datatorrent.contrib.enrich.POJOEnricher;
import com.datatorrent.contrib.parser.JsonParser;
import com.datatorrent.lib.io.ConsoleOutputOperator;
-@ApplicationAnnotation(name="EnricherAppWithJSONFile")
+@ApplicationAnnotation(name = "EnricherAppWithJSONFile")
public class EnricherAppWithJSONFile implements StreamingApplication
{
@Override
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/enricher/src/main/java/org/apache/apex/examples/enricher/LineOutputOperator.java
----------------------------------------------------------------------
diff --git a/examples/enricher/src/main/java/org/apache/apex/examples/enricher/LineOutputOperator.java b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/LineOutputOperator.java
index aca7df6..6e0c742 100644
--- a/examples/enricher/src/main/java/org/apache/apex/examples/enricher/LineOutputOperator.java
+++ b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/LineOutputOperator.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.enricher;
import java.nio.charset.Charset;
@@ -19,16 +38,25 @@ public class LineOutputOperator extends AbstractFileOutputOperator<Object>
private String baseName;
@Override
- public byte[] getBytesForTuple(Object t) {
+ public byte[] getBytesForTuple(Object t)
+ {
String result = new String(t.toString().getBytes(), CS) + NL;
return result.getBytes(CS);
- }
+ }
@Override
- protected String getFileName(Object tuple) {
+ protected String getFileName(Object tuple)
+ {
return baseName;
}
- public String getBaseName() { return baseName; }
- public void setBaseName(String v) { baseName = v; }
+ public String getBaseName()
+ {
+ return baseName;
+ }
+
+ public void setBaseName(String v)
+ {
+ baseName = v;
+ }
}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/enricher/src/main/java/org/apache/apex/examples/enricher/POJO.java
----------------------------------------------------------------------
diff --git a/examples/enricher/src/main/java/org/apache/apex/examples/enricher/POJO.java b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/POJO.java
index d48bd1a..2a61c59 100644
--- a/examples/enricher/src/main/java/org/apache/apex/examples/enricher/POJO.java
+++ b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/POJO.java
@@ -1,49 +1,68 @@
-package org.apache.apex.examples.enricher;
-
-public class POJO
-{
- private String phone;
- private String imei;
- private String imsi;
- private int circleId;
-
- public String getPhone()
- {
- return phone;
- }
-
- public void setPhone(String phone)
- {
- this.phone = phone;
- }
-
- public String getImei()
- {
- return imei;
- }
-
- public void setImei(String imei)
- {
- this.imei = imei;
- }
-
- public String getImsi()
- {
- return imsi;
- }
-
- public void setImsi(String imsi)
- {
- this.imsi = imsi;
- }
-
- public int getCircleId()
- {
- return circleId;
- }
-
- public void setCircleId(int circleId)
- {
- this.circleId = circleId;
- }
-}
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.apex.examples.enricher;
+
+public class POJO
+{
+ private String phone;
+ private String imei;
+ private String imsi;
+ private int circleId;
+
+ public String getPhone()
+ {
+ return phone;
+ }
+
+ public void setPhone(String phone)
+ {
+ this.phone = phone;
+ }
+
+ public String getImei()
+ {
+ return imei;
+ }
+
+ public void setImei(String imei)
+ {
+ this.imei = imei;
+ }
+
+ public String getImsi()
+ {
+ return imsi;
+ }
+
+ public void setImsi(String imsi)
+ {
+ this.imsi = imsi;
+ }
+
+ public int getCircleId()
+ {
+ return circleId;
+ }
+
+ public void setCircleId(int circleId)
+ {
+ this.circleId = circleId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/enricher/src/main/java/org/apache/apex/examples/enricher/POJOEnriched.java
----------------------------------------------------------------------
diff --git a/examples/enricher/src/main/java/org/apache/apex/examples/enricher/POJOEnriched.java b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/POJOEnriched.java
index df08c7f..55566c9 100644
--- a/examples/enricher/src/main/java/org/apache/apex/examples/enricher/POJOEnriched.java
+++ b/examples/enricher/src/main/java/org/apache/apex/examples/enricher/POJOEnriched.java
@@ -1,3 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.apex.examples.enricher;
public class POJOEnriched
@@ -58,7 +77,8 @@ public class POJOEnriched
this.circleName = circleName;
}
- @Override public String toString()
+ @Override
+ public String toString()
{
return "POJOEnriched{" +
"phone='" + phone + '\'' +
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/enricher/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/enricher/src/main/resources/META-INF/properties.xml b/examples/enricher/src/main/resources/META-INF/properties.xml
index 543b99a..c37be45 100644
--- a/examples/enricher/src/main/resources/META-INF/properties.xml
+++ b/examples/enricher/src/main/resources/META-INF/properties.xml
@@ -1,4 +1,24 @@
<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<configuration>
<!-- Parser -->
<property>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/enricher/src/test/java/org/apache/apex/examples/enricher/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/enricher/src/test/java/org/apache/apex/examples/enricher/ApplicationTest.java b/examples/enricher/src/test/java/org/apache/apex/examples/enricher/ApplicationTest.java
index 6b6698e..6cbf846 100644
--- a/examples/enricher/src/test/java/org/apache/apex/examples/enricher/ApplicationTest.java
+++ b/examples/enricher/src/test/java/org/apache/apex/examples/enricher/ApplicationTest.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
package org.apache.apex.examples.enricher;
import javax.validation.ConstraintViolationException;
@@ -12,10 +30,12 @@ import com.datatorrent.api.LocalMode;
/**
* Test the DAG declaration in local mode.
*/
-public class ApplicationTest {
+public class ApplicationTest
+{
@Test
- public void testApplication() throws Exception {
+ public void testApplication() throws Exception
+ {
try {
LocalMode lma = LocalMode.newInstance();
Configuration conf = new Configuration(false);
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/enricher/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/enricher/src/test/resources/log4j.properties b/examples/enricher/src/test/resources/log4j.properties
index 3bfcdc5..451cff3 100644
--- a/examples/enricher/src/test/resources/log4j.properties
+++ b/examples/enricher/src/test/resources/log4j.properties
@@ -1,3 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
log4j.rootLogger=DEBUG,CONSOLE
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/filter/XmlJavadocCommentsExtractor.xsl
----------------------------------------------------------------------
diff --git a/examples/filter/XmlJavadocCommentsExtractor.xsl b/examples/filter/XmlJavadocCommentsExtractor.xsl
deleted file mode 100644
index 08075a9..0000000
--- a/examples/filter/XmlJavadocCommentsExtractor.xsl
+++ /dev/null
@@ -1,44 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
--->
-
-<!--
- Document : XmlJavadocCommentsExtractor.xsl
- Created on : September 16, 2014, 11:30 AM
- Description:
- The transformation strips off all information except for comments and tags from xml javadoc generated by xml-doclet.
--->
-
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
- <xsl:output method="xml" standalone="yes"/>
-
- <!-- copy xml by selecting only the following nodes, attributes and text -->
- <xsl:template match="node()|text()|@*">
- <xsl:copy>
- <xsl:apply-templates select="root|package|class|interface|method|field|type|comment|tag|text()|@name|@qualified|@text"/>
- </xsl:copy>
- </xsl:template>
-
- <!-- Strip off the following paths from the selected xml -->
- <xsl:template match="//root/package/interface/interface
- |//root/package/interface/method/@qualified
- |//root/package/class/interface
- |//root/package/class/class
- |//root/package/class/method/@qualified
- |//root/package/class/field/@qualified" />
-
- <xsl:strip-space elements="*"/>
-</xsl:stylesheet>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/filter/pom.xml
----------------------------------------------------------------------
diff --git a/examples/filter/pom.xml b/examples/filter/pom.xml
index 7ef038e..265ddd5 100644
--- a/examples/filter/pom.xml
+++ b/examples/filter/pom.xml
@@ -1,4 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@@ -12,7 +32,7 @@
<packaging>jar</packaging>
<!-- change these to the appropriate values -->
- <name>Filter Operator</name>
+ <name>Apache Apex Malhar Filter Example</name>
<description>Apex application demonstrating filter operator</description>
<dependencies>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/filter/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/filter/src/assemble/appPackage.xml b/examples/filter/src/assemble/appPackage.xml
index 7ad071c..a870807 100644
--- a/examples/filter/src/assemble/appPackage.xml
+++ b/examples/filter/src/assemble/appPackage.xml
@@ -1,3 +1,23 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/filter/src/main/java/org/apache/apex/examples/filter/Application.java
----------------------------------------------------------------------
diff --git a/examples/filter/src/main/java/org/apache/apex/examples/filter/Application.java b/examples/filter/src/main/java/org/apache/apex/examples/filter/Application.java
index 5bbf159..ce832e6 100644
--- a/examples/filter/src/main/java/org/apache/apex/examples/filter/Application.java
+++ b/examples/filter/src/main/java/org/apache/apex/examples/filter/Application.java
@@ -1,8 +1,23 @@
/**
- * Copyright (c) 2016 DataTorrent, Inc.
- * All rights reserved.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
+
package org.apache.apex.examples.filter;
import org.apache.apex.malhar.lib.fs.FSRecordReaderModule;
@@ -19,7 +34,7 @@ import com.datatorrent.lib.filter.FilterOperator;
/**
* Simple application illustrating filter operator
*/
-@ApplicationAnnotation(name="FilterExample")
+@ApplicationAnnotation(name = "FilterExample")
public class Application implements StreamingApplication
{
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/filter/src/main/java/org/apache/apex/examples/filter/TransactionPOJO.java
----------------------------------------------------------------------
diff --git a/examples/filter/src/main/java/org/apache/apex/examples/filter/TransactionPOJO.java b/examples/filter/src/main/java/org/apache/apex/examples/filter/TransactionPOJO.java
index 4b0d7a9..528d8b1 100644
--- a/examples/filter/src/main/java/org/apache/apex/examples/filter/TransactionPOJO.java
+++ b/examples/filter/src/main/java/org/apache/apex/examples/filter/TransactionPOJO.java
@@ -1,8 +1,23 @@
/**
- * Copyright (c) 2016 DataTorrent, Inc.
- * All rights reserved.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
+
package org.apache.apex.examples.filter;
public class TransactionPOJO
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/filter/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/filter/src/main/resources/META-INF/properties.xml b/examples/filter/src/main/resources/META-INF/properties.xml
index 8c1a4a3..58b6e08 100644
--- a/examples/filter/src/main/resources/META-INF/properties.xml
+++ b/examples/filter/src/main/resources/META-INF/properties.xml
@@ -1,6 +1,26 @@
<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
<configuration>
- <!--
+ <!--
<property>
<name>dt.application.{appName}.operator.{opName}.prop.{propName}</name>
<value>some-default-value (if value is not specified, it is required from the user or custom config when launching)</value>
@@ -14,7 +34,7 @@
-->
<!-- these values override those set in populateDAG -->
-
+
<property>
<name>dt.application.FilterExample.operator.*.attr.MEMORY_MB</name>
<value>750</value>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/filter/src/test/java/org/apache/apex/examples/filter/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/filter/src/test/java/org/apache/apex/examples/filter/ApplicationTest.java b/examples/filter/src/test/java/org/apache/apex/examples/filter/ApplicationTest.java
index 2170753..b820643 100644
--- a/examples/filter/src/test/java/org/apache/apex/examples/filter/ApplicationTest.java
+++ b/examples/filter/src/test/java/org/apache/apex/examples/filter/ApplicationTest.java
@@ -1,8 +1,23 @@
/**
- * Copyright (c) 2016 DataTorrent, Inc.
- * All rights reserved.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
+
package org.apache.apex.examples.filter;
import java.io.File;
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/b8a83a45/examples/filter/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/filter/src/test/resources/log4j.properties b/examples/filter/src/test/resources/log4j.properties
index 98544e8..1c9776b 100644
--- a/examples/filter/src/test/resources/log4j.properties
+++ b/examples/filter/src/test/resources/log4j.properties
@@ -1,3 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
log4j.rootLogger=DEBUG,CONSOLE
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
[07/19] apex-malhar git commit: Added deduper example. Updated malhar
version to 3.5.0
Posted by th...@apache.org.
Added deduper example.
Updated malhar version to 3.5.0
Project: http://git-wip-us.apache.org/repos/asf/apex-malhar/repo
Commit: http://git-wip-us.apache.org/repos/asf/apex-malhar/commit/45735280
Tree: http://git-wip-us.apache.org/repos/asf/apex-malhar/tree/45735280
Diff: http://git-wip-us.apache.org/repos/asf/apex-malhar/diff/45735280
Branch: refs/heads/master
Commit: 4573528050751bc4d9e440bcd7e7a08560956106
Parents: 63df474
Author: bhupeshchawda <bh...@gmail.com>
Authored: Mon Jul 25 19:02:17 2016 +0530
Committer: Lakshmi Prasanna Velineni <la...@datatorrent.com>
Committed: Sun Mar 26 11:43:48 2017 -0700
----------------------------------------------------------------------
examples/dedup/README.md | 16 ++
examples/dedup/XmlJavadocCommentsExtractor.xsl | 44 +++
examples/dedup/pom.xml | 280 +++++++++++++++++++
examples/dedup/src/assemble/appPackage.xml | 43 +++
.../java/com/example/dedup/Application.java | 123 ++++++++
.../src/main/resources/META-INF/properties.xml | 40 +++
.../java/com/example/dedup/ApplicationTest.java | 38 +++
.../dedup/src/test/resources/log4j.properties | 22 ++
8 files changed, 606 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/45735280/examples/dedup/README.md
----------------------------------------------------------------------
diff --git a/examples/dedup/README.md b/examples/dedup/README.md
new file mode 100644
index 0000000..207a0c4
--- /dev/null
+++ b/examples/dedup/README.md
@@ -0,0 +1,16 @@
+This sample application shows how to use the Dedup operator for de-duplicating in a stream of incoming data.
+The operators in the application are as follows:
+1. Random data generator (RandomGenerator) which emits POJO tuples as records
+2. Dedup operator (Deduper) which accepts the POJO tuples and identifies unique and duplicate tuples.
+3. Console operator (ConsoleUnique) for unique tuples
+4. Console operator (ConsoleDuplicate) for duplicate tuples
+5. Console operator (ConsoleExpired) for expired tuples
+
+The following properties are configured for using the Application:
+1. ```dt.application.DedupExample.operator.RandomGenerator.prop.tuplesPerWindow``` - This is a limit on the number of tuples that will be generated by the Random Generator operator.
+2. ```dt.application.DedupExample.operator.Deduper.prop.keyExpression``` - This is the pseudo java expression for deriving the key fields from the incoming POJO.
+3. ```dt.application.DedupExample.operator.Deduper.prop.timeExpression``` - This is the pseudo java expression for deriving the time field in the incoming POJO. In case, ```timeExpression``` is not specified, then the System time is used to compute the expiration for the tuples.
+4. ```dt.application.DedupExample.operator.Deduper.prop.expireBefore``` - The expiry time for incoming tuples in seconds. The keys in the system expire after every ```expireBefore``` seconds.
+5. ```dt.application.DedupExample.operator.Deduper.prop.bucketSpan``` - The span of a single expiry bucket. When an expiry time elapses, the bucket as a whole is discarded from the system. This can be set keeping in mind the largest unit that can be discarded. For example, if ```expireBefore``` is set to 1 hour, and we are getting new data per minuite, it would make sense to set the ```bucketSpan``` to 1 minute or 5 minutes.
+
+Example values for these parameters have been specified in src/main/resources/META-INF/properties.xml.
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/45735280/examples/dedup/XmlJavadocCommentsExtractor.xsl
----------------------------------------------------------------------
diff --git a/examples/dedup/XmlJavadocCommentsExtractor.xsl b/examples/dedup/XmlJavadocCommentsExtractor.xsl
new file mode 100644
index 0000000..08075a9
--- /dev/null
+++ b/examples/dedup/XmlJavadocCommentsExtractor.xsl
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+
+<!--
+ Document : XmlJavadocCommentsExtractor.xsl
+ Created on : September 16, 2014, 11:30 AM
+ Description:
+ The transformation strips off all information except for comments and tags from xml javadoc generated by xml-doclet.
+-->
+
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+ <xsl:output method="xml" standalone="yes"/>
+
+ <!-- copy xml by selecting only the following nodes, attributes and text -->
+ <xsl:template match="node()|text()|@*">
+ <xsl:copy>
+ <xsl:apply-templates select="root|package|class|interface|method|field|type|comment|tag|text()|@name|@qualified|@text"/>
+ </xsl:copy>
+ </xsl:template>
+
+ <!-- Strip off the following paths from the selected xml -->
+ <xsl:template match="//root/package/interface/interface
+ |//root/package/interface/method/@qualified
+ |//root/package/class/interface
+ |//root/package/class/class
+ |//root/package/class/method/@qualified
+ |//root/package/class/field/@qualified" />
+
+ <xsl:strip-space elements="*"/>
+</xsl:stylesheet>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/45735280/examples/dedup/pom.xml
----------------------------------------------------------------------
diff --git a/examples/dedup/pom.xml b/examples/dedup/pom.xml
new file mode 100644
index 0000000..f777784
--- /dev/null
+++ b/examples/dedup/pom.xml
@@ -0,0 +1,280 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>com.example</groupId>
+ <version>1.0-SNAPSHOT</version>
+ <artifactId>dedup</artifactId>
+ <packaging>jar</packaging>
+
+ <!-- change these to the appropriate values -->
+ <name>My Apex Application</name>
+ <description>My Apex Application Description</description>
+
+ <properties>
+ <!-- change this if you desire to use a different version of Apex Core -->
+ <apex.version>3.5.0</apex.version>
+ <malhar.version>3.6.0</malhar.version>
+ <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-eclipse-plugin</artifactId>
+ <version>2.9</version>
+ <configuration>
+ <downloadSources>true</downloadSources>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.3</version>
+ <configuration>
+ <encoding>UTF-8</encoding>
+ <source>1.7</source>
+ <target>1.7</target>
+ <debug>true</debug>
+ <optimize>false</optimize>
+ <showDeprecation>true</showDeprecation>
+ <showWarnings>true</showWarnings>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.8</version>
+ <executions>
+ <execution>
+ <id>copy-dependencies</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>target/deps</outputDirectory>
+ <includeScope>runtime</includeScope>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>app-package-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <finalName>${project.artifactId}-${project.version}-apexapp</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <descriptors>
+ <descriptor>src/assemble/appPackage.xml</descriptor>
+ </descriptors>
+ <archiverConfig>
+ <defaultDirectoryMode>0755</defaultDirectoryMode>
+ </archiverConfig>
+ <archive>
+ <manifestEntries>
+ <Class-Path>${apex.apppackage.classpath}</Class-Path>
+ <DT-Engine-Version>${apex.version}</DT-Engine-Version>
+ <DT-App-Package-Group-Id>${project.groupId}</DT-App-Package-Group-Id>
+ <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name>
+ <DT-App-Package-Version>${project.version}</DT-App-Package-Version>
+ <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name>
+ <DT-App-Package-Description>${project.description}</DT-App-Package-Description>
+ </manifestEntries>
+ </archive>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <configuration>
+ <target>
+ <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar"
+ tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa" />
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ <execution>
+ <!-- create resource directory for xml javadoc-->
+ <id>createJavadocDirectory</id>
+ <phase>generate-resources</phase>
+ <configuration>
+ <tasks>
+ <delete dir="${project.build.directory}/generated-resources/xml-javadoc"/>
+ <mkdir dir="${project.build.directory}/generated-resources/xml-javadoc"/>
+ </tasks>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.9.1</version>
+ <executions>
+ <execution>
+ <id>attach-artifacts</id>
+ <phase>package</phase>
+ <goals>
+ <goal>attach-artifact</goal>
+ </goals>
+ <configuration>
+ <artifacts>
+ <artifact>
+ <file>target/${project.artifactId}-${project.version}.apa</file>
+ <type>apa</type>
+ </artifact>
+ </artifacts>
+ <skipAttach>false</skipAttach>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- generate javdoc -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <executions>
+ <!-- generate xml javadoc -->
+ <execution>
+ <id>xml-doclet</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>javadoc</goal>
+ </goals>
+ <configuration>
+ <doclet>com.github.markusbernhardt.xmldoclet.XmlDoclet</doclet>
+ <additionalparam>-d ${project.build.directory}/generated-resources/xml-javadoc -filename ${project.artifactId}-${project.version}-javadoc.xml</additionalparam>
+ <useStandardDocletOptions>false</useStandardDocletOptions>
+ <docletArtifact>
+ <groupId>com.github.markusbernhardt</groupId>
+ <artifactId>xml-doclet</artifactId>
+ <version>1.0.4</version>
+ </docletArtifact>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <!-- Transform xml javadoc to stripped down version containing only class/interface comments and tags-->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>xml-maven-plugin</artifactId>
+ <version>1.0</version>
+ <executions>
+ <execution>
+ <id>transform-xmljavadoc</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>transform</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <transformationSets>
+ <transformationSet>
+ <dir>${project.build.directory}/generated-resources/xml-javadoc</dir>
+ <includes>
+ <include>${project.artifactId}-${project.version}-javadoc.xml</include>
+ </includes>
+ <stylesheet>XmlJavadocCommentsExtractor.xsl</stylesheet>
+ <outputDir>${project.build.directory}/generated-resources/xml-javadoc</outputDir>
+ </transformationSet>
+ </transformationSets>
+ </configuration>
+ </plugin>
+ <!-- copy xml javadoc to class jar -->
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>2.6</version>
+ <executions>
+ <execution>
+ <id>copy-resources</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${basedir}/target/classes</outputDirectory>
+ <resources>
+ <resource>
+ <directory>${project.build.directory}/generated-resources/xml-javadoc</directory>
+ <includes>
+ <include>${project.artifactId}-${project.version}-javadoc.xml</include>
+ </includes>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+
+ </build>
+
+ <dependencies>
+ <!-- add your dependencies here -->
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>malhar-library</artifactId>
+ <version>${malhar.version}</version>
+ <!--
+ If you know that your application does not need transitive dependencies pulled in by malhar-library,
+ uncomment the following to reduce the size of your app package.
+ -->
+ <!--
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ -->
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>apex-common</artifactId>
+ <version>${apex.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.10</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.apex</groupId>
+ <artifactId>apex-engine</artifactId>
+ <version>${apex.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.janino</groupId>
+ <artifactId>janino</artifactId>
+ <version>2.7.8</version>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/45735280/examples/dedup/src/assemble/appPackage.xml
----------------------------------------------------------------------
diff --git a/examples/dedup/src/assemble/appPackage.xml b/examples/dedup/src/assemble/appPackage.xml
new file mode 100644
index 0000000..7ad071c
--- /dev/null
+++ b/examples/dedup/src/assemble/appPackage.xml
@@ -0,0 +1,43 @@
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>appPackage</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/target/</directory>
+ <outputDirectory>/app</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/target/deps</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/site/conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/META-INF</directory>
+ <outputDirectory>/META-INF</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/app</directory>
+ <outputDirectory>/app</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/src/main/resources/resources</directory>
+ <outputDirectory>/resources</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/45735280/examples/dedup/src/main/java/com/example/dedup/Application.java
----------------------------------------------------------------------
diff --git a/examples/dedup/src/main/java/com/example/dedup/Application.java b/examples/dedup/src/main/java/com/example/dedup/Application.java
new file mode 100644
index 0000000..cabdce2
--- /dev/null
+++ b/examples/dedup/src/main/java/com/example/dedup/Application.java
@@ -0,0 +1,123 @@
+/**
+ * Put your copyright and license info here.
+ */
+package com.example.dedup;
+
+import java.util.Date;
+import java.util.Random;
+
+import org.apache.apex.malhar.lib.dedup.TimeBasedDedupOperator;
+import org.apache.hadoop.conf.Configuration;
+
+import com.datatorrent.api.annotation.ApplicationAnnotation;
+import com.datatorrent.common.partitioner.StatelessPartitioner;
+import com.datatorrent.common.util.BaseOperator;
+import com.datatorrent.lib.io.ConsoleOutputOperator;
+import com.datatorrent.api.StreamingApplication;
+import com.datatorrent.api.Context;
+import com.datatorrent.api.DAG;
+import com.datatorrent.api.DefaultOutputPort;
+import com.datatorrent.api.InputOperator;
+
+@ApplicationAnnotation(name="DedupExample")
+public class Application implements StreamingApplication
+{
+
+ @Override
+ public void populateDAG(DAG dag, Configuration conf)
+ {
+ // Test Data Generator Operator
+ RandomDataGeneratorOperator gen = dag.addOperator("RandomGenerator", new RandomDataGeneratorOperator());
+
+ // Dedup Operator. Configuration through resources/META-INF/properties.xml
+ TimeBasedDedupOperator dedup = dag.addOperator("Deduper", new TimeBasedDedupOperator());
+
+ // Console output operator for unique tuples
+ ConsoleOutputOperator consoleUnique = dag.addOperator("ConsoleUnique", new ConsoleOutputOperator());
+
+ // Console output operator for duplicate tuples
+ ConsoleOutputOperator consoleDuplicate = dag.addOperator("ConsoleDuplicate", new ConsoleOutputOperator());
+
+ // Console output operator for duplicate tuples
+ ConsoleOutputOperator consoleExpired = dag.addOperator("ConsoleExpired", new ConsoleOutputOperator());
+
+ // Streams
+ dag.addStream("Generator to Dedup", gen.output, dedup.input);
+
+ // Connect Dedup unique to Console
+ dag.addStream("Dedup Unique to Console", dedup.unique, consoleUnique.input);
+ // Connect Dedup duplicate to Console
+ dag.addStream("Dedup Duplicate to Console", dedup.duplicate, consoleDuplicate.input);
+ // Connect Dedup expired to Console
+ dag.addStream("Dedup Expired to Console", dedup.expired, consoleExpired.input);
+
+ // Set Attribute TUPLE_CLASS for supplying schema information to the port
+ dag.setInputPortAttribute(dedup.input, Context.PortContext.TUPLE_CLASS, TestEvent.class);
+
+ // Uncomment the following line to create multiple partitions for Dedup operator. In this case: 2
+ // dag.setAttribute(dedup, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<TimeBasedDedupOperator>(2));
+ }
+
+ public static class RandomDataGeneratorOperator extends BaseOperator implements InputOperator
+ {
+
+ public final transient DefaultOutputPort<TestEvent> output = new DefaultOutputPort<>();
+ private final transient Random r = new Random();
+ private int tuplesPerWindow = 100;
+ private transient int count = 0;
+
+ @Override
+ public void beginWindow(long windowId) {
+ count = 0;
+ }
+
+ @Override
+ public void emitTuples()
+ {
+ if (count++ > tuplesPerWindow) {
+ return;
+ }
+ TestEvent event = new TestEvent();
+ event.id = r.nextInt(100);
+ event.eventTime = new Date(System.currentTimeMillis() - (r.nextInt(60 * 1000)));
+ output.emit(event);
+ }
+ }
+
+ public static class TestEvent
+ {
+ private int id;
+ private Date eventTime;
+
+ public TestEvent()
+ {
+ }
+
+ public int getId()
+ {
+ return id;
+ }
+
+ public void setId(int id)
+ {
+ this.id = id;
+ }
+
+ public Date getEventTime()
+ {
+ return eventTime;
+ }
+
+ public void setEventTime(Date eventTime)
+ {
+ this.eventTime = eventTime;
+ }
+
+ @Override
+ public String toString() {
+ return "TestEvent [id=" + id + ", eventTime=" + eventTime + "]";
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/45735280/examples/dedup/src/main/resources/META-INF/properties.xml
----------------------------------------------------------------------
diff --git a/examples/dedup/src/main/resources/META-INF/properties.xml b/examples/dedup/src/main/resources/META-INF/properties.xml
new file mode 100644
index 0000000..ffe3350
--- /dev/null
+++ b/examples/dedup/src/main/resources/META-INF/properties.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0"?>
+<configuration>
+ <property>
+ <name>dt.application.DedupExample.operator.RandomGenerator.prop.tuplesPerWindow</name>
+ <value>100</value>
+ </property>
+ <property>
+ <name>dt.application.DedupExample.operator.Deduper.prop.keyExpression</name>
+ <value>id</value>
+ </property>
+ <property>
+ <name>dt.application.DedupExample.operator.Deduper.prop.timeExpression</name>
+ <value>eventTime.getTime()</value>
+ </property>
+ <property>
+ <name>dt.application.DedupExample.operator.Deduper.prop.bucketSpan</name>
+ <value>10</value>
+ </property>
+ <property>
+ <name>dt.application.DedupExample.operator.Deduper.prop.expireBefore</name>
+ <value>60</value>
+ </property>
+ <property>
+ <name>dt.application.DedupExample.operator.ConsoleUnique.prop.stringFormat</name>
+ <value>Unique: %s</value>
+ </property>
+ <property>
+ <name>dt.application.DedupExample.operator.ConsoleDuplicate.prop.stringFormat</name>
+ <value>Duplicate: %s</value>
+ </property>
+ <property>
+ <name>dt.application.DedupExample.operator.ConsoleExpired.prop.stringFormat</name>
+ <value>Expired: %s</value>
+ </property>
+ <property>
+ <name>dt.application.DedupExample.operator.*.attr.MEMORY_MB</name>
+ <value>512</value>
+ </property>
+</configuration>
+
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/45735280/examples/dedup/src/test/java/com/example/dedup/ApplicationTest.java
----------------------------------------------------------------------
diff --git a/examples/dedup/src/test/java/com/example/dedup/ApplicationTest.java b/examples/dedup/src/test/java/com/example/dedup/ApplicationTest.java
new file mode 100644
index 0000000..9c9f17c
--- /dev/null
+++ b/examples/dedup/src/test/java/com/example/dedup/ApplicationTest.java
@@ -0,0 +1,38 @@
+/**
+ * Put your copyright and license info here.
+ */
+package com.example.dedup;
+
+import java.io.IOException;
+
+import javax.validation.ConstraintViolationException;
+
+import org.junit.Assert;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+import com.datatorrent.api.LocalMode;
+import com.example.dedup.Application;
+
+/**
+ * Test the DAG declaration in local mode.
+ */
+public class ApplicationTest {
+
+ @Test
+ public void testApplication() throws IOException, Exception {
+ try {
+ LocalMode lma = LocalMode.newInstance();
+ Configuration conf = new Configuration(false);
+ conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
+ lma.prepareDAG(new Application(), conf);
+ LocalMode.Controller lc = lma.getController();
+ lc.runAsync();
+ Thread.sleep(10 * 1000);
+ lc.shutdown();
+ } catch (ConstraintViolationException e) {
+ Assert.fail("constraint violations: " + e.getConstraintViolations());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/45735280/examples/dedup/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/dedup/src/test/resources/log4j.properties b/examples/dedup/src/test/resources/log4j.properties
new file mode 100644
index 0000000..98544e8
--- /dev/null
+++ b/examples/dedup/src/test/resources/log4j.properties
@@ -0,0 +1,22 @@
+log4j.rootLogger=DEBUG,CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+
+log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n
+log4j.appender.RFA.File=/tmp/app.log
+
+# to enable, add SYSLOG to rootLogger
+log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender
+log4j.appender.SYSLOG.syslogHost=127.0.0.1
+log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout
+log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n
+log4j.appender.SYSLOG.Facility=LOCAL1
+
+#log4j.logger.org.apache.commons.beanutils=warn
+log4j.logger.com.datatorrent=debug
+log4j.logger.org.apache.apex=debug
+log4j.logger.org=info