You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@oodt.apache.org by Radu Manole <ma...@gmail.com> on 2015/07/30 23:54:52 UTC

Review Request 36953: Full implementation of AvroRPC alongside XMLRPC in filemgr module

-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/36953/
-----------------------------------------------------------

Review request for oodt, Lewis McGibbney and Chris Mattmann.


Repository: oodt


Description
-------

Separated the FileManager logic from rpc communication createing interfaces for server/client side (FileManagerCient, FileManagerServer).Refactor existing XMLRPC server/client so it would use communication interfaces. Created Avro schemas and protocol for ipc communication, (classes that have the prefixe Avro), with a Factory class AvroTypeFactory that transforms from oodt to avro and from avro to oodt classes. Implement avro server/client. For changeability I created a Factory (RpcCommunicationFactory) that can be configurable to use either avro or xmlrpc for IPC. Because of this, anywhere in project where on instantiation of server/client rewrite to use this factory.


Diffs
-----

  trunk/app/fmbrowser/src/main/java/org/apache/oodt/cas/filemgr/browser/model/CasDB.java 1693501 
  trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FilemgrUniquenessChecker.java 1693501 
  trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/comparator/FilemgrUniquenessCheckComparator.java 1693501 
  trunk/crawler/src/site/xdoc/user/index.xml 1693501 
  trunk/filemgr/pom.xml 1693501 
  trunk/filemgr/src/main/avro/types/AvroComplexQuery.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroElement.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroExtractorSpec.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroFileTransferStatus.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroFilterAlgor.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroMetadata.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroMimeType.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroProduct.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroProductPage.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroProductType.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroQuery.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroQueryCriteria.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroQueryFilter.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroQueryResult.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroReference.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroTypeHandler.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/protocol.avdl PRE-CREATION 
  trunk/filemgr/src/main/bin/filemgr 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/AbstractDeleteProductCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/DeleteProductByIdCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/FileManagerCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetFirstPageCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetLastPageCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNextPageCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNumProductsCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetPrevPageCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetProductPercentTransferredCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/RetrieveFilesCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/AvroLocalDataTransferer.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/InPlaceDataTransferer.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/LocalCache.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/ProductMetKeys.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/extractors/examples/TikaAutoDetectExtractor.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerServer.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManager.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServer.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerServer.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerClientFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerServerFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerClientFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerServerFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerClientFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerServerFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ExpImpCatalog.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataBasedProductMover.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataDumper.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ProductDumper.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/AvroTypeFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/RpcCommunicationFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/XmlRpcStructFactory.java 1693501 
  trunk/filemgr/src/main/resources/.~lock.CAS File Manager User Guide.doc# PRE-CREATION 
  trunk/filemgr/src/main/resources/filemgr.properties 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/TestFileManagerCli.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/UseMockClientCmdLineActionStore.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java PRE-CREATION 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestAddProductTypeCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByIdCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByNameCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDumpMetadataCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransferCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransfersCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFilePercentTransferredCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFirstPageCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetLastPageCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNextPageCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNumProductsCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetPrevPageCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByIdCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByNameCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductPercentTransferredCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductTypeByNameCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestHasProductCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestIngestProductCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestLuceneQueryCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestSqlQueryCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestCachedIngester.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestLocalCache.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestRmiCache.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestStdIngester.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/structs/type/TestTypeHandler.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/MockFileManagerClient.java PRE-CREATION 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManager.java PRE-CREATION 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManagerClient.java PRE-CREATION 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/rpc/TestServerClientFactories.java PRE-CREATION 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestExpImpCatalog.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestMetadataBasedProductMover.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestAvroTypesFactory.java PRE-CREATION 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestXmlRpcStructFactory.java 1693501 
  trunk/filemgr/src/test/resources/createProduct/test.txt PRE-CREATION 
  trunk/filemgr/src/test/resources/filemgr.properties 1693501 
  trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/filemgr/src/main/resources/bin/filemgr 1693501 
  trunk/pcs/core/src/main/java/org/apache/oodt/pcs/util/FileManagerUtils.java 1693501 
  trunk/pge/src/main/java/org/apache/oodt/cas/pge/staging/FileStager.java 1693501 
  trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java 1693501 
  trunk/pge/src/test/java/org/apache/oodt/cas/pge/staging/TestFileStager.java 1693501 
  trunk/profile/src/main/java/org/apache/oodt/profile/handlers/cas/CASProfileHandler.java 1693501 
  trunk/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/filemgr/FileManagerConn.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/CASProductHandler.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DataDeliveryServlet.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DatasetDeliveryServlet.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/services/CasProductJaxrsService.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/servlets/CasProductJaxrsServlet.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFDatasetServlet.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFProductServlet.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductTransferServlet.java 1693501 

Diff: https://reviews.apache.org/r/36953/diff/


Testing
-------

Used existing tests, applied them for avro ipc.


Thanks,

Radu Manole

Re: Review Request 36953: Full implementation of AvroRPC alongside XMLRPC in filemgr module

Posted by Lewis John Mcgibbney <le...@gmail.com>.
Hi Radu,
Excellent work on this one.
Apologies for being a bit distant. Loads of work myself but glad to be
re-engaging with you now.
Let me take a look at this review request and I will get back to you as
soon as I can.
Do you have reports substantiated on the OODT wiki briefly describing the
key contributions of the entire work which is proposed here?
Thanks, and again. Great work and drive.
Lewis

On Thu, Jul 30, 2015 at 2:54 PM, Radu Manole <ma...@gmail.com>
wrote:

> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/36953/
> Review request for oodt, Lewis McGibbney and Chris Mattmann.
> By Radu Manole.
> *Repository: * oodt
> Description
>
> Separated the FileManager logic from rpc communication createing interfaces for server/client side (FileManagerCient, FileManagerServer).Refactor existing XMLRPC server/client so it would use communication interfaces. Created Avro schemas and protocol for ipc communication, (classes that have the prefixe Avro), with a Factory class AvroTypeFactory that transforms from oodt to avro and from avro to oodt classes. Implement avro server/client. For changeability I created a Factory (RpcCommunicationFactory) that can be configurable to use either avro or xmlrpc for IPC. Because of this, anywhere in project where on instantiation of server/client rewrite to use this factory.
>
> Testing
>
> Used existing tests, applied them for avro ipc.
>
> Diffs
>
>    - trunk/app/fmbrowser/src/main/java/org/apache/oodt/cas/filemgr/browser/model/CasDB.java
>    (1693501)
>    - trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FilemgrUniquenessChecker.java
>    (1693501)
>    - trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/comparator/FilemgrUniquenessCheckComparator.java
>    (1693501)
>    - trunk/crawler/src/site/xdoc/user/index.xml (1693501)
>    - trunk/filemgr/pom.xml (1693501)
>    - trunk/filemgr/src/main/avro/types/AvroComplexQuery.avsc
>    (PRE-CREATION)
>    - trunk/filemgr/src/main/avro/types/AvroElement.avsc (PRE-CREATION)
>    - trunk/filemgr/src/main/avro/types/AvroExtractorSpec.avsc
>    (PRE-CREATION)
>    - trunk/filemgr/src/main/avro/types/AvroFileTransferStatus.avsc
>    (PRE-CREATION)
>    - trunk/filemgr/src/main/avro/types/AvroFilterAlgor.avsc (PRE-CREATION)
>    - trunk/filemgr/src/main/avro/types/AvroMetadata.avsc (PRE-CREATION)
>    - trunk/filemgr/src/main/avro/types/AvroMimeType.avsc (PRE-CREATION)
>    - trunk/filemgr/src/main/avro/types/AvroProduct.avsc (PRE-CREATION)
>    - trunk/filemgr/src/main/avro/types/AvroProductPage.avsc (PRE-CREATION)
>    - trunk/filemgr/src/main/avro/types/AvroProductType.avsc (PRE-CREATION)
>    - trunk/filemgr/src/main/avro/types/AvroQuery.avsc (PRE-CREATION)
>    - trunk/filemgr/src/main/avro/types/AvroQueryCriteria.avsc
>    (PRE-CREATION)
>    - trunk/filemgr/src/main/avro/types/AvroQueryFilter.avsc (PRE-CREATION)
>    - trunk/filemgr/src/main/avro/types/AvroQueryResult.avsc (PRE-CREATION)
>    - trunk/filemgr/src/main/avro/types/AvroReference.avsc (PRE-CREATION)
>    - trunk/filemgr/src/main/avro/types/AvroTypeHandler.avsc (PRE-CREATION)
>    - trunk/filemgr/src/main/avro/types/protocol.avdl (PRE-CREATION)
>    - trunk/filemgr/src/main/bin/filemgr (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
>    (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/AbstractDeleteProductCliAction.java
>    (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/DeleteProductByIdCliAction.java
>    (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/FileManagerCliAction.java
>    (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetFirstPageCliAction.java
>    (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetLastPageCliAction.java
>    (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNextPageCliAction.java
>    (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNumProductsCliAction.java
>    (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetPrevPageCliAction.java
>    (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetProductPercentTransferredCliAction.java
>    (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java
>    (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/RetrieveFilesCliAction.java
>    (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/AvroLocalDataTransferer.java
>    (PRE-CREATION)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/InPlaceDataTransferer.java
>    (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java
>    (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java
>    (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/LocalCache.java
>    (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java
>    (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/ProductMetKeys.java
>    (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/extractors/examples/TikaAutoDetectExtractor.java
>    (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java
>    (PRE-CREATION)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerServer.java
>    (PRE-CREATION)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManager.java
>    (PRE-CREATION)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java
>    (PRE-CREATION)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServer.java
>    (PRE-CREATION)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java
>    (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerServer.java
>    (PRE-CREATION)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerClientFactory.java
>    (PRE-CREATION)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerServerFactory.java
>    (PRE-CREATION)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerClientFactory.java
>    (PRE-CREATION)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerServerFactory.java
>    (PRE-CREATION)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerClientFactory.java
>    (PRE-CREATION)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerServerFactory.java
>    (PRE-CREATION)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java
>    (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java
>    (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ExpImpCatalog.java
>    (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataBasedProductMover.java
>    (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataDumper.java
>    (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ProductDumper.java
>    (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
>    (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java
>    (1693501)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/AvroTypeFactory.java
>    (PRE-CREATION)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/RpcCommunicationFactory.java
>    (PRE-CREATION)
>    - trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/XmlRpcStructFactory.java
>    (1693501)
>    - trunk/filemgr/src/main/resources/.~lock.CAS File Manager User
>    Guide.doc# (PRE-CREATION)
>    - trunk/filemgr/src/main/resources/filemgr.properties (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/TestFileManagerCli.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/UseMockClientCmdLineActionStore.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java
>    (PRE-CREATION)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestAddProductTypeCliAction.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByIdCliAction.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByNameCliAction.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDumpMetadataCliAction.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransferCliAction.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransfersCliAction.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFilePercentTransferredCliAction.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFirstPageCliAction.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetLastPageCliAction.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNextPageCliAction.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNumProductsCliAction.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetPrevPageCliAction.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByIdCliAction.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByNameCliAction.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductPercentTransferredCliAction.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductTypeByNameCliAction.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestHasProductCliAction.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestIngestProductCliAction.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestLuceneQueryCliAction.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestSqlQueryCliAction.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestCachedIngester.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestLocalCache.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestRmiCache.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestStdIngester.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/structs/type/TestTypeHandler.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/MockFileManagerClient.java
>    (PRE-CREATION)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManager.java
>    (PRE-CREATION)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManagerClient.java
>    (PRE-CREATION)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/rpc/TestServerClientFactories.java
>    (PRE-CREATION)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestExpImpCatalog.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestMetadataBasedProductMover.java
>    (1693501)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestAvroTypesFactory.java
>    (PRE-CREATION)
>    - trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestXmlRpcStructFactory.java
>    (1693501)
>    - trunk/filemgr/src/test/resources/createProduct/test.txt
>    (PRE-CREATION)
>    - trunk/filemgr/src/test/resources/filemgr.properties (1693501)
>    - trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/filemgr/src/main/resources/bin/filemgr
>    (1693501)
>    - trunk/pcs/core/src/main/java/org/apache/oodt/pcs/util/FileManagerUtils.java
>    (1693501)
>    - trunk/pge/src/main/java/org/apache/oodt/cas/pge/staging/FileStager.java
>    (1693501)
>    - trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java
>    (1693501)
>    - trunk/pge/src/test/java/org/apache/oodt/cas/pge/staging/TestFileStager.java
>    (1693501)
>    - trunk/profile/src/main/java/org/apache/oodt/profile/handlers/cas/CASProfileHandler.java
>    (1693501)
>    - trunk/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/filemgr/FileManagerConn.java
>    (1693501)
>    - trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/CASProductHandler.java
>    (1693501)
>    - trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DataDeliveryServlet.java
>    (1693501)
>    - trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DatasetDeliveryServlet.java
>    (1693501)
>    - trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/services/CasProductJaxrsService.java
>    (1693501)
>    - trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/servlets/CasProductJaxrsServlet.java
>    (1693501)
>    - trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFDatasetServlet.java
>    (1693501)
>    - trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFProductServlet.java
>    (1693501)
>    - trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java
>    (1693501)
>    - trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductTransferServlet.java
>    (1693501)
>
> View Diff <https://reviews.apache.org/r/36953/diff/>
>



-- 
*Lewis*

Re: Review Request 36953: Full implementation of AvroRPC alongside XMLRPC in filemgr module

Posted by Radu Manole <ma...@gmail.com>.

> On Aug. 22, 2015, 4:38 p.m., Chris Mattmann wrote:
> > This was a LOT better Radu. You are getting close.
> > 
> > Some big things to remember:
> > 
> > 1. If code is commented out, you should remove it.
> > 2. We shouldn't disable tests e.g., Tika, by commenting it out. If the test isn't passing, then something is wrong (we have a regression)
> > 3. You need to update RADIX and make sure this is working with OODT RADIX.

Thanks for advices, sorry for number 2 I totally forgot, and missed it in the diff.


- Radu


-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/36953/#review96119
-----------------------------------------------------------


On Aug. 24, 2015, 5:19 p.m., Radu Manole wrote:
> 
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/36953/
> -----------------------------------------------------------
> 
> (Updated Aug. 24, 2015, 5:19 p.m.)
> 
> 
> Review request for oodt, Lewis McGibbney and Chris Mattmann.
> 
> 
> Repository: oodt
> 
> 
> Description
> -------
> 
> Separated the FileManager logic from rpc communication createing interfaces for server/client side (FileManagerCient, FileManagerServer).Refactor existing XMLRPC server/client so it would use communication interfaces. Created Avro schemas and protocol for ipc communication, (classes that have the prefixe Avro), with a Factory class AvroTypeFactory that transforms from oodt to avro and from avro to oodt classes. Implement avro server/client. For changeability I created a Factory (RpcCommunicationFactory) that can be configurable to use either avro or xmlrpc for IPC. Because of this, anywhere in project where on instantiation of server/client rewrite to use this factory.
> 
> 
> Diffs
> -----
> 
>   trunk/app/fmbrowser/src/main/java/org/apache/oodt/cas/filemgr/browser/model/CasDB.java 1693501 
>   trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FilemgrUniquenessChecker.java 1693501 
>   trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/comparator/FilemgrUniquenessCheckComparator.java 1693501 
>   trunk/filemgr/pom.xml 1693501 
>   trunk/filemgr/src/main/avro/types/AvroComplexQuery.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroElement.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroExtractorSpec.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroFileTransferStatus.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroFilterAlgor.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroMetadata.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroMimeType.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProduct.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProductPage.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProductType.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQuery.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryCriteria.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryFilter.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryResult.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroReference.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroTypeHandler.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/protocol.avdl PRE-CREATION 
>   trunk/filemgr/src/main/bin/filemgr 1693501 
>   trunk/filemgr/src/main/bin/filemgr-client 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/AbstractDeleteProductCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/DeleteProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/FileManagerCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetFirstPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetLastPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNextPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNumProductsCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetPrevPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetProductPercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/RetrieveFilesCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/AvroLocalDataTransferer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/InPlaceDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/LocalCache.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/ProductMetKeys.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManager.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClientMain.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServerMain.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ExpImpCatalog.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataBasedProductMover.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataDumper.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ProductDumper.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/AvroTypeFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/RpcCommunicationFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/XmlRpcStructFactory.java 1693501 
>   trunk/filemgr/src/main/resources/filemgr.properties 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/TestFileManagerCli.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/UseMockClientCmdLineActionStore.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestAddProductTypeCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDumpMetadataCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransferCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransfersCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFilePercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFirstPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetLastPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNextPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNumProductsCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetPrevPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductPercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductTypeByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestHasProductCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestIngestProductCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestLuceneQueryCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestSqlQueryCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestCachedIngester.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestLocalCache.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestRmiCache.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestStdIngester.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/metadata/extractors/examples/TestTikaAutoDetectExtractor.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/structs/type/TestTypeHandler.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/MockFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManager.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/rpc/TestServerClientFactories.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestExpImpCatalog.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestMetadataBasedProductMover.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestAvroTypesFactory.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestXmlRpcStructFactory.java 1693501 
>   trunk/filemgr/src/test/resources/createProduct/test.txt PRE-CREATION 
>   trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/filemgr/src/main/resources/bin/filemgr 1693501 
>   trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/filemgr/src/main/resources/bin/filemgr-client 1693501 
>   trunk/pcs/core/src/main/java/org/apache/oodt/pcs/util/FileManagerUtils.java 1693501 
>   trunk/pge/src/main/java/org/apache/oodt/cas/pge/staging/FileStager.java 1693501 
>   trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java 1693501 
>   trunk/pge/src/test/java/org/apache/oodt/cas/pge/staging/TestFileStager.java 1693501 
>   trunk/profile/src/main/java/org/apache/oodt/profile/handlers/cas/CASProfileHandler.java 1693501 
>   trunk/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/filemgr/FileManagerConn.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/CASProductHandler.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DataDeliveryServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DatasetDeliveryServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/services/CasProductJaxrsService.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/servlets/CasProductJaxrsServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFDatasetServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFProductServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductTransferServlet.java 1693501 
> 
> Diff: https://reviews.apache.org/r/36953/diff/
> 
> 
> Testing
> -------
> 
> Used existing tests, applied them for Avro RPC. 
> 
> Installed filemanager module, and started it as described in the oodt documentation. Ingested a simple Generic file. Used querry-tool.
> 
> 
> Thanks,
> 
> Radu Manole
> 
>

Re: Review Request 36953: Full implementation of AvroRPC alongside XMLRPC in filemgr module

Posted by Chris Mattmann <ma...@apache.org>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/36953/#review96119
-----------------------------------------------------------


This was a LOT better Radu. You are getting close.

Some big things to remember:

1. If code is commented out, you should remove it.
2. We shouldn't disable tests e.g., Tika, by commenting it out. If the test isn't passing, then something is wrong (we have a regression)
3. You need to update RADIX and make sure this is working with OODT RADIX.

- Chris Mattmann


On Aug. 21, 2015, 7:53 p.m., Radu Manole wrote:
> 
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/36953/
> -----------------------------------------------------------
> 
> (Updated Aug. 21, 2015, 7:53 p.m.)
> 
> 
> Review request for oodt, Lewis McGibbney and Chris Mattmann.
> 
> 
> Repository: oodt
> 
> 
> Description
> -------
> 
> Separated the FileManager logic from rpc communication createing interfaces for server/client side (FileManagerCient, FileManagerServer).Refactor existing XMLRPC server/client so it would use communication interfaces. Created Avro schemas and protocol for ipc communication, (classes that have the prefixe Avro), with a Factory class AvroTypeFactory that transforms from oodt to avro and from avro to oodt classes. Implement avro server/client. For changeability I created a Factory (RpcCommunicationFactory) that can be configurable to use either avro or xmlrpc for IPC. Because of this, anywhere in project where on instantiation of server/client rewrite to use this factory.
> 
> 
> Diffs
> -----
> 
>   trunk/app/fmbrowser/src/main/java/org/apache/oodt/cas/filemgr/browser/model/CasDB.java 1693501 
>   trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FilemgrUniquenessChecker.java 1693501 
>   trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/comparator/FilemgrUniquenessCheckComparator.java 1693501 
>   trunk/crawler/src/site/xdoc/user/index.xml 1693501 
>   trunk/filemgr/pom.xml 1693501 
>   trunk/filemgr/src/main/avro/types/AvroComplexQuery.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroElement.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroExtractorSpec.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroFileTransferStatus.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroFilterAlgor.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroMetadata.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroMimeType.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProduct.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProductPage.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProductType.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQuery.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryCriteria.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryFilter.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryResult.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroReference.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroTypeHandler.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/protocol.avdl PRE-CREATION 
>   trunk/filemgr/src/main/bin/filemgr 1693501 
>   trunk/filemgr/src/main/bin/filemgr-client 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/AbstractDeleteProductCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/DeleteProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/FileManagerCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetFirstPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetLastPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNextPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNumProductsCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetPrevPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetProductPercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/RetrieveFilesCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/AvroLocalDataTransferer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/InPlaceDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/LocalCache.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/ProductMetKeys.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManager.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClientMain.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServerMain.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ExpImpCatalog.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataBasedProductMover.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataDumper.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ProductDumper.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/AvroTypeFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/RpcCommunicationFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/XmlRpcStructFactory.java 1693501 
>   trunk/filemgr/src/main/resources/filemgr.properties 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/TestFileManagerCli.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/UseMockClientCmdLineActionStore.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestAddProductTypeCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDumpMetadataCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransferCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransfersCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFilePercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFirstPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetLastPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNextPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNumProductsCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetPrevPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductPercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductTypeByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestHasProductCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestIngestProductCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestLuceneQueryCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestSqlQueryCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestCachedIngester.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestLocalCache.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestRmiCache.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestStdIngester.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/metadata/extractors/examples/TestTikaAutoDetectExtractor.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/structs/type/TestTypeHandler.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/MockFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManager.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/rpc/TestServerClientFactories.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestExpImpCatalog.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestMetadataBasedProductMover.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestAvroTypesFactory.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestXmlRpcStructFactory.java 1693501 
>   trunk/filemgr/src/test/resources/createProduct/test.txt PRE-CREATION 
>   trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/filemgr/src/main/resources/bin/filemgr 1693501 
>   trunk/pcs/core/src/main/java/org/apache/oodt/pcs/util/FileManagerUtils.java 1693501 
>   trunk/pge/src/main/java/org/apache/oodt/cas/pge/staging/FileStager.java 1693501 
>   trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java 1693501 
>   trunk/pge/src/test/java/org/apache/oodt/cas/pge/staging/TestFileStager.java 1693501 
>   trunk/profile/src/main/java/org/apache/oodt/profile/handlers/cas/CASProfileHandler.java 1693501 
>   trunk/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/filemgr/FileManagerConn.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/CASProductHandler.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DataDeliveryServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DatasetDeliveryServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/services/CasProductJaxrsService.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/servlets/CasProductJaxrsServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFDatasetServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFProductServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductTransferServlet.java 1693501 
> 
> Diff: https://reviews.apache.org/r/36953/diff/
> 
> 
> Testing
> -------
> 
> Used existing tests, applied them for Avro RPC. 
> 
> Installed filemanager module, and started it as described in the oodt documentation. Ingested a simple Generic file. Used querry-tool.
> 
> 
> Thanks,
> 
> Radu Manole
> 
>

Re: Review Request 36953: Full implementation of AvroRPC alongside XMLRPC in filemgr module

Posted by Chris Mattmann <ma...@apache.org>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/36953/#review96223
-----------------------------------------------------------



trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/AvroLocalDataTransferer.java (line 1)
<https://reviews.apache.org/r/36953/#comment151545>

    it feels wrong that this is a whole new class, when I'd say 95% of it is the same as LocalDataTransferer. Can we refactor both into a superclass, LocalDataTransferer, and then inherit from it, and then use XmlRpc, and/or Avro, based on the sub class.



trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManager.java (line 27)
<https://reviews.apache.org/r/36953/#comment151562>

    don't use star imports, please.



trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManager.java (line 28)
<https://reviews.apache.org/r/36953/#comment151563>

    don't use star imports, please.



trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java (line 638)
<https://reviews.apache.org/r/36953/#comment151564>

    why does this method need to change names?



trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/AvroTypeFactory.java (line 21)
<https://reviews.apache.org/r/36953/#comment151565>

    don't use star imports, please.



trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/AvroTypeFactory.java (line 22)
<https://reviews.apache.org/r/36953/#comment151566>

    don't use star imports, please.


- Chris Mattmann


On Aug. 24, 2015, 5:19 p.m., Radu Manole wrote:
> 
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/36953/
> -----------------------------------------------------------
> 
> (Updated Aug. 24, 2015, 5:19 p.m.)
> 
> 
> Review request for oodt, Lewis McGibbney and Chris Mattmann.
> 
> 
> Repository: oodt
> 
> 
> Description
> -------
> 
> Separated the FileManager logic from rpc communication createing interfaces for server/client side (FileManagerCient, FileManagerServer).Refactor existing XMLRPC server/client so it would use communication interfaces. Created Avro schemas and protocol for ipc communication, (classes that have the prefixe Avro), with a Factory class AvroTypeFactory that transforms from oodt to avro and from avro to oodt classes. Implement avro server/client. For changeability I created a Factory (RpcCommunicationFactory) that can be configurable to use either avro or xmlrpc for IPC. Because of this, anywhere in project where on instantiation of server/client rewrite to use this factory.
> 
> 
> Diffs
> -----
> 
>   trunk/app/fmbrowser/src/main/java/org/apache/oodt/cas/filemgr/browser/model/CasDB.java 1693501 
>   trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FilemgrUniquenessChecker.java 1693501 
>   trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/comparator/FilemgrUniquenessCheckComparator.java 1693501 
>   trunk/filemgr/pom.xml 1693501 
>   trunk/filemgr/src/main/avro/types/AvroComplexQuery.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroElement.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroExtractorSpec.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroFileTransferStatus.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroFilterAlgor.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroMetadata.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroMimeType.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProduct.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProductPage.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProductType.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQuery.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryCriteria.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryFilter.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryResult.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroReference.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroTypeHandler.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/protocol.avdl PRE-CREATION 
>   trunk/filemgr/src/main/bin/filemgr 1693501 
>   trunk/filemgr/src/main/bin/filemgr-client 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/AbstractDeleteProductCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/DeleteProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/FileManagerCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetFirstPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetLastPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNextPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNumProductsCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetPrevPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetProductPercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/RetrieveFilesCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/AvroLocalDataTransferer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/InPlaceDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/LocalCache.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/ProductMetKeys.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManager.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClientMain.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServerMain.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ExpImpCatalog.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataBasedProductMover.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataDumper.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ProductDumper.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/AvroTypeFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/RpcCommunicationFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/XmlRpcStructFactory.java 1693501 
>   trunk/filemgr/src/main/resources/filemgr.properties 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/TestFileManagerCli.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/UseMockClientCmdLineActionStore.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestAddProductTypeCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDumpMetadataCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransferCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransfersCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFilePercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFirstPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetLastPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNextPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNumProductsCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetPrevPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductPercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductTypeByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestHasProductCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestIngestProductCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestLuceneQueryCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestSqlQueryCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestCachedIngester.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestLocalCache.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestRmiCache.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestStdIngester.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/metadata/extractors/examples/TestTikaAutoDetectExtractor.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/structs/type/TestTypeHandler.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/MockFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManager.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/rpc/TestServerClientFactories.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestExpImpCatalog.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestMetadataBasedProductMover.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestAvroTypesFactory.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestXmlRpcStructFactory.java 1693501 
>   trunk/filemgr/src/test/resources/createProduct/test.txt PRE-CREATION 
>   trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/filemgr/src/main/resources/bin/filemgr 1693501 
>   trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/filemgr/src/main/resources/bin/filemgr-client 1693501 
>   trunk/pcs/core/src/main/java/org/apache/oodt/pcs/util/FileManagerUtils.java 1693501 
>   trunk/pge/src/main/java/org/apache/oodt/cas/pge/staging/FileStager.java 1693501 
>   trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java 1693501 
>   trunk/pge/src/test/java/org/apache/oodt/cas/pge/staging/TestFileStager.java 1693501 
>   trunk/profile/src/main/java/org/apache/oodt/profile/handlers/cas/CASProfileHandler.java 1693501 
>   trunk/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/filemgr/FileManagerConn.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/CASProductHandler.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DataDeliveryServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DatasetDeliveryServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/services/CasProductJaxrsService.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/servlets/CasProductJaxrsServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFDatasetServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFProductServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductTransferServlet.java 1693501 
> 
> Diff: https://reviews.apache.org/r/36953/diff/
> 
> 
> Testing
> -------
> 
> Used existing tests, applied them for Avro RPC. 
> 
> Installed filemanager module, and started it as described in the oodt documentation. Ingested a simple Generic file. Used querry-tool.
> 
> 
> Thanks,
> 
> Radu Manole
> 
>

Re: Review Request 36953: Full implementation of AvroRPC alongside XMLRPC in filemgr module

Posted by Chris Mattmann <ma...@apache.org>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/36953/#review97952
-----------------------------------------------------------



trunk/filemgr/pom.xml (line 149)
<https://reviews.apache.org/r/36953/#comment154087>

    I don't think we should make it Avro by default.



trunk/filemgr/src/main/resources/filemgr.properties (line 19)
<https://reviews.apache.org/r/36953/#comment154088>

    avro by default?


- Chris Mattmann


On Aug. 27, 2015, 10:54 p.m., Radu Manole wrote:
> 
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/36953/
> -----------------------------------------------------------
> 
> (Updated Aug. 27, 2015, 10:54 p.m.)
> 
> 
> Review request for oodt, Lewis McGibbney and Chris Mattmann.
> 
> 
> Repository: oodt
> 
> 
> Description
> -------
> 
> Separated the FileManager logic from rpc communication createing interfaces for server/client side (FileManagerCient, FileManagerServer).Refactor existing XMLRPC server/client so it would use communication interfaces. Created Avro schemas and protocol for ipc communication, (classes that have the prefixe Avro), with a Factory class AvroTypeFactory that transforms from oodt to avro and from avro to oodt classes. Implement avro server/client. For changeability I created a Factory (RpcCommunicationFactory) that can be configurable to use either avro or xmlrpc for IPC. Because of this, anywhere in project where on instantiation of server/client rewrite to use this factory.
> 
> 
> Diffs
> -----
> 
>   trunk/app/fmbrowser/src/main/java/org/apache/oodt/cas/filemgr/browser/model/CasDB.java 1693501 
>   trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FilemgrUniquenessChecker.java 1693501 
>   trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/comparator/FilemgrUniquenessCheckComparator.java 1693501 
>   trunk/filemgr/pom.xml 1693501 
>   trunk/filemgr/src/main/avro/types/AvroComplexQuery.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroElement.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroExtractorSpec.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroFileTransferStatus.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroFilterAlgor.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroMetadata.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroMimeType.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProduct.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProductPage.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProductType.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQuery.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryCriteria.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryFilter.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryResult.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroReference.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroTypeHandler.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/protocol.avdl PRE-CREATION 
>   trunk/filemgr/src/main/bin/filemgr 1693501 
>   trunk/filemgr/src/main/bin/filemgr-client 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/AbstractDeleteProductCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/DeleteProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/FileManagerCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetFirstPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetLastPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNextPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNumProductsCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetPrevPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetProductPercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/RetrieveFilesCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/InPlaceDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/LocalCache.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/ProductMetKeys.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManager.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClientMain.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServerMain.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ExpImpCatalog.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataBasedProductMover.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataDumper.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ProductDumper.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/AvroTypeFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/RpcCommunicationFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/XmlRpcStructFactory.java 1693501 
>   trunk/filemgr/src/main/resources/filemgr.properties 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/TestFileManagerCli.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/UseMockClientCmdLineActionStore.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestAddProductTypeCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDumpMetadataCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransferCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransfersCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFilePercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFirstPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetLastPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNextPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNumProductsCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetPrevPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductPercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductTypeByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestHasProductCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestIngestProductCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestLuceneQueryCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestSqlQueryCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestCachedIngester.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestLocalCache.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestRmiCache.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestStdIngester.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/metadata/extractors/examples/TestTikaAutoDetectExtractor.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/structs/type/TestTypeHandler.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/MockFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManager.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/rpc/TestServerClientFactories.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestExpImpCatalog.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestMetadataBasedProductMover.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestAvroTypesFactory.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestXmlRpcStructFactory.java 1693501 
>   trunk/filemgr/src/test/resources/createProduct/test.txt PRE-CREATION 
>   trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/filemgr/src/main/resources/bin/filemgr 1693501 
>   trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/filemgr/src/main/resources/bin/filemgr-client 1693501 
>   trunk/pcs/core/src/main/java/org/apache/oodt/pcs/util/FileManagerUtils.java 1693501 
>   trunk/pge/src/main/java/org/apache/oodt/cas/pge/staging/FileStager.java 1693501 
>   trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java 1693501 
>   trunk/pge/src/test/java/org/apache/oodt/cas/pge/staging/TestFileStager.java 1693501 
>   trunk/profile/src/main/java/org/apache/oodt/profile/handlers/cas/CASProfileHandler.java 1693501 
>   trunk/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/filemgr/FileManagerConn.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/CASProductHandler.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DataDeliveryServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DatasetDeliveryServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/services/CasProductJaxrsService.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/servlets/CasProductJaxrsServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFDatasetServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFProductServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductTransferServlet.java 1693501 
> 
> Diff: https://reviews.apache.org/r/36953/diff/
> 
> 
> Testing
> -------
> 
> Used existing tests, applied them for Avro RPC. 
> 
> Installed filemanager module, and started it as described in the oodt documentation. Ingested a simple Generic file. Used querry-tool.
> 
> 
> Thanks,
> 
> Radu Manole
> 
>

Re: Review Request 36953: Full implementation of AvroRPC alongside XMLRPC in filemgr module

Posted by Chris Mattmann <ma...@apache.org>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/36953/#review97953
-----------------------------------------------------------

Ship it!


Ship It!

- Chris Mattmann


On Aug. 27, 2015, 10:54 p.m., Radu Manole wrote:
> 
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/36953/
> -----------------------------------------------------------
> 
> (Updated Aug. 27, 2015, 10:54 p.m.)
> 
> 
> Review request for oodt, Lewis McGibbney and Chris Mattmann.
> 
> 
> Repository: oodt
> 
> 
> Description
> -------
> 
> Separated the FileManager logic from rpc communication createing interfaces for server/client side (FileManagerCient, FileManagerServer).Refactor existing XMLRPC server/client so it would use communication interfaces. Created Avro schemas and protocol for ipc communication, (classes that have the prefixe Avro), with a Factory class AvroTypeFactory that transforms from oodt to avro and from avro to oodt classes. Implement avro server/client. For changeability I created a Factory (RpcCommunicationFactory) that can be configurable to use either avro or xmlrpc for IPC. Because of this, anywhere in project where on instantiation of server/client rewrite to use this factory.
> 
> 
> Diffs
> -----
> 
>   trunk/app/fmbrowser/src/main/java/org/apache/oodt/cas/filemgr/browser/model/CasDB.java 1693501 
>   trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FilemgrUniquenessChecker.java 1693501 
>   trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/comparator/FilemgrUniquenessCheckComparator.java 1693501 
>   trunk/filemgr/pom.xml 1693501 
>   trunk/filemgr/src/main/avro/types/AvroComplexQuery.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroElement.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroExtractorSpec.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroFileTransferStatus.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroFilterAlgor.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroMetadata.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroMimeType.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProduct.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProductPage.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProductType.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQuery.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryCriteria.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryFilter.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryResult.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroReference.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroTypeHandler.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/protocol.avdl PRE-CREATION 
>   trunk/filemgr/src/main/bin/filemgr 1693501 
>   trunk/filemgr/src/main/bin/filemgr-client 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/AbstractDeleteProductCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/DeleteProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/FileManagerCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetFirstPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetLastPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNextPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNumProductsCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetPrevPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetProductPercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/RetrieveFilesCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/InPlaceDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/LocalCache.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/ProductMetKeys.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManager.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClientMain.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServerMain.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ExpImpCatalog.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataBasedProductMover.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataDumper.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ProductDumper.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/AvroTypeFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/RpcCommunicationFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/XmlRpcStructFactory.java 1693501 
>   trunk/filemgr/src/main/resources/filemgr.properties 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/TestFileManagerCli.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/UseMockClientCmdLineActionStore.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestAddProductTypeCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDumpMetadataCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransferCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransfersCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFilePercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFirstPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetLastPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNextPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNumProductsCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetPrevPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductPercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductTypeByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestHasProductCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestIngestProductCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestLuceneQueryCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestSqlQueryCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestCachedIngester.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestLocalCache.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestRmiCache.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestStdIngester.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/metadata/extractors/examples/TestTikaAutoDetectExtractor.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/structs/type/TestTypeHandler.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/MockFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManager.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/rpc/TestServerClientFactories.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestExpImpCatalog.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestMetadataBasedProductMover.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestAvroTypesFactory.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestXmlRpcStructFactory.java 1693501 
>   trunk/filemgr/src/test/resources/createProduct/test.txt PRE-CREATION 
>   trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/filemgr/src/main/resources/bin/filemgr 1693501 
>   trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/filemgr/src/main/resources/bin/filemgr-client 1693501 
>   trunk/pcs/core/src/main/java/org/apache/oodt/pcs/util/FileManagerUtils.java 1693501 
>   trunk/pge/src/main/java/org/apache/oodt/cas/pge/staging/FileStager.java 1693501 
>   trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java 1693501 
>   trunk/pge/src/test/java/org/apache/oodt/cas/pge/staging/TestFileStager.java 1693501 
>   trunk/profile/src/main/java/org/apache/oodt/profile/handlers/cas/CASProfileHandler.java 1693501 
>   trunk/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/filemgr/FileManagerConn.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/CASProductHandler.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DataDeliveryServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DatasetDeliveryServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/services/CasProductJaxrsService.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/servlets/CasProductJaxrsServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFDatasetServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFProductServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductTransferServlet.java 1693501 
> 
> Diff: https://reviews.apache.org/r/36953/diff/
> 
> 
> Testing
> -------
> 
> Used existing tests, applied them for Avro RPC. 
> 
> Installed filemanager module, and started it as described in the oodt documentation. Ingested a simple Generic file. Used querry-tool.
> 
> 
> Thanks,
> 
> Radu Manole
> 
>

Re: Review Request 36953: Full implementation of AvroRPC alongside XMLRPC in filemgr module

Posted by Lewis McGibbney <le...@apache.org>.

> On Sept. 8, 2015, 1:27 a.m., Chris Mattmann wrote:
> > This is good enough to ship. I was wondering why it's set to Avro by default but since this will be in a branch this makes sense. Great work Radu! Lewis, please feel free to commit @lewismmc

ACK Chris.
I have an issue with the local tests which fail as described below
https://issues.apache.org/jira/browse/OODT-858


- Lewis


-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/36953/#review97954
-----------------------------------------------------------


On Aug. 27, 2015, 10:54 p.m., Radu Manole wrote:
> 
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/36953/
> -----------------------------------------------------------
> 
> (Updated Aug. 27, 2015, 10:54 p.m.)
> 
> 
> Review request for oodt, Lewis McGibbney and Chris Mattmann.
> 
> 
> Repository: oodt
> 
> 
> Description
> -------
> 
> Separated the FileManager logic from rpc communication createing interfaces for server/client side (FileManagerCient, FileManagerServer).Refactor existing XMLRPC server/client so it would use communication interfaces. Created Avro schemas and protocol for ipc communication, (classes that have the prefixe Avro), with a Factory class AvroTypeFactory that transforms from oodt to avro and from avro to oodt classes. Implement avro server/client. For changeability I created a Factory (RpcCommunicationFactory) that can be configurable to use either avro or xmlrpc for IPC. Because of this, anywhere in project where on instantiation of server/client rewrite to use this factory.
> 
> 
> Diffs
> -----
> 
>   trunk/app/fmbrowser/src/main/java/org/apache/oodt/cas/filemgr/browser/model/CasDB.java 1693501 
>   trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FilemgrUniquenessChecker.java 1693501 
>   trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/comparator/FilemgrUniquenessCheckComparator.java 1693501 
>   trunk/filemgr/pom.xml 1693501 
>   trunk/filemgr/src/main/avro/types/AvroComplexQuery.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroElement.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroExtractorSpec.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroFileTransferStatus.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroFilterAlgor.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroMetadata.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroMimeType.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProduct.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProductPage.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProductType.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQuery.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryCriteria.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryFilter.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryResult.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroReference.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroTypeHandler.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/protocol.avdl PRE-CREATION 
>   trunk/filemgr/src/main/bin/filemgr 1693501 
>   trunk/filemgr/src/main/bin/filemgr-client 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/AbstractDeleteProductCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/DeleteProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/FileManagerCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetFirstPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetLastPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNextPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNumProductsCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetPrevPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetProductPercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/RetrieveFilesCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/InPlaceDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/LocalCache.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/ProductMetKeys.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManager.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClientMain.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServerMain.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ExpImpCatalog.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataBasedProductMover.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataDumper.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ProductDumper.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/AvroTypeFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/RpcCommunicationFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/XmlRpcStructFactory.java 1693501 
>   trunk/filemgr/src/main/resources/filemgr.properties 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/TestFileManagerCli.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/UseMockClientCmdLineActionStore.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestAddProductTypeCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDumpMetadataCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransferCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransfersCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFilePercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFirstPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetLastPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNextPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNumProductsCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetPrevPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductPercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductTypeByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestHasProductCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestIngestProductCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestLuceneQueryCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestSqlQueryCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestCachedIngester.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestLocalCache.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestRmiCache.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestStdIngester.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/metadata/extractors/examples/TestTikaAutoDetectExtractor.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/structs/type/TestTypeHandler.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/MockFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManager.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/rpc/TestServerClientFactories.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestExpImpCatalog.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestMetadataBasedProductMover.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestAvroTypesFactory.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestXmlRpcStructFactory.java 1693501 
>   trunk/filemgr/src/test/resources/createProduct/test.txt PRE-CREATION 
>   trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/filemgr/src/main/resources/bin/filemgr 1693501 
>   trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/filemgr/src/main/resources/bin/filemgr-client 1693501 
>   trunk/pcs/core/src/main/java/org/apache/oodt/pcs/util/FileManagerUtils.java 1693501 
>   trunk/pge/src/main/java/org/apache/oodt/cas/pge/staging/FileStager.java 1693501 
>   trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java 1693501 
>   trunk/pge/src/test/java/org/apache/oodt/cas/pge/staging/TestFileStager.java 1693501 
>   trunk/profile/src/main/java/org/apache/oodt/profile/handlers/cas/CASProfileHandler.java 1693501 
>   trunk/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/filemgr/FileManagerConn.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/CASProductHandler.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DataDeliveryServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DatasetDeliveryServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/services/CasProductJaxrsService.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/servlets/CasProductJaxrsServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFDatasetServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFProductServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductTransferServlet.java 1693501 
> 
> Diff: https://reviews.apache.org/r/36953/diff/
> 
> 
> Testing
> -------
> 
> Used existing tests, applied them for Avro RPC. 
> 
> Installed filemanager module, and started it as described in the oodt documentation. Ingested a simple Generic file. Used querry-tool.
> 
> 
> Thanks,
> 
> Radu Manole
> 
>

Re: Review Request 36953: Full implementation of AvroRPC alongside XMLRPC in filemgr module

Posted by Chris Mattmann <ma...@apache.org>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/36953/#review97954
-----------------------------------------------------------


This is good enough to ship. I was wondering why it's set to Avro by default but since this will be in a branch this makes sense. Great work Radu! Lewis, please feel free to commit @lewismmc

- Chris Mattmann


On Aug. 27, 2015, 10:54 p.m., Radu Manole wrote:
> 
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/36953/
> -----------------------------------------------------------
> 
> (Updated Aug. 27, 2015, 10:54 p.m.)
> 
> 
> Review request for oodt, Lewis McGibbney and Chris Mattmann.
> 
> 
> Repository: oodt
> 
> 
> Description
> -------
> 
> Separated the FileManager logic from rpc communication createing interfaces for server/client side (FileManagerCient, FileManagerServer).Refactor existing XMLRPC server/client so it would use communication interfaces. Created Avro schemas and protocol for ipc communication, (classes that have the prefixe Avro), with a Factory class AvroTypeFactory that transforms from oodt to avro and from avro to oodt classes. Implement avro server/client. For changeability I created a Factory (RpcCommunicationFactory) that can be configurable to use either avro or xmlrpc for IPC. Because of this, anywhere in project where on instantiation of server/client rewrite to use this factory.
> 
> 
> Diffs
> -----
> 
>   trunk/app/fmbrowser/src/main/java/org/apache/oodt/cas/filemgr/browser/model/CasDB.java 1693501 
>   trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FilemgrUniquenessChecker.java 1693501 
>   trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/comparator/FilemgrUniquenessCheckComparator.java 1693501 
>   trunk/filemgr/pom.xml 1693501 
>   trunk/filemgr/src/main/avro/types/AvroComplexQuery.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroElement.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroExtractorSpec.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroFileTransferStatus.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroFilterAlgor.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroMetadata.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroMimeType.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProduct.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProductPage.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProductType.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQuery.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryCriteria.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryFilter.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryResult.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroReference.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroTypeHandler.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/protocol.avdl PRE-CREATION 
>   trunk/filemgr/src/main/bin/filemgr 1693501 
>   trunk/filemgr/src/main/bin/filemgr-client 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/AbstractDeleteProductCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/DeleteProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/FileManagerCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetFirstPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetLastPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNextPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNumProductsCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetPrevPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetProductPercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/RetrieveFilesCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/InPlaceDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/LocalCache.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/ProductMetKeys.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManager.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClientMain.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServerMain.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ExpImpCatalog.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataBasedProductMover.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataDumper.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ProductDumper.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/AvroTypeFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/RpcCommunicationFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/XmlRpcStructFactory.java 1693501 
>   trunk/filemgr/src/main/resources/filemgr.properties 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/TestFileManagerCli.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/UseMockClientCmdLineActionStore.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestAddProductTypeCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDumpMetadataCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransferCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransfersCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFilePercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFirstPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetLastPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNextPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNumProductsCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetPrevPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductPercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductTypeByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestHasProductCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestIngestProductCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestLuceneQueryCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestSqlQueryCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestCachedIngester.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestLocalCache.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestRmiCache.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestStdIngester.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/metadata/extractors/examples/TestTikaAutoDetectExtractor.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/structs/type/TestTypeHandler.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/MockFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManager.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/rpc/TestServerClientFactories.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestExpImpCatalog.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestMetadataBasedProductMover.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestAvroTypesFactory.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestXmlRpcStructFactory.java 1693501 
>   trunk/filemgr/src/test/resources/createProduct/test.txt PRE-CREATION 
>   trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/filemgr/src/main/resources/bin/filemgr 1693501 
>   trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/filemgr/src/main/resources/bin/filemgr-client 1693501 
>   trunk/pcs/core/src/main/java/org/apache/oodt/pcs/util/FileManagerUtils.java 1693501 
>   trunk/pge/src/main/java/org/apache/oodt/cas/pge/staging/FileStager.java 1693501 
>   trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java 1693501 
>   trunk/pge/src/test/java/org/apache/oodt/cas/pge/staging/TestFileStager.java 1693501 
>   trunk/profile/src/main/java/org/apache/oodt/profile/handlers/cas/CASProfileHandler.java 1693501 
>   trunk/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/filemgr/FileManagerConn.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/CASProductHandler.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DataDeliveryServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DatasetDeliveryServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/services/CasProductJaxrsService.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/servlets/CasProductJaxrsServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFDatasetServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFProductServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductTransferServlet.java 1693501 
> 
> Diff: https://reviews.apache.org/r/36953/diff/
> 
> 
> Testing
> -------
> 
> Used existing tests, applied them for Avro RPC. 
> 
> Installed filemanager module, and started it as described in the oodt documentation. Ingested a simple Generic file. Used querry-tool.
> 
> 
> Thanks,
> 
> Radu Manole
> 
>

Re: Review Request 36953: Full implementation of AvroRPC alongside XMLRPC in filemgr module

Posted by Radu Manole <ma...@gmail.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/36953/
-----------------------------------------------------------

(Updated Aug. 27, 2015, 10:54 p.m.)


Review request for oodt, Lewis McGibbney and Chris Mattmann.


Changes
-------

Fixed bugs, tested and looked throught all components for minor changes.


Repository: oodt


Description
-------

Separated the FileManager logic from rpc communication createing interfaces for server/client side (FileManagerCient, FileManagerServer).Refactor existing XMLRPC server/client so it would use communication interfaces. Created Avro schemas and protocol for ipc communication, (classes that have the prefixe Avro), with a Factory class AvroTypeFactory that transforms from oodt to avro and from avro to oodt classes. Implement avro server/client. For changeability I created a Factory (RpcCommunicationFactory) that can be configurable to use either avro or xmlrpc for IPC. Because of this, anywhere in project where on instantiation of server/client rewrite to use this factory.


Diffs (updated)
-----

  trunk/app/fmbrowser/src/main/java/org/apache/oodt/cas/filemgr/browser/model/CasDB.java 1693501 
  trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FilemgrUniquenessChecker.java 1693501 
  trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/comparator/FilemgrUniquenessCheckComparator.java 1693501 
  trunk/filemgr/pom.xml 1693501 
  trunk/filemgr/src/main/avro/types/AvroComplexQuery.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroElement.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroExtractorSpec.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroFileTransferStatus.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroFilterAlgor.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroMetadata.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroMimeType.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroProduct.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroProductPage.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroProductType.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroQuery.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroQueryCriteria.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroQueryFilter.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroQueryResult.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroReference.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroTypeHandler.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/protocol.avdl PRE-CREATION 
  trunk/filemgr/src/main/bin/filemgr 1693501 
  trunk/filemgr/src/main/bin/filemgr-client 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/AbstractDeleteProductCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/DeleteProductByIdCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/FileManagerCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetFirstPageCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetLastPageCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNextPageCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNumProductsCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetPrevPageCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetProductPercentTransferredCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/RetrieveFilesCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/InPlaceDataTransferer.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/LocalCache.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/ProductMetKeys.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerServer.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManager.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClientMain.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServer.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServerMain.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerServer.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerClientFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerServerFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerClientFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerServerFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerClientFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerServerFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ExpImpCatalog.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataBasedProductMover.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataDumper.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ProductDumper.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/AvroTypeFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/RpcCommunicationFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/XmlRpcStructFactory.java 1693501 
  trunk/filemgr/src/main/resources/filemgr.properties 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/TestFileManagerCli.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/UseMockClientCmdLineActionStore.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java PRE-CREATION 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestAddProductTypeCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByIdCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByNameCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDumpMetadataCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransferCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransfersCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFilePercentTransferredCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFirstPageCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetLastPageCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNextPageCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNumProductsCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetPrevPageCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByIdCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByNameCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductPercentTransferredCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductTypeByNameCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestHasProductCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestIngestProductCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestLuceneQueryCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestSqlQueryCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestCachedIngester.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestLocalCache.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestRmiCache.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestStdIngester.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/metadata/extractors/examples/TestTikaAutoDetectExtractor.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/structs/type/TestTypeHandler.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/MockFileManagerClient.java PRE-CREATION 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManager.java PRE-CREATION 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManagerClient.java PRE-CREATION 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/rpc/TestServerClientFactories.java PRE-CREATION 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestExpImpCatalog.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestMetadataBasedProductMover.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestAvroTypesFactory.java PRE-CREATION 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestXmlRpcStructFactory.java 1693501 
  trunk/filemgr/src/test/resources/createProduct/test.txt PRE-CREATION 
  trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/filemgr/src/main/resources/bin/filemgr 1693501 
  trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/filemgr/src/main/resources/bin/filemgr-client 1693501 
  trunk/pcs/core/src/main/java/org/apache/oodt/pcs/util/FileManagerUtils.java 1693501 
  trunk/pge/src/main/java/org/apache/oodt/cas/pge/staging/FileStager.java 1693501 
  trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java 1693501 
  trunk/pge/src/test/java/org/apache/oodt/cas/pge/staging/TestFileStager.java 1693501 
  trunk/profile/src/main/java/org/apache/oodt/profile/handlers/cas/CASProfileHandler.java 1693501 
  trunk/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/filemgr/FileManagerConn.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/CASProductHandler.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DataDeliveryServlet.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DatasetDeliveryServlet.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/services/CasProductJaxrsService.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/servlets/CasProductJaxrsServlet.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFDatasetServlet.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFProductServlet.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductTransferServlet.java 1693501 

Diff: https://reviews.apache.org/r/36953/diff/


Testing
-------

Used existing tests, applied them for Avro RPC. 

Installed filemanager module, and started it as described in the oodt documentation. Ingested a simple Generic file. Used querry-tool.


Thanks,

Radu Manole

Re: Review Request 36953: Full implementation of AvroRPC alongside XMLRPC in filemgr module

Posted by Lewis McGibbney <le...@apache.org>.

> On Aug. 24, 2015, 11:38 p.m., Chris Mattmann wrote:
> > This looks a lot better, Radu. A couple more things:
> > 
> > 1. Please don't use star imports. Makes it too hard to track which classes are imported. Please use FQDN class names.
> > 2. What about the curator and other places the File Manager is used, e.g., pcs-services, etc. It seems like these changes would break those portions of the system. Have you tried to build all of OODT with these changes?
> > 3. As you noted, still need to update RADIX.

I am not sure that the curator was scoped for inclusion in this portion of the work.
RADIX needs to be updated and I am prepared to help out to get that done.


- Lewis


-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/36953/#review96236
-----------------------------------------------------------


On Aug. 24, 2015, 5:19 p.m., Radu Manole wrote:
> 
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/36953/
> -----------------------------------------------------------
> 
> (Updated Aug. 24, 2015, 5:19 p.m.)
> 
> 
> Review request for oodt, Lewis McGibbney and Chris Mattmann.
> 
> 
> Repository: oodt
> 
> 
> Description
> -------
> 
> Separated the FileManager logic from rpc communication createing interfaces for server/client side (FileManagerCient, FileManagerServer).Refactor existing XMLRPC server/client so it would use communication interfaces. Created Avro schemas and protocol for ipc communication, (classes that have the prefixe Avro), with a Factory class AvroTypeFactory that transforms from oodt to avro and from avro to oodt classes. Implement avro server/client. For changeability I created a Factory (RpcCommunicationFactory) that can be configurable to use either avro or xmlrpc for IPC. Because of this, anywhere in project where on instantiation of server/client rewrite to use this factory.
> 
> 
> Diffs
> -----
> 
>   trunk/app/fmbrowser/src/main/java/org/apache/oodt/cas/filemgr/browser/model/CasDB.java 1693501 
>   trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FilemgrUniquenessChecker.java 1693501 
>   trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/comparator/FilemgrUniquenessCheckComparator.java 1693501 
>   trunk/filemgr/pom.xml 1693501 
>   trunk/filemgr/src/main/avro/types/AvroComplexQuery.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroElement.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroExtractorSpec.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroFileTransferStatus.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroFilterAlgor.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroMetadata.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroMimeType.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProduct.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProductPage.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProductType.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQuery.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryCriteria.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryFilter.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryResult.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroReference.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroTypeHandler.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/protocol.avdl PRE-CREATION 
>   trunk/filemgr/src/main/bin/filemgr 1693501 
>   trunk/filemgr/src/main/bin/filemgr-client 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/AbstractDeleteProductCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/DeleteProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/FileManagerCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetFirstPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetLastPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNextPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNumProductsCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetPrevPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetProductPercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/RetrieveFilesCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/AvroLocalDataTransferer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/InPlaceDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/LocalCache.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/ProductMetKeys.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManager.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClientMain.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServerMain.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ExpImpCatalog.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataBasedProductMover.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataDumper.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ProductDumper.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/AvroTypeFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/RpcCommunicationFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/XmlRpcStructFactory.java 1693501 
>   trunk/filemgr/src/main/resources/filemgr.properties 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/TestFileManagerCli.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/UseMockClientCmdLineActionStore.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestAddProductTypeCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDumpMetadataCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransferCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransfersCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFilePercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFirstPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetLastPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNextPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNumProductsCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetPrevPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductPercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductTypeByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestHasProductCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestIngestProductCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestLuceneQueryCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestSqlQueryCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestCachedIngester.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestLocalCache.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestRmiCache.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestStdIngester.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/metadata/extractors/examples/TestTikaAutoDetectExtractor.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/structs/type/TestTypeHandler.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/MockFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManager.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/rpc/TestServerClientFactories.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestExpImpCatalog.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestMetadataBasedProductMover.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestAvroTypesFactory.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestXmlRpcStructFactory.java 1693501 
>   trunk/filemgr/src/test/resources/createProduct/test.txt PRE-CREATION 
>   trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/filemgr/src/main/resources/bin/filemgr 1693501 
>   trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/filemgr/src/main/resources/bin/filemgr-client 1693501 
>   trunk/pcs/core/src/main/java/org/apache/oodt/pcs/util/FileManagerUtils.java 1693501 
>   trunk/pge/src/main/java/org/apache/oodt/cas/pge/staging/FileStager.java 1693501 
>   trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java 1693501 
>   trunk/pge/src/test/java/org/apache/oodt/cas/pge/staging/TestFileStager.java 1693501 
>   trunk/profile/src/main/java/org/apache/oodt/profile/handlers/cas/CASProfileHandler.java 1693501 
>   trunk/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/filemgr/FileManagerConn.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/CASProductHandler.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DataDeliveryServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DatasetDeliveryServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/services/CasProductJaxrsService.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/servlets/CasProductJaxrsServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFDatasetServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFProductServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductTransferServlet.java 1693501 
> 
> Diff: https://reviews.apache.org/r/36953/diff/
> 
> 
> Testing
> -------
> 
> Used existing tests, applied them for Avro RPC. 
> 
> Installed filemanager module, and started it as described in the oodt documentation. Ingested a simple Generic file. Used querry-tool.
> 
> 
> Thanks,
> 
> Radu Manole
> 
>

Re: Review Request 36953: Full implementation of AvroRPC alongside XMLRPC in filemgr module

Posted by Chris Mattmann <ma...@apache.org>.

> On Aug. 24, 2015, 11:38 p.m., Chris Mattmann wrote:
> > This looks a lot better, Radu. A couple more things:
> > 
> > 1. Please don't use star imports. Makes it too hard to track which classes are imported. Please use FQDN class names.
> > 2. What about the curator and other places the File Manager is used, e.g., pcs-services, etc. It seems like these changes would break those portions of the system. Have you tried to build all of OODT with these changes?
> > 3. As you noted, still need to update RADIX.
> 
> Lewis McGibbney wrote:
>     I am not sure that the curator was scoped for inclusion in this portion of the work.
>     RADIX needs to be updated and I am prepared to help out to get that done.

Thanks Lewis. Whether the curator is scoped or not, this code cannot be committed without updating the curator since it depends on the FM APIs.


- Chris


-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/36953/#review96236
-----------------------------------------------------------


On Aug. 24, 2015, 5:19 p.m., Radu Manole wrote:
> 
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/36953/
> -----------------------------------------------------------
> 
> (Updated Aug. 24, 2015, 5:19 p.m.)
> 
> 
> Review request for oodt, Lewis McGibbney and Chris Mattmann.
> 
> 
> Repository: oodt
> 
> 
> Description
> -------
> 
> Separated the FileManager logic from rpc communication createing interfaces for server/client side (FileManagerCient, FileManagerServer).Refactor existing XMLRPC server/client so it would use communication interfaces. Created Avro schemas and protocol for ipc communication, (classes that have the prefixe Avro), with a Factory class AvroTypeFactory that transforms from oodt to avro and from avro to oodt classes. Implement avro server/client. For changeability I created a Factory (RpcCommunicationFactory) that can be configurable to use either avro or xmlrpc for IPC. Because of this, anywhere in project where on instantiation of server/client rewrite to use this factory.
> 
> 
> Diffs
> -----
> 
>   trunk/app/fmbrowser/src/main/java/org/apache/oodt/cas/filemgr/browser/model/CasDB.java 1693501 
>   trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FilemgrUniquenessChecker.java 1693501 
>   trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/comparator/FilemgrUniquenessCheckComparator.java 1693501 
>   trunk/filemgr/pom.xml 1693501 
>   trunk/filemgr/src/main/avro/types/AvroComplexQuery.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroElement.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroExtractorSpec.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroFileTransferStatus.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroFilterAlgor.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroMetadata.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroMimeType.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProduct.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProductPage.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProductType.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQuery.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryCriteria.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryFilter.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryResult.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroReference.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroTypeHandler.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/protocol.avdl PRE-CREATION 
>   trunk/filemgr/src/main/bin/filemgr 1693501 
>   trunk/filemgr/src/main/bin/filemgr-client 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/AbstractDeleteProductCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/DeleteProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/FileManagerCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetFirstPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetLastPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNextPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNumProductsCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetPrevPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetProductPercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/RetrieveFilesCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/AvroLocalDataTransferer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/InPlaceDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/LocalCache.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/ProductMetKeys.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManager.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClientMain.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServerMain.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ExpImpCatalog.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataBasedProductMover.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataDumper.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ProductDumper.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/AvroTypeFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/RpcCommunicationFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/XmlRpcStructFactory.java 1693501 
>   trunk/filemgr/src/main/resources/filemgr.properties 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/TestFileManagerCli.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/UseMockClientCmdLineActionStore.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestAddProductTypeCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDumpMetadataCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransferCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransfersCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFilePercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFirstPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetLastPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNextPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNumProductsCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetPrevPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductPercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductTypeByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestHasProductCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestIngestProductCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestLuceneQueryCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestSqlQueryCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestCachedIngester.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestLocalCache.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestRmiCache.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestStdIngester.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/metadata/extractors/examples/TestTikaAutoDetectExtractor.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/structs/type/TestTypeHandler.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/MockFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManager.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/rpc/TestServerClientFactories.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestExpImpCatalog.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestMetadataBasedProductMover.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestAvroTypesFactory.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestXmlRpcStructFactory.java 1693501 
>   trunk/filemgr/src/test/resources/createProduct/test.txt PRE-CREATION 
>   trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/filemgr/src/main/resources/bin/filemgr 1693501 
>   trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/filemgr/src/main/resources/bin/filemgr-client 1693501 
>   trunk/pcs/core/src/main/java/org/apache/oodt/pcs/util/FileManagerUtils.java 1693501 
>   trunk/pge/src/main/java/org/apache/oodt/cas/pge/staging/FileStager.java 1693501 
>   trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java 1693501 
>   trunk/pge/src/test/java/org/apache/oodt/cas/pge/staging/TestFileStager.java 1693501 
>   trunk/profile/src/main/java/org/apache/oodt/profile/handlers/cas/CASProfileHandler.java 1693501 
>   trunk/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/filemgr/FileManagerConn.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/CASProductHandler.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DataDeliveryServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DatasetDeliveryServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/services/CasProductJaxrsService.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/servlets/CasProductJaxrsServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFDatasetServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFProductServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductTransferServlet.java 1693501 
> 
> Diff: https://reviews.apache.org/r/36953/diff/
> 
> 
> Testing
> -------
> 
> Used existing tests, applied them for Avro RPC. 
> 
> Installed filemanager module, and started it as described in the oodt documentation. Ingested a simple Generic file. Used querry-tool.
> 
> 
> Thanks,
> 
> Radu Manole
> 
>

Re: Review Request 36953: Full implementation of AvroRPC alongside XMLRPC in filemgr module

Posted by Chris Mattmann <ma...@apache.org>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/36953/#review96236
-----------------------------------------------------------


This looks a lot better, Radu. A couple more things:

1. Please don't use star imports. Makes it too hard to track which classes are imported. Please use FQDN class names.
2. What about the curator and other places the File Manager is used, e.g., pcs-services, etc. It seems like these changes would break those portions of the system. Have you tried to build all of OODT with these changes?
3. As you noted, still need to update RADIX.

- Chris Mattmann


On Aug. 24, 2015, 5:19 p.m., Radu Manole wrote:
> 
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/36953/
> -----------------------------------------------------------
> 
> (Updated Aug. 24, 2015, 5:19 p.m.)
> 
> 
> Review request for oodt, Lewis McGibbney and Chris Mattmann.
> 
> 
> Repository: oodt
> 
> 
> Description
> -------
> 
> Separated the FileManager logic from rpc communication createing interfaces for server/client side (FileManagerCient, FileManagerServer).Refactor existing XMLRPC server/client so it would use communication interfaces. Created Avro schemas and protocol for ipc communication, (classes that have the prefixe Avro), with a Factory class AvroTypeFactory that transforms from oodt to avro and from avro to oodt classes. Implement avro server/client. For changeability I created a Factory (RpcCommunicationFactory) that can be configurable to use either avro or xmlrpc for IPC. Because of this, anywhere in project where on instantiation of server/client rewrite to use this factory.
> 
> 
> Diffs
> -----
> 
>   trunk/app/fmbrowser/src/main/java/org/apache/oodt/cas/filemgr/browser/model/CasDB.java 1693501 
>   trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FilemgrUniquenessChecker.java 1693501 
>   trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/comparator/FilemgrUniquenessCheckComparator.java 1693501 
>   trunk/filemgr/pom.xml 1693501 
>   trunk/filemgr/src/main/avro/types/AvroComplexQuery.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroElement.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroExtractorSpec.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroFileTransferStatus.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroFilterAlgor.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroMetadata.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroMimeType.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProduct.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProductPage.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProductType.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQuery.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryCriteria.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryFilter.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryResult.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroReference.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroTypeHandler.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/protocol.avdl PRE-CREATION 
>   trunk/filemgr/src/main/bin/filemgr 1693501 
>   trunk/filemgr/src/main/bin/filemgr-client 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/AbstractDeleteProductCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/DeleteProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/FileManagerCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetFirstPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetLastPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNextPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNumProductsCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetPrevPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetProductPercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/RetrieveFilesCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/AvroLocalDataTransferer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/InPlaceDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/LocalCache.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/ProductMetKeys.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManager.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClientMain.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServerMain.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ExpImpCatalog.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataBasedProductMover.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataDumper.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ProductDumper.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/AvroTypeFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/RpcCommunicationFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/XmlRpcStructFactory.java 1693501 
>   trunk/filemgr/src/main/resources/filemgr.properties 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/TestFileManagerCli.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/UseMockClientCmdLineActionStore.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestAddProductTypeCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDumpMetadataCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransferCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransfersCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFilePercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFirstPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetLastPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNextPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNumProductsCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetPrevPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductPercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductTypeByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestHasProductCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestIngestProductCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestLuceneQueryCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestSqlQueryCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestCachedIngester.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestLocalCache.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestRmiCache.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestStdIngester.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/metadata/extractors/examples/TestTikaAutoDetectExtractor.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/structs/type/TestTypeHandler.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/MockFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManager.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/rpc/TestServerClientFactories.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestExpImpCatalog.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestMetadataBasedProductMover.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestAvroTypesFactory.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestXmlRpcStructFactory.java 1693501 
>   trunk/filemgr/src/test/resources/createProduct/test.txt PRE-CREATION 
>   trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/filemgr/src/main/resources/bin/filemgr 1693501 
>   trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/filemgr/src/main/resources/bin/filemgr-client 1693501 
>   trunk/pcs/core/src/main/java/org/apache/oodt/pcs/util/FileManagerUtils.java 1693501 
>   trunk/pge/src/main/java/org/apache/oodt/cas/pge/staging/FileStager.java 1693501 
>   trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java 1693501 
>   trunk/pge/src/test/java/org/apache/oodt/cas/pge/staging/TestFileStager.java 1693501 
>   trunk/profile/src/main/java/org/apache/oodt/profile/handlers/cas/CASProfileHandler.java 1693501 
>   trunk/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/filemgr/FileManagerConn.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/CASProductHandler.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DataDeliveryServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DatasetDeliveryServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/services/CasProductJaxrsService.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/servlets/CasProductJaxrsServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFDatasetServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFProductServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductTransferServlet.java 1693501 
> 
> Diff: https://reviews.apache.org/r/36953/diff/
> 
> 
> Testing
> -------
> 
> Used existing tests, applied them for Avro RPC. 
> 
> Installed filemanager module, and started it as described in the oodt documentation. Ingested a simple Generic file. Used querry-tool.
> 
> 
> Thanks,
> 
> Radu Manole
> 
>

Re: Review Request 36953: Full implementation of AvroRPC alongside XMLRPC in filemgr module

Posted by Radu Manole <ma...@gmail.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/36953/
-----------------------------------------------------------

(Updated Aug. 24, 2015, 5:19 p.m.)


Review request for oodt, Lewis McGibbney and Chris Mattmann.


Changes
-------

Fixed review issues. RADIX scripts not yet updated.


Repository: oodt


Description
-------

Separated the FileManager logic from rpc communication createing interfaces for server/client side (FileManagerCient, FileManagerServer).Refactor existing XMLRPC server/client so it would use communication interfaces. Created Avro schemas and protocol for ipc communication, (classes that have the prefixe Avro), with a Factory class AvroTypeFactory that transforms from oodt to avro and from avro to oodt classes. Implement avro server/client. For changeability I created a Factory (RpcCommunicationFactory) that can be configurable to use either avro or xmlrpc for IPC. Because of this, anywhere in project where on instantiation of server/client rewrite to use this factory.


Diffs (updated)
-----

  trunk/app/fmbrowser/src/main/java/org/apache/oodt/cas/filemgr/browser/model/CasDB.java 1693501 
  trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FilemgrUniquenessChecker.java 1693501 
  trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/comparator/FilemgrUniquenessCheckComparator.java 1693501 
  trunk/filemgr/pom.xml 1693501 
  trunk/filemgr/src/main/avro/types/AvroComplexQuery.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroElement.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroExtractorSpec.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroFileTransferStatus.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroFilterAlgor.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroMetadata.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroMimeType.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroProduct.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroProductPage.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroProductType.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroQuery.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroQueryCriteria.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroQueryFilter.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroQueryResult.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroReference.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroTypeHandler.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/protocol.avdl PRE-CREATION 
  trunk/filemgr/src/main/bin/filemgr 1693501 
  trunk/filemgr/src/main/bin/filemgr-client 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/AbstractDeleteProductCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/DeleteProductByIdCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/FileManagerCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetFirstPageCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetLastPageCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNextPageCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNumProductsCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetPrevPageCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetProductPercentTransferredCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/RetrieveFilesCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/AvroLocalDataTransferer.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/InPlaceDataTransferer.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/LocalCache.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/ProductMetKeys.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerServer.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManager.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClientMain.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServer.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServerMain.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerServer.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerClientFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerServerFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerClientFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerServerFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerClientFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerServerFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ExpImpCatalog.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataBasedProductMover.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataDumper.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ProductDumper.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/AvroTypeFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/RpcCommunicationFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/XmlRpcStructFactory.java 1693501 
  trunk/filemgr/src/main/resources/filemgr.properties 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/TestFileManagerCli.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/UseMockClientCmdLineActionStore.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java PRE-CREATION 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestAddProductTypeCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByIdCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByNameCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDumpMetadataCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransferCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransfersCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFilePercentTransferredCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFirstPageCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetLastPageCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNextPageCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNumProductsCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetPrevPageCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByIdCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByNameCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductPercentTransferredCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductTypeByNameCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestHasProductCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestIngestProductCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestLuceneQueryCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestSqlQueryCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestCachedIngester.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestLocalCache.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestRmiCache.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestStdIngester.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/metadata/extractors/examples/TestTikaAutoDetectExtractor.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/structs/type/TestTypeHandler.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/MockFileManagerClient.java PRE-CREATION 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManager.java PRE-CREATION 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManagerClient.java PRE-CREATION 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/rpc/TestServerClientFactories.java PRE-CREATION 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestExpImpCatalog.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestMetadataBasedProductMover.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestAvroTypesFactory.java PRE-CREATION 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestXmlRpcStructFactory.java 1693501 
  trunk/filemgr/src/test/resources/createProduct/test.txt PRE-CREATION 
  trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/filemgr/src/main/resources/bin/filemgr 1693501 
  trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/filemgr/src/main/resources/bin/filemgr-client 1693501 
  trunk/pcs/core/src/main/java/org/apache/oodt/pcs/util/FileManagerUtils.java 1693501 
  trunk/pge/src/main/java/org/apache/oodt/cas/pge/staging/FileStager.java 1693501 
  trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java 1693501 
  trunk/pge/src/test/java/org/apache/oodt/cas/pge/staging/TestFileStager.java 1693501 
  trunk/profile/src/main/java/org/apache/oodt/profile/handlers/cas/CASProfileHandler.java 1693501 
  trunk/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/filemgr/FileManagerConn.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/CASProductHandler.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DataDeliveryServlet.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DatasetDeliveryServlet.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/services/CasProductJaxrsService.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/servlets/CasProductJaxrsServlet.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFDatasetServlet.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFProductServlet.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductTransferServlet.java 1693501 

Diff: https://reviews.apache.org/r/36953/diff/


Testing
-------

Used existing tests, applied them for Avro RPC. 

Installed filemanager module, and started it as described in the oodt documentation. Ingested a simple Generic file. Used querry-tool.


Thanks,

Radu Manole

Re: Review Request 36953: Full implementation of AvroRPC alongside XMLRPC in filemgr module

Posted by Chris Mattmann <ma...@apache.org>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/36953/#review96118
-----------------------------------------------------------



trunk/app/fmbrowser/src/main/java/org/apache/oodt/cas/filemgr/browser/model/CasDB.java (line 55)
<https://reviews.apache.org/r/36953/#comment151334>

    commented out: remove



trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FilemgrUniquenessChecker.java (line 52)
<https://reviews.apache.org/r/36953/#comment151335>

    remove commented out code.



trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/comparator/FilemgrUniquenessCheckComparator.java (line 54)
<https://reviews.apache.org/r/36953/#comment151336>

    if it's commented out remove it.



trunk/crawler/src/site/xdoc/user/index.xml (line 405)
<https://reviews.apache.org/r/36953/#comment151337>

    why change this?



trunk/filemgr/src/main/bin/filemgr (line 66)
<https://reviews.apache.org/r/36953/#comment151338>

    You'll also need to update the scripts in RADIX.



trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java (line 92)
<https://reviews.apache.org/r/36953/#comment151339>

    >>> bad merge



trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/metadata/extractors/examples/TestTikaAutoDetectExtractor.java (line 48)
<https://reviews.apache.org/r/36953/#comment151340>

    these need to be added back in - why are they commented out?



trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java (line 498)
<https://reviews.apache.org/r/36953/#comment151341>

    this shouldn't be commented out



trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DatasetDeliveryServlet.java (line 261)
<https://reviews.apache.org/r/36953/#comment151342>

    remove if it's commented out, remove it.



trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/servlets/CasProductJaxrsServlet.java (line 103)
<https://reviews.apache.org/r/36953/#comment151343>

    remove it's commented out.



trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFDatasetServlet.java (line 122)
<https://reviews.apache.org/r/36953/#comment151344>

    remove



trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFProductServlet.java (line 127)
<https://reviews.apache.org/r/36953/#comment151345>

    remove



trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java (line 323)
<https://reviews.apache.org/r/36953/#comment151346>

    remove



trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductTransferServlet.java (line 125)
<https://reviews.apache.org/r/36953/#comment151347>

    remove


- Chris Mattmann


On Aug. 21, 2015, 7:53 p.m., Radu Manole wrote:
> 
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/36953/
> -----------------------------------------------------------
> 
> (Updated Aug. 21, 2015, 7:53 p.m.)
> 
> 
> Review request for oodt, Lewis McGibbney and Chris Mattmann.
> 
> 
> Repository: oodt
> 
> 
> Description
> -------
> 
> Separated the FileManager logic from rpc communication createing interfaces for server/client side (FileManagerCient, FileManagerServer).Refactor existing XMLRPC server/client so it would use communication interfaces. Created Avro schemas and protocol for ipc communication, (classes that have the prefixe Avro), with a Factory class AvroTypeFactory that transforms from oodt to avro and from avro to oodt classes. Implement avro server/client. For changeability I created a Factory (RpcCommunicationFactory) that can be configurable to use either avro or xmlrpc for IPC. Because of this, anywhere in project where on instantiation of server/client rewrite to use this factory.
> 
> 
> Diffs
> -----
> 
>   trunk/app/fmbrowser/src/main/java/org/apache/oodt/cas/filemgr/browser/model/CasDB.java 1693501 
>   trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FilemgrUniquenessChecker.java 1693501 
>   trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/comparator/FilemgrUniquenessCheckComparator.java 1693501 
>   trunk/crawler/src/site/xdoc/user/index.xml 1693501 
>   trunk/filemgr/pom.xml 1693501 
>   trunk/filemgr/src/main/avro/types/AvroComplexQuery.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroElement.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroExtractorSpec.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroFileTransferStatus.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroFilterAlgor.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroMetadata.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroMimeType.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProduct.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProductPage.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProductType.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQuery.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryCriteria.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryFilter.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryResult.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroReference.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroTypeHandler.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/protocol.avdl PRE-CREATION 
>   trunk/filemgr/src/main/bin/filemgr 1693501 
>   trunk/filemgr/src/main/bin/filemgr-client 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/AbstractDeleteProductCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/DeleteProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/FileManagerCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetFirstPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetLastPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNextPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNumProductsCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetPrevPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetProductPercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/RetrieveFilesCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/AvroLocalDataTransferer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/InPlaceDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/LocalCache.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/ProductMetKeys.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManager.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClientMain.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServerMain.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ExpImpCatalog.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataBasedProductMover.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataDumper.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ProductDumper.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/AvroTypeFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/RpcCommunicationFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/XmlRpcStructFactory.java 1693501 
>   trunk/filemgr/src/main/resources/filemgr.properties 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/TestFileManagerCli.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/UseMockClientCmdLineActionStore.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestAddProductTypeCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDumpMetadataCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransferCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransfersCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFilePercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFirstPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetLastPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNextPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNumProductsCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetPrevPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductPercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductTypeByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestHasProductCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestIngestProductCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestLuceneQueryCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestSqlQueryCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestCachedIngester.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestLocalCache.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestRmiCache.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestStdIngester.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/metadata/extractors/examples/TestTikaAutoDetectExtractor.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/structs/type/TestTypeHandler.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/MockFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManager.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/rpc/TestServerClientFactories.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestExpImpCatalog.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestMetadataBasedProductMover.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestAvroTypesFactory.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestXmlRpcStructFactory.java 1693501 
>   trunk/filemgr/src/test/resources/createProduct/test.txt PRE-CREATION 
>   trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/filemgr/src/main/resources/bin/filemgr 1693501 
>   trunk/pcs/core/src/main/java/org/apache/oodt/pcs/util/FileManagerUtils.java 1693501 
>   trunk/pge/src/main/java/org/apache/oodt/cas/pge/staging/FileStager.java 1693501 
>   trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java 1693501 
>   trunk/pge/src/test/java/org/apache/oodt/cas/pge/staging/TestFileStager.java 1693501 
>   trunk/profile/src/main/java/org/apache/oodt/profile/handlers/cas/CASProfileHandler.java 1693501 
>   trunk/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/filemgr/FileManagerConn.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/CASProductHandler.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DataDeliveryServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DatasetDeliveryServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/services/CasProductJaxrsService.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/servlets/CasProductJaxrsServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFDatasetServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFProductServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductTransferServlet.java 1693501 
> 
> Diff: https://reviews.apache.org/r/36953/diff/
> 
> 
> Testing
> -------
> 
> Used existing tests, applied them for Avro RPC. 
> 
> Installed filemanager module, and started it as described in the oodt documentation. Ingested a simple Generic file. Used querry-tool.
> 
> 
> Thanks,
> 
> Radu Manole
> 
>

Re: Review Request 36953: Full implementation of AvroRPC alongside XMLRPC in filemgr module

Posted by Radu Manole <ma...@gmail.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/36953/
-----------------------------------------------------------

(Updated Aug. 21, 2015, 7:53 p.m.)


Review request for oodt, Lewis McGibbney and Chris Mattmann.


Changes
-------

Added documetation for main classes of Avro RPC implementation. Added posibility to use avro rpc FileManager in bash scrips, which are configurable in filemanager.proprierties file.


Repository: oodt


Description
-------

Separated the FileManager logic from rpc communication createing interfaces for server/client side (FileManagerCient, FileManagerServer).Refactor existing XMLRPC server/client so it would use communication interfaces. Created Avro schemas and protocol for ipc communication, (classes that have the prefixe Avro), with a Factory class AvroTypeFactory that transforms from oodt to avro and from avro to oodt classes. Implement avro server/client. For changeability I created a Factory (RpcCommunicationFactory) that can be configurable to use either avro or xmlrpc for IPC. Because of this, anywhere in project where on instantiation of server/client rewrite to use this factory.


Diffs (updated)
-----

  trunk/app/fmbrowser/src/main/java/org/apache/oodt/cas/filemgr/browser/model/CasDB.java 1693501 
  trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FilemgrUniquenessChecker.java 1693501 
  trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/comparator/FilemgrUniquenessCheckComparator.java 1693501 
  trunk/crawler/src/site/xdoc/user/index.xml 1693501 
  trunk/filemgr/pom.xml 1693501 
  trunk/filemgr/src/main/avro/types/AvroComplexQuery.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroElement.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroExtractorSpec.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroFileTransferStatus.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroFilterAlgor.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroMetadata.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroMimeType.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroProduct.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroProductPage.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroProductType.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroQuery.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroQueryCriteria.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroQueryFilter.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroQueryResult.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroReference.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/AvroTypeHandler.avsc PRE-CREATION 
  trunk/filemgr/src/main/avro/types/protocol.avdl PRE-CREATION 
  trunk/filemgr/src/main/bin/filemgr 1693501 
  trunk/filemgr/src/main/bin/filemgr-client 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/AbstractDeleteProductCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/DeleteProductByIdCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/FileManagerCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetFirstPageCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetLastPageCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNextPageCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNumProductsCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetPrevPageCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetProductPercentTransferredCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/RetrieveFilesCliAction.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/AvroLocalDataTransferer.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/InPlaceDataTransferer.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/LocalCache.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/ProductMetKeys.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerServer.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManager.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClientMain.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServer.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServerMain.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerServer.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerClientFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerServerFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerClientFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerServerFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerClientFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerServerFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ExpImpCatalog.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataBasedProductMover.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataDumper.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ProductDumper.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java 1693501 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/AvroTypeFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/RpcCommunicationFactory.java PRE-CREATION 
  trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/XmlRpcStructFactory.java 1693501 
  trunk/filemgr/src/main/resources/filemgr.properties 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/TestFileManagerCli.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/UseMockClientCmdLineActionStore.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java PRE-CREATION 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestAddProductTypeCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByIdCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByNameCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDumpMetadataCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransferCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransfersCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFilePercentTransferredCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFirstPageCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetLastPageCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNextPageCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNumProductsCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetPrevPageCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByIdCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByNameCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductPercentTransferredCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductTypeByNameCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestHasProductCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestIngestProductCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestLuceneQueryCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestSqlQueryCliAction.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestCachedIngester.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestLocalCache.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestRmiCache.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestStdIngester.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/metadata/extractors/examples/TestTikaAutoDetectExtractor.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/structs/type/TestTypeHandler.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/MockFileManagerClient.java PRE-CREATION 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManager.java PRE-CREATION 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManagerClient.java PRE-CREATION 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/rpc/TestServerClientFactories.java PRE-CREATION 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestExpImpCatalog.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestMetadataBasedProductMover.java 1693501 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestAvroTypesFactory.java PRE-CREATION 
  trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestXmlRpcStructFactory.java 1693501 
  trunk/filemgr/src/test/resources/createProduct/test.txt PRE-CREATION 
  trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/filemgr/src/main/resources/bin/filemgr 1693501 
  trunk/pcs/core/src/main/java/org/apache/oodt/pcs/util/FileManagerUtils.java 1693501 
  trunk/pge/src/main/java/org/apache/oodt/cas/pge/staging/FileStager.java 1693501 
  trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java 1693501 
  trunk/pge/src/test/java/org/apache/oodt/cas/pge/staging/TestFileStager.java 1693501 
  trunk/profile/src/main/java/org/apache/oodt/profile/handlers/cas/CASProfileHandler.java 1693501 
  trunk/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/filemgr/FileManagerConn.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/CASProductHandler.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DataDeliveryServlet.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DatasetDeliveryServlet.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/services/CasProductJaxrsService.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/servlets/CasProductJaxrsServlet.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFDatasetServlet.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFProductServlet.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java 1693501 
  trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductTransferServlet.java 1693501 

Diff: https://reviews.apache.org/r/36953/diff/


Testing (updated)
-------

Used existing tests, applied them for Avro RPC. 

Installed filemanager module, and started it as described in the oodt documentation. Ingested a simple Generic file. Used querry-tool.


Thanks,

Radu Manole

Re: Review Request 36953: Full implementation of AvroRPC alongside XMLRPC in filemgr module

Posted by Chris Mattmann <ma...@apache.org>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/36953/#review95352
-----------------------------------------------------------



trunk/filemgr/pom.xml (line 60)
<https://reviews.apache.org/r/36953/#comment150261>

    probably should remove this commented line.



trunk/filemgr/pom.xml (line 211)
<https://reviews.apache.org/r/36953/#comment150262>

    do we need SLF4J?



trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java (line 271)
<https://reviews.apache.org/r/36953/#comment150273>

    orthogonal



trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java 
<https://reviews.apache.org/r/36953/#comment150274>

    orthogonal change. Please remove.



trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/FileManagerCliAction.java (line 53)
<https://reviews.apache.org/r/36953/#comment150275>

    remove



trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/AvroLocalDataTransferer.java (line 1)
<https://reviews.apache.org/r/36953/#comment150276>

    need ALv2 header.



trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/InPlaceDataTransferer.java (line 51)
<https://reviews.apache.org/r/36953/#comment150277>

    merge error.



trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java (line 179)
<https://reviews.apache.org/r/36953/#comment150278>

    merge error.



trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java (line 188)
<https://reviews.apache.org/r/36953/#comment150279>

    whitespace change. Remove.



trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java (line 215)
<https://reviews.apache.org/r/36953/#comment150280>

    whitespace change remove.



trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java (line 262)
<https://reviews.apache.org/r/36953/#comment150281>

    whitespace change remove.



trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/extractors/examples/TikaAutoDetectExtractor.java 
<https://reviews.apache.org/r/36953/#comment150282>

    please remove these types of whitepsace changes.



trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/extractors/examples/TikaAutoDetectExtractor.java 
<https://reviews.apache.org/r/36953/#comment150283>

    orthogonal whitespace change remove.



trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java (line 160)
<https://reviews.apache.org/r/36953/#comment150284>

    merge error.



trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java (line 175)
<https://reviews.apache.org/r/36953/#comment150285>

    whitespace.



trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java (line 710)
<https://reviews.apache.org/r/36953/#comment150286>

    merge error.



trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java (line 831)
<https://reviews.apache.org/r/36953/#comment150287>

    merge error.



trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java (line 1220)
<https://reviews.apache.org/r/36953/#comment150288>

    merge error.



trunk/filemgr/src/main/resources/.~lock.CAS File Manager User Guide.doc# (line 1)
<https://reviews.apache.org/r/36953/#comment150289>

    remove.



trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/rpc/TestServerClientFactories.java (line 1)
<https://reviews.apache.org/r/36953/#comment150290>

    ALv2 header.



trunk/filemgr/src/test/resources/filemgr.properties 
<https://reviews.apache.org/r/36953/#comment150291>

    why was this removed?


- Chris Mattmann


On July 30, 2015, 9:54 p.m., Radu Manole wrote:
> 
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/36953/
> -----------------------------------------------------------
> 
> (Updated July 30, 2015, 9:54 p.m.)
> 
> 
> Review request for oodt, Lewis McGibbney and Chris Mattmann.
> 
> 
> Repository: oodt
> 
> 
> Description
> -------
> 
> Separated the FileManager logic from rpc communication createing interfaces for server/client side (FileManagerCient, FileManagerServer).Refactor existing XMLRPC server/client so it would use communication interfaces. Created Avro schemas and protocol for ipc communication, (classes that have the prefixe Avro), with a Factory class AvroTypeFactory that transforms from oodt to avro and from avro to oodt classes. Implement avro server/client. For changeability I created a Factory (RpcCommunicationFactory) that can be configurable to use either avro or xmlrpc for IPC. Because of this, anywhere in project where on instantiation of server/client rewrite to use this factory.
> 
> 
> Diffs
> -----
> 
>   trunk/app/fmbrowser/src/main/java/org/apache/oodt/cas/filemgr/browser/model/CasDB.java 1693501 
>   trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/action/FilemgrUniquenessChecker.java 1693501 
>   trunk/crawler/src/main/java/org/apache/oodt/cas/crawl/comparator/FilemgrUniquenessCheckComparator.java 1693501 
>   trunk/crawler/src/site/xdoc/user/index.xml 1693501 
>   trunk/filemgr/pom.xml 1693501 
>   trunk/filemgr/src/main/avro/types/AvroComplexQuery.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroElement.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroExtractorSpec.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroFileTransferStatus.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroFilterAlgor.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroMetadata.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroMimeType.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProduct.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProductPage.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroProductType.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQuery.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryCriteria.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryFilter.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroQueryResult.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroReference.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/AvroTypeHandler.avsc PRE-CREATION 
>   trunk/filemgr/src/main/avro/types/protocol.avdl PRE-CREATION 
>   trunk/filemgr/src/main/bin/filemgr 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/AbstractDeleteProductCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/DeleteProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/FileManagerCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetFirstPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetLastPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNextPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetNumProductsCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetPrevPageCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/GetProductPercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/RetrieveFilesCliAction.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/AvroLocalDataTransferer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/InPlaceDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/RemoteDataTransferer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/LocalCache.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/ProductMetKeys.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/extractors/examples/TikaAutoDetectExtractor.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManager.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerServer.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/AvroFileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/FileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerClientFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/rpc/XmlRpcFileManagerServerFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ExpImpCatalog.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataBasedProductMover.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataDumper.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ProductDumper.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java 1693501 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/AvroTypeFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/RpcCommunicationFactory.java PRE-CREATION 
>   trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/util/XmlRpcStructFactory.java 1693501 
>   trunk/filemgr/src/main/resources/.~lock.CAS File Manager User Guide.doc# PRE-CREATION 
>   trunk/filemgr/src/main/resources/filemgr.properties 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/TestFileManagerCli.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/UseMockClientCmdLineActionStore.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestAddProductTypeCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDeleteProductByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestDumpMetadataCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransferCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetCurrentTransfersCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFilePercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetFirstPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetLastPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNextPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetNumProductsCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetPrevPageCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByIdCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductPercentTransferredCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestGetProductTypeByNameCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestHasProductCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestIngestProductCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestLuceneQueryCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/TestSqlQueryCliAction.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestCachedIngester.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestLocalCache.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestRmiCache.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/ingest/TestStdIngester.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/structs/type/TestTypeHandler.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/MockFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManager.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/TestRpcFileManagerClient.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/rpc/TestServerClientFactories.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestExpImpCatalog.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/tools/TestMetadataBasedProductMover.java 1693501 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestAvroTypesFactory.java PRE-CREATION 
>   trunk/filemgr/src/test/java/org/apache/oodt/cas/filemgr/util/TestXmlRpcStructFactory.java 1693501 
>   trunk/filemgr/src/test/resources/createProduct/test.txt PRE-CREATION 
>   trunk/filemgr/src/test/resources/filemgr.properties 1693501 
>   trunk/mvn/archetypes/radix/src/main/resources/archetype-resources/filemgr/src/main/resources/bin/filemgr 1693501 
>   trunk/pcs/core/src/main/java/org/apache/oodt/pcs/util/FileManagerUtils.java 1693501 
>   trunk/pge/src/main/java/org/apache/oodt/cas/pge/staging/FileStager.java 1693501 
>   trunk/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java 1693501 
>   trunk/pge/src/test/java/org/apache/oodt/cas/pge/staging/TestFileStager.java 1693501 
>   trunk/profile/src/main/java/org/apache/oodt/profile/handlers/cas/CASProfileHandler.java 1693501 
>   trunk/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/filemgr/FileManagerConn.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/CASProductHandler.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DataDeliveryServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DatasetDeliveryServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/services/CasProductJaxrsService.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/servlets/CasProductJaxrsServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFDatasetServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFProductServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java 1693501 
>   trunk/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductTransferServlet.java 1693501 
> 
> Diff: https://reviews.apache.org/r/36953/diff/
> 
> 
> Testing
> -------
> 
> Used existing tests, applied them for avro ipc.
> 
> 
> Thanks,
> 
> Radu Manole
> 
>