You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by am...@apache.org on 2016/01/03 18:41:19 UTC
[21/21] incubator-asterixdb git commit: First stage of external data
cleanup
First stage of external data cleanup
In this change, different parts of external data were refactored.
The goal was to make it more modular, easier to maintain and allow
higher flexibility for extension in addition to reducing code redundancy.
Change-Id: I04a8c4e494d8d1363992b6fe0bdbe6b2b3b7b767
Reviewed-on: https://asterix-gerrit.ics.uci.edu/566
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <hu...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/commit/284590ed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/tree/284590ed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/diff/284590ed
Branch: refs/heads/master
Commit: 284590ed9c30349058fdc448852ebb6dc698912f
Parents: 1d5cf64
Author: Abdullah Alamoudi <ba...@gmail.com>
Authored: Sun Jan 3 15:42:18 2016 +0300
Committer: abdullah alamoudi <ba...@gmail.com>
Committed: Sun Jan 3 09:35:48 2016 -0800
----------------------------------------------------------------------
asterix-algebra/pom.xml | 1 -
asterix-app/pom.xml | 12 +-
.../asterix/aql/translator/QueryTranslator.java | 16 +-
.../file/ExternalIndexingOperations.java | 132 +-
.../org/apache/asterix/file/FeedOperations.java | 13 +-
.../file/SecondaryIndexOperationsHelper.java | 2 +-
.../bootstrap/ExternalLibraryBootstrap.java | 4 +-
.../asterix/test/optimizer/OptimizerTest.java | 4 +-
.../asterix/test/runtime/ExecutionTest.java | 10 +-
.../asterix/test/runtime/ExecutionTestUtil.java | 4 +-
.../metadata/results/basic/meta15/meta15.1.adm | 15 +-
.../common/api/IAsterixAppRuntimeContext.java | 1 -
.../common/feeds/BasicMonitoredBuffer.java | 2 +
.../feeds/ComputeSideMonitoredBuffer.java | 2 +
.../common/feeds/IFramePostProcessor.java | 28 -
.../common/feeds/IFramePreprocessor.java | 26 -
.../common/feeds/IntakeSideMonitoredBuffer.java | 2 +
.../asterix/common/feeds/MonitoredBuffer.java | 2 +
.../feeds/StorageSideMonitoredBuffer.java | 2 +
.../feeds/api/IAdapterRuntimeManager.java | 4 +-
.../common/feeds/api/IDataSourceAdapter.java | 60 +
.../common/feeds/api/IDatasourceAdapter.java | 47 -
.../asterix/common/feeds/api/IFeedAdapter.java | 60 -
.../common/feeds/api/IFramePostProcessor.java | 28 +
.../common/feeds/api/IFramePreprocessor.java | 26 +
.../feeds/api/ITupleTrackingFeedAdapter.java | 2 +-
.../common/parse/IAsterixTupleParser.java | 29 -
.../common/parse/ITupleForwardPolicy.java | 48 -
.../asterix/common/parse/ITupleForwarder.java | 46 +
.../apache/asterix/test/aql/TestExecutor.java | 9 +-
.../asterix/event/service/ZooKeeperService.java | 39 +-
asterix-external-data/pom.xml | 91 ++
.../adapter/factory/CNNFeedAdapterFactory.java | 153 ---
.../adapter/factory/GenericAdapterFactory.java | 101 ++
.../adapter/factory/HDFSAdapterFactory.java | 343 -----
.../factory/HDFSIndexingAdapterFactory.java | 208 ---
.../adapter/factory/HiveAdapterFactory.java | 127 --
.../adapter/factory/IAdapterFactory.java | 98 --
.../factory/IControlledAdapterFactory.java | 36 -
.../adapter/factory/IFeedAdapterFactory.java | 29 -
.../adapter/factory/LookupAdapterFactory.java | 111 ++
.../factory/NCFileSystemAdapterFactory.java | 166 ---
.../PullBasedAzureTwitterAdapterFactory.java | 132 --
.../factory/PullBasedTwitterAdapterFactory.java | 118 --
.../factory/PushBasedTwitterAdapterFactory.java | 108 --
.../adapter/factory/RSSFeedAdapterFactory.java | 114 --
.../factory/StreamBasedAdapterFactory.java | 49 -
.../asterix/external/api/IAdapterFactory.java | 79 ++
.../external/api/IDataFlowController.java | 55 +
.../asterix/external/api/IDataParser.java | 127 ++
.../external/api/IDataParserFactory.java | 52 +
.../api/IExternalDataSourceFactory.java | 64 +
.../asterix/external/api/IExternalFunction.java | 27 +
.../asterix/external/api/IExternalIndexer.java | 53 +
.../external/api/IExternalScalarFunction.java | 25 +
.../asterix/external/api/IFunctionFactory.java | 25 +
.../asterix/external/api/IFunctionHelper.java | 39 +
.../api/IIndexibleExternalDataSource.java | 34 +
.../external/api/IIndexingAdapterFactory.java | 27 +
.../external/api/IIndexingDatasource.java | 25 +
.../external/api/IInputStreamProvider.java | 25 +
.../api/IInputStreamProviderFactory.java | 26 +
.../asterix/external/api/IJListAccessor.java | 31 +
.../apache/asterix/external/api/IJObject.java | 37 +
.../asterix/external/api/IJObjectAccessor.java | 29 +
.../asterix/external/api/IJRecordAccessor.java | 34 +
.../org/apache/asterix/external/api/IJType.java | 29 +
.../external/api/ILookupReaderFactory.java | 27 +
.../external/api/ILookupRecordReader.java | 40 +
.../asterix/external/api/INodeResolver.java | 37 +
.../external/api/INodeResolverFactory.java | 34 +
.../apache/asterix/external/api/IRawRecord.java | 51 +
.../asterix/external/api/IRecordDataParser.java | 36 +
.../external/api/IRecordDataParserFactory.java | 32 +
.../external/api/IRecordFlowController.java | 27 +
.../asterix/external/api/IRecordReader.java | 66 +
.../external/api/IRecordReaderFactory.java | 29 +
.../asterix/external/api/IResultCollector.java | 45 +
.../asterix/external/api/IStreamDataParser.java | 39 +
.../external/api/IStreamDataParserFactory.java | 29 +
.../external/api/IStreamFlowController.java | 23 +
.../dataflow/AbstractDataFlowController.java | 55 +
.../dataflow/CounterTimerTupleForwarder.java | 157 +++
.../dataflow/FrameFullTupleForwarder.java | 72 ++
.../dataflow/IndexingDataFlowController.java | 40 +
.../dataflow/RateControlledTupleForwarder.java | 87 ++
.../dataflow/RecordDataFlowController.java | 77 ++
.../dataflow/StreamDataFlowController.java | 64 +
.../dataset/adapter/AzureTweetEntity.java | 51 -
.../adapter/AzureTweetMetadataEntity.java | 95 --
.../dataset/adapter/ClientBasedFeedAdapter.java | 155 ---
.../external/dataset/adapter/FeedClient.java | 174 ---
.../dataset/adapter/FileSystemBasedAdapter.java | 62 -
.../dataset/adapter/GenericAdapter.java | 48 +
.../external/dataset/adapter/HDFSAdapter.java | 97 --
.../dataset/adapter/HDFSIndexingAdapter.java | 78 --
.../external/dataset/adapter/HiveAdapter.java | 49 -
.../dataset/adapter/IControlledAdapter.java | 65 -
.../external/dataset/adapter/IFeedClient.java | 46 -
.../dataset/adapter/IFeedClientFactory.java | 40 -
.../dataset/adapter/IPullBasedFeedClient.java | 46 -
.../external/dataset/adapter/LookupAdapter.java | 162 +++
.../dataset/adapter/NCFileSystemAdapter.java | 69 -
.../dataset/adapter/PullBasedAdapter.java | 162 ---
.../adapter/PullBasedAzureFeedClient.java | 190 ---
.../adapter/PullBasedAzureTwitterAdapter.java | 103 --
.../adapter/PullBasedTwitterAdapter.java | 80 --
.../adapter/PullBasedTwitterFeedClient.java | 105 --
.../adapter/PushBasedTwitterAdapter.java | 70 --
.../adapter/PushBasedTwitterFeedClient.java | 111 --
.../dataset/adapter/RSSFeedAdapter.java | 98 --
.../external/dataset/adapter/RSSFeedClient.java | 161 ---
.../dataset/adapter/StreamBasedAdapter.java | 11 +-
.../external/feeds/IPullBasedFeedAdapter.java | 34 -
.../asterix/external/indexing/ExternalFile.java | 10 +
.../indexing/ExternalFileIndexAccessor.java | 59 +-
.../indexing/FileIndexTupleTranslator.java | 91 ++
.../external/indexing/FileOffsetIndexer.java | 74 ++
.../external/indexing/IndexingScheduler.java | 348 +++++
.../external/indexing/RCRecordIdReader.java | 43 +
.../indexing/RecordColumnarIndexer.java | 84 ++
.../asterix/external/indexing/RecordId.java | 54 +
.../external/indexing/RecordIdReader.java | 78 ++
.../indexing/RecordIdReaderFactory.java | 38 +
.../dataflow/AbstractIndexingTupleParser.java | 97 --
.../AdmOrDelimitedControlledTupleParser.java | 239 ----
.../AdmOrDelimitedIndexingTupleParser.java | 105 --
.../dataflow/FileIndexTupleTranslator.java | 95 --
.../dataflow/HDFSIndexingParserFactory.java | 140 ---
.../indexing/dataflow/HDFSLookupAdapter.java | 187 ---
.../dataflow/HDFSLookupAdapterFactory.java | 83 --
.../dataflow/HDFSObjectTupleParser.java | 81 --
.../dataflow/HDFSObjectTupleParserFactory.java | 69 -
.../indexing/dataflow/HiveObjectParser.java | 426 -------
.../dataflow/IAsterixHDFSRecordParser.java | 55 -
.../dataflow/IControlledTupleParser.java | 44 -
.../dataflow/IControlledTupleParserFactory.java | 23 -
.../indexing/dataflow/IndexingScheduler.java | 348 -----
.../dataflow/RCFileControlledTupleParser.java | 197 ---
.../dataflow/RCFileIndexingTupleParser.java | 63 -
.../dataflow/SeqOrTxtControlledTupleParser.java | 190 ---
.../dataflow/TextOrSeqIndexingTupleParser.java | 47 -
.../input/AbstractHDFSLookupInputStream.java | 117 --
.../indexing/input/AbstractHDFSReader.java | 107 --
.../input/GenericFileAwareRecordReader.java | 127 --
.../indexing/input/GenericRecordReader.java | 135 --
.../indexing/input/HDFSSeekableLineReader.java | 232 ----
.../external/indexing/input/ILookupReader.java | 24 -
.../indexing/input/RCFileDataReader.java | 178 ---
.../indexing/input/RCFileLookupReader.java | 106 --
.../input/SequenceFileLookupInputStream.java | 75 --
.../input/SequenceFileLookupReader.java | 109 --
.../input/TextFileLookupInputStream.java | 69 -
.../indexing/input/TextFileLookupReader.java | 97 --
.../indexing/input/TextualDataReader.java | 234 ----
.../input/TextualFullScanDataReader.java | 209 ---
...xternalDatasetIndexesOperatorDescriptor.java | 122 --
...alDatasetIndexesAbortOperatorDescriptor.java | 56 -
...lDatasetIndexesCommitOperatorDescriptor.java | 65 -
...DatasetIndexesRecoverOperatorDescriptor.java | 57 -
...ternalIndexBulkModifyOperatorDescriptor.java | 69 -
...rnalIndexBulkModifyOperatorNodePushable.java | 119 --
.../ExternalLoopkupOperatorDiscriptor.java | 141 ---
.../operators/IndexInfoOperatorDescriptor.java | 126 --
.../external/input/HDFSDataSourceFactory.java | 198 +++
.../external/input/record/CharArrayRecord.java | 106 ++
.../external/input/record/GenericRecord.java | 62 +
.../reader/AbstractCharRecordLookupReader.java | 78 ++
.../reader/AbstractHDFSLookupRecordReader.java | 113 ++
.../reader/AbstractStreamRecordReader.java | 75 ++
.../AbstractStreamRecordReaderFactory.java | 99 ++
.../input/record/reader/EmptyRecordReader.java | 56 +
.../input/record/reader/HDFSRecordReader.java | 194 +++
.../input/record/reader/HDFSTextLineReader.java | 234 ++++
.../input/record/reader/LineRecordReader.java | 108 ++
.../reader/LookupReaderFactoryProvider.java | 44 +
.../record/reader/QuotedLineRecordReader.java | 115 ++
.../input/record/reader/RCLookupReader.java | 92 ++
.../input/record/reader/RSSRecordReader.java | 177 +++
.../reader/SemiStructuredRecordReader.java | 147 +++
.../record/reader/SequenceLookupReader.java | 71 ++
.../input/record/reader/TextLookupReader.java | 64 +
.../record/reader/TwitterPullRecordReader.java | 106 ++
.../record/reader/TwitterPushRecordReader.java | 122 ++
.../reader/factory/HDFSLookupReaderFactory.java | 90 ++
.../reader/factory/LineRecordReaderFactory.java | 55 +
.../reader/factory/RSSRecordReaderFactory.java | 87 ++
.../SemiStructuredRecordReaderFactory.java | 46 +
.../factory/TwitterRecordReaderFactory.java | 136 ++
.../external/input/stream/AInputStream.java | 28 +
.../input/stream/AInputStreamReader.java | 34 +
.../external/input/stream/BasicInputStream.java | 86 ++
.../input/stream/HDFSInputStreamProvider.java | 118 ++
.../stream/LocalFSInputStreamProvider.java | 56 +
.../input/stream/SocketInputStream.java | 101 ++
.../input/stream/SocketInputStreamProvider.java | 36 +
.../TwitterFirehoseInputStreamProvider.java | 183 +++
.../LocalFSInputStreamProviderFactory.java | 139 ++
.../SocketInputStreamProviderFactory.java | 130 ++
.../TwitterFirehoseStreamProviderFactory.java | 96 ++
.../external/library/ExternalFunction.java | 3 +
.../library/ExternalFunctionProvider.java | 3 +
.../external/library/IExternalFunction.java | 27 -
.../library/IExternalScalarFunction.java | 25 -
.../external/library/IFunctionFactory.java | 25 -
.../external/library/IFunctionHelper.java | 40 -
.../external/library/IResultCollector.java | 45 -
.../external/library/JTypeObjectFactory.java | 2 +-
.../external/library/JavaFunctionHelper.java | 3 +-
.../external/library/ResultCollector.java | 1 +
.../asterix/external/library/TypeInfo.java | 2 +-
.../external/library/java/IJListAccessor.java | 30 -
.../asterix/external/library/java/IJObject.java | 37 -
.../external/library/java/IJObjectAccessor.java | 29 -
.../external/library/java/IJRecordAccessor.java | 33 -
.../asterix/external/library/java/IJType.java | 29 -
.../external/library/java/JObjectAccessors.java | 4 +
.../library/java/JObjectPointableVisitor.java | 4 +
.../external/library/java/JObjectUtil.java | 1 +
.../asterix/external/library/java/JObjects.java | 1 +
...xternalDatasetIndexesOperatorDescriptor.java | 121 ++
.../ExternalBTreeSearchOperatorDescriptor.java | 60 +
...ExternalBTreeSearchOperatorNodePushable.java | 87 ++
...alDatasetIndexesAbortOperatorDescriptor.java | 56 +
...lDatasetIndexesCommitOperatorDescriptor.java | 65 +
...DatasetIndexesRecoverOperatorDescriptor.java | 57 +
.../ExternalFilesIndexOperatorDescriptor.java | 165 +++
...ternalIndexBulkModifyOperatorDescriptor.java | 69 +
...rnalIndexBulkModifyOperatorNodePushable.java | 119 ++
.../ExternalLookupOperatorDescriptor.java | 132 ++
.../ExternalRTreeSearchOperatorDescriptor.java | 57 +
...ExternalRTreeSearchOperatorNodePushable.java | 87 ++
.../operators/IndexInfoOperatorDescriptor.java | 126 ++
.../asterix/external/parser/ADMDataParser.java | 1148 +++++++++++++++++
.../external/parser/AbstractDataParser.java | 522 ++++++++
.../external/parser/DelimitedDataParser.java | 208 +++
.../external/parser/HiveRecordParser.java | 385 ++++++
.../asterix/external/parser/RSSParser.java | 83 ++
.../asterix/external/parser/TweetParser.java | 144 +++
.../parser/factory/ADMDataParserFactory.java | 56 +
.../AbstractRecordStreamParserFactory.java | 51 +
.../factory/DelimitedDataParserFactory.java | 95 ++
.../parser/factory/HiveDataParserFactory.java | 67 +
.../parser/factory/RSSParserFactory.java | 68 +
.../parser/factory/TweetParserFactory.java | 67 +
.../provider/AdapterFactoryProvider.java | 114 ++
.../provider/DataflowControllerProvider.java | 96 ++
.../provider/DatasourceFactoryProvider.java | 101 ++
.../provider/ExternalIndexerProvider.java | 45 +
.../provider/ParserFactoryProvider.java | 65 +
.../asterix/external/runtime/DataGenerator.java | 1188 ++++++++++++++++++
.../runtime/GenericSocketFeedAdapter.java | 118 ++
.../GenericSocketFeedAdapterFactory.java | 154 +++
.../external/runtime/SocketClientAdapter.java | 106 ++
.../runtime/SocketClientAdapterFactory.java | 77 ++
.../external/runtime/TweetGenerator.java | 156 +++
.../asterix/external/util/DNSResolver.java | 54 -
.../external/util/DNSResolverFactory.java | 7 +-
.../asterix/external/util/DataflowUtils.java | 69 +
.../util/ExternalDataCompatibilityUtils.java | 85 ++
.../external/util/ExternalDataConstants.java | 176 +++
.../util/ExternalDataExceptionUtils.java | 32 +
.../external/util/ExternalDataUtils.java | 222 ++++
.../apache/asterix/external/util/HDFSUtils.java | 232 ++++
.../asterix/external/util/INodeResolver.java | 37 -
.../external/util/INodeResolverFactory.java | 34 -
.../external/util/IdentitiyResolverFactory.java | 3 +
.../asterix/external/util/IdentityResolver.java | 2 +
.../asterix/external/util/NodeResolver.java | 98 ++
.../src/main/resources/adm.grammar | 86 ++
.../external/library/AddHashTagsFactory.java | 4 +-
.../external/library/AddHashTagsFunction.java | 2 +
.../library/AddHashTagsInPlaceFactory.java | 2 +
.../library/AddHashTagsInPlaceFunction.java | 2 +
.../external/library/AllTypesFactory.java | 4 +-
.../external/library/AllTypesFunction.java | 2 +
.../external/library/CapitalFinderFactory.java | 4 +-
.../external/library/CapitalFinderFunction.java | 2 +
.../external/library/EchoDelayFactory.java | 2 +
.../external/library/EchoDelayFunction.java | 2 +
.../external/library/ParseTweetFactory.java | 4 +-
.../external/library/ParseTweetFunction.java | 2 +
.../asterix/external/library/SumFactory.java | 2 +
.../asterix/external/library/SumFunction.java | 4 +-
.../external/library/UpperCaseFactory.java | 4 +-
.../external/library/UpperCaseFunction.java | 2 +
.../library/adapter/TestTypedAdapter.java | 13 +-
.../adapter/TestTypedAdapterFactory.java | 85 +-
.../operator/file/ADMDataParserTest.java | 116 ++
.../installer/test/AbstractExecutionIT.java | 16 +-
.../test/AsterixExternalLibraryIT.java | 25 +-
.../test/AsterixInstallerIntegrationUtil.java | 1 +
.../installer/test/ClusterExecutionIT.java | 20 +-
.../installer/test/ManagixExecutionIT.java | 17 -
.../installer/test/ManagixSqlppExecutionIT.java | 13 +-
.../aql/statement/SubscribeFeedStatement.java | 4 +-
.../src/main/resources/Lexer.java | 27 +-
.../metadata/bootstrap/MetadataBootstrap.java | 38 +-
.../metadata/declared/AqlMetadataProvider.java | 223 +---
.../declared/FieldExtractingAdapter.java | 170 ---
.../declared/FieldExtractingAdapterFactory.java | 86 --
.../metadata/declared/PKGeneratingAdapter.java | 184 ---
.../declared/PKGeneratingAdapterFactory.java | 86 --
.../feeds/AbstractDatasourceAdapter.java | 4 +-
.../feeds/AbstractFeedDatasourceAdapter.java | 4 +-
.../asterix/metadata/feeds/AdapterExecutor.java | 6 +-
.../metadata/feeds/AdapterRuntimeManager.java | 11 +-
.../ExternalDataScanOperatorDescriptor.java | 6 +-
.../feeds/FeedIntakeOperatorDescriptor.java | 20 +-
.../feeds/FeedIntakeOperatorNodePushable.java | 21 +-
.../apache/asterix/metadata/feeds/FeedUtil.java | 42 +-
.../metadata/feeds/ITypedAdapterFactory.java | 2 +-
.../asterix/formats/base/IDataFormat.java | 10 +-
.../asterix/om/util/AsterixRuntimeUtil.java | 4 +
asterix-runtime/pom.xml | 226 +---
.../ExternalBTreeSearchOperatorDescriptor.java | 60 -
...ExternalBTreeSearchOperatorNodePushable.java | 87 --
.../ExternalRTreeSearchOperatorDescriptor.java | 57 -
...ExternalRTreeSearchOperatorNodePushable.java | 87 --
.../runtime/formats/NonTaggedDataFormat.java | 162 ++-
.../runtime/operators/file/ADMDataParser.java | 1100 ----------------
.../operators/file/AbstractDataParser.java | 521 --------
.../operators/file/AbstractTupleParser.java | 81 --
.../file/AsterixTupleParserFactory.java | 272 ----
.../file/CounterTimerTupleForwardPolicy.java | 158 ---
.../operators/file/DelimitedDataParser.java | 176 ---
.../file/FrameFullTupleForwardPolicy.java | 76 --
.../runtime/operators/file/IDataParser.java | 59 -
.../file/RateContolledParserPolicy.java | 97 --
.../file/RateControlledTupleForwardPolicy.java | 88 --
asterix-runtime/src/main/resources/adm.grammar | 86 --
.../operator/file/ADMDataParserTest.java | 116 --
.../tools/external/data/DataGenerator.java | 1188 ------------------
.../ExternalFilesIndexOperatorDescriptor.java | 165 ---
.../tools/external/data/GULongIDGenerator.java | 50 -
.../external/data/GenericSocketFeedAdapter.java | 123 --
.../data/GenericSocketFeedAdapterFactory.java | 172 ---
.../RateControlledFileSystemBasedAdapter.java | 69 -
...ControlledFileSystemBasedAdapterFactory.java | 131 --
.../external/data/SocketClientAdapter.java | 110 --
.../data/SocketClientAdapterFactory.java | 92 --
.../tools/external/data/TweetGenerator.java | 156 ---
.../data/TwitterFirehoseFeedAdapter.java | 191 ---
.../data/TwitterFirehoseFeedAdapterFactory.java | 125 --
344 files changed, 15151 insertions(+), 16580 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-algebra/pom.xml
----------------------------------------------------------------------
diff --git a/asterix-algebra/pom.xml b/asterix-algebra/pom.xml
index 3e76d43..bf38393 100644
--- a/asterix-algebra/pom.xml
+++ b/asterix-algebra/pom.xml
@@ -112,7 +112,6 @@
<groupId>org.apache.asterix</groupId>
<artifactId>asterix-external-data</artifactId>
<version>0.8.8-SNAPSHOT</version>
- <scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.asterix</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-app/pom.xml
----------------------------------------------------------------------
diff --git a/asterix-app/pom.xml b/asterix-app/pom.xml
index 0360529..9b87d9f 100644
--- a/asterix-app/pom.xml
+++ b/asterix-app/pom.xml
@@ -187,7 +187,6 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
- <type>jar</type>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -196,6 +195,12 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
<groupId>xerces</groupId>
<artifactId>xercesImpl</artifactId>
<version>2.9.1</version>
@@ -233,5 +238,10 @@
<version>0.8.8-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-external-data</artifactId>
+ <version>0.8.8-SNAPSHOT</version>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
index 08b92e7..7c055e2 100644
--- a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
+++ b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
@@ -63,7 +63,7 @@ import org.apache.asterix.common.feeds.api.IFeedLifecycleEventSubscriber.FeedLif
import org.apache.asterix.common.feeds.api.IFeedLifecycleListener.ConnectionLocation;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.compiler.provider.ILangCompilationProvider;
-import org.apache.asterix.external.adapter.factory.IFeedAdapterFactory;
+import org.apache.asterix.external.api.IAdapterFactory;
import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.feeds.CentralFeedManager;
import org.apache.asterix.feeds.FeedJoint;
@@ -238,13 +238,12 @@ public class QueryTranslator extends AbstractLangTranslator {
/**
* Compiles and submits for execution a list of AQL statements.
- *
* @param hcc
- * A Hyracks client connection that is used to submit a jobspec to Hyracks.
+ * A Hyracks client connection that is used to submit a jobspec to Hyracks.
* @param hdc
- * A Hyracks dataset client object that is used to read the results.
+ * A Hyracks dataset client object that is used to read the results.
* @param resultDelivery
- * True if the results should be read asynchronously or false if we should wait for results to be read.
+ * True if the results should be read asynchronously or false if we should wait for results to be read.
* @return A List<QueryResult> containing a QueryResult instance corresponding to each submitted query.
* @throws Exception
*/
@@ -2133,7 +2132,7 @@ public class QueryTranslator extends AbstractLangTranslator {
FeedId feedId = connectionRequest.getFeedJointKey().getFeedId();
PrimaryFeed primaryFeed = (PrimaryFeed) MetadataManager.INSTANCE.getFeed(mdTxnCtx,
feedId.getDataverse(), feedId.getFeedName());
- Pair<JobSpecification, IFeedAdapterFactory> pair = FeedOperations.buildFeedIntakeJobSpec(primaryFeed,
+ Pair<JobSpecification, IAdapterFactory> pair = FeedOperations.buildFeedIntakeJobSpec(primaryFeed,
metadataProvider, policyAccessor);
// adapter configuration are valid at this stage
// register the feed joints (these are auto-de-registered)
@@ -2141,11 +2140,13 @@ public class QueryTranslator extends AbstractLangTranslator {
FeedLifecycleListener.INSTANCE.registerFeedJoint(fj);
}
JobUtils.runJob(hcc, pair.first, false);
- IFeedAdapterFactory adapterFactory = pair.second;
+ /* TODO: Fix record tracking
+ * IFeedAdapterFactory adapterFactory = pair.second;
if (adapterFactory.isRecordTrackingEnabled()) {
FeedLifecycleListener.INSTANCE.registerFeedIntakeProgressTracker(feedConnId,
adapterFactory.createIntakeProgressTracker());
}
+ */
eventSubscriber.assertEvent(FeedLifecycleEvent.FEED_INTAKE_STARTED);
} else {
for (IFeedJoint fj : triple.third) {
@@ -2186,7 +2187,6 @@ public class QueryTranslator extends AbstractLangTranslator {
/**
* Generates a subscription request corresponding to a connect feed request. In addition, provides a boolean
* flag indicating if feed intake job needs to be started (source primary feed not found to be active).
- *
* @param dataverse
* @param feed
* @param dataset
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-app/src/main/java/org/apache/asterix/file/ExternalIndexingOperations.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/ExternalIndexingOperations.java b/asterix-app/src/main/java/org/apache/asterix/file/ExternalIndexingOperations.java
index d3fade5..595fe4c 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/ExternalIndexingOperations.java
+++ b/asterix-app/src/main/java/org/apache/asterix/file/ExternalIndexingOperations.java
@@ -27,11 +27,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
import org.apache.asterix.common.api.ILocalResourceMetadata;
import org.apache.asterix.common.config.AsterixStorageProperties;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
@@ -45,16 +40,17 @@ import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactor
import org.apache.asterix.common.ioopcallbacks.LSMBTreeWithBuddyIOOperationCallbackFactory;
import org.apache.asterix.common.ioopcallbacks.LSMRTreeIOOperationCallbackFactory;
import org.apache.asterix.dataflow.data.nontagged.valueproviders.AqlPrimitiveValueProviderFactory;
-import org.apache.asterix.external.adapter.factory.HDFSAdapterFactory;
-import org.apache.asterix.external.adapter.factory.HDFSIndexingAdapterFactory;
-import org.apache.asterix.external.adapter.factory.HiveAdapterFactory;
+import org.apache.asterix.external.api.IAdapterFactory;
import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.external.indexing.FilesIndexDescription;
import org.apache.asterix.external.indexing.IndexingConstants;
-import org.apache.asterix.external.indexing.operators.ExternalDatasetIndexesAbortOperatorDescriptor;
-import org.apache.asterix.external.indexing.operators.ExternalDatasetIndexesCommitOperatorDescriptor;
-import org.apache.asterix.external.indexing.operators.ExternalDatasetIndexesRecoverOperatorDescriptor;
-import org.apache.asterix.external.indexing.operators.IndexInfoOperatorDescriptor;
+import org.apache.asterix.external.operators.ExternalDatasetIndexesAbortOperatorDescriptor;
+import org.apache.asterix.external.operators.ExternalDatasetIndexesCommitOperatorDescriptor;
+import org.apache.asterix.external.operators.ExternalDatasetIndexesRecoverOperatorDescriptor;
+import org.apache.asterix.external.operators.ExternalFilesIndexOperatorDescriptor;
+import org.apache.asterix.external.operators.IndexInfoOperatorDescriptor;
+import org.apache.asterix.external.provider.AdapterFactoryProvider;
+import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
import org.apache.asterix.formats.nontagged.AqlTypeTraitProvider;
@@ -73,13 +69,17 @@ import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.util.AsterixAppContextInfo;
import org.apache.asterix.om.util.NonTaggedFormatUtil;
-import org.apache.asterix.tools.external.data.ExternalFilesIndexOperatorDescriptor;
import org.apache.asterix.transaction.management.opcallbacks.SecondaryIndexOperationTrackerProvider;
import org.apache.asterix.transaction.management.resource.ExternalBTreeLocalResourceMetadata;
import org.apache.asterix.transaction.management.resource.PersistentLocalResourceFactoryProvider;
import org.apache.asterix.transaction.management.service.transaction.AsterixRuntimeComponentsProvider;
import org.apache.asterix.translator.CompiledStatements.CompiledCreateIndexStatement;
import org.apache.asterix.translator.CompiledStatements.CompiledIndexDropStatement;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -107,6 +107,7 @@ public class ExternalIndexingOperations {
public static final List<List<String>> FILE_INDEX_FIELD_NAMES = new ArrayList<List<String>>();
public static final ArrayList<IAType> FILE_INDEX_FIELD_TYPES = new ArrayList<IAType>();
+
static {
FILE_INDEX_FIELD_NAMES.add(new ArrayList<String>(Arrays.asList("")));
FILE_INDEX_FIELD_TYPES.add(BuiltinType.ASTRING);
@@ -114,9 +115,7 @@ public class ExternalIndexingOperations {
public static boolean isIndexible(ExternalDatasetDetails ds) {
String adapter = ds.getAdapter();
- if (adapter.equalsIgnoreCase("hdfs") || adapter.equalsIgnoreCase("hive")
- || adapter.equalsIgnoreCase("org.apache.asterix.external.dataset.adapter.HDFSAdapter")
- || adapter.equalsIgnoreCase("org.apache.asterix.external.dataset.adapter.HIVEAdapter")) {
+ if (adapter.equalsIgnoreCase(ExternalDataConstants.ALIAS_HDFS_ADAPTER)) {
return true;
}
return false;
@@ -126,12 +125,6 @@ public class ExternalIndexingOperations {
return ds.getState() != ExternalDatasetTransactionState.COMMIT;
}
- public static boolean datasetUsesHiveAdapter(ExternalDatasetDetails ds) {
- String adapter = ds.getAdapter();
- return (adapter.equalsIgnoreCase("hive") || adapter
- .equalsIgnoreCase("org.apache.asterix.external.dataset.adapter.HIVEAdapter"));
- }
-
public static boolean isValidIndexName(String datasetName, String indexName) {
return (!datasetName.concat(IndexingConstants.EXTERNAL_FILE_INDEX_NAME_SUFFIX).equals(indexName));
}
@@ -154,18 +147,15 @@ public class ExternalIndexingOperations {
return IndexingConstants.getBuddyBtreeComparatorFactories();
}
- public static ArrayList<ExternalFile> getSnapshotFromExternalFileSystem(Dataset dataset) throws AlgebricksException {
+ public static ArrayList<ExternalFile> getSnapshotFromExternalFileSystem(Dataset dataset)
+ throws AlgebricksException {
ArrayList<ExternalFile> files = new ArrayList<ExternalFile>();
ExternalDatasetDetails datasetDetails = (ExternalDatasetDetails) dataset.getDatasetDetails();
try {
// Create the file system object
FileSystem fs = getFileSystemObject(datasetDetails.getProperties());
- // If dataset uses hive adapter, add path to the dataset properties
- if (datasetUsesHiveAdapter(datasetDetails)) {
- HiveAdapterFactory.populateConfiguration(datasetDetails.getProperties());
- }
// Get paths of dataset
- String path = datasetDetails.getProperties().get(HDFSAdapterFactory.KEY_PATH);
+ String path = datasetDetails.getProperties().get(ExternalDataConstants.KEY_PATH);
String[] paths = path.split(",");
// Add fileStatuses to files
@@ -176,9 +166,9 @@ public class ExternalIndexingOperations {
if (fileStatuses[i].isDirectory()) {
listSubFiles(dataset, fs, fileStatuses[i], files);
} else {
- files.add(new ExternalFile(dataset.getDataverseName(), dataset.getDatasetName(),
- nextFileNumber, fileStatuses[i].getPath().toUri().getPath(), new Date(fileStatuses[i]
- .getModificationTime()), fileStatuses[i].getLen(),
+ files.add(new ExternalFile(dataset.getDataverseName(), dataset.getDatasetName(), nextFileNumber,
+ fileStatuses[i].getPath().toUri().getPath(),
+ new Date(fileStatuses[i].getModificationTime()), fileStatuses[i].getLen(),
ExternalFilePendingOp.PENDING_NO_OP));
}
}
@@ -216,14 +206,14 @@ public class ExternalIndexingOperations {
public static FileSystem getFileSystemObject(Map<String, String> map) throws IOException {
Configuration conf = new Configuration();
- conf.set("fs.default.name", map.get(HDFSAdapterFactory.KEY_HDFS_URL).trim());
- conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
+ conf.set(ExternalDataConstants.KEY_HADOOP_FILESYSTEM_URI, map.get(ExternalDataConstants.KEY_HDFS_URL).trim());
+ conf.set(ExternalDataConstants.KEY_HADOOP_FILESYSTEM_CLASS, DistributedFileSystem.class.getName());
return FileSystem.get(conf);
}
public static JobSpecification buildFilesIndexReplicationJobSpec(Dataset dataset,
ArrayList<ExternalFile> externalFilesSnapshot, AqlMetadataProvider metadataProvider, boolean createIndex)
- throws MetadataException, AlgebricksException {
+ throws MetadataException, AlgebricksException {
JobSpecification spec = JobSpecificationUtils.createJobSpecification();
IAsterixPropertiesProvider asterixPropertiesProvider = AsterixAppContextInfo.getInstance();
AsterixStorageProperties storageProperties = asterixPropertiesProvider.getStorageProperties();
@@ -232,20 +222,20 @@ public class ExternalIndexingOperations {
ILSMMergePolicyFactory mergePolicyFactory = compactionInfo.first;
Map<String, String> mergePolicyFactoryProperties = compactionInfo.second;
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> secondarySplitsAndConstraint = metadataProvider
- .splitProviderAndPartitionConstraintsForFilesIndex(dataset.getDataverseName(),
- dataset.getDatasetName(), getFilesIndexName(dataset.getDatasetName()), true);
+ .splitProviderAndPartitionConstraintsForFilesIndex(dataset.getDataverseName(), dataset.getDatasetName(),
+ getFilesIndexName(dataset.getDatasetName()), true);
IFileSplitProvider secondaryFileSplitProvider = secondarySplitsAndConstraint.first;
FilesIndexDescription filesIndexDescription = new FilesIndexDescription();
ILocalResourceMetadata localResourceMetadata = new ExternalBTreeLocalResourceMetadata(
- filesIndexDescription.EXTERNAL_FILE_INDEX_TYPE_TRAITS,
- filesIndexDescription.FILES_INDEX_COMP_FACTORIES, new int[] { 0 }, false, dataset.getDatasetId(),
- mergePolicyFactory, mergePolicyFactoryProperties);
+ filesIndexDescription.EXTERNAL_FILE_INDEX_TYPE_TRAITS, filesIndexDescription.FILES_INDEX_COMP_FACTORIES,
+ new int[] { 0 }, false, dataset.getDatasetId(), mergePolicyFactory, mergePolicyFactoryProperties);
PersistentLocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.ExternalBTreeResource);
ExternalBTreeDataflowHelperFactory indexDataflowHelperFactory = new ExternalBTreeDataflowHelperFactory(
- mergePolicyFactory, mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(
- dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
- LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties.getBloomFilterFalsePositiveRate(),
+ mergePolicyFactory, mergePolicyFactoryProperties,
+ new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+ AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
+ storageProperties.getBloomFilterFalsePositiveRate(),
ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true);
ExternalFilesIndexOperatorDescriptor externalFilesOp = new ExternalFilesIndexOperatorDescriptor(spec,
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
@@ -260,7 +250,6 @@ public class ExternalIndexingOperations {
/**
* This method create an indexing operator that index records in HDFS
- *
* @param jobSpec
* @param itemType
* @param dataset
@@ -272,10 +261,10 @@ public class ExternalIndexingOperations {
private static Pair<ExternalDataScanOperatorDescriptor, AlgebricksPartitionConstraint> getExternalDataIndexingOperator(
JobSpecification jobSpec, IAType itemType, Dataset dataset, List<ExternalFile> files,
RecordDescriptor indexerDesc, AqlMetadataProvider metadataProvider) throws Exception {
- HDFSIndexingAdapterFactory adapterFactory = new HDFSIndexingAdapterFactory();
- adapterFactory.setFiles(files);
- adapterFactory.configure(((ExternalDatasetDetails) dataset.getDatasetDetails()).getProperties(),
- (ARecordType) itemType);
+ ExternalDatasetDetails externalDatasetDetails = (ExternalDatasetDetails) dataset.getDatasetDetails();
+ Map<String, String> configuration = externalDatasetDetails.getProperties();
+ IAdapterFactory adapterFactory = AdapterFactoryProvider.getAdapterFactory(externalDatasetDetails.getAdapter(),
+ configuration, (ARecordType) itemType, files, true);
return new Pair<ExternalDataScanOperatorDescriptor, AlgebricksPartitionConstraint>(
new ExternalDataScanOperatorDescriptor(jobSpec, indexerDesc, adapterFactory),
adapterFactory.getPartitionConstraint());
@@ -297,7 +286,6 @@ public class ExternalIndexingOperations {
* deleteedFiles should contain files that are no longer there in the file system
* appendedFiles should have the new file information of existing files
* The method should return false in case of zero delta
- *
* @param dataset
* @param metadataFiles
* @param addedFiles
@@ -309,7 +297,7 @@ public class ExternalIndexingOperations {
*/
public static boolean isDatasetUptodate(Dataset dataset, List<ExternalFile> metadataFiles,
List<ExternalFile> addedFiles, List<ExternalFile> deletedFiles, List<ExternalFile> appendedFiles)
- throws MetadataException, AlgebricksException {
+ throws MetadataException, AlgebricksException {
boolean uptodate = true;
int newFileNumber = metadataFiles.get(metadataFiles.size() - 1).getFileNumber() + 1;
@@ -340,9 +328,10 @@ public class ExternalIndexingOperations {
} else {
// Same file name, Different file mod date -> delete and add
metadataFile.setPendingOp(ExternalFilePendingOp.PENDING_DROP_OP);
- deletedFiles.add(new ExternalFile(metadataFile.getDataverseName(), metadataFile
- .getDatasetName(), 0, metadataFile.getFileName(), metadataFile.getLastModefiedTime(),
- metadataFile.getSize(), ExternalFilePendingOp.PENDING_DROP_OP));
+ deletedFiles
+ .add(new ExternalFile(metadataFile.getDataverseName(), metadataFile.getDatasetName(), 0,
+ metadataFile.getFileName(), metadataFile.getLastModefiedTime(),
+ metadataFile.getSize(), ExternalFilePendingOp.PENDING_DROP_OP));
fileSystemFile.setPendingOp(ExternalFilePendingOp.PENDING_ADD_OP);
fileSystemFile.setFileNumber(newFileNumber);
addedFiles.add(fileSystemFile);
@@ -382,8 +371,8 @@ public class ExternalIndexingOperations {
if (metadataFile.getPendingOp() == ExternalFilePendingOp.PENDING_NO_OP) {
metadataFile.setPendingOp(ExternalFilePendingOp.PENDING_DROP_OP);
deletedFiles.add(new ExternalFile(metadataFile.getDataverseName(), metadataFile.getDatasetName(),
- newFileNumber, metadataFile.getFileName(), metadataFile.getLastModefiedTime(), metadataFile
- .getSize(), metadataFile.getPendingOp()));
+ newFileNumber, metadataFile.getFileName(), metadataFile.getLastModefiedTime(),
+ metadataFile.getSize(), metadataFile.getPendingOp()));
newFileNumber++;
uptodate = false;
}
@@ -421,13 +410,14 @@ public class ExternalIndexingOperations {
metadataProvider.getMetadataTxnContext());
IndexDropOperatorDescriptor btreeDrop = new IndexDropOperatorDescriptor(spec,
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
- splitsAndConstraint.first, new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(
- dataset.getDatasetId()), compactionInfo.first, compactionInfo.second,
+ splitsAndConstraint.first,
+ new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()),
+ compactionInfo.first, compactionInfo.second,
new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), false, null, null, null, null, !temp));
- AlgebricksPartitionConstraintHelper
- .setPartitionConstraintInJobSpec(spec, btreeDrop, splitsAndConstraint.second);
+ AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, btreeDrop,
+ splitsAndConstraint.second);
spec.addRoot(btreeDrop);
return spec;
@@ -443,9 +433,9 @@ public class ExternalIndexingOperations {
else if (file.getPendingOp() == ExternalFilePendingOp.PENDING_APPEND_OP) {
for (ExternalFile appendedFile : appendedFiles) {
if (appendedFile.getFileName().equals(file.getFileName())) {
- files.add(new ExternalFile(file.getDataverseName(), file.getDatasetName(),
- file.getFileNumber(), file.getFileName(), file.getLastModefiedTime(), appendedFile
- .getSize(), ExternalFilePendingOp.PENDING_NO_OP));
+ files.add(new ExternalFile(file.getDataverseName(), file.getDatasetName(), file.getFileNumber(),
+ file.getFileName(), file.getLastModefiedTime(), appendedFile.getSize(),
+ ExternalFilePendingOp.PENDING_NO_OP));
}
}
}
@@ -557,8 +547,7 @@ public class ExternalIndexingOperations {
AsterixStorageProperties storageProperties, JobSpecification spec) {
return new ExternalBTreeWithBuddyDataflowHelperFactory(mergePolicyFactory, mergePolicyFactoryProperties,
new SecondaryIndexOperationTrackerProvider(ds.getDatasetId()),
- AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
- LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE,
+ AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), new int[] { index.getKeyFieldNames().size() },
ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(ds), true);
}
@@ -567,7 +556,7 @@ public class ExternalIndexingOperations {
private static ExternalRTreeDataflowHelperFactory getRTreeDataflowHelperFactory(Dataset ds, Index index,
ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyFactoryProperties,
AsterixStorageProperties storageProperties, AqlMetadataProvider metadataProvider, JobSpecification spec)
- throws AlgebricksException, AsterixException {
+ throws AlgebricksException, AsterixException {
int numPrimaryKeys = getRIDSize(ds);
List<List<String>> secondaryKeyFields = index.getKeyFieldNames();
secondaryKeyFields.size();
@@ -594,8 +583,8 @@ public class ExternalIndexingOperations {
.getSerializerDeserializer(nestedKeyType);
secondaryRecFields[i] = keySerde;
- secondaryComparatorFactories[i] = AqlBinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(
- nestedKeyType, true);
+ secondaryComparatorFactories[i] = AqlBinaryComparatorFactoryProvider.INSTANCE
+ .getBinaryComparatorFactory(nestedKeyType, true);
secondaryTypeTraits[i] = AqlTypeTraitProvider.INSTANCE.getTypeTrait(nestedKeyType);
valueProviderFactories[i] = AqlPrimitiveValueProviderFactory.INSTANCE;
}
@@ -743,13 +732,14 @@ public class ExternalIndexingOperations {
ILSMMergePolicyFactory mergePolicyFactory = compactionInfo.first;
Map<String, String> mergePolicyFactoryProperties = compactionInfo.second;
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> secondarySplitsAndConstraint = metadataProvider
- .splitProviderAndPartitionConstraintsForFilesIndex(dataset.getDataverseName(),
- dataset.getDatasetName(), getFilesIndexName(dataset.getDatasetName()), true);
+ .splitProviderAndPartitionConstraintsForFilesIndex(dataset.getDataverseName(), dataset.getDatasetName(),
+ getFilesIndexName(dataset.getDatasetName()), true);
IFileSplitProvider secondaryFileSplitProvider = secondarySplitsAndConstraint.first;
ExternalBTreeDataflowHelperFactory indexDataflowHelperFactory = new ExternalBTreeDataflowHelperFactory(
- mergePolicyFactory, mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(
- dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
- LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties.getBloomFilterFalsePositiveRate(),
+ mergePolicyFactory, mergePolicyFactoryProperties,
+ new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+ AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
+ storageProperties.getBloomFilterFalsePositiveRate(),
ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true);
FilesIndexDescription filesIndexDescription = new FilesIndexDescription();
LSMTreeIndexCompactOperatorDescriptor compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-app/src/main/java/org/apache/asterix/file/FeedOperations.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/FeedOperations.java b/asterix-app/src/main/java/org/apache/asterix/file/FeedOperations.java
index c7bb0e3..cb55c5f 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/FeedOperations.java
+++ b/asterix-app/src/main/java/org/apache/asterix/file/FeedOperations.java
@@ -33,7 +33,7 @@ import org.apache.asterix.common.feeds.api.IFeedMessage;
import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
import org.apache.asterix.common.feeds.message.EndFeedMessage;
import org.apache.asterix.common.feeds.message.ThrottlingEnabledFeedMessage;
-import org.apache.asterix.external.adapter.factory.IFeedAdapterFactory;
+import org.apache.asterix.external.api.IAdapterFactory;
import org.apache.asterix.feeds.FeedLifecycleListener;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
import org.apache.asterix.metadata.entities.PrimaryFeed;
@@ -58,23 +58,22 @@ public class FeedOperations {
/**
* Builds the job spec for ingesting a (primary) feed from its external source via the feed adaptor.
- *
* @param primaryFeed
* @param metadataProvider
* @return JobSpecification the Hyracks job specification for receiving data from external source
* @throws Exception
*/
- public static Pair<JobSpecification, IFeedAdapterFactory> buildFeedIntakeJobSpec(PrimaryFeed primaryFeed,
+ public static Pair<JobSpecification, IAdapterFactory> buildFeedIntakeJobSpec(PrimaryFeed primaryFeed,
AqlMetadataProvider metadataProvider, FeedPolicyAccessor policyAccessor) throws Exception {
JobSpecification spec = JobSpecificationUtils.createJobSpecification();
spec.setFrameSize(FeedConstants.JobConstants.DEFAULT_FRAME_SIZE);
- IFeedAdapterFactory adapterFactory = null;
+ IAdapterFactory adapterFactory = null;
IOperatorDescriptor feedIngestor;
AlgebricksPartitionConstraint ingesterPc;
try {
- Triple<IOperatorDescriptor, AlgebricksPartitionConstraint, IFeedAdapterFactory> t = metadataProvider
+ Triple<IOperatorDescriptor, AlgebricksPartitionConstraint, IAdapterFactory> t = metadataProvider
.buildFeedIntakeRuntime(spec, primaryFeed, policyAccessor);
feedIngestor = t.first;
ingesterPc = t.second;
@@ -90,7 +89,7 @@ public class FeedOperations {
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, nullSink, ingesterPc);
spec.connect(new OneToOneConnectorDescriptor(spec), feedIngestor, 0, nullSink, 0);
spec.addRoot(nullSink);
- return new Pair<JobSpecification, IFeedAdapterFactory>(spec, adapterFactory);
+ return new Pair<JobSpecification, IAdapterFactory>(spec, adapterFactory);
}
public static JobSpecification buildDiscontinueFeedSourceSpec(AqlMetadataProvider metadataProvider, FeedId feedId)
@@ -247,7 +246,7 @@ public class FeedOperations {
private static Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildDisconnectFeedMessengerRuntime(
JobSpecification jobSpec, FeedConnectionId feedConenctionId, List<String> locations,
FeedRuntimeType sourceFeedRuntimeType, boolean completeDisconnection, FeedId sourceFeedId)
- throws AlgebricksException {
+ throws AlgebricksException {
IFeedMessage feedMessage = new EndFeedMessage(feedConenctionId, sourceFeedRuntimeType, sourceFeedId,
completeDisconnection, EndFeedMessage.EndMessageType.DISCONNECT_FEED);
return buildSendFeedMessageRuntime(jobSpec, feedConenctionId, feedMessage, locations);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
index 4eb6944..44af0ff 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
+++ b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
@@ -38,7 +38,7 @@ import org.apache.asterix.common.transactions.IRecoveryManager.ResourceType;
import org.apache.asterix.common.transactions.JobId;
import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.external.indexing.IndexingConstants;
-import org.apache.asterix.external.indexing.operators.ExternalIndexBulkModifyOperatorDescriptor;
+import org.apache.asterix.external.operators.ExternalIndexBulkModifyOperatorDescriptor;
import org.apache.asterix.formats.nontagged.AqlBinaryBooleanInspectorImpl;
import org.apache.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java
index 267be3d..01775ab 100755
--- a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java
+++ b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java
@@ -46,6 +46,7 @@ import org.apache.asterix.metadata.api.IMetadataEntity;
import org.apache.asterix.metadata.entities.DatasourceAdapter;
import org.apache.asterix.metadata.entities.DatasourceAdapter.AdapterType;
import org.apache.asterix.metadata.entities.Dataverse;
+import org.apache.asterix.metadata.entities.Library;
import org.apache.asterix.metadata.feeds.AdapterIdentifier;
import org.apache.asterix.runtime.formats.NonTaggedDataFormat;
@@ -221,8 +222,7 @@ public class ExternalLibraryBootstrap {
LOGGER.info("Installed adapters contain in library :" + libraryName);
}
- MetadataManager.INSTANCE.addLibrary(mdTxnCtx,
- new org.apache.asterix.metadata.entities.Library(dataverse, libraryName));
+ MetadataManager.INSTANCE.addLibrary(mdTxnCtx, new Library(dataverse, libraryName));
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Added library " + libraryName + "to Metadata");
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java b/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
index 2f8a910..9eee818 100644
--- a/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
+++ b/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
@@ -34,7 +34,7 @@ import org.apache.asterix.common.config.GlobalConfig;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.compiler.provider.AqlCompilationProvider;
import org.apache.asterix.compiler.provider.ILangCompilationProvider;
-import org.apache.asterix.external.dataset.adapter.FileSystemBasedAdapter;
+import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.IdentitiyResolverFactory;
import org.apache.asterix.test.base.AsterixTestHelper;
import org.apache.asterix.test.common.TestHelper;
@@ -80,7 +80,7 @@ public class OptimizerTest {
AsterixHyracksIntegrationUtil.init(true);
// Set the node resolver to be the identity resolver that expects node names
// to be node controller ids; a valid assumption in test environment.
- System.setProperty(FileSystemBasedAdapter.NODE_RESOLVER_FACTORY_PROPERTY,
+ System.setProperty(ExternalDataConstants.NODE_RESOLVER_FACTORY_PROPERTY,
IdentitiyResolverFactory.class.getName());
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java b/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
index 7a55c90..922486f 100644
--- a/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
+++ b/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
@@ -23,18 +23,14 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.logging.Logger;
-import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil;
import org.apache.asterix.common.config.AsterixTransactionProperties;
import org.apache.asterix.test.aql.TestExecutor;
import org.apache.asterix.testframework.context.TestCaseContext;
import org.apache.asterix.testframework.xml.TestGroup;
import org.apache.commons.lang3.StringUtils;
-import org.apache.hyracks.api.lifecycle.ILifeCycleComponent;
-import org.apache.hyracks.api.lifecycle.ILifeCycleComponentManager;
-import org.apache.hyracks.control.nc.NodeControllerService;
-import org.apache.hyracks.control.nc.application.NCApplicationContext;
-import org.apache.hyracks.storage.common.buffercache.BufferCache;
-import org.junit.*;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java b/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java
index b072e55..5e76ecb 100644
--- a/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java
+++ b/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java
@@ -26,7 +26,7 @@ import java.util.logging.Logger;
import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil;
import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
import org.apache.asterix.common.config.GlobalConfig;
-import org.apache.asterix.external.dataset.adapter.FileSystemBasedAdapter;
+import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.IdentitiyResolverFactory;
import org.apache.asterix.testframework.xml.TestGroup;
import org.apache.asterix.testframework.xml.TestSuite;
@@ -64,7 +64,7 @@ public class ExecutionTestUtil {
// Set the node resolver to be the identity resolver that expects node
// names
// to be node controller ids; a valid assumption in test environment.
- System.setProperty(FileSystemBasedAdapter.NODE_RESOLVER_FACTORY_PROPERTY,
+ System.setProperty(ExternalDataConstants.NODE_RESOLVER_FACTORY_PROPERTY,
IdentitiyResolverFactory.class.getName());
FailedGroup = new TestGroup();
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-app/src/test/resources/metadata/results/basic/meta15/meta15.1.adm
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/metadata/results/basic/meta15/meta15.1.adm b/asterix-app/src/test/resources/metadata/results/basic/meta15/meta15.1.adm
index 8a23d78..c4dde05 100644
--- a/asterix-app/src/test/resources/metadata/results/basic/meta15/meta15.1.adm
+++ b/asterix-app/src/test/resources/metadata/results/basic/meta15/meta15.1.adm
@@ -1,12 +1,3 @@
-{ "DataverseName": "Metadata", "Name": "azure_twitter", "Classname": "org.apache.asterix.external.adapter.factory.PullBasedAzureTwitterAdapterFactory", "Type": "INTERNAL", "Timestamp": "Thu Oct 24 01:39:27 PDT 2013" }
-{ "DataverseName": "Metadata", "Name": "cnn_feed", "Classname": "org.apache.asterix.external.adapter.factory.CNNFeedAdapterFactory", "Type": "INTERNAL", "Timestamp": "Tue Jul 16 22:38:45 PDT 2013" }
-{ "DataverseName": "Metadata", "Name": "file_feed", "Classname": "org.apache.asterix.tools.external.data.RateControlledFileSystemBasedAdapterFactory", "Type": "INTERNAL", "Timestamp": "Tue Jul 16 22:38:45 PDT 2013" }
-{ "DataverseName": "Metadata", "Name": "hdfs", "Classname": "org.apache.asterix.external.adapter.factory.HDFSAdapterFactory", "Type": "INTERNAL", "Timestamp": "Tue Jul 16 22:38:45 PDT 2013" }
-{ "DataverseName": "Metadata", "Name": "hive", "Classname": "org.apache.asterix.external.adapter.factory.HiveAdapterFactory", "Type": "INTERNAL", "Timestamp": "Tue Jul 16 22:38:45 PDT 2013" }
-{ "DataverseName": "Metadata", "Name": "localfs", "Classname": "org.apache.asterix.external.adapter.factory.NCFileSystemAdapterFactory", "Type": "INTERNAL", "Timestamp": "Tue Jul 16 22:38:45 PDT 2013" }
-{ "DataverseName": "Metadata", "Name": "pull_twitter", "Classname": "org.apache.asterix.external.adapter.factory.PullBasedTwitterAdapterFactory", "Type": "INTERNAL", "Timestamp": "Tue Jul 16 22:38:45 PDT 2013" }
-{ "DataverseName": "Metadata", "Name": "push_twitter", "Classname": "org.apache.asterix.external.adapter.factory.PushBasedTwitterAdapterFactory", "Type": "INTERNAL", "Timestamp": "Tue Jul 16 22:38:45 PDT 2013" }
-{ "DataverseName": "Metadata", "Name": "rss_feed", "Classname": "org.apache.asterix.external.adapter.factory.RSSFeedAdapterFactory", "Type": "INTERNAL", "Timestamp": "Tue Jul 16 22:38:45 PDT 2013" }
-{ "DataverseName": "Metadata", "Name": "socket_adapter", "Classname": "org.apache.asterix.tools.external.data.GenericSocketFeedAdapterFactory", "Type": "INTERNAL", "Timestamp": "Tue Jul 16 22:38:45 PDT 2013" }
-{ "DataverseName": "Metadata", "Name": "socket_client", "Classname": "org.apache.asterix.tools.external.data.SocketClientAdapterFactory", "Type": "INTERNAL", "Timestamp": "Wed Nov 20 14:45:58 IST 2013" }
-{ "DataverseName": "Metadata", "Name": "twitter_firehose", "Classname": "org.apache.asterix.tools.external.data.TwitterFirehoseFeedAdapterFactory", "Type": "INTERNAL", "Timestamp": "Tue Jul 16 22:38:45 PDT 2013" }
+{ "DataverseName": "Metadata", "Name": "adapter", "Classname": "org.apache.asterix.external.adapter.factory.GenericAdapterFactory", "Type": "INTERNAL", "Timestamp": "Sun Jan 03 15:39:35 AST 2016" }
+{ "DataverseName": "Metadata", "Name": "socket_adapter", "Classname": "org.apache.asterix.external.runtime.GenericSocketFeedAdapterFactory", "Type": "INTERNAL", "Timestamp": "Sun Jan 03 15:39:35 AST 2016" }
+{ "DataverseName": "Metadata", "Name": "socket_client", "Classname": "org.apache.asterix.external.runtime.SocketClientAdapterFactory", "Type": "INTERNAL", "Timestamp": "Sun Jan 03 15:39:35 AST 2016" }
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
----------------------------------------------------------------------
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java b/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
index cd829e7..b8c3f2f 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
@@ -40,7 +40,6 @@ import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.storage.common.file.ILocalResourceRepository;
import org.apache.hyracks.storage.common.file.IResourceIdFactory;
-import org.apache.hyracks.storage.common.file.ResourceIdFactory;
public interface IAsterixAppRuntimeContext {
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-common/src/main/java/org/apache/asterix/common/feeds/BasicMonitoredBuffer.java
----------------------------------------------------------------------
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/BasicMonitoredBuffer.java b/asterix-common/src/main/java/org/apache/asterix/common/feeds/BasicMonitoredBuffer.java
index c5594db..70833fc 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/BasicMonitoredBuffer.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/feeds/BasicMonitoredBuffer.java
@@ -21,6 +21,8 @@ package org.apache.asterix.common.feeds;
import org.apache.asterix.common.feeds.api.IExceptionHandler;
import org.apache.asterix.common.feeds.api.IFeedMetricCollector;
import org.apache.asterix.common.feeds.api.IFrameEventCallback;
+import org.apache.asterix.common.feeds.api.IFramePostProcessor;
+import org.apache.asterix.common.feeds.api.IFramePreprocessor;
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-common/src/main/java/org/apache/asterix/common/feeds/ComputeSideMonitoredBuffer.java
----------------------------------------------------------------------
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/ComputeSideMonitoredBuffer.java b/asterix-common/src/main/java/org/apache/asterix/common/feeds/ComputeSideMonitoredBuffer.java
index 4ae288d..7ec3fdf 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/ComputeSideMonitoredBuffer.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/feeds/ComputeSideMonitoredBuffer.java
@@ -21,6 +21,8 @@ package org.apache.asterix.common.feeds;
import org.apache.asterix.common.feeds.api.IExceptionHandler;
import org.apache.asterix.common.feeds.api.IFeedMetricCollector;
import org.apache.asterix.common.feeds.api.IFrameEventCallback;
+import org.apache.asterix.common.feeds.api.IFramePostProcessor;
+import org.apache.asterix.common.feeds.api.IFramePreprocessor;
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-common/src/main/java/org/apache/asterix/common/feeds/IFramePostProcessor.java
----------------------------------------------------------------------
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/IFramePostProcessor.java b/asterix-common/src/main/java/org/apache/asterix/common/feeds/IFramePostProcessor.java
deleted file mode 100644
index 1dfbee9..0000000
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/IFramePostProcessor.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.common.feeds;
-
-import java.nio.ByteBuffer;
-
-import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-
-public interface IFramePostProcessor {
-
- public void postProcessFrame(ByteBuffer frame, FrameTupleAccessor frameAccessor);
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-common/src/main/java/org/apache/asterix/common/feeds/IFramePreprocessor.java
----------------------------------------------------------------------
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/IFramePreprocessor.java b/asterix-common/src/main/java/org/apache/asterix/common/feeds/IFramePreprocessor.java
deleted file mode 100644
index f602656..0000000
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/IFramePreprocessor.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.common.feeds;
-
-import java.nio.ByteBuffer;
-
-public interface IFramePreprocessor {
-
- public void preProcess(ByteBuffer frame) throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-common/src/main/java/org/apache/asterix/common/feeds/IntakeSideMonitoredBuffer.java
----------------------------------------------------------------------
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/IntakeSideMonitoredBuffer.java b/asterix-common/src/main/java/org/apache/asterix/common/feeds/IntakeSideMonitoredBuffer.java
index ed1e943..10b7ddb 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/IntakeSideMonitoredBuffer.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/feeds/IntakeSideMonitoredBuffer.java
@@ -21,6 +21,8 @@ package org.apache.asterix.common.feeds;
import org.apache.asterix.common.feeds.api.IExceptionHandler;
import org.apache.asterix.common.feeds.api.IFeedMetricCollector;
import org.apache.asterix.common.feeds.api.IFrameEventCallback;
+import org.apache.asterix.common.feeds.api.IFramePostProcessor;
+import org.apache.asterix.common.feeds.api.IFramePreprocessor;
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-common/src/main/java/org/apache/asterix/common/feeds/MonitoredBuffer.java
----------------------------------------------------------------------
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/MonitoredBuffer.java b/asterix-common/src/main/java/org/apache/asterix/common/feeds/MonitoredBuffer.java
index 5761944..e5a22b5 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/MonitoredBuffer.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/feeds/MonitoredBuffer.java
@@ -35,6 +35,8 @@ import org.apache.asterix.common.feeds.api.IFeedMetricCollector.ValueType;
import org.apache.asterix.common.feeds.api.IFeedRuntime.Mode;
import org.apache.asterix.common.feeds.api.IFrameEventCallback;
import org.apache.asterix.common.feeds.api.IFrameEventCallback.FrameEvent;
+import org.apache.asterix.common.feeds.api.IFramePostProcessor;
+import org.apache.asterix.common.feeds.api.IFramePreprocessor;
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-common/src/main/java/org/apache/asterix/common/feeds/StorageSideMonitoredBuffer.java
----------------------------------------------------------------------
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/StorageSideMonitoredBuffer.java b/asterix-common/src/main/java/org/apache/asterix/common/feeds/StorageSideMonitoredBuffer.java
index d545b09..4027237 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/StorageSideMonitoredBuffer.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/feeds/StorageSideMonitoredBuffer.java
@@ -24,6 +24,8 @@ import org.apache.asterix.common.feeds.FeedConstants.StatisticsConstants;
import org.apache.asterix.common.feeds.api.IExceptionHandler;
import org.apache.asterix.common.feeds.api.IFeedMetricCollector;
import org.apache.asterix.common.feeds.api.IFrameEventCallback;
+import org.apache.asterix.common.feeds.api.IFramePostProcessor;
+import org.apache.asterix.common.feeds.api.IFramePreprocessor;
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IAdapterRuntimeManager.java
----------------------------------------------------------------------
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IAdapterRuntimeManager.java b/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IAdapterRuntimeManager.java
index 86e0a73..2eb6caa 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IAdapterRuntimeManager.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IAdapterRuntimeManager.java
@@ -45,14 +45,12 @@ public interface IAdapterRuntimeManager {
/**
* Start feed ingestion
- *
* @throws Exception
*/
public void start() throws Exception;
/**
* Stop feed ingestion.
- *
* @throws Exception
*/
public void stop() throws Exception;
@@ -65,7 +63,7 @@ public interface IAdapterRuntimeManager {
/**
* @return the instance of the feed adapter (an implementation of {@code IFeedAdapter}) in use.
*/
- public IFeedAdapter getFeedAdapter();
+ public IDataSourceAdapter getFeedAdapter();
/**
* @return state associated with the AdapterRuntimeManager. See {@code State}.
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IDataSourceAdapter.java
----------------------------------------------------------------------
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IDataSourceAdapter.java b/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IDataSourceAdapter.java
new file mode 100644
index 0000000..9dd4e76
--- /dev/null
+++ b/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IDataSourceAdapter.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.feeds.api;
+
+import java.io.Serializable;
+
+import org.apache.hyracks.api.comm.IFrameWriter;
+
+/**
+ * A super interface implemented by a data source adapter. An adapter can be a
+ * pull based or push based. This interface provides all common APIs that need
+ * to be implemented by each adapter irrespective of the the kind of
+ * adapter(pull or push).
+ */
+public interface IDataSourceAdapter extends Serializable {
+
+ /**
+ * Triggers the adapter to begin ingesting data from the external source.
+ *
+ * @param partition
+ * The adapter could be running with a degree of parallelism.
+ * partition corresponds to the i'th parallel instance.
+ * @param writer
+ * The instance of frame writer that is used by the adapter to
+ * write frame to. Adapter packs the fetched bytes (from external source),
+ * packs them into frames and forwards the frames to an upstream receiving
+ * operator using the instance of IFrameWriter.
+ * @throws Exception
+ */
+ public void start(int partition, IFrameWriter writer) throws Exception;
+
+ /**
+ * Discontinue the ingestion of data.
+ *
+ * @throws Exception
+ */
+ public boolean stop() throws Exception;
+
+ /**
+ * @param e
+ * @return true if the ingestion should continue post the exception else false
+ */
+ public boolean handleException(Throwable e);
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IDatasourceAdapter.java
----------------------------------------------------------------------
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IDatasourceAdapter.java b/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IDatasourceAdapter.java
deleted file mode 100644
index c1ee0f7..0000000
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IDatasourceAdapter.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.common.feeds.api;
-
-import java.io.Serializable;
-
-import org.apache.hyracks.api.comm.IFrameWriter;
-
-/**
- * A super interface implemented by a data source adapter. An adapter can be a
- * pull based or push based. This interface provides all common APIs that need
- * to be implemented by each adapter irrespective of the the kind of
- * adapter(pull or push).
- */
-public interface IDatasourceAdapter extends Serializable {
-
- /**
- * Triggers the adapter to begin ingesting data from the external source.
- *
- * @param partition
- * The adapter could be running with a degree of parallelism.
- * partition corresponds to the i'th parallel instance.
- * @param writer
- * The instance of frame writer that is used by the adapter to
- * write frame to. Adapter packs the fetched bytes (from external source),
- * packs them into frames and forwards the frames to an upstream receiving
- * operator using the instance of IFrameWriter.
- * @throws Exception
- */
- public void start(int partition, IFrameWriter writer) throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedAdapter.java
----------------------------------------------------------------------
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedAdapter.java b/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedAdapter.java
deleted file mode 100644
index 2307285..0000000
--- a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFeedAdapter.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.common.feeds.api;
-
-/**
- * Interface implemented by a feed adapter.
- */
-public interface IFeedAdapter extends IDatasourceAdapter {
-
- public enum DataExchangeMode {
- /**
- * PULL model requires the adaptor to make a separate request each time to receive data
- **/
- PULL,
-
- /**
- * PUSH mode involves the use o just one initial request (handshake) by the adaptor
- * to the datasource for setting up the connection and providing any protocol-specific
- * parameters. Once a connection is established, the data source "pushes" data to the adaptor.
- **/
- PUSH
- }
-
- /**
- * Returns the data exchange mode (PULL/PUSH) associated with the flow.
- *
- * @return
- */
- public DataExchangeMode getDataExchangeMode();
-
- /**
- * Discontinue the ingestion of data and end the feed.
- *
- * @throws Exception
- */
- public void stop() throws Exception;
-
- /**
- * @param e
- * @return true if the feed ingestion should continue post the exception else false
- */
- public boolean handleException(Exception e);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFramePostProcessor.java
----------------------------------------------------------------------
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFramePostProcessor.java b/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFramePostProcessor.java
new file mode 100644
index 0000000..ed74037
--- /dev/null
+++ b/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFramePostProcessor.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.feeds.api;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+
+public interface IFramePostProcessor {
+
+ public void postProcessFrame(ByteBuffer frame, FrameTupleAccessor frameAccessor);
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFramePreprocessor.java
----------------------------------------------------------------------
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFramePreprocessor.java b/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFramePreprocessor.java
new file mode 100644
index 0000000..59a6c97
--- /dev/null
+++ b/asterix-common/src/main/java/org/apache/asterix/common/feeds/api/IFramePreprocessor.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.feeds.api;
+
+import java.nio.ByteBuffer;
+
+public interface IFramePreprocessor {
+
+ public void preProcess(ByteBuffer frame) throws Exception;
+}