You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by am...@apache.org on 2015/12/23 07:17:54 UTC
[4/4] incubator-asterixdb git commit: Refactored External Data
Refactored External Data
This change re-arrange asterix module's order. asterix-
external-data is moved in front of asterix-metadata.
Change-Id: I46b60b5e1cc37fd59adc0dd89f374d96502091b2
Reviewed-on: https://asterix-gerrit.ics.uci.edu/559
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <ba...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/commit/486e47ad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/tree/486e47ad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/diff/486e47ad
Branch: refs/heads/master
Commit: 486e47ad7f341eb9828061158ca47a9c7302d60e
Parents: fc7272c
Author: Abdullah Alamoudi <ba...@gmail.com>
Authored: Wed Dec 23 08:17:57 2015 +0300
Committer: abdullah alamoudi <ba...@gmail.com>
Committed: Tue Dec 22 22:12:57 2015 -0800
----------------------------------------------------------------------
.../physical/ExternalDataLookupPOperator.java | 11 +-
.../optimizer/rules/am/AccessMethodUtils.java | 12 +-
.../asterix/aql/translator/QueryTranslator.java | 4 +-
.../file/ExternalIndexingOperations.java | 6 +-
.../org/apache/asterix/file/FeedOperations.java | 2 +-
.../apache/asterix/file/IndexOperations.java | 2 +-
.../file/SecondaryBTreeOperationsHelper.java | 2 +-
.../file/SecondaryIndexOperationsHelper.java | 4 +-
.../file/SecondaryRTreeOperationsHelper.java | 2 +-
.../bootstrap/AsterixGlobalRecoveryManager.java | 2 +-
.../bootstrap/ExternalLibraryBootstrap.java | 2 +-
asterix-external-data/pom.xml | 344 ++++++++--------
.../adapter/factory/CNNFeedAdapterFactory.java | 1 -
.../adapter/factory/HDFSAdapterFactory.java | 90 ++---
.../factory/HDFSIndexingAdapterFactory.java | 14 +-
.../adapter/factory/HiveAdapterFactory.java | 24 +-
.../adapter/factory/IAdapterFactory.java | 98 +++++
.../factory/IControlledAdapterFactory.java | 36 ++
.../adapter/factory/IFeedAdapterFactory.java | 29 ++
.../factory/NCFileSystemAdapterFactory.java | 20 +-
.../PullBasedAzureTwitterAdapterFactory.java | 45 +--
.../factory/PullBasedTwitterAdapterFactory.java | 1 -
.../factory/PushBasedTwitterAdapterFactory.java | 1 -
.../adapter/factory/RSSFeedAdapterFactory.java | 3 +-
.../factory/StreamBasedAdapterFactory.java | 4 +-
.../dataset/adapter/ClientBasedFeedAdapter.java | 3 +-
.../external/dataset/adapter/HDFSAdapter.java | 2 +-
.../dataset/adapter/HDFSIndexingAdapter.java | 7 +-
.../dataset/adapter/IControlledAdapter.java | 65 +++
.../dataset/adapter/PullBasedAdapter.java | 7 +-
.../external/feeds/FeedPolicyEnforcer.java | 50 +++
.../external/feeds/IPullBasedFeedAdapter.java | 34 ++
.../asterix/external/indexing/ExternalFile.java | 133 ++++++
.../indexing/ExternalFileIndexAccessor.java | 151 +++++++
.../indexing/FilesIndexDescription.java | 104 +++++
.../external/indexing/IndexingConstants.java | 205 ++++++++++
.../AdmOrDelimitedControlledTupleParser.java | 31 +-
.../dataflow/FileIndexTupleTranslator.java | 31 +-
.../indexing/dataflow/HDFSLookupAdapter.java | 4 +-
.../dataflow/HDFSLookupAdapterFactory.java | 105 +----
.../dataflow/RCFileControlledTupleParser.java | 24 +-
.../dataflow/SeqOrTxtControlledTupleParser.java | 25 +-
.../input/AbstractHDFSLookupInputStream.java | 4 +-
.../input/GenericFileAwareRecordReader.java | 2 +-
.../indexing/input/RCFileDataReader.java | 2 +-
.../indexing/input/RCFileLookupReader.java | 9 +-
.../input/SequenceFileLookupInputStream.java | 16 +-
.../input/SequenceFileLookupReader.java | 7 +-
.../input/TextFileLookupInputStream.java | 3 +-
.../indexing/input/TextFileLookupReader.java | 12 +-
.../indexing/input/TextualDataReader.java | 2 +-
...rnalIndexBulkModifyOperatorNodePushable.java | 2 +-
.../ExternalLoopkupOperatorDiscriptor.java | 141 +++++++
.../external/library/ExternalFunction.java | 1 -
.../library/ExternalLibraryManager.java | 58 +++
.../adapter/TestTypedAdapterFactory.java | 3 +-
.../aql/statement/SubscribeFeedStatement.java | 2 +-
asterix-metadata/pom.xml | 138 +++----
.../asterix/metadata/MetadataManager.java | 2 +-
.../apache/asterix/metadata/MetadataNode.java | 2 +-
.../asterix/metadata/api/IMetadataManager.java | 2 +-
.../asterix/metadata/api/IMetadataNode.java | 2 +-
.../metadata/bootstrap/MetadataBootstrap.java | 4 +-
.../metadata/declared/AqlMetadataProvider.java | 402 +++++++++++--------
.../declared/FieldExtractingAdapterFactory.java | 4 +-
.../declared/PKGeneratingAdapterFactory.java | 2 +-
.../asterix/metadata/entities/ExternalFile.java | 146 -------
.../ExternalFileTupleTranslator.java | 2 +-
.../external/ExternalFileIndexAccessor.java | 148 -------
.../ExternalLoopkupOperatorDiscriptor.java | 137 -------
.../external/FilesIndexDescription.java | 106 -----
.../metadata/external/IAdapterFactory.java | 98 -----
.../metadata/external/IControlledAdapter.java | 65 ---
.../external/IControlledAdapterFactory.java | 34 --
.../metadata/external/IndexingConstants.java | 207 ----------
.../feeds/AbstractFeedDatasourceAdapter.java | 1 +
.../ExternalDataScanOperatorDescriptor.java | 2 +-
.../feeds/FeedIntakeOperatorDescriptor.java | 3 +-
.../feeds/FeedIntakeOperatorNodePushable.java | 1 +
.../feeds/FeedMetaComputeNodePushable.java | 1 +
.../metadata/feeds/FeedMetaNodePushable.java | 1 +
.../feeds/FeedMetaStoreNodePushable.java | 1 +
.../metadata/feeds/FeedPolicyEnforcer.java | 50 ---
.../apache/asterix/metadata/feeds/FeedUtil.java | 5 +-
.../metadata/feeds/IFeedAdapterFactory.java | 30 --
.../metadata/feeds/IPullBasedFeedAdapter.java | 34 --
.../metadata/feeds/ITypedAdapterFactory.java | 2 +-
.../functions/ExternalLibraryManager.java | 58 ---
.../asterix/metadata/utils/DatasetUtils.java | 12 +-
.../ExternalFilesIndexOperatorDescriptor.java | 4 +-
.../data/GenericSocketFeedAdapterFactory.java | 4 +-
...ControlledFileSystemBasedAdapterFactory.java | 4 +-
.../data/SocketClientAdapterFactory.java | 2 +-
.../data/TwitterFirehoseFeedAdapterFactory.java | 2 +-
94 files changed, 1823 insertions(+), 1899 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java b/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
index ac8e775..9b85674 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
@@ -21,12 +21,11 @@ package org.apache.asterix.algebra.operators.physical;
import java.util.ArrayList;
import java.util.List;
-import org.apache.asterix.external.indexing.dataflow.HDFSLookupAdapterFactory;
import org.apache.asterix.metadata.declared.AqlDataSource;
+import org.apache.asterix.metadata.declared.AqlDataSource.AqlDataSourceType;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
import org.apache.asterix.metadata.declared.AqlSourceId;
import org.apache.asterix.metadata.declared.DatasetDataSource;
-import org.apache.asterix.metadata.declared.AqlDataSource.AqlDataSourceType;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
@@ -70,7 +69,8 @@ public class ExternalDataLookupPOperator extends AbstractScanPOperator {
private boolean retainNull;
public ExternalDataLookupPOperator(AqlSourceId datasetId, Dataset dataset, ARecordType recordType,
- Index secondaryIndex, List<LogicalVariable> ridVarList, boolean requiresBroadcast, boolean retainInput, boolean retainNull) {
+ Index secondaryIndex, List<LogicalVariable> ridVarList, boolean requiresBroadcast, boolean retainInput,
+ boolean retainNull) {
this.datasetId = datasetId;
this.dataset = dataset;
this.recordType = recordType;
@@ -123,7 +123,7 @@ public class ExternalDataLookupPOperator extends AbstractScanPOperator {
@Override
public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
- throws AlgebricksException {
+ throws AlgebricksException {
ExternalDataLookupOperator edabro = (ExternalDataLookupOperator) op;
ILogicalExpression expr = edabro.getExpressionRef().getValue();
if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
@@ -144,7 +144,7 @@ public class ExternalDataLookupPOperator extends AbstractScanPOperator {
}
AqlMetadataProvider metadataProvider = (AqlMetadataProvider) context.getMetadataProvider();
- Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> externalLoopup = HDFSLookupAdapterFactory
+ Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> externalLoopup = AqlMetadataProvider
.buildExternalDataLookupRuntime(builder.getJobSpec(), dataset, secondaryIndex, ridIndexes, retainInput,
typeEnv, outputVars, opSchema, context, metadataProvider, retainNull);
builder.contributeHyracksOperator(edabro, externalLoopup.first);
@@ -164,6 +164,7 @@ public class ExternalDataLookupPOperator extends AbstractScanPOperator {
return keyIndexes;
}
+ @Override
public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
IPhysicalPropertiesVector reqdByParent) {
if (requiresBroadcast) {
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
index d2c36cc..1ff3df6 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
@@ -29,11 +29,12 @@ import org.apache.asterix.algebra.operators.physical.ExternalDataLookupPOperator
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.external.indexing.IndexingConstants;
import org.apache.asterix.lang.common.util.FunctionUtil;
import org.apache.asterix.metadata.declared.AqlSourceId;
import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
import org.apache.asterix.metadata.entities.Index;
-import org.apache.asterix.metadata.external.IndexingConstants;
import org.apache.asterix.metadata.utils.DatasetUtils;
import org.apache.asterix.om.base.ABoolean;
import org.apache.asterix.om.base.AInt32;
@@ -278,7 +279,8 @@ public class AccessMethodUtils {
throws AlgebricksException {
int numPrimaryKeys = 0;
if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
- numPrimaryKeys = IndexingConstants.getRIDSize(dataset);
+ numPrimaryKeys = IndexingConstants
+ .getRIDSize(((ExternalDatasetDetails) dataset.getDatasetDetails()).getProperties());
} else {
numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size();
}
@@ -293,7 +295,8 @@ public class AccessMethodUtils {
ILogicalOperator unnestMapOp) {
int numPrimaryKeys;
if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
- numPrimaryKeys = IndexingConstants.getRIDSize(dataset);
+ numPrimaryKeys = IndexingConstants
+ .getRIDSize(((ExternalDatasetDetails) dataset.getDatasetDetails()).getProperties());
} else {
numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size();
}
@@ -567,7 +570,8 @@ public class AccessMethodUtils {
}
private static void appendExternalRecPrimaryKeys(Dataset dataset, List<Object> target) throws AsterixException {
- int numPrimaryKeys = IndexingConstants.getRIDSize(dataset);
+ int numPrimaryKeys = IndexingConstants
+ .getRIDSize(((ExternalDatasetDetails) dataset.getDatasetDetails()).getProperties());
for (int i = 0; i < numPrimaryKeys; i++) {
target.add(IndexingConstants.getFieldType(i));
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
index 51e1612..714e05c 100644
--- a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
+++ b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
@@ -63,6 +63,8 @@ import org.apache.asterix.common.feeds.api.IFeedLifecycleEventSubscriber.FeedLif
import org.apache.asterix.common.feeds.api.IFeedLifecycleListener.ConnectionLocation;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.compiler.provider.ILangCompilationProvider;
+import org.apache.asterix.external.adapter.factory.IFeedAdapterFactory;
+import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.feeds.CentralFeedManager;
import org.apache.asterix.feeds.FeedJoint;
import org.apache.asterix.feeds.FeedLifecycleListener;
@@ -127,7 +129,6 @@ import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Datatype;
import org.apache.asterix.metadata.entities.Dataverse;
import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
-import org.apache.asterix.metadata.entities.ExternalFile;
import org.apache.asterix.metadata.entities.Feed;
import org.apache.asterix.metadata.entities.Feed.FeedType;
import org.apache.asterix.metadata.entities.FeedPolicy;
@@ -139,7 +140,6 @@ import org.apache.asterix.metadata.entities.PrimaryFeed;
import org.apache.asterix.metadata.entities.SecondaryFeed;
import org.apache.asterix.metadata.feeds.FeedLifecycleEventSubscriber;
import org.apache.asterix.metadata.feeds.FeedUtil;
-import org.apache.asterix.metadata.feeds.IFeedAdapterFactory;
import org.apache.asterix.metadata.utils.DatasetUtils;
import org.apache.asterix.metadata.utils.ExternalDatasetsRegistry;
import org.apache.asterix.metadata.utils.MetadataLockManager;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-app/src/main/java/org/apache/asterix/file/ExternalIndexingOperations.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/ExternalIndexingOperations.java b/asterix-app/src/main/java/org/apache/asterix/file/ExternalIndexingOperations.java
index 1b7bd1e..d3fade5 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/ExternalIndexingOperations.java
+++ b/asterix-app/src/main/java/org/apache/asterix/file/ExternalIndexingOperations.java
@@ -48,6 +48,9 @@ import org.apache.asterix.dataflow.data.nontagged.valueproviders.AqlPrimitiveVal
import org.apache.asterix.external.adapter.factory.HDFSAdapterFactory;
import org.apache.asterix.external.adapter.factory.HDFSIndexingAdapterFactory;
import org.apache.asterix.external.adapter.factory.HiveAdapterFactory;
+import org.apache.asterix.external.indexing.ExternalFile;
+import org.apache.asterix.external.indexing.FilesIndexDescription;
+import org.apache.asterix.external.indexing.IndexingConstants;
import org.apache.asterix.external.indexing.operators.ExternalDatasetIndexesAbortOperatorDescriptor;
import org.apache.asterix.external.indexing.operators.ExternalDatasetIndexesCommitOperatorDescriptor;
import org.apache.asterix.external.indexing.operators.ExternalDatasetIndexesRecoverOperatorDescriptor;
@@ -60,10 +63,7 @@ import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
-import org.apache.asterix.metadata.entities.ExternalFile;
import org.apache.asterix.metadata.entities.Index;
-import org.apache.asterix.metadata.external.FilesIndexDescription;
-import org.apache.asterix.metadata.external.IndexingConstants;
import org.apache.asterix.metadata.feeds.ExternalDataScanOperatorDescriptor;
import org.apache.asterix.metadata.utils.DatasetUtils;
import org.apache.asterix.metadata.utils.ExternalDatasetsRegistry;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-app/src/main/java/org/apache/asterix/file/FeedOperations.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/FeedOperations.java b/asterix-app/src/main/java/org/apache/asterix/file/FeedOperations.java
index cd2f304..c7bb0e3 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/FeedOperations.java
+++ b/asterix-app/src/main/java/org/apache/asterix/file/FeedOperations.java
@@ -33,11 +33,11 @@ import org.apache.asterix.common.feeds.api.IFeedMessage;
import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
import org.apache.asterix.common.feeds.message.EndFeedMessage;
import org.apache.asterix.common.feeds.message.ThrottlingEnabledFeedMessage;
+import org.apache.asterix.external.adapter.factory.IFeedAdapterFactory;
import org.apache.asterix.feeds.FeedLifecycleListener;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
import org.apache.asterix.metadata.entities.PrimaryFeed;
import org.apache.asterix.metadata.feeds.FeedMessageOperatorDescriptor;
-import org.apache.asterix.metadata.feeds.IFeedAdapterFactory;
import org.apache.asterix.metadata.feeds.PrepareStallMessage;
import org.apache.asterix.metadata.feeds.TerminateDataFlowMessage;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-app/src/main/java/org/apache/asterix/file/IndexOperations.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/IndexOperations.java b/asterix-app/src/main/java/org/apache/asterix/file/IndexOperations.java
index c5870a6..4cb8a0f 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/IndexOperations.java
+++ b/asterix-app/src/main/java/org/apache/asterix/file/IndexOperations.java
@@ -26,10 +26,10 @@ import org.apache.asterix.common.config.OptimizationConfUtil;
import org.apache.asterix.common.context.AsterixVirtualBufferCacheProvider;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
+import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.metadata.MetadataException;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
import org.apache.asterix.metadata.entities.Dataset;
-import org.apache.asterix.metadata.entities.ExternalFile;
import org.apache.asterix.metadata.utils.DatasetUtils;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.util.AsterixAppContextInfo;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-app/src/main/java/org/apache/asterix/file/SecondaryBTreeOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryBTreeOperationsHelper.java b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryBTreeOperationsHelper.java
index 7c14e5d..a579d2c 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryBTreeOperationsHelper.java
+++ b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryBTreeOperationsHelper.java
@@ -30,9 +30,9 @@ import org.apache.asterix.common.context.AsterixVirtualBufferCacheProvider;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
import org.apache.asterix.common.ioopcallbacks.LSMBTreeWithBuddyIOOperationCallbackFactory;
+import org.apache.asterix.external.indexing.IndexingConstants;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
import org.apache.asterix.metadata.entities.Index;
-import org.apache.asterix.metadata.external.IndexingConstants;
import org.apache.asterix.metadata.feeds.ExternalDataScanOperatorDescriptor;
import org.apache.asterix.metadata.utils.ExternalDatasetsRegistry;
import org.apache.asterix.om.types.IAType;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
index 0cc137b..4eb6944 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
+++ b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
@@ -36,6 +36,8 @@ import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
import org.apache.asterix.common.transactions.IRecoveryManager.ResourceType;
import org.apache.asterix.common.transactions.JobId;
+import org.apache.asterix.external.indexing.ExternalFile;
+import org.apache.asterix.external.indexing.IndexingConstants;
import org.apache.asterix.external.indexing.operators.ExternalIndexBulkModifyOperatorDescriptor;
import org.apache.asterix.formats.nontagged.AqlBinaryBooleanInspectorImpl;
import org.apache.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
@@ -44,8 +46,6 @@ import org.apache.asterix.formats.nontagged.AqlTypeTraitProvider;
import org.apache.asterix.metadata.MetadataException;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
import org.apache.asterix.metadata.entities.Dataset;
-import org.apache.asterix.metadata.entities.ExternalFile;
-import org.apache.asterix.metadata.external.IndexingConstants;
import org.apache.asterix.metadata.feeds.ExternalDataScanOperatorDescriptor;
import org.apache.asterix.metadata.utils.DatasetUtils;
import org.apache.asterix.om.types.ARecordType;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java
index 6777d82..4d887dc 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java
+++ b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java
@@ -30,12 +30,12 @@ import org.apache.asterix.common.context.AsterixVirtualBufferCacheProvider;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.ioopcallbacks.LSMRTreeIOOperationCallbackFactory;
import org.apache.asterix.dataflow.data.nontagged.valueproviders.AqlPrimitiveValueProviderFactory;
+import org.apache.asterix.external.indexing.IndexingConstants;
import org.apache.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
import org.apache.asterix.formats.nontagged.AqlTypeTraitProvider;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
import org.apache.asterix.metadata.entities.Index;
-import org.apache.asterix.metadata.external.IndexingConstants;
import org.apache.asterix.metadata.feeds.ExternalDataScanOperatorDescriptor;
import org.apache.asterix.metadata.utils.ExternalDatasetsRegistry;
import org.apache.asterix.om.types.ATypeTag;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/AsterixGlobalRecoveryManager.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/AsterixGlobalRecoveryManager.java b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/AsterixGlobalRecoveryManager.java
index 7cb2fd7..c6e04df 100644
--- a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/AsterixGlobalRecoveryManager.java
+++ b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/AsterixGlobalRecoveryManager.java
@@ -29,6 +29,7 @@ import org.apache.asterix.common.api.IClusterManagementWorkResponse;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.ExternalDatasetTransactionState;
import org.apache.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
+import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.feeds.CentralFeedManager;
import org.apache.asterix.file.ExternalIndexingOperations;
import org.apache.asterix.metadata.MetadataManager;
@@ -38,7 +39,6 @@ import org.apache.asterix.metadata.declared.AqlMetadataProvider;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Dataverse;
import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
-import org.apache.asterix.metadata.entities.ExternalFile;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.om.util.AsterixClusterProperties;
import org.apache.asterix.common.api.IClusterManagementWork.ClusterState;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java
index 51d857a..267be3d 100755
--- a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java
+++ b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java
@@ -37,6 +37,7 @@ import org.apache.asterix.common.exceptions.ACIDException;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.external.library.ExternalLibrary;
+import org.apache.asterix.external.library.ExternalLibraryManager;
import org.apache.asterix.external.library.LibraryAdapter;
import org.apache.asterix.external.library.LibraryFunction;
import org.apache.asterix.metadata.MetadataManager;
@@ -46,7 +47,6 @@ import org.apache.asterix.metadata.entities.DatasourceAdapter;
import org.apache.asterix.metadata.entities.DatasourceAdapter.AdapterType;
import org.apache.asterix.metadata.entities.Dataverse;
import org.apache.asterix.metadata.feeds.AdapterIdentifier;
-import org.apache.asterix.metadata.functions.ExternalLibraryManager;
import org.apache.asterix.runtime.formats.NonTaggedDataFormat;
public class ExternalLibraryBootstrap {
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/pom.xml
----------------------------------------------------------------------
diff --git a/asterix-external-data/pom.xml b/asterix-external-data/pom.xml
index 86e466e..4062b23 100644
--- a/asterix-external-data/pom.xml
+++ b/asterix-external-data/pom.xml
@@ -17,184 +17,174 @@
! under the License.
!-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <artifactId>asterix</artifactId>
- <groupId>org.apache.asterix</groupId>
- <version>0.8.8-SNAPSHOT</version>
- </parent>
-
- <licenses>
- <license>
- <name>Apache License, Version 2.0</name>
- <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
- <distribution>repo</distribution>
- <comments>A business-friendly OSS license</comments>
- </license>
- </licenses>
-
- <artifactId>asterix-external-data</artifactId>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.jvnet.jaxb2.maven2</groupId>
- <artifactId>maven-jaxb2-plugin</artifactId>
- <version>0.9.0</version>
- <executions>
- <execution>
- <id>configuration</id>
- <goals>
- <goal>generate</goal>
- </goals>
- <configuration>
- <schemaDirectory>src/main/resources/schema</schemaDirectory>
- <schemaIncludes>
- <include>library.xsd</include>
- </schemaIncludes>
- <generatePackage>org.apache.asterix.external.library</generatePackage>
- <generateDirectory>${project.build.directory}/generated-sources/configuration</generateDirectory>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <version>2.4</version>
- <configuration>
- <includes>
- <include>**/*.class</include>
- <include>**/*.txt</include>
- </includes>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- <phase>package</phase>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <version>2.2-beta-5</version>
- <executions>
- <execution>
- <configuration>
- <descriptor>src/main/assembly/binary-assembly-libzip.xml</descriptor>
- <finalName>testlib-zip</finalName>
- </configuration>
- <phase>package</phase>
- <goals>
- <goal>attached</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- <dependencies>
- <dependency>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- <type>jar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.asterix</groupId>
- <artifactId>asterix-om</artifactId>
- <version>0.8.8-SNAPSHOT</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.asterix</groupId>
- <artifactId>asterix-runtime</artifactId>
- <version>0.8.8-SNAPSHOT</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.asterix</groupId>
- <artifactId>asterix-metadata</artifactId>
- <version>0.8.8-SNAPSHOT</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>algebricks-compiler</artifactId>
- </dependency>
- <dependency>
- <groupId>com.kenai.nbpwr</groupId>
- <artifactId>org-apache-commons-io</artifactId>
- <version>1.3.1-201002241208</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.twitter4j</groupId>
- <artifactId>twitter4j-core</artifactId>
- <version>[4.0,)</version>
- </dependency>
- <dependency>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>asterix</artifactId>
+ <groupId>org.apache.asterix</groupId>
+ <version>0.8.8-SNAPSHOT</version>
+ </parent>
+ <licenses>
+ <license>
+ <name>Apache License, Version 2.0</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ <distribution>repo</distribution>
+ <comments>A business-friendly OSS license</comments>
+ </license>
+ </licenses>
+ <artifactId>asterix-external-data</artifactId>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.jvnet.jaxb2.maven2</groupId>
+ <artifactId>maven-jaxb2-plugin</artifactId>
+ <version>0.9.0</version>
+ <executions>
+ <execution>
+ <id>configuration</id>
+ <goals>
+ <goal>generate</goal>
+ </goals>
+ <configuration>
+ <schemaDirectory>src/main/resources/schema</schemaDirectory>
+ <schemaIncludes>
+ <include>library.xsd</include>
+ </schemaIncludes>
+ <generatePackage>org.apache.asterix.external.library</generatePackage>
+ <generateDirectory>${project.build.directory}/generated-sources/configuration</generateDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>2.4</version>
+ <configuration>
+ <includes>
+ <include>**/*.class</include>
+ <include>**/*.txt</include>
+ </includes>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ <phase>package</phase>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.2-beta-5</version>
+ <executions>
+ <execution>
+ <configuration>
+ <descriptor>src/main/assembly/binary-assembly-libzip.xml</descriptor>
+ <finalName>testlib-zip</finalName>
+ </configuration>
+ <phase>package</phase>
+ <goals>
+ <goal>attached</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ <type>jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-om</artifactId>
+ <version>0.8.8-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-runtime</artifactId>
+ <version>0.8.8-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>algebricks-compiler</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.kenai.nbpwr</groupId>
+ <artifactId>org-apache-commons-io</artifactId>
+ <version>1.3.1-201002241208</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.twitter4j</groupId>
+ <artifactId>twitter4j-core</artifactId>
+ <version>[4.0,)</version>
+ </dependency>
+ <dependency>
<groupId>org.twitter4j</groupId>
<artifactId>twitter4j-stream</artifactId>
<version>[4.0,)</version>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>net.java.dev.rome</groupId>
- <artifactId>rome-fetcher</artifactId>
- <version>1.0.0</version>
- <type>jar</type>
- <scope>compile</scope>
- <exclusions>
- <exclusion>
- <artifactId>rome</artifactId>
- <groupId>net.java.dev.rome</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>rome</groupId>
- <artifactId>rome</artifactId>
- <version>1.0.1-modified-01</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hyracks</groupId>
- <artifactId>hyracks-hdfs-core</artifactId>
- <version>${hyracks.version}</version>
- </dependency>
- <dependency>
- <groupId>jdom</groupId>
- <artifactId>jdom</artifactId>
- <version>1.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.asterix</groupId>
- <artifactId>asterix-common</artifactId>
- <version>0.8.8-SNAPSHOT</version>
- </dependency>
- <dependency>
- <groupId>com.microsoft.windowsazure</groupId>
- <artifactId>microsoft-windowsazure-api</artifactId>
- <version>0.4.4</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-exec</artifactId>
- <version>0.13.0</version>
- </dependency>
- <dependency>
- <groupId>javax.jdo</groupId>
- <artifactId>jdo2-api</artifactId>
- <version>2.3-20090302111651</version>
- </dependency>
- </dependencies>
-</project>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>net.java.dev.rome</groupId>
+ <artifactId>rome-fetcher</artifactId>
+ <version>1.0.0</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>rome</artifactId>
+ <groupId>net.java.dev.rome</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>rome</groupId>
+ <artifactId>rome</artifactId>
+ <version>1.0.1-modified-01</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-hdfs-core</artifactId>
+ <version>${hyracks.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>jdom</groupId>
+ <artifactId>jdom</artifactId>
+ <version>1.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-common</artifactId>
+ <version>0.8.8-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>com.microsoft.windowsazure</groupId>
+ <artifactId>microsoft-windowsazure-api</artifactId>
+ <version>0.4.4</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <version>0.13.0</version>
+ </dependency>
+ <dependency>
+ <groupId>javax.jdo</groupId>
+ <artifactId>jdo2-api</artifactId>
+ <version>2.3-20090302111651</version>
+ </dependency>
+ </dependencies>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/CNNFeedAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/CNNFeedAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/CNNFeedAdapterFactory.java
index a74ae4d..8b7b6d5 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/CNNFeedAdapterFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/CNNFeedAdapterFactory.java
@@ -26,7 +26,6 @@ import java.util.Map;
import org.apache.asterix.common.feeds.api.IDatasourceAdapter;
import org.apache.asterix.common.feeds.api.IIntakeProgressTracker;
import org.apache.asterix.external.dataset.adapter.RSSFeedAdapter;
-import org.apache.asterix.metadata.feeds.IFeedAdapterFactory;
import org.apache.asterix.om.types.ARecordType;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksCountPartitionConstraint;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HDFSAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HDFSAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HDFSAdapterFactory.java
index cc4e2ef..ebf41cc 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HDFSAdapterFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HDFSAdapterFactory.java
@@ -26,26 +26,24 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.hadoop.fs.BlockLocation;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.FileSplit;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
-
import org.apache.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
import org.apache.asterix.common.feeds.api.IDatasourceAdapter;
import org.apache.asterix.external.dataset.adapter.HDFSAdapter;
+import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.external.indexing.dataflow.HDFSObjectTupleParserFactory;
-import org.apache.asterix.metadata.entities.ExternalFile;
-import org.apache.asterix.metadata.external.IAdapterFactory;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.util.AsterixAppContextInfo;
import org.apache.asterix.om.util.AsterixClusterProperties;
import org.apache.asterix.runtime.operators.file.AsterixTupleParserFactory;
import org.apache.asterix.runtime.operators.file.AsterixTupleParserFactory.InputDataFormat;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.FileSplit;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.api.context.ICCContext;
@@ -106,8 +104,8 @@ public class HDFSAdapterFactory extends StreamBasedAdapterFactory implements IAd
ICCContext ccContext = AsterixAppContextInfo.getInstance().getCCApplicationContext().getCCContext();
Scheduler scheduler = null;
try {
- scheduler = new Scheduler(ccContext.getClusterControllerInfo().getClientNetAddress(), ccContext
- .getClusterControllerInfo().getClientNetPort());
+ scheduler = new Scheduler(ccContext.getClusterControllerInfo().getClientNetAddress(),
+ ccContext.getClusterControllerInfo().getClientNetPort());
} catch (HyracksException e) {
throw new IllegalStateException("Cannot obtain hdfs scheduler");
}
@@ -145,15 +143,15 @@ public class HDFSAdapterFactory extends StreamBasedAdapterFactory implements IAd
public static JobConf configureJobConf(Map<String, String> configuration) throws Exception {
JobConf conf = new JobConf();
- String formatClassName = (String) formatClassNames.get(((String) configuration.get(KEY_INPUT_FORMAT)).trim());
- String localShortCircuitSocketPath = (String) configuration.get(KEY_LOCAL_SOCKET_PATH);
+ String formatClassName = formatClassNames.get(configuration.get(KEY_INPUT_FORMAT).trim());
+ String localShortCircuitSocketPath = configuration.get(KEY_LOCAL_SOCKET_PATH);
if (formatClassName == null) {
- formatClassName = ((String) configuration.get(KEY_INPUT_FORMAT)).trim();
+ formatClassName = configuration.get(KEY_INPUT_FORMAT).trim();
}
- conf.set(KEY_HADOOP_FILESYSTEM_URI, ((String) configuration.get(KEY_HDFS_URL)).trim());
+ conf.set(KEY_HADOOP_FILESYSTEM_URI, configuration.get(KEY_HDFS_URL).trim());
conf.set(KEY_HADOOP_FILESYSTEM_CLASS, CLASS_NAME_HDFS_FILESYSTEM);
conf.setClassLoader(HDFSAdapter.class.getClassLoader());
- conf.set(KEY_HADOOP_INPUT_DIR, ((String) configuration.get(KEY_PATH)).trim());
+ conf.set(KEY_HADOOP_INPUT_DIR, configuration.get(KEY_PATH).trim());
conf.set(KEY_HADOOP_INPUT_FORMAT, formatClassName);
// Enable local short circuit reads if user supplied the parameters
@@ -169,7 +167,7 @@ public class HDFSAdapterFactory extends StreamBasedAdapterFactory implements IAd
if (!configured) {
throw new IllegalStateException("Adapter factory has not been configured yet");
}
- return (AlgebricksPartitionConstraint) clusterLocations;
+ return clusterLocations;
}
@Override
@@ -199,7 +197,7 @@ public class HDFSAdapterFactory extends StreamBasedAdapterFactory implements IAd
Arrays.fill(executed, false);
configured = true;
- atype = (IAType) outputType;
+ atype = outputType;
configureFormat(atype);
}
@@ -241,26 +239,26 @@ public class HDFSAdapterFactory extends StreamBasedAdapterFactory implements IAd
* if data is text data (adm or delimited text), it will use a text tuple parser,
* otherwise it will use hdfs record object parser
*/
+ @Override
protected void configureFormat(IAType sourceDatatype) throws Exception {
- String specifiedFormat = (String) configuration.get(AsterixTupleParserFactory.KEY_FORMAT);
- if (specifiedFormat == null) {
- throw new IllegalArgumentException(" Unspecified data format");
- }
-
- if(AsterixTupleParserFactory.FORMAT_BINARY.equalsIgnoreCase(specifiedFormat)){
- parserFactory = new HDFSObjectTupleParserFactory((ARecordType) atype, this, configuration);
- } else {
- InputDataFormat inputFormat = InputDataFormat.UNKNOWN;
- if (AsterixTupleParserFactory.FORMAT_DELIMITED_TEXT.equalsIgnoreCase(specifiedFormat)) {
- inputFormat = InputDataFormat.DELIMITED;
- } else if (AsterixTupleParserFactory.FORMAT_ADM.equalsIgnoreCase(specifiedFormat)) {
- inputFormat = InputDataFormat.ADM;
- }
- parserFactory = new AsterixTupleParserFactory(configuration, (ARecordType) sourceDatatype
- , inputFormat);
- }
-
- }
+ String specifiedFormat = configuration.get(AsterixTupleParserFactory.KEY_FORMAT);
+ if (specifiedFormat == null) {
+ throw new IllegalArgumentException(" Unspecified data format");
+ }
+
+ if (AsterixTupleParserFactory.FORMAT_BINARY.equalsIgnoreCase(specifiedFormat)) {
+ parserFactory = new HDFSObjectTupleParserFactory((ARecordType) atype, this, configuration);
+ } else {
+ InputDataFormat inputFormat = InputDataFormat.UNKNOWN;
+ if (AsterixTupleParserFactory.FORMAT_DELIMITED_TEXT.equalsIgnoreCase(specifiedFormat)) {
+ inputFormat = InputDataFormat.DELIMITED;
+ } else if (AsterixTupleParserFactory.FORMAT_ADM.equalsIgnoreCase(specifiedFormat)) {
+ inputFormat = InputDataFormat.ADM;
+ }
+ parserFactory = new AsterixTupleParserFactory(configuration, (ARecordType) sourceDatatype, inputFormat);
+ }
+
+ }
/**
* Instead of creating the split using the input format, we do it manually
@@ -296,9 +294,11 @@ public class HDFSAdapterFactory extends StreamBasedAdapterFactory implements IAd
// Create a split per block
for (BlockLocation block : fileBlocks) {
if (block.getOffset() < file.getSize()) {
- fileSplits.add(new FileSplit(filePath, block.getOffset(), (block.getLength() + block
- .getOffset()) < file.getSize() ? block.getLength() : (file.getSize() - block
- .getOffset()), block.getHosts()));
+ fileSplits
+ .add(new FileSplit(filePath,
+ block.getOffset(), (block.getLength() + block.getOffset()) < file.getSize()
+ ? block.getLength() : (file.getSize() - block.getOffset()),
+ block.getHosts()));
orderedExternalFiles.add(file);
}
}
@@ -322,11 +322,11 @@ public class HDFSAdapterFactory extends StreamBasedAdapterFactory implements IAd
if (block.getOffset() < newSize) {
// Block interact with delta -> Create a split
long startCut = (block.getOffset() > oldSize) ? 0L : oldSize - block.getOffset();
- long endCut = (block.getOffset() + block.getLength() < newSize) ? 0L : block.getOffset()
- + block.getLength() - newSize;
+ long endCut = (block.getOffset() + block.getLength() < newSize) ? 0L
+ : block.getOffset() + block.getLength() - newSize;
long splitLength = block.getLength() - startCut - endCut;
- fileSplits.add(new FileSplit(filePath, block.getOffset() + startCut, splitLength, block
- .getHosts()));
+ fileSplits.add(new FileSplit(filePath, block.getOffset() + startCut, splitLength,
+ block.getHosts()));
orderedExternalFiles.add(file);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HDFSIndexingAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HDFSIndexingAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HDFSIndexingAdapterFactory.java
index 0033e5c..11e2b96 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HDFSIndexingAdapterFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HDFSIndexingAdapterFactory.java
@@ -23,9 +23,6 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
-
import org.apache.asterix.common.feeds.api.IDatasourceAdapter;
import org.apache.asterix.external.dataset.adapter.HDFSIndexingAdapter;
import org.apache.asterix.external.indexing.dataflow.HDFSIndexingParserFactory;
@@ -39,6 +36,8 @@ import org.apache.asterix.om.util.AsterixClusterProperties;
import org.apache.asterix.om.util.NonTaggedFormatUtil;
import org.apache.asterix.runtime.operators.file.AsterixTupleParserFactory;
import org.apache.asterix.runtime.operators.file.DelimitedDataParser;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException;
@@ -75,8 +74,8 @@ public class HDFSIndexingAdapterFactory extends HDFSAdapterFactory {
ICCContext ccContext = AsterixAppContextInfo.getInstance().getCCApplicationContext().getCCContext();
IndexingScheduler scheduler = null;
try {
- scheduler = new IndexingScheduler(ccContext.getClusterControllerInfo().getClientNetAddress(), ccContext
- .getClusterControllerInfo().getClientNetPort());
+ scheduler = new IndexingScheduler(ccContext.getClusterControllerInfo().getClientNetAddress(),
+ ccContext.getClusterControllerInfo().getClientNetPort());
} catch (HyracksException e) {
throw new IllegalStateException("Cannot obtain hdfs scheduler");
}
@@ -98,7 +97,7 @@ public class HDFSIndexingAdapterFactory extends HDFSAdapterFactory {
if (!configured) {
throw new IllegalStateException("Adapter factory has not been configured yet");
}
- return (AlgebricksPartitionConstraint) clusterLocations;
+ return clusterLocations;
}
@Override
@@ -131,11 +130,12 @@ public class HDFSIndexingAdapterFactory extends HDFSAdapterFactory {
executed = new boolean[readSchedule.length];
Arrays.fill(executed, false);
configured = true;
- atype = (IAType) outputType;
+ atype = outputType;
// The function below is overwritten to create indexing adapter factory instead of regular adapter factory
configureFormat(atype);
}
+ @Override
protected void configureFormat(IAType sourceDatatype) throws Exception {
char delimiter = AsterixTupleParserFactory.getDelimiter(configuration);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HiveAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HiveAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HiveAdapterFactory.java
index b61f5f3..553682e 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HiveAdapterFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HiveAdapterFactory.java
@@ -24,8 +24,7 @@ import java.util.Map;
import org.apache.asterix.common.feeds.api.IDatasourceAdapter;
import org.apache.asterix.external.dataset.adapter.HDFSAdapter;
import org.apache.asterix.external.dataset.adapter.HiveAdapter;
-import org.apache.asterix.metadata.entities.ExternalFile;
-import org.apache.asterix.metadata.external.IAdapterFactory;
+import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.runtime.operators.file.AsterixTupleParserFactory;
@@ -67,7 +66,6 @@ public class HiveAdapterFactory extends StreamBasedAdapterFactory implements IAd
return "hive";
}
-
@Override
public SupportedOperation getSupportedOperations() {
return SupportedOperation.READ;
@@ -78,13 +76,13 @@ public class HiveAdapterFactory extends StreamBasedAdapterFactory implements IAd
if (!configured) {
populateConfiguration(configuration);
hdfsAdapterFactory.configure(configuration, outputType);
- this.atype = (IAType) outputType;
+ this.atype = outputType;
}
}
public static void populateConfiguration(Map<String, String> configuration) throws Exception {
/** configure hive */
- String database = (String) configuration.get(HIVE_DATABASE);
+ String database = configuration.get(HIVE_DATABASE);
String tablePath = null;
if (database == null) {
tablePath = configuration.get(HIVE_WAREHOUSE_DIR) + "/" + configuration.get(HIVE_TABLE);
@@ -93,14 +91,17 @@ public class HiveAdapterFactory extends StreamBasedAdapterFactory implements IAd
+ configuration.get(HIVE_TABLE);
}
configuration.put(HDFSAdapterFactory.KEY_PATH, tablePath);
- if (!configuration.get(AsterixTupleParserFactory.KEY_FORMAT).equals(AsterixTupleParserFactory.FORMAT_DELIMITED_TEXT)) {
- throw new IllegalArgumentException("format" + configuration.get(AsterixTupleParserFactory.KEY_FORMAT) + " is not supported");
+ if (!configuration.get(AsterixTupleParserFactory.KEY_FORMAT)
+ .equals(AsterixTupleParserFactory.FORMAT_DELIMITED_TEXT)) {
+ throw new IllegalArgumentException(
+ "format" + configuration.get(AsterixTupleParserFactory.KEY_FORMAT) + " is not supported");
}
- if (!(configuration.get(HDFSAdapterFactory.KEY_INPUT_FORMAT).equals(HDFSAdapterFactory.INPUT_FORMAT_TEXT) || configuration
- .get(HDFSAdapterFactory.KEY_INPUT_FORMAT).equals(HDFSAdapterFactory.INPUT_FORMAT_SEQUENCE))) {
- throw new IllegalArgumentException("file input format"
- + configuration.get(HDFSAdapterFactory.KEY_INPUT_FORMAT) + " is not supported");
+ if (!(configuration.get(HDFSAdapterFactory.KEY_INPUT_FORMAT).equals(HDFSAdapterFactory.INPUT_FORMAT_TEXT)
+ || configuration.get(HDFSAdapterFactory.KEY_INPUT_FORMAT)
+ .equals(HDFSAdapterFactory.INPUT_FORMAT_SEQUENCE))) {
+ throw new IllegalArgumentException(
+ "file input format" + configuration.get(HDFSAdapterFactory.KEY_INPUT_FORMAT) + " is not supported");
}
}
@@ -109,7 +110,6 @@ public class HiveAdapterFactory extends StreamBasedAdapterFactory implements IAd
return hdfsAdapterFactory.getPartitionConstraint();
}
-
@Override
public ARecordType getAdapterOutputType() {
return (ARecordType) atype;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/IAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/IAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/IAdapterFactory.java
new file mode 100644
index 0000000..b8005cd
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/IAdapterFactory.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.adapter.factory;
+
+import java.io.Serializable;
+import java.util.Map;
+
+import org.apache.asterix.common.feeds.api.IDatasourceAdapter;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+
+/**
+ * Base interface for IGenericDatasetAdapterFactory and ITypedDatasetAdapterFactory.
+ * Acts as a marker interface indicating that the implementation provides functionality
+ * for creating an adapter.
+ */
+public interface IAdapterFactory extends Serializable {
+
+ public static final String KEY_TYPE_NAME = "type-name";
+
+ public enum SupportedOperation {
+ READ,
+ WRITE,
+ READ_WRITE
+ }
+
+ /**
+ * Returns the type of adapter indicating if the adapter can be used for
+ * reading from an external data source or writing to an external data
+ * source or can be used for both purposes.
+ *
+ * @see SupportedOperation
+ * @return
+ */
+ public SupportedOperation getSupportedOperations();
+
+ /**
+ * Returns the display name corresponding to the Adapter type that is created by the factory.
+ *
+ * @return the display name
+ */
+ public String getName();
+
+ /**
+ * Gets a list of partition constraints. A partition constraint can be a
+ * requirement to execute at a particular location or could be cardinality
+ * constraints indicating the number of instances that need to run in
+ * parallel. example, a IDatasourceAdapter implementation written for data
+ * residing on the local file system of a node cannot run on any other node
+ * and thus has a location partition constraint. The location partition
+ * constraint can be expressed as a node IP address or a node controller id.
+ * In the former case, the IP address is translated to a node controller id
+ * running on the node with the given IP address.
+ */
+ public AlgebricksPartitionConstraint getPartitionConstraint() throws Exception;
+
+ /**
+ * Creates an instance of IDatasourceAdapter.
+ *
+ * @param HyracksTaskContext
+ * @param partition
+ * @return An instance of IDatasourceAdapter.
+ * @throws Exception
+ */
+ public IDatasourceAdapter createAdapter(IHyracksTaskContext ctx, int partition) throws Exception;
+
+ /**
+ * @param configuration
+ * @param outputType
+ * @throws Exception
+ */
+ public void configure(Map<String, String> configuration, ARecordType outputType) throws Exception;
+
+ /**
+ * Gets the record type associated with the output of the adapter
+ *
+ * @return
+ */
+ public ARecordType getAdapterOutputType();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/IControlledAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/IControlledAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/IControlledAdapterFactory.java
new file mode 100644
index 0000000..0de6fad
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/IControlledAdapterFactory.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.adapter.factory;
+
+import java.io.Serializable;
+import java.util.Map;
+
+import org.apache.asterix.external.dataset.adapter.IControlledAdapter;
+import org.apache.asterix.external.indexing.ExternalFileIndexAccessor;
+import org.apache.asterix.om.types.IAType;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+
+public interface IControlledAdapterFactory extends Serializable {
+ public IControlledAdapter createAdapter(IHyracksTaskContext ctx, ExternalFileIndexAccessor fileIndexAccessor,
+ RecordDescriptor inRecDesc);
+
+ public void configure(IAType atype, boolean propagateInput, int[] ridFields,
+ Map<String, String> adapterConfiguration, boolean retainNull);
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/IFeedAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/IFeedAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/IFeedAdapterFactory.java
new file mode 100644
index 0000000..9358a52
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/IFeedAdapterFactory.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.adapter.factory;
+
+import org.apache.asterix.common.feeds.api.IIntakeProgressTracker;
+
+public interface IFeedAdapterFactory extends IAdapterFactory {
+
+ public boolean isRecordTrackingEnabled();
+
+ public IIntakeProgressTracker createIntakeProgressTracker();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/NCFileSystemAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/NCFileSystemAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/NCFileSystemAdapterFactory.java
index 61bf4a3..251d69a 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/NCFileSystemAdapterFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/NCFileSystemAdapterFactory.java
@@ -26,11 +26,10 @@ import java.util.logging.Level;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.feeds.api.IDatasourceAdapter;
import org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter;
+import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.external.util.DNSResolverFactory;
import org.apache.asterix.external.util.INodeResolver;
import org.apache.asterix.external.util.INodeResolverFactory;
-import org.apache.asterix.metadata.entities.ExternalFile;
-import org.apache.asterix.metadata.external.IAdapterFactory;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.runtime.operators.file.AsterixTupleParserFactory;
@@ -58,7 +57,6 @@ public class NCFileSystemAdapterFactory extends StreamBasedAdapterFactory implem
private FileSplit[] fileSplits;
private ARecordType outputType;
-
@Override
public IDatasourceAdapter createAdapter(IHyracksTaskContext ctx, int partition) throws Exception {
NCFileSystemAdapter fsAdapter = new NCFileSystemAdapter(fileSplits, parserFactory, sourceDatatype, ctx);
@@ -70,7 +68,6 @@ public class NCFileSystemAdapterFactory extends StreamBasedAdapterFactory implem
return NC_FILE_SYSTEM_ADAPTER_NAME;
}
-
@Override
public SupportedOperation getSupportedOperations() {
return SupportedOperation.READ;
@@ -80,8 +77,8 @@ public class NCFileSystemAdapterFactory extends StreamBasedAdapterFactory implem
public void configure(Map<String, String> configuration, ARecordType outputType) throws Exception {
this.configuration = configuration;
this.outputType = outputType;
- String[] splits = ((String) configuration.get(AsterixTupleParserFactory.KEY_PATH)).split(",");
- IAType sourceDatatype = (IAType) outputType;
+ String[] splits = configuration.get(AsterixTupleParserFactory.KEY_PATH).split(",");
+ IAType sourceDatatype = outputType;
configureFileSplits(splits);
configureFormat(sourceDatatype);
@@ -102,8 +99,8 @@ public class NCFileSystemAdapterFactory extends StreamBasedAdapterFactory implem
for (String splitPath : splits) {
trimmedValue = splitPath.trim();
if (!trimmedValue.contains("://")) {
- throw new AsterixException("Invalid path: " + splitPath
- + "\nUsage- path=\"Host://Absolute File Path\"");
+ throw new AsterixException(
+ "Invalid path: " + splitPath + "\nUsage- path=\"Host://Absolute File Path\"");
}
nodeName = trimmedValue.split(":")[0];
nodeLocalPath = trimmedValue.split("://")[1];
@@ -132,7 +129,8 @@ public class NCFileSystemAdapterFactory extends StreamBasedAdapterFactory implem
private static INodeResolver initializeNodeResolver() {
INodeResolver nodeResolver = null;
- String configuredNodeResolverFactory = System.getProperty(AsterixTupleParserFactory.NODE_RESOLVER_FACTORY_PROPERTY);
+ String configuredNodeResolverFactory = System
+ .getProperty(AsterixTupleParserFactory.NODE_RESOLVER_FACTORY_PROPERTY);
if (configuredNodeResolverFactory != null) {
try {
nodeResolver = ((INodeResolverFactory) (Class.forName(configuredNodeResolverFactory).newInstance()))
@@ -150,12 +148,12 @@ public class NCFileSystemAdapterFactory extends StreamBasedAdapterFactory implem
}
return nodeResolver;
}
-
+
@Override
public ARecordType getAdapterOutputType() {
return outputType;
}
-
+
@Override
public InputDataFormat getInputDataFormat() {
return InputDataFormat.UNKNOWN;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PullBasedAzureTwitterAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PullBasedAzureTwitterAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PullBasedAzureTwitterAdapterFactory.java
index bf92480..fbde1b4 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PullBasedAzureTwitterAdapterFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PullBasedAzureTwitterAdapterFactory.java
@@ -25,13 +25,7 @@ import org.apache.asterix.common.feeds.FeedPolicyAccessor;
import org.apache.asterix.common.feeds.api.IDatasourceAdapter;
import org.apache.asterix.common.feeds.api.IIntakeProgressTracker;
import org.apache.asterix.external.dataset.adapter.PullBasedAzureTwitterAdapter;
-import org.apache.asterix.metadata.MetadataManager;
-import org.apache.asterix.metadata.MetadataTransactionContext;
-import org.apache.asterix.metadata.entities.Datatype;
-import org.apache.asterix.metadata.feeds.IFeedAdapterFactory;
import org.apache.asterix.om.types.ARecordType;
-import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.api.context.IHyracksTaskContext;
@@ -42,7 +36,6 @@ public class PullBasedAzureTwitterAdapterFactory implements IFeedAdapterFactory
private static final String INGESTOR_LOCATIONS_KEY = "ingestor-locations";
private static final String PARTITIONS_KEY = "partitions";
- private static final String OUTPUT_TYPE_KEY = "output-type";
private static final String TABLE_NAME_KEY = "table-name";
private static final String ACCOUNT_NAME_KEY = "account-name";
private static final String ACCOUNT_KEY_KEY = "account-key";
@@ -78,8 +71,8 @@ public class PullBasedAzureTwitterAdapterFactory implements IFeedAdapterFactory
@Override
public IDatasourceAdapter createAdapter(IHyracksTaskContext ctx, int partition) throws Exception {
- return new PullBasedAzureTwitterAdapter(azureAccountName, azureAccountKey, tableName, partitions,
- configuration, ctx, outputType);
+ return new PullBasedAzureTwitterAdapter(azureAccountName, azureAccountKey, tableName, partitions, configuration,
+ ctx, outputType);
}
@Override
@@ -120,38 +113,6 @@ public class PullBasedAzureTwitterAdapterFactory implements IFeedAdapterFactory
throw new AsterixException("Invalid adapter configuration: number of ingestion-locations ("
+ nIngestLocations + ") must be the same as the number of partitions (" + nPartitions + ")");
}
- configureType();
- }
-
- private void configureType() throws Exception {
- String fqOutputType = configuration.get(OUTPUT_TYPE_KEY);
-
- if (fqOutputType == null) {
- throw new IllegalArgumentException("No output type specified");
- }
- String[] dataverseAndType = fqOutputType.split("[.]");
- String dataverseName = dataverseAndType[0];
- String datatypeName = dataverseAndType[1];
-
- MetadataTransactionContext ctx = null;
- MetadataManager.INSTANCE.acquireReadLatch();
- try {
- ctx = MetadataManager.INSTANCE.beginTransaction();
- Datatype t = MetadataManager.INSTANCE.getDatatype(ctx, dataverseName, datatypeName);
- IAType type = t.getDatatype();
- if (type.getTypeTag() != ATypeTag.RECORD) {
- throw new IllegalStateException();
- }
- outputType = (ARecordType) t.getDatatype();
- MetadataManager.INSTANCE.commitTransaction(ctx);
- } catch (Exception e) {
- if (ctx != null) {
- MetadataManager.INSTANCE.abortTransaction(ctx);
- }
- throw e;
- } finally {
- MetadataManager.INSTANCE.releaseReadLatch();
- }
}
@Override
@@ -167,7 +128,5 @@ public class PullBasedAzureTwitterAdapterFactory implements IFeedAdapterFactory
public FeedPolicyAccessor getIngestionPolicy() {
return ingestionPolicy;
}
-
-
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PullBasedTwitterAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PullBasedTwitterAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PullBasedTwitterAdapterFactory.java
index c05df33..7d2dd73 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PullBasedTwitterAdapterFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PullBasedTwitterAdapterFactory.java
@@ -28,7 +28,6 @@ import org.apache.asterix.common.feeds.api.IIntakeProgressTracker;
import org.apache.asterix.external.dataset.adapter.PullBasedTwitterAdapter;
import org.apache.asterix.external.util.TwitterUtil;
import org.apache.asterix.external.util.TwitterUtil.SearchAPIConstants;
-import org.apache.asterix.metadata.feeds.IFeedAdapterFactory;
import org.apache.asterix.om.types.ARecordType;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksCountPartitionConstraint;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PushBasedTwitterAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PushBasedTwitterAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PushBasedTwitterAdapterFactory.java
index 2606063..5bfdbcf 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PushBasedTwitterAdapterFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PushBasedTwitterAdapterFactory.java
@@ -25,7 +25,6 @@ import org.apache.asterix.common.feeds.api.IIntakeProgressTracker;
import org.apache.asterix.external.dataset.adapter.PushBasedTwitterAdapter;
import org.apache.asterix.external.util.TwitterUtil;
import org.apache.asterix.external.util.TwitterUtil.AuthenticationConstants;
-import org.apache.asterix.metadata.feeds.IFeedAdapterFactory;
import org.apache.asterix.om.types.ARecordType;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksCountPartitionConstraint;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/RSSFeedAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/RSSFeedAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/RSSFeedAdapterFactory.java
index 8f47aa2..4d893fe 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/RSSFeedAdapterFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/RSSFeedAdapterFactory.java
@@ -26,7 +26,6 @@ import org.apache.asterix.common.feeds.FeedPolicyAccessor;
import org.apache.asterix.common.feeds.api.IDatasourceAdapter;
import org.apache.asterix.common.feeds.api.IIntakeProgressTracker;
import org.apache.asterix.external.dataset.adapter.RSSFeedAdapter;
-import org.apache.asterix.metadata.feeds.IFeedAdapterFactory;
import org.apache.asterix.om.types.ARecordType;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksCountPartitionConstraint;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
@@ -68,7 +67,7 @@ public class RSSFeedAdapterFactory implements IFeedAdapterFactory {
public void configure(Map<String, String> configuration, ARecordType outputType) throws Exception {
this.configuration = configuration;
this.outputType = outputType;
- String rssURLProperty = (String) configuration.get(KEY_RSS_URL);
+ String rssURLProperty = configuration.get(KEY_RSS_URL);
if (rssURLProperty == null) {
throw new IllegalArgumentException("no rss url provided");
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/StreamBasedAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/StreamBasedAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/StreamBasedAdapterFactory.java
index e37cf3e..c7e582f 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/StreamBasedAdapterFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/StreamBasedAdapterFactory.java
@@ -22,7 +22,6 @@ import java.util.Map;
import java.util.logging.Logger;
import org.apache.asterix.external.util.INodeResolver;
-import org.apache.asterix.metadata.external.IAdapterFactory;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.runtime.operators.file.AsterixTupleParserFactory;
@@ -42,7 +41,8 @@ public abstract class StreamBasedAdapterFactory implements IAdapterFactory {
public abstract InputDataFormat getInputDataFormat();
protected void configureFormat(IAType sourceDatatype) throws Exception {
- parserFactory = new AsterixTupleParserFactory(configuration, (ARecordType) sourceDatatype, getInputDataFormat());
+ parserFactory = new AsterixTupleParserFactory(configuration, (ARecordType) sourceDatatype,
+ getInputDataFormat());
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/ClientBasedFeedAdapter.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/ClientBasedFeedAdapter.java b/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/ClientBasedFeedAdapter.java
index 26d4d8f..a197368 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/ClientBasedFeedAdapter.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/ClientBasedFeedAdapter.java
@@ -25,7 +25,7 @@ import java.util.logging.Logger;
import org.apache.asterix.common.feeds.api.IFeedAdapter;
import org.apache.asterix.common.parse.ITupleForwardPolicy;
import org.apache.asterix.external.dataset.adapter.IFeedClient.InflowState;
-import org.apache.asterix.metadata.feeds.FeedPolicyEnforcer;
+import org.apache.asterix.external.feeds.FeedPolicyEnforcer;
import org.apache.asterix.om.types.ARecordType;
import org.apache.hyracks.api.comm.IFrame;
import org.apache.hyracks.api.comm.IFrameWriter;
@@ -138,6 +138,7 @@ public abstract class ClientBasedFeedAdapter implements IFeedAdapter {
*
* @throws Exception
*/
+ @Override
public void stop() throws Exception {
continueIngestion = false;
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/HDFSAdapter.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/HDFSAdapter.java b/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/HDFSAdapter.java
index df0926b..5f1b1ae 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/HDFSAdapter.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/HDFSAdapter.java
@@ -23,11 +23,11 @@ import java.io.InputStream;
import java.util.List;
import java.util.Map;
+import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.external.indexing.input.GenericFileAwareRecordReader;
import org.apache.asterix.external.indexing.input.GenericRecordReader;
import org.apache.asterix.external.indexing.input.TextualDataReader;
import org.apache.asterix.external.indexing.input.TextualFullScanDataReader;
-import org.apache.asterix.metadata.entities.ExternalFile;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.runtime.operators.file.AsterixTupleParserFactory;
import org.apache.hadoop.mapred.InputSplit;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/HDFSIndexingAdapter.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/HDFSIndexingAdapter.java b/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/HDFSIndexingAdapter.java
index 7da182a..92a049d0 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/HDFSIndexingAdapter.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/HDFSIndexingAdapter.java
@@ -22,16 +22,15 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.List;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
-
import org.apache.asterix.external.adapter.factory.HDFSAdapterFactory;
+import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.external.indexing.input.GenericFileAwareRecordReader;
import org.apache.asterix.external.indexing.input.RCFileDataReader;
import org.apache.asterix.external.indexing.input.TextualDataReader;
-import org.apache.asterix.metadata.entities.ExternalFile;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.runtime.operators.file.AsterixTupleParserFactory;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.dataflow.std.file.ITupleParserFactory;