You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by am...@apache.org on 2016/01/03 18:41:17 UTC
[19/21] incubator-asterixdb git commit: First stage of external data
cleanup
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PullBasedAzureTwitterAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PullBasedAzureTwitterAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PullBasedAzureTwitterAdapterFactory.java
deleted file mode 100644
index fbde1b4..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PullBasedAzureTwitterAdapterFactory.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.adapter.factory;
-
-import java.util.Map;
-
-import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.feeds.FeedPolicyAccessor;
-import org.apache.asterix.common.feeds.api.IDatasourceAdapter;
-import org.apache.asterix.common.feeds.api.IIntakeProgressTracker;
-import org.apache.asterix.external.dataset.adapter.PullBasedAzureTwitterAdapter;
-import org.apache.asterix.om.types.ARecordType;
-import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
-import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-
-public class PullBasedAzureTwitterAdapterFactory implements IFeedAdapterFactory {
-
- private static final long serialVersionUID = 1L;
-
- private static final String INGESTOR_LOCATIONS_KEY = "ingestor-locations";
- private static final String PARTITIONS_KEY = "partitions";
- private static final String TABLE_NAME_KEY = "table-name";
- private static final String ACCOUNT_NAME_KEY = "account-name";
- private static final String ACCOUNT_KEY_KEY = "account-key";
-
- private ARecordType outputType;
- private Map<String, String> configuration;
- private String tableName;
- private String azureAccountName;
- private String azureAccountKey;
- private String[] locations;
- private String[] partitions;
- private FeedPolicyAccessor ingestionPolicy;
-
- @Override
- public SupportedOperation getSupportedOperations() {
- return SupportedOperation.READ;
- }
-
- @Override
- public String getName() {
- return "azure_twitter";
- }
-
- @Override
- public AlgebricksPartitionConstraint getPartitionConstraint() throws Exception {
- String locationsStr = configuration.get(INGESTOR_LOCATIONS_KEY);
- if (locationsStr == null) {
- return null;
- }
- String[] locations = locationsStr.split(",");
- return new AlgebricksAbsolutePartitionConstraint(locations);
- }
-
- @Override
- public IDatasourceAdapter createAdapter(IHyracksTaskContext ctx, int partition) throws Exception {
- return new PullBasedAzureTwitterAdapter(azureAccountName, azureAccountKey, tableName, partitions, configuration,
- ctx, outputType);
- }
-
- @Override
- public ARecordType getAdapterOutputType() {
- return outputType;
- }
-
- @Override
- public void configure(Map<String, String> configuration, ARecordType outputType) throws Exception {
- this.configuration = configuration;
- this.outputType = outputType;
-
- tableName = configuration.get(TABLE_NAME_KEY);
- if (tableName == null) {
- throw new AsterixException("You must specify a valid table name");
- }
- azureAccountName = configuration.get(ACCOUNT_NAME_KEY);
- azureAccountKey = configuration.get(ACCOUNT_KEY_KEY);
- if (azureAccountName == null || azureAccountKey == null) {
- throw new AsterixException("You must specify a valid Azure account name and key");
- }
-
- int nIngestLocations = 1;
- String locationsStr = configuration.get(INGESTOR_LOCATIONS_KEY);
- if (locationsStr != null) {
- locations = locationsStr.split(",");
- nIngestLocations = locations.length;
- }
-
- int nPartitions = 1;
- String partitionsStr = configuration.get(PARTITIONS_KEY);
- if (partitionsStr != null) {
- partitions = partitionsStr.split(",");
- nPartitions = partitions.length;
- }
-
- if (nIngestLocations != nPartitions) {
- throw new AsterixException("Invalid adapter configuration: number of ingestion-locations ("
- + nIngestLocations + ") must be the same as the number of partitions (" + nPartitions + ")");
- }
- }
-
- @Override
- public boolean isRecordTrackingEnabled() {
- return false;
- }
-
- @Override
- public IIntakeProgressTracker createIntakeProgressTracker() {
- return null;
- }
-
- public FeedPolicyAccessor getIngestionPolicy() {
- return ingestionPolicy;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PullBasedTwitterAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PullBasedTwitterAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PullBasedTwitterAdapterFactory.java
deleted file mode 100644
index 7d2dd73..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PullBasedTwitterAdapterFactory.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.adapter.factory;
-
-import java.util.Map;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.feeds.api.IDatasourceAdapter;
-import org.apache.asterix.common.feeds.api.IIntakeProgressTracker;
-import org.apache.asterix.external.dataset.adapter.PullBasedTwitterAdapter;
-import org.apache.asterix.external.util.TwitterUtil;
-import org.apache.asterix.external.util.TwitterUtil.SearchAPIConstants;
-import org.apache.asterix.om.types.ARecordType;
-import org.apache.hyracks.algebricks.common.constraints.AlgebricksCountPartitionConstraint;
-import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-
-/**
- * Factory class for creating an instance of PullBasedTwitterAdapter.
- * This adapter provides the functionality of fetching tweets from Twitter service
- * via pull-based Twitter API.
- */
-public class PullBasedTwitterAdapterFactory implements IFeedAdapterFactory {
- private static final long serialVersionUID = 1L;
- private static final Logger LOGGER = Logger.getLogger(PullBasedTwitterAdapterFactory.class.getName());
-
- public static final String PULL_BASED_TWITTER_ADAPTER_NAME = "pull_twitter";
-
- private static final String DEFAULT_INTERVAL = "10"; // 10 seconds
- private static final int INTAKE_CARDINALITY = 1; // degree of parallelism at intake stage
-
- private ARecordType outputType;
-
- private Map<String, String> configuration;
-
- @Override
- public IDatasourceAdapter createAdapter(IHyracksTaskContext ctx, int partition) throws Exception {
- return new PullBasedTwitterAdapter(configuration, outputType, ctx);
- }
-
- @Override
- public String getName() {
- return PULL_BASED_TWITTER_ADAPTER_NAME;
- }
-
- @Override
- public SupportedOperation getSupportedOperations() {
- return SupportedOperation.READ;
- }
-
- @Override
- public void configure(Map<String, String> configuration, ARecordType outputType) throws Exception {
- this.outputType = outputType;
- this.configuration = configuration;
- TwitterUtil.initializeConfigurationWithAuthInfo(configuration);
-
- if (configuration.get(SearchAPIConstants.QUERY) == null) {
- throw new AsterixException(
- "parameter " + SearchAPIConstants.QUERY + " not specified as part of adaptor configuration");
- }
-
- String interval = configuration.get(SearchAPIConstants.INTERVAL);
- if (interval != null) {
- try {
- Integer.parseInt(interval);
- } catch (NumberFormatException nfe) {
- throw new IllegalArgumentException(
- "parameter " + SearchAPIConstants.INTERVAL + " is defined incorrectly, expecting a number");
- }
- } else {
- configuration.put(SearchAPIConstants.INTERVAL, DEFAULT_INTERVAL);
- if (LOGGER.isLoggable(Level.WARNING)) {
- LOGGER.warning(" Parameter " + SearchAPIConstants.INTERVAL + " not defined, using default ("
- + DEFAULT_INTERVAL + ")");
- }
- }
-
- }
-
- @Override
- public AlgebricksPartitionConstraint getPartitionConstraint() throws Exception {
- return new AlgebricksCountPartitionConstraint(INTAKE_CARDINALITY);
- }
-
- @Override
- public boolean isRecordTrackingEnabled() {
- return false;
- }
-
- @Override
- public IIntakeProgressTracker createIntakeProgressTracker() {
- return null;
- }
-
- @Override
- public ARecordType getAdapterOutputType() {
- return outputType;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PushBasedTwitterAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PushBasedTwitterAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PushBasedTwitterAdapterFactory.java
deleted file mode 100644
index 5bfdbcf..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PushBasedTwitterAdapterFactory.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.adapter.factory;
-
-import java.util.Map;
-
-import org.apache.asterix.common.feeds.api.IDatasourceAdapter;
-import org.apache.asterix.common.feeds.api.IIntakeProgressTracker;
-import org.apache.asterix.external.dataset.adapter.PushBasedTwitterAdapter;
-import org.apache.asterix.external.util.TwitterUtil;
-import org.apache.asterix.external.util.TwitterUtil.AuthenticationConstants;
-import org.apache.asterix.om.types.ARecordType;
-import org.apache.hyracks.algebricks.common.constraints.AlgebricksCountPartitionConstraint;
-import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-
-public class PushBasedTwitterAdapterFactory implements IFeedAdapterFactory {
-
- private static final long serialVersionUID = 1L;
-
- private static final String NAME = "push_twitter";
-
- private ARecordType outputType;
-
- private Map<String, String> configuration;
-
- @Override
- public SupportedOperation getSupportedOperations() {
- return SupportedOperation.READ;
- }
-
- @Override
- public String getName() {
- return NAME;
- }
-
- @Override
- public AlgebricksPartitionConstraint getPartitionConstraint() throws Exception {
- return new AlgebricksCountPartitionConstraint(1);
- }
-
- @Override
- public IDatasourceAdapter createAdapter(IHyracksTaskContext ctx, int partition) throws Exception {
- PushBasedTwitterAdapter twitterAdapter = new PushBasedTwitterAdapter(configuration, outputType, ctx);
- return twitterAdapter;
- }
-
- @Override
- public void configure(Map<String, String> configuration, ARecordType outputType) throws Exception {
- this.outputType = outputType;
- this.configuration = configuration;
- TwitterUtil.initializeConfigurationWithAuthInfo(configuration);
- boolean requiredParamsSpecified = validateConfiguration(configuration);
- if (!requiredParamsSpecified) {
- StringBuilder builder = new StringBuilder();
- builder.append("One or more parameters are missing from adapter configuration\n");
- builder.append(AuthenticationConstants.OAUTH_CONSUMER_KEY + "\n");
- builder.append(AuthenticationConstants.OAUTH_CONSUMER_SECRET + "\n");
- builder.append(AuthenticationConstants.OAUTH_ACCESS_TOKEN + "\n");
- builder.append(AuthenticationConstants.OAUTH_ACCESS_TOKEN_SECRET + "\n");
- throw new Exception(builder.toString());
- }
- }
-
- @Override
- public ARecordType getAdapterOutputType() {
- return outputType;
- }
-
- @Override
- public boolean isRecordTrackingEnabled() {
- return false;
- }
-
- @Override
- public IIntakeProgressTracker createIntakeProgressTracker() {
- return null;
- }
-
- private boolean validateConfiguration(Map<String, String> configuration) {
- String consumerKey = configuration.get(AuthenticationConstants.OAUTH_CONSUMER_KEY);
- String consumerSecret = configuration.get(AuthenticationConstants.OAUTH_CONSUMER_SECRET);
- String accessToken = configuration.get(AuthenticationConstants.OAUTH_ACCESS_TOKEN);
- String tokenSecret = configuration.get(AuthenticationConstants.OAUTH_ACCESS_TOKEN_SECRET);
-
- if (consumerKey == null || consumerSecret == null || accessToken == null || tokenSecret == null) {
- return false;
- }
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/RSSFeedAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/RSSFeedAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/RSSFeedAdapterFactory.java
deleted file mode 100644
index 4d893fe..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/RSSFeedAdapterFactory.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.adapter.factory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.asterix.common.feeds.FeedPolicyAccessor;
-import org.apache.asterix.common.feeds.api.IDatasourceAdapter;
-import org.apache.asterix.common.feeds.api.IIntakeProgressTracker;
-import org.apache.asterix.external.dataset.adapter.RSSFeedAdapter;
-import org.apache.asterix.om.types.ARecordType;
-import org.apache.hyracks.algebricks.common.constraints.AlgebricksCountPartitionConstraint;
-import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-
-/**
- * Factory class for creating an instance of @see {RSSFeedAdapter}.
- * RSSFeedAdapter provides the functionality of fetching an RSS based feed.
- */
-public class RSSFeedAdapterFactory implements IFeedAdapterFactory {
- private static final long serialVersionUID = 1L;
- public static final String RSS_FEED_ADAPTER_NAME = "rss_feed";
-
- public static final String KEY_RSS_URL = "url";
- public static final String KEY_INTERVAL = "interval";
-
- private Map<String, String> configuration;
- private ARecordType outputType;
- private List<String> feedURLs = new ArrayList<String>();
- private FeedPolicyAccessor ingestionPolicy;
-
- @Override
- public IDatasourceAdapter createAdapter(IHyracksTaskContext ctx, int partition) throws Exception {
- RSSFeedAdapter rssFeedAdapter = new RSSFeedAdapter(configuration, outputType, ctx);
- return rssFeedAdapter;
- }
-
- @Override
- public String getName() {
- return "rss_feed";
- }
-
- @Override
- public SupportedOperation getSupportedOperations() {
- return SupportedOperation.READ;
- }
-
- @Override
- public void configure(Map<String, String> configuration, ARecordType outputType) throws Exception {
- this.configuration = configuration;
- this.outputType = outputType;
- String rssURLProperty = configuration.get(KEY_RSS_URL);
- if (rssURLProperty == null) {
- throw new IllegalArgumentException("no rss url provided");
- }
- initializeFeedURLs(rssURLProperty);
- configurePartitionConstraints();
- }
-
- @Override
- public AlgebricksPartitionConstraint getPartitionConstraint() throws Exception {
- return new AlgebricksCountPartitionConstraint(feedURLs.size());
- }
-
- private void initializeFeedURLs(String rssURLProperty) {
- feedURLs.clear();
- String[] feedURLProperty = rssURLProperty.split(",");
- for (String feedURL : feedURLProperty) {
- feedURLs.add(feedURL);
- }
- }
-
- protected void configurePartitionConstraints() {
-
- }
-
- @Override
- public ARecordType getAdapterOutputType() {
- return outputType;
- }
-
- @Override
- public boolean isRecordTrackingEnabled() {
- return false;
- }
-
- @Override
- public IIntakeProgressTracker createIntakeProgressTracker() {
- return null;
- }
-
- public FeedPolicyAccessor getIngestionPolicy() {
- return ingestionPolicy;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/StreamBasedAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/StreamBasedAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/StreamBasedAdapterFactory.java
deleted file mode 100644
index c7e582f..0000000
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/StreamBasedAdapterFactory.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.adapter.factory;
-
-import java.util.Map;
-import java.util.logging.Logger;
-
-import org.apache.asterix.external.util.INodeResolver;
-import org.apache.asterix.om.types.ARecordType;
-import org.apache.asterix.om.types.IAType;
-import org.apache.asterix.runtime.operators.file.AsterixTupleParserFactory;
-import org.apache.asterix.runtime.operators.file.AsterixTupleParserFactory.InputDataFormat;
-import org.apache.hyracks.dataflow.std.file.ITupleParserFactory;
-
-public abstract class StreamBasedAdapterFactory implements IAdapterFactory {
-
- private static final long serialVersionUID = 1L;
- protected static final Logger LOGGER = Logger.getLogger(StreamBasedAdapterFactory.class.getName());
-
- protected static INodeResolver nodeResolver;
-
- protected Map<String, String> configuration;
- protected ITupleParserFactory parserFactory;
-
- public abstract InputDataFormat getInputDataFormat();
-
- protected void configureFormat(IAType sourceDatatype) throws Exception {
- parserFactory = new AsterixTupleParserFactory(configuration, (ARecordType) sourceDatatype,
- getInputDataFormat());
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/IAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IAdapterFactory.java
new file mode 100644
index 0000000..9539278
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IAdapterFactory.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+import java.io.Serializable;
+import java.util.Map;
+
+import org.apache.asterix.common.feeds.api.IDataSourceAdapter;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+
+/**
+ * Base interface for IGenericDatasetAdapterFactory and ITypedDatasetAdapterFactory.
+ * Acts as a marker interface indicating that the implementation provides functionality
+ * for creating an adapter.
+ */
+public interface IAdapterFactory extends Serializable {
+
+ /**
+ * Returns the display name corresponding to the Adapter type that is created by the factory.
+ *
+ * @return the display name
+ */
+ public String getAlias();
+
+ /**
+ * Gets a list of partition constraints. A partition constraint can be a
+ * requirement to execute at a particular location or could be cardinality
+ * constraints indicating the number of instances that need to run in
+ * parallel. example, a IDatasourceAdapter implementation written for data
+ * residing on the local file system of a node cannot run on any other node
+ * and thus has a location partition constraint. The location partition
+ * constraint can be expressed as a node IP address or a node controller id.
+ * In the former case, the IP address is translated to a node controller id
+ * running on the node with the given IP address.
+ */
+ public AlgebricksPartitionConstraint getPartitionConstraint() throws Exception;
+
+ /**
+ * Creates an instance of IDatasourceAdapter.
+ *
+ * @param HyracksTaskContext
+ * @param partition
+ * @return An instance of IDatasourceAdapter.
+ * @throws Exception
+ */
+ public IDataSourceAdapter createAdapter(IHyracksTaskContext ctx, int partition) throws Exception;
+
+ /**
+ * @param configuration
+ * @param outputType
+ * @throws Exception
+ */
+ public void configure(Map<String, String> configuration, ARecordType outputType) throws Exception;
+
+ /**
+ * Gets the record type associated with the output of the adapter
+ *
+ * @return
+ */
+ public ARecordType getAdapterOutputType();
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/IDataFlowController.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IDataFlowController.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IDataFlowController.java
new file mode 100644
index 0000000..f5f47ec
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IDataFlowController.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.asterix.common.parse.ITupleForwarder;
+import org.apache.hyracks.api.comm.IFrameWriter;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public interface IDataFlowController {
+
+ /**
+ * Order of calls:
+ * 1. Constructor()
+ * 2. if record flow controller
+ * |-a. Set record reader
+ * |-b. Set record parser
+ * else
+ * |-a. Set stream parser
+ * 3. setTupleForwarder(forwarder)
+ * 4. configure(configuration,ctx)
+ * 5. start(writer)
+ */
+
+ public void start(IFrameWriter writer) throws HyracksDataException;
+
+ public boolean stop();
+
+ public boolean handleException(Throwable th);
+
+ public ITupleForwarder getTupleForwarder();
+
+ public void setTupleForwarder(ITupleForwarder forwarder);
+
+ public void configure(Map<String, String> configuration, IHyracksTaskContext ctx) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/IDataParser.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IDataParser.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IDataParser.java
new file mode 100644
index 0000000..a4a5a43
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IDataParser.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.asterix.builders.IARecordBuilder;
+import org.apache.asterix.builders.OrderedListBuilder;
+import org.apache.asterix.builders.RecordBuilder;
+import org.apache.asterix.builders.UnorderedListBuilder;
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.external.api.IExternalDataSourceFactory.DataSourceType;
+import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
+import org.apache.asterix.om.base.AMutableOrderedList;
+import org.apache.asterix.om.base.AMutableRecord;
+import org.apache.asterix.om.base.AMutableUnorderedList;
+import org.apache.asterix.om.base.IACursor;
+import org.apache.asterix.om.base.IAObject;
+import org.apache.asterix.om.types.AOrderedListType;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.AUnorderedListType;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+
+public interface IDataParser {
+
+ /**
+ * @return The supported data sources
+ */
+ public DataSourceType getDataSourceType();
+
+ /**
+ * @param configuration
+ * a set of configurations that comes from two sources.
+ * 1. The create adapter statement.
+ * 2. The query compiler.
+ * @param recordType
+ * The expected record type
+ * @throws HyracksDataException
+ * @throws IOException
+ */
+ public void configure(Map<String, String> configuration, ARecordType recordType)
+ throws HyracksDataException, IOException;
+
+ /*
+ * The following two static methods are expensive. right now, they are used by RSSFeeds and Twitter feed
+ * TODO: Get rid of them
+ */
+ public static void writeRecord(AMutableRecord record, DataOutput dataOutput, IARecordBuilder recordBuilder)
+ throws IOException, AsterixException {
+ ArrayBackedValueStorage fieldValue = new ArrayBackedValueStorage();
+ int numFields = record.getType().getFieldNames().length;
+ for (int pos = 0; pos < numFields; pos++) {
+ fieldValue.reset();
+ IAObject obj = record.getValueByPos(pos);
+ IDataParser.writeObject(obj, fieldValue.getDataOutput());
+ recordBuilder.addField(pos, fieldValue);
+ }
+ recordBuilder.write(dataOutput, true);
+ }
+
+ @SuppressWarnings("unchecked")
+ public static void writeObject(IAObject obj, DataOutput dataOutput) throws IOException, AsterixException {
+ switch (obj.getType().getTypeTag()) {
+ case RECORD: {
+ IARecordBuilder recordBuilder = new RecordBuilder();
+ recordBuilder.reset((ARecordType) obj.getType());
+ recordBuilder.init();
+ writeRecord((AMutableRecord) obj, dataOutput, recordBuilder);
+ break;
+ }
+
+ case ORDEREDLIST: {
+ OrderedListBuilder listBuilder = new OrderedListBuilder();
+ listBuilder.reset((AOrderedListType) ((AMutableOrderedList) obj).getType());
+ IACursor cursor = ((AMutableOrderedList) obj).getCursor();
+ ArrayBackedValueStorage listItemValue = new ArrayBackedValueStorage();
+ while (cursor.next()) {
+ listItemValue.reset();
+ IAObject item = cursor.get();
+ writeObject(item, listItemValue.getDataOutput());
+ listBuilder.addItem(listItemValue);
+ }
+ listBuilder.write(dataOutput, true);
+ break;
+ }
+
+ case UNORDEREDLIST: {
+ UnorderedListBuilder listBuilder = new UnorderedListBuilder();
+ listBuilder.reset((AUnorderedListType) ((AMutableUnorderedList) obj).getType());
+ IACursor cursor = ((AMutableUnorderedList) obj).getCursor();
+ ArrayBackedValueStorage listItemValue = new ArrayBackedValueStorage();
+ while (cursor.next()) {
+ listItemValue.reset();
+ IAObject item = cursor.get();
+ writeObject(item, listItemValue.getDataOutput());
+ listBuilder.addItem(listItemValue);
+ }
+ listBuilder.write(dataOutput, true);
+ break;
+ }
+
+ default:
+ AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(obj.getType()).serialize(obj,
+ dataOutput);
+ break;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/IDataParserFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IDataParserFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IDataParserFactory.java
new file mode 100644
index 0000000..5c3845c
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IDataParserFactory.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+import java.io.Serializable;
+import java.util.Map;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.external.api.IExternalDataSourceFactory.DataSourceType;
+import org.apache.asterix.om.types.ARecordType;
+
+public interface IDataParserFactory extends Serializable {
+
+ /**
+ * @return The expected data source type {STREAM or RECORDS}
+ * The data source type for a parser and a data source must match.
+ * an instance of IDataParserFactory with RECORDS data source type must implement IRecordDataParserFactory
+ * <T>
+ * an instance of IDataParserFactory with STREAM data source type must implement IStreamDataParserFactory
+ * @throws AsterixException
+ */
+ public DataSourceType getDataSourceType() throws AsterixException;
+
+ /**
+ * Configure the data parser factory. The passed map contains key value pairs from the
+ * submitted AQL statement and any additional pairs added by the compiler
+ * @param configuration
+ */
+ public void configure(Map<String, String> configuration) throws Exception;
+
+ /**
+ * Set the record type expected to be produced by parsers created by this factory
+ * @param recordType
+ */
+ public void setRecordType(ARecordType recordType);
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalDataSourceFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalDataSourceFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalDataSourceFactory.java
new file mode 100644
index 0000000..580ac99
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalDataSourceFactory.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+import java.io.Serializable;
+import java.util.Map;
+
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+
+public interface IExternalDataSourceFactory extends Serializable {
+
+ /**
+ * The data source type indicates whether the data source produces a continuous stream or
+ * a set of records
+ * @author amoudi
+ */
+ public enum DataSourceType {
+ STREAM,
+ RECORDS
+ }
+
+ /**
+ * @return The data source type {STREAM or RECORDS}
+ */
+ public DataSourceType getDataSourceType();
+
+ /**
+ * Specifies on which locations this data source is expected to run.
+ * @return
+ * @throws Exception
+ */
+ public AlgebricksPartitionConstraint getPartitionConstraint() throws Exception;
+
+ /**
+ * Configure the data parser factory. The passed map contains key value pairs from the
+ * submitted AQL statement and any additional pairs added by the compiler
+ * @param configuration
+ * @throws Exception
+ */
+ public void configure(Map<String, String> configuration) throws Exception;
+
+ /**
+ * Specify whether the external data source can be indexed
+ * @return
+ */
+ public boolean isIndexible();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalFunction.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalFunction.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalFunction.java
new file mode 100755
index 0000000..9c25c7f
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalFunction.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+public interface IExternalFunction {
+
+ public void initialize(IFunctionHelper functionHelper) throws Exception;
+
+ public void deinitialize();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalIndexer.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalIndexer.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalIndexer.java
new file mode 100644
index 0000000..0b4277e
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalIndexer.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+import java.io.Serializable;
+
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+
+/**
+ * @author amoudi
+ * This Interface represents the component responsible for adding record ids to tuples when indexing external data
+ */
+public interface IExternalIndexer extends Serializable {
+
+ /**
+ * This method is called by an indexible datasource when the external source reader have been updated.
+ * this gives a chance for the indexer to update its reader specific values (i,e. file name)
+ * @param reader
+ * the new reader
+ * @throws Exception
+ */
+ public void reset(IRecordReader<?> reader) throws Exception;
+
+ /**
+ * This method is called by the dataflow controller with each tuple. the indexer is expected to append record ids to the tuple.
+ * @param tb
+ * @throws Exception
+ */
+ public void index(ArrayTupleBuilder tb) throws Exception;
+
+ /**
+ * This method returns the number of fields in the record id. It is used by tuple appender at the initialization step.
+ * @return
+ * @throws Exception
+ */
+ public int getNumberOfFields() throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalScalarFunction.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalScalarFunction.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalScalarFunction.java
new file mode 100755
index 0000000..ecdb833
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalScalarFunction.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+public interface IExternalScalarFunction extends IExternalFunction {
+
+ public void evaluate(IFunctionHelper functionHelper) throws Exception;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/IFunctionFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IFunctionFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IFunctionFactory.java
new file mode 100755
index 0000000..5c5bcd0
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IFunctionFactory.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+public interface IFunctionFactory {
+
+ public IExternalFunction getExternalFunction();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/IFunctionHelper.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IFunctionHelper.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IFunctionHelper.java
new file mode 100755
index 0000000..ebd0757
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IFunctionHelper.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.external.library.java.JTypeTag;
+
+import java.io.IOException;
+
+public interface IFunctionHelper {
+
+ public IJObject getArgument(int index);
+
+ public IJObject getResultObject();
+
+ public void setResult(IJObject result) throws IOException, AsterixException;
+
+ public boolean isValidResult();
+
+ public IJObject getObject(JTypeTag jtypeTag);
+
+ public void reset();
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexibleExternalDataSource.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexibleExternalDataSource.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexibleExternalDataSource.java
new file mode 100644
index 0000000..fe30b38
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexibleExternalDataSource.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+import java.util.List;
+
+import org.apache.asterix.external.indexing.ExternalFile;
+
+public interface IIndexibleExternalDataSource extends IExternalDataSourceFactory {
+ public void setSnapshot(List<ExternalFile> files, boolean indexingOp) throws Exception;
+
+ /**
+ * Specify whether the external data source is configured for indexing
+ *
+ * @return
+ */
+ public boolean isIndexingOp();
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingAdapterFactory.java
new file mode 100644
index 0000000..37cc1cf
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingAdapterFactory.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+import java.util.List;
+
+import org.apache.asterix.external.indexing.ExternalFile;
+
+public interface IIndexingAdapterFactory extends IAdapterFactory {
+ public void setSnapshot(List<ExternalFile> files, boolean indexingOp);
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java
new file mode 100644
index 0000000..ed5e7b5
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IIndexingDatasource.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+public interface IIndexingDatasource {
+ public IExternalIndexer getIndexer();
+
+ public void setIndexer(IExternalIndexer indexer);
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
new file mode 100644
index 0000000..3dabb29
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProvider.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+import org.apache.asterix.external.input.stream.AInputStream;
+
+public interface IInputStreamProvider {
+ public AInputStream getInputStream() throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProviderFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProviderFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProviderFactory.java
new file mode 100644
index 0000000..3cc31dc
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IInputStreamProviderFactory.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+
+public interface IInputStreamProviderFactory extends IExternalDataSourceFactory {
+
+ public IInputStreamProvider createInputStreamProvider(IHyracksTaskContext ctx, int partition) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/IJListAccessor.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IJListAccessor.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IJListAccessor.java
new file mode 100644
index 0000000..70665db
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IJListAccessor.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+import org.apache.asterix.external.library.java.JObjectPointableVisitor;
+import org.apache.asterix.om.pointables.AListVisitablePointable;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.util.container.IObjectPool;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public interface IJListAccessor {
+
+ IJObject access(AListVisitablePointable pointable, IObjectPool<IJObject, IAType> objectPool, IAType listType,
+ JObjectPointableVisitor pointableVisitor) throws HyracksDataException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/IJObject.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IJObject.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IJObject.java
new file mode 100644
index 0000000..a88f47d
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IJObject.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+import java.io.DataOutput;
+
+import org.apache.asterix.om.base.IAObject;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public interface IJObject {
+
+ public ATypeTag getTypeTag();
+
+ public IAObject getIAObject();
+
+ public void serialize(DataOutput dataOutput, boolean writeTypeTag) throws HyracksDataException;
+
+ public void reset() throws AlgebricksException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/IJObjectAccessor.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IJObjectAccessor.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IJObjectAccessor.java
new file mode 100644
index 0000000..7b10af1
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IJObjectAccessor.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+import org.apache.asterix.om.pointables.base.IVisitablePointable;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.util.container.IObjectPool;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public interface IJObjectAccessor {
+ IJObject access(IVisitablePointable pointable, IObjectPool<IJObject, IAType> obj) throws HyracksDataException;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/IJRecordAccessor.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IJRecordAccessor.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IJRecordAccessor.java
new file mode 100644
index 0000000..08c5dde
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IJRecordAccessor.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+import org.apache.asterix.external.library.java.JObjectPointableVisitor;
+import org.apache.asterix.external.library.java.JObjects.JRecord;
+import org.apache.asterix.om.pointables.ARecordVisitablePointable;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.util.container.IObjectPool;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public interface IJRecordAccessor {
+
+ public JRecord access(ARecordVisitablePointable pointable, IObjectPool<IJObject, IAType> objectPool,
+ ARecordType recordType, JObjectPointableVisitor pointableVisitor) throws HyracksDataException;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/IJType.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IJType.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IJType.java
new file mode 100644
index 0000000..9c0ebae
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IJType.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+import org.apache.asterix.om.base.IAObject;
+import org.apache.asterix.om.types.ATypeTag;
+
+public interface IJType {
+
+ public ATypeTag getTypeTag();
+
+ public IAObject getIAObject();
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/ILookupReaderFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/ILookupReaderFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/ILookupReaderFactory.java
new file mode 100644
index 0000000..c0baead
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/ILookupReaderFactory.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+import org.apache.asterix.external.indexing.ExternalFileIndexAccessor;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+
+public interface ILookupReaderFactory<T> extends IExternalDataSourceFactory {
+ public ILookupRecordReader<? extends T> createRecordReader(IHyracksTaskContext ctx, int partition,
+ ExternalFileIndexAccessor snapshotAccessor) throws Exception;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/ILookupRecordReader.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/ILookupRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/ILookupRecordReader.java
new file mode 100644
index 0000000..fd03a07
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/ILookupRecordReader.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.asterix.external.indexing.RecordId;
+
+public interface ILookupRecordReader<T> {
+
+ public void configure(Map<String, String> configuration) throws Exception;
+
+ public Class<?> getRecordClass() throws IOException;
+
+ public IRawRecord<T> read(RecordId rid) throws IOException, Exception;
+
+ public void open() throws IOException;
+
+ public void fail() throws IOException;
+
+ public void close() throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/INodeResolver.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/INodeResolver.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/INodeResolver.java
new file mode 100644
index 0000000..831cd69
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/INodeResolver.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+
+/**
+ * A policy for resolving a name to a node controller id.
+ */
+public interface INodeResolver {
+
+ /**
+ * Resolve a passed-in value to a node controller id.
+ *
+ * @param value
+ * string to be resolved
+ * @return resolved result (a node controller id)
+ * @throws AsterixException
+ */
+ public String resolveNode(String value) throws AsterixException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/INodeResolverFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/INodeResolverFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/INodeResolverFactory.java
new file mode 100644
index 0000000..ce49596
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/INodeResolverFactory.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+/**
+ * Factory for creating an instance of INodeResolver
+ *
+ * @see INodeResolver
+ */
+public interface INodeResolverFactory {
+
+ /**
+ * Create an instance of {@link INodeResolver}
+ *
+ * @return an instance of INodeResolver
+ */
+ public INodeResolver createNodeResolver();
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRawRecord.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRawRecord.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRawRecord.java
new file mode 100644
index 0000000..92b500d
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRawRecord.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+/**
+ * This interface represents a raw record that is not parsed yet.
+ * @param <T>
+ */
+public interface IRawRecord<T> {
+ /**
+ * @return the bytes representing this record. This is intended to be used for passing raw records in frames and
+ * performing lazy deserialization on them. If the record can't be serialized, this method returns null.
+ */
+ public byte[] getBytes();
+
+ /**
+ * @return the java object of the record.
+ */
+ public T get();
+
+ /**
+ * @return The class of the record objects.
+ */
+ public Class<?> getRecordClass();
+
+ /**
+ * Resets the object to prepare it for another write operation.
+ */
+ public void reset();
+
+ /**
+ * @return The size of the valid bytes of the object. If the object can't be serialized, this method returns -1
+ */
+ int size();
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordDataParser.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordDataParser.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordDataParser.java
new file mode 100644
index 0000000..cc24847
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordDataParser.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+import java.io.DataOutput;
+
+public interface IRecordDataParser<T> extends IDataParser {
+
+ /**
+ * @param record
+ * @param out
+ * @throws Exception
+ */
+ public void parse(IRawRecord<? extends T> record, DataOutput out) throws Exception;
+
+ /**
+ * @return the record class
+ */
+ public Class<? extends T> getRecordClass();
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordDataParserFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordDataParserFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordDataParserFactory.java
new file mode 100644
index 0000000..993d947
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordDataParserFactory.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+import java.io.IOException;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public interface IRecordDataParserFactory<T> extends IDataParserFactory {
+ public IRecordDataParser<T> createRecordParser(IHyracksTaskContext ctx)
+ throws HyracksDataException, AsterixException, IOException;
+
+ public Class<? extends T> getRecordClass();
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordFlowController.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordFlowController.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordFlowController.java
new file mode 100644
index 0000000..c3bdc56
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordFlowController.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+public interface IRecordFlowController<T> extends IDataFlowController {
+
+ public void setRecordParser(IRecordDataParser<T> dataParser);
+
+ public void setRecordReader(IRecordReader<T> recordReader) throws Exception;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
new file mode 100644
index 0000000..019fe8f
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReader.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * This interface represents a record reader that reads data from external source as a set of records
+ * @param <T>
+ */
+public interface IRecordReader<T> extends Closeable {
+
+ /**
+ * Configure the reader with the set of key/value pairs passed by the compiler
+ * @param configuration
+ * the set of key/value pairs
+ * @throws Exception
+ * when the reader can't be configured (i,e. due to incorrect configuration, unreachable source, etc.)
+ */
+ public void configure(Map<String, String> configuration) throws Exception;
+
+ /**
+ * @return true if the reader has more records remaining, false, otherwise.
+ * @throws Exception
+ * if an error takes place
+ */
+ public boolean hasNext() throws Exception;
+
+ /**
+ * @return the object representing the next record.
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public IRawRecord<T> next() throws IOException, InterruptedException;
+
+ /**
+ * @return the class of the java objects representing the records. used to check compatibility between readers and
+ * parsers.
+ * @throws IOException
+ */
+ public Class<? extends T> getRecordClass() throws IOException;
+
+ /**
+ * used to stop reader from producing more records.
+ * @return true if the connection to the external source has been suspended, false otherwise.
+ */
+ public boolean stop();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/284590ed/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReaderFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReaderFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReaderFactory.java
new file mode 100644
index 0000000..adb2602
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReaderFactory.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.api;
+
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+
+public interface IRecordReaderFactory<T> extends IExternalDataSourceFactory {
+
+ public IRecordReader<? extends T> createRecordReader(IHyracksTaskContext ctx, int partition) throws Exception;
+
+ public Class<? extends T> getRecordClass();
+
+}