You are viewing a plain text version of this content. The canonical link for it is here.

Posted to notifications@asterixdb.apache.org by "Xikui Wang (Code Review)" <do...@asterixdb.incubator.apache.org> on 2017/01/11 06:01:12 UTC

Change in asterixdb[master]: WIP - Modularize feed adaptors

Xikui Wang has uploaded a new change for review.

  https://asterix-gerrit.ics.uci.edu/1430

Change subject: WIP - Modularize feed adaptors
......................................................................

WIP - Modularize feed adaptors

Modularize the feed adaptors and move 'license-unfriendly' adaptors to
separate package.

Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
---
M asterixdb/asterix-external-data/pom.xml
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReaderFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReaderFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
A asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
A asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/feed/test/DatasourceFactoryProviderTest.java
M asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/RecordWithPKTestReaderFactory.java
M asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java
A asterixdb/asterix-villain/pom.xml
R asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/input/record/reader/twitter/TwitterPullRecordReader.java
R asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/input/record/reader/twitter/TwitterPushRecordReader.java
R asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/input/record/reader/twitter/TwitterRecordReaderFactory.java
R asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/util/TwitterUtil.java
A asterixdb/asterix-villain/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
M asterixdb/pom.xml
17 files changed, 215 insertions(+), 62 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/30/1430/1

diff --git a/asterixdb/asterix-external-data/pom.xml b/asterixdb/asterix-external-data/pom.xml
index 72c7997..3796a0d 100644
--- a/asterixdb/asterix-external-data/pom.xml
+++ b/asterixdb/asterix-external-data/pom.xml
@@ -106,6 +106,7 @@
             <include>**/NOTICE</include>
             <include>**/LICENSE</include>
             <include>**/DEPENDENCIES</include>
+            <include>**/services/**</include>
           </includes>
         </configuration>
         <executions>
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReaderFactory.java
index c6adbc4..627bd65 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReaderFactory.java
@@ -32,4 +32,7 @@
     public default DataSourceType getDataSourceType() {
         return DataSourceType.RECORDS;
     }
+
+    public String[] getRecordReaderNames();
+
 }
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
index 2b899d9..65d89cb 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
@@ -74,6 +74,7 @@
     private InputSplit[] inputSplits;
     private String nodeName;
     private Format format;
+    private String[] recordReaderNames = { "hdfs" };
 
     @Override
     public void configure(Map<String, String> configuration) throws AsterixException {
@@ -226,4 +227,9 @@
     public boolean isIndexingOp() {
         return ((files != null) && indexingOp);
     }
+
+    @Override
+    public String[] getRecordReaderNames() {
+        return recordReaderNames;
+    }
 }
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReaderFactory.java
index 2ded3fb..6f7db89 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReaderFactory.java
@@ -39,6 +39,7 @@
     private static final long serialVersionUID = 1L;
     private final List<String> urls = new ArrayList<String>();
     private transient AlgebricksAbsolutePartitionConstraint clusterLocations;
+    private static String[] recordReaderNames = { "rss_feed" };
 
     @Override
     public DataSourceType getDataSourceType() {
@@ -89,4 +90,9 @@
         return SyndEntryImpl.class;
     }
 
+    @Override
+    public String[] getRecordReaderNames() {
+        return recordReaderNames;
+    }
+
 }
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java
index 4649559..4a9fadb 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java
@@ -23,8 +23,12 @@
 import org.apache.asterix.external.api.IInputStreamFactory;
 import org.apache.asterix.external.api.IRecordReader;
 import org.apache.asterix.external.api.IRecordReaderFactory;
+import org.apache.asterix.external.input.stream.factory.LocalFSInputStreamFactory;
+import org.apache.asterix.external.input.stream.factory.SocketClientInputStreamFactory;
+import org.apache.asterix.external.input.stream.factory.SocketServerInputStreamFactory;
 import org.apache.asterix.external.provider.StreamRecordReaderProvider;
 import org.apache.asterix.external.provider.StreamRecordReaderProvider.Format;
+import org.apache.asterix.external.util.ExternalDataConstants;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.api.context.IHyracksTaskContext;
@@ -33,13 +37,13 @@
 public class StreamRecordReaderFactory implements IRecordReaderFactory<char[]> {
 
     private static final long serialVersionUID = 1L;
-    protected final IInputStreamFactory streamFactory;
+    protected IInputStreamFactory streamFactory;
     protected Map<String, String> configuration;
     protected Format format;
-
-    public StreamRecordReaderFactory(IInputStreamFactory inputStreamFactory) {
-        this.streamFactory = inputStreamFactory;
-    }
+    protected String recordReaderName;
+    private String[] recordReaderNames = { ExternalDataConstants.ALIAS_LOCALFS_ADAPTER,
+            ExternalDataConstants.ALIAS_SOCKET_ADAPTER, ExternalDataConstants.SOCKET,
+            ExternalDataConstants.STREAM_SOCKET_CLIENT };
 
     @Override
     public DataSourceType getDataSourceType() {
@@ -60,6 +64,19 @@
     @Override
     public void configure(Map<String, String> configuration) throws HyracksDataException, AlgebricksException {
         this.configuration = configuration;
+        recordReaderName = configuration.get(ExternalDataConstants.KEY_READER);
+        switch (recordReaderName) {
+            case ExternalDataConstants.ALIAS_LOCALFS_ADAPTER:
+                this.streamFactory = new LocalFSInputStreamFactory();
+                break;
+            case ExternalDataConstants.ALIAS_SOCKET_ADAPTER:
+            case ExternalDataConstants.SOCKET:
+                this.streamFactory = new SocketServerInputStreamFactory();
+                break;
+            case ExternalDataConstants.STREAM_SOCKET_CLIENT:
+                this.streamFactory = new SocketClientInputStreamFactory();
+                break;
+        }
         streamFactory.configure(configuration);
         format = StreamRecordReaderProvider.getReaderFormat(configuration);
     }
@@ -70,4 +87,9 @@
         return StreamRecordReaderProvider.createRecordReader(format, streamFactory.createInputStream(ctx, partition),
                 configuration);
     }
+
+    @Override
+    public String[] getRecordReaderNames() {
+        return recordReaderNames;
+    }
 }
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
index 1428863..4eed8ea 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
@@ -18,8 +18,17 @@
  */
 package org.apache.asterix.external.provider;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.reflect.InvocationTargetException;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.HashMap;
 import java.util.Map;
 
+import org.apache.asterix.common.exceptions.AsterixException;
 import org.apache.asterix.common.exceptions.ErrorCode;
 import org.apache.asterix.common.exceptions.RuntimeDataException;
 import org.apache.asterix.common.library.ILibraryManager;
@@ -27,25 +36,23 @@
 import org.apache.asterix.external.api.IExternalDataSourceFactory.DataSourceType;
 import org.apache.asterix.external.api.IInputStreamFactory;
 import org.apache.asterix.external.api.IRecordReaderFactory;
-import org.apache.asterix.external.input.HDFSDataSourceFactory;
-import org.apache.asterix.external.input.record.reader.rss.RSSRecordReaderFactory;
-import org.apache.asterix.external.input.record.reader.stream.StreamRecordReaderFactory;
-import org.apache.asterix.external.input.record.reader.twitter.TwitterRecordReaderFactory;
 import org.apache.asterix.external.input.stream.factory.LocalFSInputStreamFactory;
-import org.apache.asterix.external.input.stream.factory.SocketClientInputStreamFactory;
 import org.apache.asterix.external.input.stream.factory.SocketServerInputStreamFactory;
 import org.apache.asterix.external.util.ExternalDataConstants;
 import org.apache.asterix.external.util.ExternalDataUtils;
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.commons.io.IOUtils;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 
 public class DatasourceFactoryProvider {
+
+    private static final String RESOURCE = "META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory";
+    private static Map<String, Class> factories = null;
 
     private DatasourceFactoryProvider() {
     }
 
     public static IExternalDataSourceFactory getExternalDataSourceFactory(ILibraryManager libraryManager,
-            Map<String, String> configuration) throws HyracksDataException {
+            Map<String, String> configuration) throws HyracksDataException, AsterixException {
         if (ExternalDataUtils.getDataSourceType(configuration).equals(DataSourceType.RECORDS)) {
             String reader = configuration.get(ExternalDataConstants.KEY_READER);
             return DatasourceFactoryProvider.getRecordReaderFactory(libraryManager, reader, configuration);
@@ -88,41 +95,60 @@
         return streamSourceFactory;
     }
 
-    public static IRecordReaderFactory<?> getRecordReaderFactory(ILibraryManager libraryManager, String reader,
-            Map<String, String> configuration) throws HyracksDataException {
-        if (reader.equals(ExternalDataConstants.EXTERNAL)) {
-            try {
-                return ExternalDataUtils.createExternalRecordReaderFactory(libraryManager, configuration);
-            } catch (AlgebricksException e) {
-                // Not sure whether this is the right way to handle AlgebricksException  (xikui)
-                throw new HyracksDataException(e);
-            }
+    protected static IRecordReaderFactory getInstance(Class clazz) throws AsterixException {
+        try {
+            return (IRecordReaderFactory) clazz.newInstance();
+        } catch (InstantiationException | IllegalAccessException | ClassCastException e) {
+            throw new AsterixException("Cannot create: " + clazz.getSimpleName(), e);
         }
-        switch (reader) {
-            case ExternalDataConstants.READER_HDFS:
-                return new HDFSDataSourceFactory();
-            case ExternalDataConstants.ALIAS_LOCALFS_ADAPTER:
-                return new StreamRecordReaderFactory(new LocalFSInputStreamFactory());
-            case ExternalDataConstants.READER_TWITTER_PULL:
-            case ExternalDataConstants.READER_TWITTER_PUSH:
-            case ExternalDataConstants.READER_PUSH_TWITTER:
-            case ExternalDataConstants.READER_PULL_TWITTER:
-            case ExternalDataConstants.READER_USER_STREAM_TWITTER:
-                return new TwitterRecordReaderFactory();
-            case ExternalDataConstants.ALIAS_SOCKET_ADAPTER:
-            case ExternalDataConstants.SOCKET:
-                return new StreamRecordReaderFactory(new SocketServerInputStreamFactory());
-            case ExternalDataConstants.STREAM_SOCKET_CLIENT:
-                return new StreamRecordReaderFactory(new SocketClientInputStreamFactory());
-            case ExternalDataConstants.READER_RSS:
-                return new RSSRecordReaderFactory();
-            default:
-                try {
-                    return (IRecordReaderFactory<?>) Class.forName(reader).newInstance();
-                } catch (IllegalAccessException | ClassNotFoundException | InstantiationException
-                        | ClassCastException e) {
-                    throw new RuntimeDataException(ErrorCode.UNKNOWN_RECORD_READER_FACTORY, e,reader);
+    }
+
+    public static IRecordReaderFactory<?> getRecordReaderFactory(ILibraryManager libraryManager, String adaptorName,
+            Map<String, String> configuration) throws HyracksDataException, AsterixException {
+        if (adaptorName.equals(ExternalDataConstants.EXTERNAL)) {
+            return ExternalDataUtils.createExternalRecordReaderFactory(libraryManager, configuration);
+        }
+
+        if (factories == null) {
+            factories = initFactories();
+        }
+
+        if (factories.containsKey(adaptorName)) {
+            return getInstance(factories.get(adaptorName));
+        }
+
+        try {
+            return (IRecordReaderFactory) Class.forName(adaptorName).newInstance();
+        } catch (IllegalAccessException | ClassNotFoundException | InstantiationException | ClassCastException e) {
+            throw new RuntimeDataException(ErrorCode.UNKNOWN_RECORD_READER_FACTORY, e, adaptorName);
+        }
+    }
+
+    protected static Map<String, Class> initFactories() throws AsterixException {
+        Map<String, Class> factories = new HashMap<>();
+        ClassLoader cl = ParserFactoryProvider.class.getClassLoader();
+        final Charset encoding = Charset.forName("UTF-8");
+        try {
+            Enumeration<URL> urls = cl.getResources(RESOURCE);
+            for (URL url : Collections.list(urls)) {
+                InputStream is = url.openStream();
+                String config = IOUtils.toString(is, encoding);
+                is.close();
+                String[] classNames = config.split("\n");
+                for (String className : classNames) {
+                    final Class<?> clazz = Class.forName(className);
+                    String[] formats = ((IRecordReaderFactory) clazz.newInstance()).getRecordReaderNames();
+                    for (String format : formats) {
+                        if (factories.containsKey(format)) {
+                            throw new AsterixException("Duplicate format " + format);
+                        }
+                        factories.put(format, clazz);
+                    }
                 }
+            }
+        } catch (IOException | ClassNotFoundException | InstantiationException | IllegalAccessException e) {
+            throw new AsterixException(e);
         }
+        return factories;
     }
 }
diff --git a/asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory b/asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
new file mode 100644
index 0000000..f13421c
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
@@ -0,0 +1,3 @@
+org.apache.asterix.external.input.HDFSDataSourceFactory
+org.apache.asterix.external.input.record.reader.stream.StreamRecordReaderFactory
+org.apache.asterix.external.input.record.reader.rss.RSSRecordReaderFactory
\ No newline at end of file
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/feed/test/DatasourceFactoryProviderTest.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/feed/test/DatasourceFactoryProviderTest.java
new file mode 100644
index 0000000..3923015
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/feed/test/DatasourceFactoryProviderTest.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.feed.test;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.external.api.IRecordReaderFactory;
+import org.apache.asterix.external.input.HDFSDataSourceFactory;
+import org.apache.asterix.external.provider.DatasourceFactoryProvider;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class DatasourceFactoryProviderTest {
+
+    IRecordReaderFactory factory = null;
+
+    @Test
+    public void test() throws HyracksDataException, AsterixException {
+        factory = DatasourceFactoryProvider.getRecordReaderFactory(null, "hdfs", null);
+        Assert.assertTrue(factory instanceof HDFSDataSourceFactory);
+    }
+}
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/RecordWithPKTestReaderFactory.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/RecordWithPKTestReaderFactory.java
index 3467411..bf8a616 100644
--- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/RecordWithPKTestReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/RecordWithPKTestReaderFactory.java
@@ -33,6 +33,7 @@
     private static final long serialVersionUID = 1L;
     private transient AlgebricksAbsolutePartitionConstraint clusterLocations;
 
+
     @Override
     public AlgebricksAbsolutePartitionConstraint getPartitionConstraint() throws AlgebricksException {
         clusterLocations = IExternalDataSourceFactory.getPartitionConstraints(clusterLocations, 1);
@@ -53,4 +54,8 @@
     public Class<?> getRecordClass() {
         return RecordWithPK.class;
     }
+
+    @Override public String[] getRecordReaderNames() {
+        return new String[0];
+    }
 }
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java
index 6841e6b..b9bba1f 100644
--- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java
@@ -77,4 +77,8 @@
     public Class<?> getRecordClass() {
         return DCPRequest.class;
     }
+
+    @Override public String[] getRecordReaderNames() {
+        return new String[0];
+    }
 }
diff --git a/asterixdb/asterix-villain/pom.xml b/asterixdb/asterix-villain/pom.xml
new file mode 100644
index 0000000..c4ea6de
--- /dev/null
+++ b/asterixdb/asterix-villain/pom.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements.  See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership.  The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License.  You may obtain a copy of the License at
+ !
+ !   http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied.  See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <artifactId>apache-asterixdb</artifactId>
+        <groupId>org.apache.asterix</groupId>
+        <version>0.8.9-SNAPSHOT</version>
+    </parent>
+    <artifactId>asterix-villain</artifactId>
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.asterix</groupId>
+            <artifactId>asterix-external-data</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+    </dependencies>
+</project>
\ No newline at end of file
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java b/asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/input/record/reader/twitter/TwitterPullRecordReader.java
similarity index 97%
rename from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
rename to asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/input/record/reader/twitter/TwitterPullRecordReader.java
index 5a7b4b9..fde685b 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPullRecordReader.java
+++ b/asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/input/record/reader/twitter/TwitterPullRecordReader.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.asterix.external.input.record.reader.twitter;
+package org.apache.asterix.villain.external.input.record.reader.twitter;
 
 import java.io.IOException;
 import java.util.List;
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java b/asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/input/record/reader/twitter/TwitterPushRecordReader.java
similarity index 89%
rename from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
rename to asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/input/record/reader/twitter/TwitterPushRecordReader.java
index ffffbd7..610bf37 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
+++ b/asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/input/record/reader/twitter/TwitterPushRecordReader.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.asterix.external.input.record.reader.twitter;
+package org.apache.asterix.villain.external.input.record.reader.twitter;
 
 import java.io.IOException;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -26,18 +26,9 @@
 import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
 import org.apache.asterix.external.input.record.GenericRecord;
 import org.apache.asterix.external.util.FeedLogManager;
-import org.apache.asterix.external.util.TwitterUtil;
-import twitter4j.DirectMessage;
+import org.apache.asterix.villain.external.util.TwitterUtil;
 import twitter4j.FilterQuery;
-import twitter4j.StallWarning;
-import twitter4j.Status;
-import twitter4j.StatusDeletionNotice;
-import twitter4j.StatusListener;
-import twitter4j.TwitterObjectFactory;
 import twitter4j.TwitterStream;
-import twitter4j.User;
-import twitter4j.UserList;
-import twitter4j.UserStreamListener;
 
 public class TwitterPushRecordReader implements IRecordReader<String> {
     private LinkedBlockingQueue<String> inputQ;
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterRecordReaderFactory.java b/asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/input/record/reader/twitter/TwitterRecordReaderFactory.java
similarity index 90%
rename from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterRecordReaderFactory.java
rename to asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/input/record/reader/twitter/TwitterRecordReaderFactory.java
index 570155c..d795905 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterRecordReaderFactory.java
+++ b/asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/input/record/reader/twitter/TwitterRecordReaderFactory.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.asterix.external.input.record.reader.twitter;
+package org.apache.asterix.villain.external.input.record.reader.twitter;
 
 import java.util.Map;
 import java.util.logging.Level;
@@ -27,9 +27,9 @@
 import org.apache.asterix.external.api.IRecordReader;
 import org.apache.asterix.external.api.IRecordReaderFactory;
 import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.asterix.external.util.TwitterUtil;
-import org.apache.asterix.external.util.TwitterUtil.AuthenticationConstants;
-import org.apache.asterix.external.util.TwitterUtil.SearchAPIConstants;
+import org.apache.asterix.villain.external.util.TwitterUtil;
+import org.apache.asterix.villain.external.util.TwitterUtil.AuthenticationConstants;
+import org.apache.asterix.villain.external.util.TwitterUtil.SearchAPIConstants;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.api.context.IHyracksTaskContext;
@@ -47,6 +47,14 @@
 
     private Map<String, String> configuration;
     private transient AlgebricksAbsolutePartitionConstraint clusterLocations;
+    private static final String[] recordReaderNames = { ExternalDataConstants.READER_TWITTER_PULL,
+            ExternalDataConstants.READER_TWITTER_PUSH, ExternalDataConstants.READER_PUSH_TWITTER,
+            ExternalDataConstants.READER_PULL_TWITTER, ExternalDataConstants.READER_USER_STREAM_TWITTER };
+
+    @Override
+    public String[] getRecordReaderNames() {
+        return recordReaderNames;
+    }
 
     @Override
     public DataSourceType getDataSourceType() {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/TwitterUtil.java b/asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/util/TwitterUtil.java
similarity index 98%
rename from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/TwitterUtil.java
rename to asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/util/TwitterUtil.java
index bd8e52d..ef244c5 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/TwitterUtil.java
+++ b/asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/util/TwitterUtil.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.asterix.external.util;
+package org.apache.asterix.villain.external.util;
 
 import java.io.InputStream;
 import java.util.HashMap;
@@ -30,6 +30,7 @@
 
 import org.apache.asterix.common.exceptions.AsterixException;
 
+import org.apache.asterix.external.util.ExternalDataConstants;
 import twitter4j.DirectMessage;
 import twitter4j.FilterQuery;
 import twitter4j.StallWarning;
diff --git a/asterixdb/asterix-villain/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory b/asterixdb/asterix-villain/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
new file mode 100644
index 0000000..b876491
--- /dev/null
+++ b/asterixdb/asterix-villain/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
@@ -0,0 +1 @@
+org.apache.asterix.villain.external.input.record.reader.twitter.TwitterRecordReaderFactory
\ No newline at end of file
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index 675d549..37600b8 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -605,6 +605,7 @@
     <module>asterix-coverage</module>
     <module>asterix-active</module>
     <module>asterix-client-helper</module>
+    <module>asterix-villain</module>
   </modules>
 
   <repositories>

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: Modularize feed adaptors
......................................................................


Patch Set 9:

Integration Tests Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/1716/

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 9
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: WIP - Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: WIP - Modularize feed adaptors
......................................................................


Patch Set 2:

Integration Tests Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/1584/

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: WIP - Modularize feed adaptors

Posted by "Xikui Wang (Code Review)" <do...@asterixdb.incubator.apache.org>.

Hello Jenkins,

I'd like you to reexamine a change.  Please visit

    https://asterix-gerrit.ics.uci.edu/1430

to look at the new patch set (#2).

Change subject: WIP - Modularize feed adaptors
......................................................................

WIP - Modularize feed adaptors

Modularize the feed adaptors and move 'license-unfriendly' adaptors to
separate package.

Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
---
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
M asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
M asterixdb/asterix-external-data/pom.xml
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReaderFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/rss/RSSRecordReaderFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
A asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
A asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/feed/test/DatasourceFactoryProviderTest.java
M asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/RecordWithPKTestReaderFactory.java
M asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java
A asterixdb/asterix-villain/pom.xml
R asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/input/record/reader/twitter/TwitterPullRecordReader.java
R asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/input/record/reader/twitter/TwitterPushRecordReader.java
R asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/input/record/reader/twitter/TwitterRecordReaderFactory.java
R asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/util/TwitterUtil.java
A asterixdb/asterix-villain/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
A asterixdb/asterix-villain/src/test/java/org/apache/asterix/villain/test/VillainExecutionTest.java
A asterixdb/asterix-villain/src/test/resources/runtimets/only.xml
R asterixdb/asterix-villain/src/test/resources/runtimets/queries/feeds/revised-tweet-parser/revised-tweet-parser.1.ddl.aql
R asterixdb/asterix-villain/src/test/resources/runtimets/queries/feeds/revised-tweet-parser/revised-tweet-parser.2.update.aql
R asterixdb/asterix-villain/src/test/resources/runtimets/queries/feeds/twitter-feed/twitter-feed.1.ddl.aql
R asterixdb/asterix-villain/src/test/resources/runtimets/queries/feeds/twitter-feed/twitter-feed.2.update.aql
A asterixdb/asterix-villain/src/test/resources/runtimets/testsuite.xml
M asterixdb/pom.xml
27 files changed, 482 insertions(+), 95 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/30/1430/2
-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newpatchset
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>

Change in asterixdb[master]: WIP - Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: WIP - Modularize feed adaptors
......................................................................


Patch Set 2: Integration-Tests-1

Integration Tests Timed Out

https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/1577/ : ABORTED

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: WIP - Modularize feed adaptors

Posted by "Michael Blow (Code Review)" <do...@asterixdb.incubator.apache.org>.

Michael Blow has posted comments on this change.

Change subject: WIP - Modularize feed adaptors
......................................................................


Patch Set 2:

Is this change ready to merge to master?  Please fix the Commit Msg to remove the WIP...

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: WIP - Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: WIP - Modularize feed adaptors
......................................................................


Patch Set 2: BAD+1

BAD Compatibility Tests Successful

https://asterix-jenkins.ics.uci.edu/job/asterixbad-compat/272/ : SUCCESS

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: Modularize feed adaptors
......................................................................


Patch Set 8: BAD+1

BAD Compatibility Tests Successful

https://asterix-jenkins.ics.uci.edu/job/asterixbad-compat/386/ : SUCCESS

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 8
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: Modularize feed adaptors
......................................................................


Patch Set 4: BAD+1

BAD Compatibility Tests Successful

https://asterix-jenkins.ics.uci.edu/job/asterixbad-compat/346/ : SUCCESS

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 4
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: WIP - Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: WIP - Modularize feed adaptors
......................................................................


Patch Set 2:

BAD Compatibility Tests Started https://asterix-jenkins.ics.uci.edu/job/asterixbad-compat/272/

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: Modularize feed adaptors
......................................................................


Patch Set 9: BAD+1

BAD Compatibility Tests Successful

https://asterix-jenkins.ics.uci.edu/job/asterixbad-compat/390/ : SUCCESS

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 9
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: Modularize feed adaptors
......................................................................


Patch Set 9: Integration-Tests+1

Integration Tests Successful

https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/1716/ : SUCCESS

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 9
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: Modularize feed adaptors
......................................................................


Patch Set 4:

Integration Tests Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/1665/

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 4
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: Modularize feed adaptors
......................................................................


Patch Set 3:

Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/3965/

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: Modularize feed adaptors
......................................................................


Patch Set 4:

Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/3998/

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 4
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Xikui Wang (Code Review)" <do...@asterixdb.incubator.apache.org>.

Hello Jenkins,

I'd like you to reexamine a change.  Please visit

    https://asterix-gerrit.ics.uci.edu/1430

to look at the new patch set (#7).

Change subject: Modularize feed adaptors
......................................................................

Modularize feed adaptors

1. Load adaptors from class-path.
2. Move rss&twitter adaptors/parsers to the new module, as well as related test
cases. All future parsers/adaptors that depend on various libraries should
be placed in this new module.

Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
---
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
M asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
M asterixdb/asterix-external-data/pom.xml
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReaderFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/Datatypes.java
M asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IDataParserFactory
A asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
A asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/feed/test/DatasourceFactoryProviderTest.java
M asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/RecordWithPKTestReaderFactory.java
M asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java
D asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/ParserFactoryProviderLoadParserTest.java
A asterixdb/asterix-villain/adaptor-rss/pom.xml
R asterixdb/asterix-villain/adaptor-rss/src/main/java/org/apache/asterix/villain/external/parser/RSSParser.java
R asterixdb/asterix-villain/adaptor-rss/src/main/java/org/apache/asterix/villain/external/parser/RSSParserFactory.java
R asterixdb/asterix-villain/adaptor-rss/src/main/java/org/apache/asterix/villain/external/reader/RSSRecordReader.java
R asterixdb/asterix-villain/adaptor-rss/src/main/java/org/apache/asterix/villain/external/reader/RSSRecordReaderFactory.java
A asterixdb/asterix-villain/adaptor-rss/src/main/resources/META-INF/services/org.apache.asterix.external.api.IDataParserFactory
A asterixdb/asterix-villain/adaptor-rss/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
A asterixdb/asterix-villain/adaptor-twitter/pom.xml
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/parser/TweetParser.java
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/parser/TweetParserFactory.java
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/reader/TwitterPullRecordReader.java
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/reader/TwitterPushRecordReader.java
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/reader/TwitterRecordReaderFactory.java
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/util/TwitterUtil.java
A asterixdb/asterix-villain/adaptor-twitter/src/main/resources/META-INF/services/org.apache.asterix.external.api.IDataParserFactory
A asterixdb/asterix-villain/adaptor-twitter/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
A asterixdb/asterix-villain/pom.xml
A asterixdb/asterix-villain/villain-test-framework/pom.xml
A asterixdb/asterix-villain/villain-test-framework/src/test/java/org/apache/asterix/villain/test/VillainExecutionTest.java
A asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/only.xml
R asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/queries/feeds/revised-tweet-parser/revised-tweet-parser.1.ddl.aql
R asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/queries/feeds/revised-tweet-parser/revised-tweet-parser.2.update.aql
R asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/queries/feeds/twitter-feed/twitter-feed.1.ddl.aql
R asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/queries/feeds/twitter-feed/twitter-feed.2.update.aql
A asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/testsuite.xml
M asterixdb/pom.xml
41 files changed, 652 insertions(+), 196 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/30/1430/7
-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newpatchset
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 7
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: Modularize feed adaptors
......................................................................


Patch Set 4: Integration-Tests+1

Integration Tests Successful

https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/1665/ : SUCCESS

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 4
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: Modularize feed adaptors
......................................................................


Patch Set 8:

Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/4052/

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 8
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Xikui Wang (Code Review)" <do...@asterixdb.incubator.apache.org>.

Hello Jenkins,

I'd like you to reexamine a change.  Please visit

    https://asterix-gerrit.ics.uci.edu/1430

to look at the new patch set (#3).

Change subject: Modularize feed adaptors
......................................................................

Modularize feed adaptors

1. Load adaptors from class-path.
2. Move rss&twitter adaptors/parsers to the new module, as well as related test
cases. All future parsers/adaptors that depend on various libraries should
be placed in this new module.

Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
---
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
M asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
M asterixdb/asterix-external-data/pom.xml
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReaderFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/Datatypes.java
M asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IDataParserFactory
A asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
A asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/feed/test/DatasourceFactoryProviderTest.java
M asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/RecordWithPKTestReaderFactory.java
M asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java
D asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/ParserFactoryProviderLoadParserTest.java
A asterixdb/asterix-villain/pom.xml
R asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/parser/rss/RSSParser.java
R asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/parser/rss/RSSParserFactory.java
R asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/parser/twitter/TweetParser.java
R asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/parser/twitter/TweetParserFactory.java
R asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/reader/rss/RSSRecordReader.java
R asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/reader/rss/RSSRecordReaderFactory.java
R asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/reader/twitter/TwitterPullRecordReader.java
R asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/reader/twitter/TwitterPushRecordReader.java
R asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/reader/twitter/TwitterRecordReaderFactory.java
R asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/util/TwitterUtil.java
A asterixdb/asterix-villain/src/main/resources/META-INF/services/org.apache.asterix.external.api.IDataParserFactory
A asterixdb/asterix-villain/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
A asterixdb/asterix-villain/src/test/java/org/apache/asterix/villain/test/VillainExecutionTest.java
A asterixdb/asterix-villain/src/test/resources/runtimets/only.xml
R asterixdb/asterix-villain/src/test/resources/runtimets/queries/feeds/revised-tweet-parser/revised-tweet-parser.1.ddl.aql
R asterixdb/asterix-villain/src/test/resources/runtimets/queries/feeds/revised-tweet-parser/revised-tweet-parser.2.update.aql
R asterixdb/asterix-villain/src/test/resources/runtimets/queries/feeds/twitter-feed/twitter-feed.1.ddl.aql
R asterixdb/asterix-villain/src/test/resources/runtimets/queries/feeds/twitter-feed/twitter-feed.2.update.aql
A asterixdb/asterix-villain/src/test/resources/runtimets/testsuite.xml
M asterixdb/pom.xml
36 files changed, 519 insertions(+), 193 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/30/1430/3
-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newpatchset
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Michael Blow (Code Review)" <do...@asterixdb.incubator.apache.org>.

Michael Blow has posted comments on this change.

Change subject: Modularize feed adaptors
......................................................................


Patch Set 4:

(7 comments)

https://asterix-gerrit.ics.uci.edu/#/c/1430/4/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
File asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java:

Line 235:         return Collections.unmodifiableList(Arrays.asList(recordReaderNames));
Comments which apply to all of the impls:
- Should we just store the collection instead of the array, to avoid rebuilding it each time?
- Make it static final?
- Should this be a Set<> instead of a List<>?  Seems these are unique, if order matters, could use SortedSet<>?


https://asterix-gerrit.ics.uci.edu/#/c/1430/4/asterixdb/asterix-villain/pom.xml
File asterixdb/asterix-villain/pom.xml:

Line 84:             <version>0.8.9-SNAPSHOT</version>
Use ${project.version} here (same as other 0.8.9-SNAPSHOTs below)


Line 126:             <version>0.2.18-SNAPSHOT</version>
Use ${hyracks.version} here (same as other 0.2.18-SNAPSHOTs below)


Line 141:             <version>1.2.17</version>
I believe this version is already defined in depdendencyManagement, and should not be redefined here.  Please check the other deps with versions for the same.  (Not sure about Eclipse, but IDEA indicates in the IDE for this pom when the definition is being overridden)


https://asterix-gerrit.ics.uci.edu/#/c/1430/4/asterixdb/asterix-villain/src/main/resources/META-INF/services/org.apache.asterix.external.api.IDataParserFactory
File asterixdb/asterix-villain/src/main/resources/META-INF/services/org.apache.asterix.external.api.IDataParserFactory:

Line 2: org.apache.asterix.villain.external.parser.rss.RSSParserFactory
These are both nice to make modular, but probably should each be in their own module, not combined into one.


https://asterix-gerrit.ics.uci.edu/#/c/1430/4/asterixdb/asterix-villain/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
File asterixdb/asterix-villain/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory:

Line 2: org.apache.asterix.villain.external.reader.rss.RSSRecordReaderFactory
These are both nice to make modular, but probably should each be in their own module, not combined into one.


https://asterix-gerrit.ics.uci.edu/#/c/1430/4/asterixdb/asterix-villain/src/test/resources/runtimets/testsuite.xml
File asterixdb/asterix-villain/src/test/resources/runtimets/testsuite.xml:

Line 26:              <!ENTITY TemporalQueries SYSTEM "queries/temporal/TemporalQueries.xml">
All of these entity decls are unused here and should be removed.


-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 4
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: Yes

Change in asterixdb[master]: WIP - Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: WIP - Modularize feed adaptors
......................................................................


Patch Set 2: Integration-Tests+1

Integration Tests Successful

https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/1584/ : SUCCESS

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: Modularize feed adaptors
......................................................................


Patch Set 8: Integration-Tests+1

Integration Tests Successful

https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/1710/ : SUCCESS

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 8
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: Modularize feed adaptors
......................................................................


Patch Set 5:

Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/4049/

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 5
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Xikui Wang (Code Review)" <do...@asterixdb.incubator.apache.org>.

Hello Jenkins,

I'd like you to reexamine a change.  Please visit

    https://asterix-gerrit.ics.uci.edu/1430

to look at the new patch set (#4).

Change subject: Modularize feed adaptors
......................................................................

Modularize feed adaptors

1. Load adaptors from class-path.
2. Move rss&twitter adaptors/parsers to the new module, as well as related test
cases. All future parsers/adaptors that depend on various libraries should
be placed in this new module.

Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
---
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
M asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
M asterixdb/asterix-external-data/pom.xml
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReaderFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/Datatypes.java
M asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IDataParserFactory
A asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
A asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/feed/test/DatasourceFactoryProviderTest.java
M asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/RecordWithPKTestReaderFactory.java
M asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java
D asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/ParserFactoryProviderLoadParserTest.java
A asterixdb/asterix-villain/pom.xml
R asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/parser/rss/RSSParser.java
R asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/parser/rss/RSSParserFactory.java
R asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/parser/twitter/TweetParser.java
R asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/parser/twitter/TweetParserFactory.java
R asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/reader/rss/RSSRecordReader.java
R asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/reader/rss/RSSRecordReaderFactory.java
R asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/reader/twitter/TwitterPullRecordReader.java
R asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/reader/twitter/TwitterPushRecordReader.java
R asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/reader/twitter/TwitterRecordReaderFactory.java
R asterixdb/asterix-villain/src/main/java/org/apache/asterix/villain/external/util/TwitterUtil.java
A asterixdb/asterix-villain/src/main/resources/META-INF/services/org.apache.asterix.external.api.IDataParserFactory
A asterixdb/asterix-villain/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
A asterixdb/asterix-villain/src/test/java/org/apache/asterix/villain/test/VillainExecutionTest.java
A asterixdb/asterix-villain/src/test/resources/runtimets/only.xml
R asterixdb/asterix-villain/src/test/resources/runtimets/queries/feeds/revised-tweet-parser/revised-tweet-parser.1.ddl.aql
R asterixdb/asterix-villain/src/test/resources/runtimets/queries/feeds/revised-tweet-parser/revised-tweet-parser.2.update.aql
R asterixdb/asterix-villain/src/test/resources/runtimets/queries/feeds/twitter-feed/twitter-feed.1.ddl.aql
R asterixdb/asterix-villain/src/test/resources/runtimets/queries/feeds/twitter-feed/twitter-feed.2.update.aql
A asterixdb/asterix-villain/src/test/resources/runtimets/testsuite.xml
M asterixdb/pom.xml
36 files changed, 562 insertions(+), 196 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/30/1430/4
-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newpatchset
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 4
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Xikui Wang (Code Review)" <do...@asterixdb.incubator.apache.org>.

Hello Jenkins,

I'd like you to reexamine a change.  Please visit

    https://asterix-gerrit.ics.uci.edu/1430

to look at the new patch set (#9).

Change subject: Modularize feed adaptors
......................................................................

Modularize feed adaptors

1. Load adaptors from class-path.
2. Move rss&twitter adaptors/parsers to the new module, as well as related test
cases. All future parsers/adaptors that depend on various libraries should
be placed in this new module.

Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
---
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
M asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
M asterixdb/asterix-external-data/pom.xml
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReaderFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/Datatypes.java
M asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IDataParserFactory
A asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
A asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/feed/test/DatasourceFactoryProviderTest.java
M asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/RecordWithPKTestReaderFactory.java
M asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java
D asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/ParserFactoryProviderLoadParserTest.java
A asterixdb/asterix-villain/adaptor-rss/pom.xml
R asterixdb/asterix-villain/adaptor-rss/src/main/java/org/apache/asterix/villain/external/parser/RSSParser.java
R asterixdb/asterix-villain/adaptor-rss/src/main/java/org/apache/asterix/villain/external/parser/RSSParserFactory.java
R asterixdb/asterix-villain/adaptor-rss/src/main/java/org/apache/asterix/villain/external/reader/RSSRecordReader.java
R asterixdb/asterix-villain/adaptor-rss/src/main/java/org/apache/asterix/villain/external/reader/RSSRecordReaderFactory.java
A asterixdb/asterix-villain/adaptor-rss/src/main/resources/META-INF/services/org.apache.asterix.external.api.IDataParserFactory
A asterixdb/asterix-villain/adaptor-rss/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
A asterixdb/asterix-villain/adaptor-twitter/pom.xml
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/parser/TweetParser.java
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/parser/TweetParserFactory.java
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/reader/TwitterPullRecordReader.java
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/reader/TwitterPushRecordReader.java
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/reader/TwitterRecordReaderFactory.java
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/util/TwitterUtil.java
A asterixdb/asterix-villain/adaptor-twitter/src/main/resources/META-INF/services/org.apache.asterix.external.api.IDataParserFactory
A asterixdb/asterix-villain/adaptor-twitter/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
A asterixdb/asterix-villain/pom.xml
A asterixdb/asterix-villain/villain-test-framework/pom.xml
A asterixdb/asterix-villain/villain-test-framework/src/test/java/org/apache/asterix/villain/test/VillainExecutionTest.java
A asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/only.xml
R asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/queries/feeds/revised-tweet-parser/revised-tweet-parser.1.ddl.aql
R asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/queries/feeds/revised-tweet-parser/revised-tweet-parser.2.update.aql
R asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/queries/feeds/twitter-feed/twitter-feed.1.ddl.aql
R asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/queries/feeds/twitter-feed/twitter-feed.2.update.aql
A asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/testsuite.xml
M asterixdb/pom.xml
41 files changed, 650 insertions(+), 196 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/30/1430/9
-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newpatchset
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 9
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Xikui Wang (Code Review)" <do...@asterixdb.incubator.apache.org>.

Hello Jenkins,

I'd like you to reexamine a change.  Please visit

    https://asterix-gerrit.ics.uci.edu/1430

to look at the new patch set (#6).

Change subject: Modularize feed adaptors
......................................................................

Modularize feed adaptors

1. Load adaptors from class-path.
2. Move rss&twitter adaptors/parsers to the new module, as well as related test
cases. All future parsers/adaptors that depend on various libraries should
be placed in this new module.

Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
---
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
M asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
M asterixdb/asterix-external-data/pom.xml
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReaderFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/Datatypes.java
M asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IDataParserFactory
A asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
A asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/feed/test/DatasourceFactoryProviderTest.java
M asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/RecordWithPKTestReaderFactory.java
M asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java
D asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/ParserFactoryProviderLoadParserTest.java
A asterixdb/asterix-villain/adaptor-rss/pom.xml
R asterixdb/asterix-villain/adaptor-rss/src/main/java/org/apache/asterix/villain/external/parser/RSSParser.java
R asterixdb/asterix-villain/adaptor-rss/src/main/java/org/apache/asterix/villain/external/parser/RSSParserFactory.java
R asterixdb/asterix-villain/adaptor-rss/src/main/java/org/apache/asterix/villain/external/reader/RSSRecordReader.java
R asterixdb/asterix-villain/adaptor-rss/src/main/java/org/apache/asterix/villain/external/reader/RSSRecordReaderFactory.java
A asterixdb/asterix-villain/adaptor-rss/src/main/resources/META-INF/services/org.apache.asterix.external.api.IDataParserFactory
A asterixdb/asterix-villain/adaptor-rss/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
A asterixdb/asterix-villain/adaptor-twitter/pom.xml
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/parser/TweetParser.java
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/parser/TweetParserFactory.java
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/reader/TwitterPullRecordReader.java
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/reader/TwitterPushRecordReader.java
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/reader/TwitterRecordReaderFactory.java
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/util/TwitterUtil.java
A asterixdb/asterix-villain/adaptor-twitter/src/main/resources/META-INF/services/org.apache.asterix.external.api.IDataParserFactory
A asterixdb/asterix-villain/adaptor-twitter/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
A asterixdb/asterix-villain/pom.xml
A asterixdb/asterix-villain/villain-test-framework/pom.xml
A asterixdb/asterix-villain/villain-test-framework/src/test/java/org/apache/asterix/villain/test/VillainExecutionTest.java
A asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/only.xml
R asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/queries/feeds/revised-tweet-parser/revised-tweet-parser.1.ddl.aql
R asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/queries/feeds/revised-tweet-parser/revised-tweet-parser.2.update.aql
R asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/queries/feeds/twitter-feed/twitter-feed.1.ddl.aql
R asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/queries/feeds/twitter-feed/twitter-feed.2.update.aql
A asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/testsuite.xml
M asterixdb/pom.xml
41 files changed, 652 insertions(+), 196 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/30/1430/6
-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newpatchset
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 6
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: Modularize feed adaptors
......................................................................


Patch Set 9:

Integration Tests Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/1714/

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 9
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Michael Blow (Code Review)" <do...@asterixdb.incubator.apache.org>.

Michael Blow has posted comments on this change.

Change subject: Modularize feed adaptors
......................................................................


Patch Set 8:

(9 comments)

https://asterix-gerrit.ics.uci.edu/#/c/1430/8/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
File asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java:

Line 79:     private static final List recordReaderNames = Collections.unmodifiableList(Arrays.asList("hdfs"));
List<String>?


Line 234:     public List getRecordReaderNames() {
List<String>?


https://asterix-gerrit.ics.uci.edu/#/c/1430/8/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java
File asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java:

Line 49:     private static final List<String> recordReaderNames = Collections.unmodifiableList(Arrays.asList(ExternalDataConstants.ALIAS_LOCALFS_ADAPTER,
> MAJOR SonarQube violation:
+1


https://asterix-gerrit.ics.uci.edu/#/c/1430/8/asterixdb/asterix-villain/adaptor-rss/pom.xml
File asterixdb/asterix-villain/adaptor-rss/pom.xml:

Line 38:             <version>1.2.17</version>
i think this version can be inherited


Line 53:             <version>${hyracks.version}</version>
i think these hyracks & algebricks versions can be removed (inherited)


https://asterix-gerrit.ics.uci.edu/#/c/1430/8/asterixdb/asterix-villain/adaptor-twitter/pom.xml
File asterixdb/asterix-villain/adaptor-twitter/pom.xml:

Line 29:             <version>2.8.4</version>
inherit this version?


https://asterix-gerrit.ics.uci.edu/#/c/1430/8/asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/reader/TwitterRecordReaderFactory.java
File asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/reader/TwitterRecordReaderFactory.java:

Line 57:                     ExternalDataConstants.READER_PULL_TWITTER, ExternalDataConstants.READER_USER_STREAM_TWITTER));
would this be more readable with one per line?  e.g.

private static final List<String> recordReaderNames = Collections.unmodifiableList(Arrays.asList(
        ExternalDataConstants.READER_TWITTER_PULL,
        ExternalDataConstants.READER_TWITTER_PUSH,
        ExternalDataConstants.READER_PUSH_TWITTER,
        ExternalDataConstants.READER_TWITTER_PUSH,
        ExternalDataConstants.READER_PUSH_TWITTER,
        ExternalDataConstants.READER_PULL_TWITTER,
        ExternalDataConstants.READER_USER_STREAM_TWITTER,
        ExternalDataConstants.READER_PULL_TWITTER, 
        ExternalDataConstants.READER_USER_STREAM_TWITTER
));


https://asterix-gerrit.ics.uci.edu/#/c/1430/8/asterixdb/asterix-villain/villain-test-framework/pom.xml
File asterixdb/asterix-villain/villain-test-framework/pom.xml:

Line 74:             <version>${hadoop.version}</version>
i think this version can be inherited


Line 81:             <version>4.12</version>
inherit version


-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 8
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: Yes

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Xikui Wang (Code Review)" <do...@asterixdb.incubator.apache.org>.

Xikui Wang has posted comments on this change.

Change subject: Modularize feed adaptors
......................................................................


Patch Set 9:

@Mike B, The further modularization comment turns out to be very useful! Quite some knowledge about Maven is gained from this process.. The quirky part about this patch is, I can't really have adaptor related dependencies resides at the villain module, because that will cause unused lib issue for the villain test framework part. So I duplicated some of them in each of the twitter-adaptor module and rss-adaptor module.

I guess the right answer would be use the AQLTestExcution load scripts from villain package and add villain-module as its dependency. Currently, These executionTests are included in asterix-app which will cause recursive dependency issue... I guess there is no better solution for that now?

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 9
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: Modularize feed adaptors
......................................................................


Patch Set 8:

Integration Tests Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/1710/

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 8
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: Modularize feed adaptors
......................................................................


Patch Set 9: Integration-Tests-1

Integration Tests Failed

https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/1714/ : UNSTABLE

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 9
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: Modularize feed adaptors
......................................................................


Patch Set 9:

BAD Compatibility Tests Started https://asterix-jenkins.ics.uci.edu/job/asterixbad-compat/390/

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 9
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: Modularize feed adaptors
......................................................................


Patch Set 7:

Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/4051/

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 7
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Xikui Wang (Code Review)" <do...@asterixdb.incubator.apache.org>.

Hello Jenkins,

I'd like you to reexamine a change.  Please visit

    https://asterix-gerrit.ics.uci.edu/1430

to look at the new patch set (#5).

Change subject: Modularize feed adaptors
......................................................................

Modularize feed adaptors

1. Load adaptors from class-path.
2. Move rss&twitter adaptors/parsers to the new module, as well as related test
cases. All future parsers/adaptors that depend on various libraries should
be placed in this new module.

Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
---
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
M asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
M asterixdb/asterix-external-data/pom.xml
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReaderFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/Datatypes.java
M asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IDataParserFactory
A asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
A asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/feed/test/DatasourceFactoryProviderTest.java
M asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/RecordWithPKTestReaderFactory.java
M asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java
D asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/ParserFactoryProviderLoadParserTest.java
A asterixdb/asterix-villain/adaptor-rss/pom.xml
R asterixdb/asterix-villain/adaptor-rss/src/main/java/org/apache/asterix/villain/external/parser/RSSParser.java
R asterixdb/asterix-villain/adaptor-rss/src/main/java/org/apache/asterix/villain/external/parser/RSSParserFactory.java
R asterixdb/asterix-villain/adaptor-rss/src/main/java/org/apache/asterix/villain/external/reader/RSSRecordReader.java
R asterixdb/asterix-villain/adaptor-rss/src/main/java/org/apache/asterix/villain/external/reader/RSSRecordReaderFactory.java
A asterixdb/asterix-villain/adaptor-rss/src/main/resources/META-INF/services/org.apache.asterix.external.api.IDataParserFactory
A asterixdb/asterix-villain/adaptor-rss/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
A asterixdb/asterix-villain/adaptor-twitter/pom.xml
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/parser/TweetParser.java
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/parser/TweetParserFactory.java
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/reader/TwitterPullRecordReader.java
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/reader/TwitterPushRecordReader.java
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/reader/TwitterRecordReaderFactory.java
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/util/TwitterUtil.java
A asterixdb/asterix-villain/adaptor-twitter/src/main/resources/META-INF/services/org.apache.asterix.external.api.IDataParserFactory
A asterixdb/asterix-villain/adaptor-twitter/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
A asterixdb/asterix-villain/pom.xml
A asterixdb/asterix-villain/villain-test-framework/pom.xml
A asterixdb/asterix-villain/villain-test-framework/src/test/java/org/apache/asterix/villain/test/VillainExecutionTest.java
A asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/only.xml
R asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/queries/feeds/revised-tweet-parser/revised-tweet-parser.1.ddl.aql
R asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/queries/feeds/revised-tweet-parser/revised-tweet-parser.2.update.aql
R asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/queries/feeds/twitter-feed/twitter-feed.1.ddl.aql
R asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/queries/feeds/twitter-feed/twitter-feed.2.update.aql
A asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/testsuite.xml
M asterixdb/pom.xml
41 files changed, 652 insertions(+), 196 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/30/1430/5
-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newpatchset
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 5
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Xikui Wang (Code Review)" <do...@asterixdb.incubator.apache.org>.

Xikui Wang has abandoned this change.

Change subject: Modularize feed adaptors
......................................................................


Abandoned

useful parts are picked up in another patch. The separate package for twitter/rss feeds is delayed for future better ideas.

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: abandon
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 9
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>

Change in asterixdb[master]: WIP - Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: WIP - Modularize feed adaptors
......................................................................


Patch Set 2:

Integration Tests Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/1577/

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: WIP - Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: WIP - Modularize feed adaptors
......................................................................


Patch Set 1:

Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/3872/

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No

Change in asterixdb[master]: WIP - Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: WIP - Modularize feed adaptors
......................................................................


Patch Set 2:

Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/3906/

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: Modularize feed adaptors
......................................................................


Patch Set 9:

Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/4058/

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 9
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: WIP - Modularize feed adaptors

Posted by "Michael Blow (Code Review)" <do...@asterixdb.incubator.apache.org>.

Michael Blow has posted comments on this change.

Change subject: WIP - Modularize feed adaptors
......................................................................


Patch Set 1:

(1 comment)

Looks pretty good to me, a minor request on the API, and please consider the SQ comments.

For testing, probably the twitter tests should be moved to a new test suite which runs from the villain module.  I think you can just add a new test in your module which is a copy of AqlExecutionTest, and move the twitter related tests into the testsuite.xml within that module.  Hopefully that makes sense.

https://asterix-gerrit.ics.uci.edu/#/c/1430/1/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReaderFactory.java
File asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReaderFactory.java:

Line 36:     public String[] getRecordReaderNames();
Would prefer this to be something immutable, such as Set<String>, where the implementations return an unmodifiable set.


-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: Yes

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: Modularize feed adaptors
......................................................................


Patch Set 8:

BAD Compatibility Tests Started https://asterix-jenkins.ics.uci.edu/job/asterixbad-compat/386/

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 8
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: WIP - Modularize feed adaptors

Posted by "Xikui Wang (Code Review)" <do...@asterixdb.incubator.apache.org>.

Xikui Wang has posted comments on this change.

Change subject: WIP - Modularize feed adaptors
......................................................................


Patch Set 1:

@Till

I added one package with temporary name for the Twitter4j related classes. I tried to move all adaptors into the new package, but that introduces a lot of cyclic reference issue. So I upload this one first...

Please help me verify the project configuration... As you may see from the test results, the twitter-related test cases are failing. AFAIK we need to add jars to classpath for enable extensions. Is there a way to enable that via Maven? I am guessing that probably I need to create a villain test executor to handle these test cases?

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: Modularize feed adaptors
......................................................................


Patch Set 6:

Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/4050/

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 6
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Michael Blow (Code Review)" <do...@asterixdb.incubator.apache.org>.

Michael Blow has posted comments on this change.

Change subject: Modularize feed adaptors
......................................................................


Patch Set 9: Code-Review+2

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 9
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Xikui Wang (Code Review)" <do...@asterixdb.incubator.apache.org>.

Hello Jenkins,

I'd like you to reexamine a change.  Please visit

    https://asterix-gerrit.ics.uci.edu/1430

to look at the new patch set (#8).

Change subject: Modularize feed adaptors
......................................................................

Modularize feed adaptors

1. Load adaptors from class-path.
2. Move rss&twitter adaptors/parsers to the new module, as well as related test
cases. All future parsers/adaptors that depend on various libraries should
be placed in this new module.

Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
---
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
M asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
M asterixdb/asterix-external-data/pom.xml
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReaderFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/StreamRecordReaderFactory.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/Datatypes.java
M asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IDataParserFactory
A asterixdb/asterix-external-data/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
A asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/feed/test/DatasourceFactoryProviderTest.java
M asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/RecordWithPKTestReaderFactory.java
M asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java
D asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/ParserFactoryProviderLoadParserTest.java
A asterixdb/asterix-villain/adaptor-rss/pom.xml
R asterixdb/asterix-villain/adaptor-rss/src/main/java/org/apache/asterix/villain/external/parser/RSSParser.java
R asterixdb/asterix-villain/adaptor-rss/src/main/java/org/apache/asterix/villain/external/parser/RSSParserFactory.java
R asterixdb/asterix-villain/adaptor-rss/src/main/java/org/apache/asterix/villain/external/reader/RSSRecordReader.java
R asterixdb/asterix-villain/adaptor-rss/src/main/java/org/apache/asterix/villain/external/reader/RSSRecordReaderFactory.java
A asterixdb/asterix-villain/adaptor-rss/src/main/resources/META-INF/services/org.apache.asterix.external.api.IDataParserFactory
A asterixdb/asterix-villain/adaptor-rss/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
A asterixdb/asterix-villain/adaptor-twitter/pom.xml
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/parser/TweetParser.java
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/parser/TweetParserFactory.java
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/reader/TwitterPullRecordReader.java
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/reader/TwitterPushRecordReader.java
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/reader/TwitterRecordReaderFactory.java
R asterixdb/asterix-villain/adaptor-twitter/src/main/java/org/apache/asterix/villain/external/util/TwitterUtil.java
A asterixdb/asterix-villain/adaptor-twitter/src/main/resources/META-INF/services/org.apache.asterix.external.api.IDataParserFactory
A asterixdb/asterix-villain/adaptor-twitter/src/main/resources/META-INF/services/org.apache.asterix.external.api.IRecordReaderFactory
A asterixdb/asterix-villain/pom.xml
A asterixdb/asterix-villain/villain-test-framework/pom.xml
A asterixdb/asterix-villain/villain-test-framework/src/test/java/org/apache/asterix/villain/test/VillainExecutionTest.java
A asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/only.xml
R asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/queries/feeds/revised-tweet-parser/revised-tweet-parser.1.ddl.aql
R asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/queries/feeds/revised-tweet-parser/revised-tweet-parser.2.update.aql
R asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/queries/feeds/twitter-feed/twitter-feed.1.ddl.aql
R asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/queries/feeds/twitter-feed/twitter-feed.2.update.aql
A asterixdb/asterix-villain/villain-test-framework/src/test/resources/runtimets/testsuite.xml
M asterixdb/pom.xml
41 files changed, 652 insertions(+), 196 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/30/1430/8
-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newpatchset
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 8
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>

Change in asterixdb[master]: Modularize feed adaptors

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: Modularize feed adaptors
......................................................................


Patch Set 4:

BAD Compatibility Tests Started https://asterix-jenkins.ics.uci.edu/job/asterixbad-compat/346/

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1430
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Ic4f95255f5493a813ee1f875b63a62e74bc47602
Gerrit-PatchSet: 4
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xk...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Xikui Wang <xk...@gmail.com>
Gerrit-HasComments: No