You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rya.apache.org by ca...@apache.org on 2017/09/11 16:14:34 UTC
[1/7] incubator-rya git commit: RYA-355 Refactored the periodic
notification service structure. Closes #221.
Repository: incubator-rya
Updated Branches:
refs/heads/master 28b0a52d0 -> de365c179
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.notification/src/test/java/org/apache/rya/periodic/notification/serialization/CommandNotificationSerializerTest.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.notification/src/test/java/org/apache/rya/periodic/notification/serialization/CommandNotificationSerializerTest.java b/extras/rya.periodic.service/periodic.service.notification/src/test/java/org/apache/rya/periodic/notification/serialization/CommandNotificationSerializerTest.java
deleted file mode 100644
index 4aad1c6..0000000
--- a/extras/rya.periodic.service/periodic.service.notification/src/test/java/org/apache/rya/periodic/notification/serialization/CommandNotificationSerializerTest.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.serialization;
-
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.rya.periodic.notification.notification.BasicNotification;
-import org.apache.rya.periodic.notification.notification.CommandNotification;
-import org.apache.rya.periodic.notification.notification.CommandNotification.Command;
-import org.apache.rya.periodic.notification.notification.PeriodicNotification;
-import org.apache.rya.periodic.notification.serialization.CommandNotificationSerializer;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class CommandNotificationSerializerTest {
-
- private CommandNotificationSerializer serializer = new CommandNotificationSerializer();
- private static final String topic = "topic";
-
- @Test
- public void basicSerializationTest() {
- PeriodicNotification notification = PeriodicNotification.builder().id(UUID.randomUUID().toString()).period(24)
- .timeUnit(TimeUnit.DAYS).initialDelay(1).build();
- CommandNotification command = new CommandNotification(Command.ADD, notification);
- Assert.assertEquals(command, serializer.deserialize(topic,serializer.serialize(topic, command)));
-
- PeriodicNotification notification1 = PeriodicNotification.builder().id(UUID.randomUUID().toString()).period(32)
- .timeUnit(TimeUnit.SECONDS).initialDelay(15).build();
- CommandNotification command1 = new CommandNotification(Command.ADD, notification1);
- Assert.assertEquals(command1, serializer.deserialize(topic,serializer.serialize(topic,command1)));
-
- PeriodicNotification notification2 = PeriodicNotification.builder().id(UUID.randomUUID().toString()).period(32)
- .timeUnit(TimeUnit.SECONDS).initialDelay(15).build();
- CommandNotification command2 = new CommandNotification(Command.ADD, notification2);
- Assert.assertEquals(command2, serializer.deserialize(topic,serializer.serialize(topic,command2)));
-
- BasicNotification notification3 = new BasicNotification(UUID.randomUUID().toString());
- CommandNotification command3 = new CommandNotification(Command.ADD, notification3);
- Assert.assertEquals(command3, serializer.deserialize(topic,serializer.serialize(topic,command3)));
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/pom.xml
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/pom.xml b/extras/rya.periodic.service/pom.xml
deleted file mode 100644
index 22ee1aa..0000000
--- a/extras/rya.periodic.service/pom.xml
+++ /dev/null
@@ -1,40 +0,0 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
- <modelVersion>4.0.0</modelVersion>
- <artifactId>rya.periodic.service</artifactId>
-
- <parent>
- <groupId>org.apache.rya</groupId>
- <artifactId>rya.extras</artifactId>
- <version>3.2.11-incubating-SNAPSHOT</version>
- </parent>
-
- <name>Apache Rya Periodic Service</name>
- <description>Parent class for Rya Periodic Service</description>
-
- <packaging>pom</packaging>
-
- <modules>
- <module>periodic.service.notification</module>
- <module>periodic.service.integration.tests</module>
- <module>periodic.service.api</module>
- </modules>
-
-</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7a0bf2c..761f992 100644
--- a/pom.xml
+++ b/pom.xml
@@ -244,24 +244,19 @@ under the License.
<artifactId>rya.pcj.fluo.app</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.rya</groupId>
- <artifactId>rya.periodic.service</artifactId>
- <version>${project.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.rya</groupId>
- <artifactId>rya.periodic.service.api</artifactId>
+ <artifactId>rya.periodic.notification.api</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.rya</groupId>
- <artifactId>rya.periodic.service.notification</artifactId>
+ <artifactId>rya.periodic.notification.service</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.rya</groupId>
- <artifactId>rya.periodic.service.integration.tests</artifactId>
+ <artifactId>rya.periodic.notification.tests</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
[2/7] incubator-rya git commit: RYA-355 Refactored the periodic
notification service structure. Closes #221.
Posted by ca...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplicationConfiguration.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplicationConfiguration.java b/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplicationConfiguration.java
deleted file mode 100644
index d69efe5..0000000
--- a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplicationConfiguration.java
+++ /dev/null
@@ -1,254 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.application;
-
-import java.util.Properties;
-
-import org.apache.rya.accumulo.AccumuloRdfConfiguration;
-
-import jline.internal.Preconditions;
-
-/**
- * Configuration object for creating a {@link PeriodicNotificationApplication}.
- */
-public class PeriodicNotificationApplicationConfiguration extends AccumuloRdfConfiguration {
-
- public static String FLUO_APP_NAME = "fluo.app.name";
- public static String FLUO_TABLE_NAME = "fluo.table.name";
- public static String KAFKA_BOOTSTRAP_SERVERS = "kafka.bootstrap.servers";
- public static String NOTIFICATION_TOPIC = "kafka.notification.topic";
- public static String NOTIFICATION_GROUP_ID = "kafka.notification.group.id";
- public static String NOTIFICATION_CLIENT_ID = "kafka.notification.client.id";
- public static String COORDINATOR_THREADS = "cep.coordinator.threads";
- public static String PRODUCER_THREADS = "cep.producer.threads";
- public static String EXPORTER_THREADS = "cep.exporter.threads";
- public static String PROCESSOR_THREADS = "cep.processor.threads";
- public static String PRUNER_THREADS = "cep.pruner.threads";
-
- public PeriodicNotificationApplicationConfiguration() {}
-
- /**
- * Creates an PeriodicNotificationApplicationConfiguration object from a Properties file. This method assumes
- * that all values in the Properties file are Strings and that the Properties file uses the keys below.
- * See rya.cep/cep.integration.tests/src/test/resources/properties/notification.properties for an example.
- * <br>
- * <ul>
- * <li>"accumulo.auths" - String of Accumulo authorizations. Default is empty String.
- * <li>"accumulo.instance" - Accumulo instance name (required)
- * <li>"accumulo.user" - Accumulo user (required)
- * <li>"accumulo.password" - Accumulo password (required)
- * <li>"accumulo.rya.prefix" - Prefix for Accumulo backed Rya instance. Default is "rya_"
- * <li>"accumulo.zookeepers" - Zookeepers for underlying Accumulo instance (required)
- * <li>"fluo.app.name" - Name of Fluo Application (required)
- * <li>"fluo.table.name" - Name of Fluo Table (required)
- * <li>"kafka.bootstrap.servers" - Kafka Bootstrap servers for Producers and Consumers (required)
- * <li>"kafka.notification.topic" - Topic to which new Periodic Notifications are published. Default is "notifications".
- * <li>"kafka.notification.client.id" - Client Id for notification topic. Default is "consumer0"
- * <li>"kafka.notification.group.id" - Group Id for notification topic. Default is "group0"
- * <li>"cep.coordinator.threads" - Number of threads used by coordinator. Default is 1.
- * <li>"cep.producer.threads" - Number of threads used by producer. Default is 1.
- * <li>"cep.exporter.threads" - Number of threads used by exporter. Default is 1.
- * <li>"cep.processor.threads" - Number of threads used by processor. Default is 1.
- * <li>"cep.pruner.threads" - Number of threads used by pruner. Default is 1.
- * </ul>
- * <br>
- * @param props - Properties file containing Accumulo specific configuration parameters
- * @return AccumumuloRdfConfiguration with properties set
- */
- public PeriodicNotificationApplicationConfiguration(Properties props) {
- super(fromProperties(props));
- setFluoAppName(props.getProperty(FLUO_APP_NAME));
- setFluoTableName(props.getProperty(FLUO_TABLE_NAME));
- setBootStrapServers(props.getProperty(KAFKA_BOOTSTRAP_SERVERS));
- setNotificationClientId(props.getProperty(NOTIFICATION_CLIENT_ID, "consumer0"));
- setNotificationTopic(props.getProperty(NOTIFICATION_TOPIC, "notifications"));
- setNotificationGroupId(props.getProperty(NOTIFICATION_GROUP_ID, "group0"));
- setProducerThreads(Integer.parseInt(props.getProperty(PRODUCER_THREADS, "1")));
- setProcessorThreads(Integer.parseInt(props.getProperty(PROCESSOR_THREADS, "1")));
- setExporterThreads(Integer.parseInt(props.getProperty(EXPORTER_THREADS, "1")));
- setPrunerThreads(Integer.parseInt(props.getProperty(PRUNER_THREADS, "1")));
- setCoordinatorThreads(Integer.parseInt(props.getProperty(COORDINATOR_THREADS, "1")));
- }
-
- /**
- * Sets the name of the Fluo Application
- * @param fluoAppName
- */
- public void setFluoAppName(String fluoAppName) {
- set(FLUO_APP_NAME, Preconditions.checkNotNull(fluoAppName));
- }
-
- /**
- * Sets the name of the Fluo table
- * @param fluoTableName
- */
- public void setFluoTableName(String fluoTableName) {
- set(FLUO_TABLE_NAME, Preconditions.checkNotNull(fluoTableName));
- }
-
- /**
- * Sets the Kafka bootstrap servers
- * @param bootStrapServers
- */
- public void setBootStrapServers(String bootStrapServers) {
- set(KAFKA_BOOTSTRAP_SERVERS, Preconditions.checkNotNull(bootStrapServers));
- }
-
- /**
- * Sets the Kafka topic name for new notification requests
- * @param notificationTopic
- */
- public void setNotificationTopic(String notificationTopic) {
- set(NOTIFICATION_TOPIC, Preconditions.checkNotNull(notificationTopic));
- }
-
- /**
- * Sets the GroupId for new notification request topic
- * @param notificationGroupId
- */
- public void setNotificationGroupId(String notificationGroupId) {
- set(NOTIFICATION_GROUP_ID, Preconditions.checkNotNull(notificationGroupId));
- }
-
- /**
- * Sets the ClientId for the Kafka notification topic
- * @param notificationClientId
- */
- public void setNotificationClientId(String notificationClientId) {
- set(NOTIFICATION_GROUP_ID, Preconditions.checkNotNull(notificationClientId));
- }
-
- /**
- * Sets the number of threads for the coordinator
- * @param threads
- */
- public void setCoordinatorThreads(int threads) {
- setInt(COORDINATOR_THREADS, threads);
- }
-
- /**
- * Sets the number of threads for the exporter
- * @param threads
- */
- public void setExporterThreads(int threads) {
- setInt(EXPORTER_THREADS, threads);
- }
-
- /**
- * Sets the number of threads for the producer for reading new periodic notifications
- * @param threads
- */
- public void setProducerThreads(int threads) {
- setInt(PRODUCER_THREADS, threads);
- }
-
- /**
- * Sets the number of threads for the bin pruner
- * @param threads
- */
- public void setPrunerThreads(int threads) {
- setInt(PRUNER_THREADS, threads);
- }
-
- /**
- * Sets the number of threads for the Notification processor
- * @param threads
- */
- public void setProcessorThreads(int threads) {
- setInt(PROCESSOR_THREADS, threads);
- }
-
- /**
- * @return name of the Fluo application
- */
- public String getFluoAppName() {
- return get(FLUO_APP_NAME);
- }
-
- /**
- * @return name of the Fluo table
- */
- public String getFluoTableName() {
- return get(FLUO_TABLE_NAME);
- }
-
- /**
- * @return Kafka bootstrap servers
- */
- public String getBootStrapServers() {
- return get(KAFKA_BOOTSTRAP_SERVERS);
- }
-
- /**
- * @return notification topic
- */
- public String getNotificationTopic() {
- return get(NOTIFICATION_TOPIC, "notifications");
- }
-
- /**
- * @return Kafka GroupId for the notificaton topic
- */
- public String getNotificationGroupId() {
- return get(NOTIFICATION_GROUP_ID, "group0");
- }
-
- /**
- * @return Kafka ClientId for the notification topic
- */
- public String getNotificationClientId() {
- return get(NOTIFICATION_CLIENT_ID, "consumer0");
- }
-
- /**
- * @return the number of threads for the coordinator
- */
- public int getCoordinatorThreads() {
- return getInt(COORDINATOR_THREADS, 1);
- }
-
- /**
- * @return the number of threads for the exporter
- */
- public int getExporterThreads() {
- return getInt(EXPORTER_THREADS, 1);
- }
-
- /**
- * @return the number of threads for the notification producer
- */
- public int getProducerThreads() {
- return getInt(PRODUCER_THREADS, 1);
- }
-
- /**
- * @return the number of threads for the bin pruner
- */
- public int getPrunerThreads() {
- return getInt(PRUNER_THREADS, 1);
- }
-
- /**
- * @return number of threads for the processor
- */
- public int getProcessorThreads() {
- return getInt(PROCESSOR_THREADS, 1);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplicationFactory.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplicationFactory.java b/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplicationFactory.java
deleted file mode 100644
index 771a4ab..0000000
--- a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplicationFactory.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.application;
-
-import java.util.Optional;
-import java.util.Properties;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
-import org.apache.accumulo.core.client.security.tokens.PasswordToken;
-import org.apache.fluo.api.client.FluoClient;
-import org.apache.fluo.api.client.Snapshot;
-import org.apache.kafka.clients.consumer.ConsumerConfig;
-import org.apache.kafka.clients.producer.KafkaProducer;
-import org.apache.kafka.common.serialization.StringDeserializer;
-import org.apache.kafka.common.serialization.StringSerializer;
-import org.apache.rya.indexing.pcj.fluo.app.util.FluoClientFactory;
-import org.apache.rya.indexing.pcj.storage.PeriodicQueryResultStorage;
-import org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPeriodicQueryResultStorage;
-import org.apache.rya.periodic.notification.api.BindingSetRecord;
-import org.apache.rya.periodic.notification.api.NodeBin;
-import org.apache.rya.periodic.notification.api.NotificationCoordinatorExecutor;
-import org.apache.rya.periodic.notification.coordinator.PeriodicNotificationCoordinatorExecutor;
-import org.apache.rya.periodic.notification.exporter.KafkaExporterExecutor;
-import org.apache.rya.periodic.notification.notification.TimestampedNotification;
-import org.apache.rya.periodic.notification.processor.NotificationProcessorExecutor;
-import org.apache.rya.periodic.notification.pruner.PeriodicQueryPrunerExecutor;
-import org.apache.rya.periodic.notification.recovery.PeriodicNotificationProvider;
-import org.apache.rya.periodic.notification.registration.kafka.KafkaNotificationProvider;
-import org.apache.rya.periodic.notification.serialization.BindingSetSerDe;
-import org.apache.rya.periodic.notification.serialization.CommandNotificationSerializer;
-import org.openrdf.query.BindingSet;
-
-/**
- * Factory for creating a {@link PeriodicNotificationApplication}.
- */
-public class PeriodicNotificationApplicationFactory {
-
- /**
- * Create a PeriodicNotificationApplication.
- * @param props - Properties file that specifies the parameters needed to create the application
- * @return PeriodicNotificationApplication to periodically poll Rya Fluo for new results
- * @throws PeriodicApplicationException
- */
- public static PeriodicNotificationApplication getPeriodicApplication(Properties props) throws PeriodicApplicationException {
- PeriodicNotificationApplicationConfiguration conf = new PeriodicNotificationApplicationConfiguration(props);
- Properties kafkaProps = getKafkaProperties(conf);
-
- BlockingQueue<TimestampedNotification> notifications = new LinkedBlockingQueue<>();
- BlockingQueue<NodeBin> bins = new LinkedBlockingQueue<>();
- BlockingQueue<BindingSetRecord> bindingSets = new LinkedBlockingQueue<>();
-
- FluoClient fluo = null;
- try {
- PeriodicQueryResultStorage storage = getPeriodicQueryResultStorage(conf);
- fluo = FluoClientFactory.getFluoClient(conf.getFluoAppName(), Optional.of(conf.getFluoTableName()), conf);
- NotificationCoordinatorExecutor coordinator = getCoordinator(conf.getCoordinatorThreads(), notifications);
- addRegisteredNotices(coordinator, fluo.newSnapshot());
- KafkaExporterExecutor exporter = getExporter(conf.getExporterThreads(), kafkaProps, bindingSets);
- PeriodicQueryPrunerExecutor pruner = getPruner(storage, fluo, conf.getPrunerThreads(), bins);
- NotificationProcessorExecutor processor = getProcessor(storage, notifications, bins, bindingSets, conf.getProcessorThreads());
- KafkaNotificationProvider provider = getProvider(conf.getProducerThreads(), conf.getNotificationTopic(), coordinator, kafkaProps);
- return PeriodicNotificationApplication.builder().setCoordinator(coordinator).setProvider(provider).setExporter(exporter)
- .setProcessor(processor).setPruner(pruner).build();
- } catch (AccumuloException | AccumuloSecurityException e) {
- throw new PeriodicApplicationException(e.getMessage());
- }
- }
-
- private static void addRegisteredNotices(NotificationCoordinatorExecutor coord, Snapshot sx) {
- coord.start();
- PeriodicNotificationProvider provider = new PeriodicNotificationProvider();
- provider.processRegisteredNotifications(coord, sx);
- }
-
- private static NotificationCoordinatorExecutor getCoordinator(int numThreads, BlockingQueue<TimestampedNotification> notifications) {
- return new PeriodicNotificationCoordinatorExecutor(numThreads, notifications);
- }
-
- private static KafkaExporterExecutor getExporter(int numThreads, Properties props, BlockingQueue<BindingSetRecord> bindingSets) {
- KafkaProducer<String, BindingSet> producer = new KafkaProducer<>(props, new StringSerializer(), new BindingSetSerDe());
- return new KafkaExporterExecutor(producer, numThreads, bindingSets);
- }
-
- private static PeriodicQueryPrunerExecutor getPruner(PeriodicQueryResultStorage storage, FluoClient fluo, int numThreads,
- BlockingQueue<NodeBin> bins) {
- return new PeriodicQueryPrunerExecutor(storage, fluo, numThreads, bins);
- }
-
- private static NotificationProcessorExecutor getProcessor(PeriodicQueryResultStorage periodicStorage,
- BlockingQueue<TimestampedNotification> notifications, BlockingQueue<NodeBin> bins, BlockingQueue<BindingSetRecord> bindingSets,
- int numThreads) {
- return new NotificationProcessorExecutor(periodicStorage, notifications, bins, bindingSets, numThreads);
- }
-
- private static KafkaNotificationProvider getProvider(int numThreads, String topic, NotificationCoordinatorExecutor coord,
- Properties props) {
- return new KafkaNotificationProvider(topic, new StringDeserializer(), new CommandNotificationSerializer(), props, coord,
- numThreads);
- }
-
- private static PeriodicQueryResultStorage getPeriodicQueryResultStorage(PeriodicNotificationApplicationConfiguration conf)
- throws AccumuloException, AccumuloSecurityException {
- Instance instance = new ZooKeeperInstance(conf.getAccumuloInstance(), conf.getAccumuloZookeepers());
- Connector conn = instance.getConnector(conf.getAccumuloUser(), new PasswordToken(conf.getAccumuloPassword()));
- String ryaInstance = conf.getTablePrefix();
- return new AccumuloPeriodicQueryResultStorage(conn, ryaInstance);
- }
-
- private static Properties getKafkaProperties(PeriodicNotificationApplicationConfiguration conf) {
- Properties kafkaProps = new Properties();
- kafkaProps.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, conf.getBootStrapServers());
- kafkaProps.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, conf.getNotificationClientId());
- kafkaProps.setProperty(ConsumerConfig.GROUP_ID_CONFIG, conf.getNotificationGroupId());
- kafkaProps.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
- return kafkaProps;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/coordinator/PeriodicNotificationCoordinatorExecutor.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/coordinator/PeriodicNotificationCoordinatorExecutor.java b/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/coordinator/PeriodicNotificationCoordinatorExecutor.java
deleted file mode 100644
index 0486244..0000000
--- a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/coordinator/PeriodicNotificationCoordinatorExecutor.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.coordinator;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.ReentrantLock;
-
-import org.apache.rya.periodic.notification.api.Notification;
-import org.apache.rya.periodic.notification.api.NotificationCoordinatorExecutor;
-import org.apache.rya.periodic.notification.api.NotificationProcessor;
-import org.apache.rya.periodic.notification.notification.CommandNotification;
-import org.apache.rya.periodic.notification.notification.PeriodicNotification;
-import org.apache.rya.periodic.notification.notification.TimestampedNotification;
-import org.apache.rya.periodic.notification.notification.CommandNotification.Command;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-
-/**
- * Implementation of {@link NotificationCoordinatorExecutor} that generates regular notifications
- * as indicated by {@link PeriodicNotification}s that are registered with this Object. When notifications
- * are generated they are placed on a work queue to be processed by the {@link NotificationProcessor}.
- *
- */
-public class PeriodicNotificationCoordinatorExecutor implements NotificationCoordinatorExecutor {
-
- private static final Logger LOG = LoggerFactory.getLogger(PeriodicNotificationCoordinatorExecutor.class);
- private int numThreads;
- private ScheduledExecutorService producerThreadPool;
- private Map<String, ScheduledFuture<?>> serviceMap = new HashMap<>();
- private BlockingQueue<TimestampedNotification> notifications;
- private final ReentrantLock lock = new ReentrantLock(true);
- private boolean running = false;
-
- public PeriodicNotificationCoordinatorExecutor(int numThreads, BlockingQueue<TimestampedNotification> notifications) {
- this.numThreads = numThreads;
- this.notifications = notifications;
- }
-
- @Override
- public void processNextCommandNotification(CommandNotification notification) {
- lock.lock();
- try {
- processNotification(notification);
- } finally {
- lock.unlock();
- }
- }
-
- @Override
- public void start() {
- if (!running) {
- producerThreadPool = Executors.newScheduledThreadPool(numThreads);
- running = true;
- }
- }
-
- @Override
- public void stop() {
-
- if (producerThreadPool != null) {
- producerThreadPool.shutdown();
- }
-
- running = false;
-
- try {
- if (!producerThreadPool.awaitTermination(5000, TimeUnit.MILLISECONDS)) {
- producerThreadPool.shutdownNow();
- }
- } catch (Exception e) {
- LOG.info("Service Executor Shutdown has been called. Terminating NotificationRunnable");
- }
- }
-
- private void processNotification(CommandNotification notification) {
- Command command = notification.getCommand();
- Notification periodic = notification.getNotification();
- switch (command) {
- case ADD:
- addNotification(periodic);
- break;
- case DELETE:
- deleteNotification(periodic);
- break;
- }
- }
-
- private void addNotification(Notification notification) {
- Preconditions.checkArgument(notification instanceof PeriodicNotification);
- PeriodicNotification notify = (PeriodicNotification) notification;
- if (!serviceMap.containsKey(notification.getId())) {
- ScheduledFuture<?> future = producerThreadPool.scheduleAtFixedRate(new NotificationProducer(notify), notify.getInitialDelay(),
- notify.getPeriod(), notify.getTimeUnit());
- serviceMap.put(notify.getId(), future);
- }
- }
-
- private boolean deleteNotification(Notification notification) {
- if (serviceMap.containsKey(notification.getId())) {
- ScheduledFuture<?> future = serviceMap.remove(notification.getId());
- future.cancel(true);
- return true;
- }
- return false;
- }
-
- /**
- * Scheduled Task that places a {@link PeriodicNotification}
- * in the work queue at regular intervals.
- *
- */
- class NotificationProducer implements Runnable {
-
- private PeriodicNotification notification;
-
- public NotificationProducer(PeriodicNotification notification) {
- this.notification = notification;
- }
-
- public void run() {
- try {
- notifications.put(new TimestampedNotification(notification));
- } catch (InterruptedException e) {
- LOG.info("Unable to add notification. Process interrupted. ");
- throw new RuntimeException(e);
- }
- }
-
- }
-
- @Override
- public boolean currentlyRunning() {
- return running;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/exporter/KafkaExporterExecutor.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/exporter/KafkaExporterExecutor.java b/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/exporter/KafkaExporterExecutor.java
deleted file mode 100644
index c2e5ebf..0000000
--- a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/exporter/KafkaExporterExecutor.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.exporter;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.kafka.clients.producer.KafkaProducer;
-import org.apache.log4j.Logger;
-import org.apache.rya.periodic.notification.api.BindingSetExporter;
-import org.apache.rya.periodic.notification.api.BindingSetRecord;
-import org.apache.rya.periodic.notification.api.LifeCycle;
-import org.openrdf.query.BindingSet;
-
-import jline.internal.Preconditions;
-
-/**
- * Executor service that runs {@link KafkaPeriodicBindingSetExporter}s.
- *
- */
-public class KafkaExporterExecutor implements LifeCycle {
-
- private static final Logger log = Logger.getLogger(BindingSetExporter.class);
- private KafkaProducer<String, BindingSet> producer;
- private BlockingQueue<BindingSetRecord> bindingSets;
- private ExecutorService executor;
- private List<KafkaPeriodicBindingSetExporter> exporters;
- private int num_Threads;
- private boolean running = false;
-
- /**
- * Creates a KafkaExporterExecutor for exporting periodic query results to Kafka.
- * @param producer for publishing results to Kafka
- * @param num_Threads number of threads used to publish results
- * @param bindingSets - work queue containing {@link BindingSet}s to be published
- */
- public KafkaExporterExecutor(KafkaProducer<String, BindingSet> producer, int num_Threads, BlockingQueue<BindingSetRecord> bindingSets) {
- Preconditions.checkNotNull(producer);
- Preconditions.checkNotNull(bindingSets);
- this.producer = producer;
- this.bindingSets = bindingSets;
- this.num_Threads = num_Threads;
- this.exporters = new ArrayList<>();
- }
-
- @Override
- public void start() {
- if (!running) {
- executor = Executors.newFixedThreadPool(num_Threads);
-
- for (int threadNumber = 0; threadNumber < num_Threads; threadNumber++) {
- log.info("Creating exporter:" + threadNumber);
- KafkaPeriodicBindingSetExporter exporter = new KafkaPeriodicBindingSetExporter(producer, threadNumber, bindingSets);
- exporters.add(exporter);
- executor.submit(exporter);
- }
- running = true;
- }
- }
-
- @Override
- public void stop() {
- if (executor != null) {
- executor.shutdown();
- }
-
- if (exporters != null && exporters.size() > 0) {
- exporters.forEach(x -> x.shutdown());
- }
-
- if (producer != null) {
- producer.close();
- }
-
- running = false;
- try {
- if (!executor.awaitTermination(5000, TimeUnit.MILLISECONDS)) {
- log.info("Timed out waiting for consumer threads to shut down, exiting uncleanly");
- executor.shutdownNow();
- }
- } catch (InterruptedException e) {
- log.info("Interrupted during shutdown, exiting uncleanly");
- }
- }
-
- @Override
- public boolean currentlyRunning() {
- return running;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/exporter/KafkaPeriodicBindingSetExporter.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/exporter/KafkaPeriodicBindingSetExporter.java b/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/exporter/KafkaPeriodicBindingSetExporter.java
deleted file mode 100644
index 8a0322f..0000000
--- a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/exporter/KafkaPeriodicBindingSetExporter.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.exporter;
-
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.kafka.clients.producer.KafkaProducer;
-import org.apache.kafka.clients.producer.ProducerRecord;
-import org.apache.kafka.clients.producer.RecordMetadata;
-import org.apache.log4j.Logger;
-import org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants;
-import org.apache.rya.periodic.notification.api.BindingSetExporter;
-import org.apache.rya.periodic.notification.api.BindingSetRecord;
-import org.apache.rya.periodic.notification.api.BindingSetRecordExportException;
-import org.openrdf.model.Literal;
-import org.openrdf.query.BindingSet;
-
-import jline.internal.Preconditions;
-
-/**
- * Object that exports {@link BindingSet}s to the Kafka topic indicated by
- * the {@link BindingSetRecord}.
- *
- */
-public class KafkaPeriodicBindingSetExporter implements BindingSetExporter, Runnable {
-
- private static final Logger log = Logger.getLogger(BindingSetExporter.class);
- private KafkaProducer<String, BindingSet> producer;
- private BlockingQueue<BindingSetRecord> bindingSets;
- private AtomicBoolean closed = new AtomicBoolean(false);
- private int threadNumber;
-
- public KafkaPeriodicBindingSetExporter(KafkaProducer<String, BindingSet> producer, int threadNumber,
- BlockingQueue<BindingSetRecord> bindingSets) {
- Preconditions.checkNotNull(producer);
- Preconditions.checkNotNull(bindingSets);
- this.threadNumber = threadNumber;
- this.producer = producer;
- this.bindingSets = bindingSets;
- }
-
- /**
- * Exports BindingSets to Kafka. The BindingSet and topic are extracted from
- * the indicated BindingSetRecord and the BindingSet is then exported to the topic.
- */
- @Override
- public void exportNotification(BindingSetRecord record) throws BindingSetRecordExportException {
- String bindingName = IncrementalUpdateConstants.PERIODIC_BIN_ID;
- BindingSet bindingSet = record.getBindingSet();
- String topic = record.getTopic();
- long binId = ((Literal) bindingSet.getValue(bindingName)).longValue();
- final Future<RecordMetadata> future = producer
- .send(new ProducerRecord<String, BindingSet>(topic, Long.toString(binId), bindingSet));
- try {
- //wait for confirmation that results have been received
- future.get(5, TimeUnit.SECONDS);
- } catch (InterruptedException | ExecutionException | TimeoutException e) {
- throw new BindingSetRecordExportException(e.getMessage());
- }
- }
-
- @Override
- public void run() {
- try {
- while (!closed.get()) {
- exportNotification(bindingSets.take());
- }
- } catch (InterruptedException | BindingSetRecordExportException e) {
- log.trace("Thread " + threadNumber + " is unable to process message.");
- }
- }
-
-
- public void shutdown() {
- closed.set(true);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/processor/NotificationProcessorExecutor.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/processor/NotificationProcessorExecutor.java b/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/processor/NotificationProcessorExecutor.java
deleted file mode 100644
index a9a5ad1..0000000
--- a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/processor/NotificationProcessorExecutor.java
+++ /dev/null
@@ -1,114 +0,0 @@
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */package org.apache.rya.periodic.notification.processor;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.log4j.Logger;
-import org.apache.rya.indexing.pcj.storage.PeriodicQueryResultStorage;
-import org.apache.rya.periodic.notification.api.BindingSetRecord;
-import org.apache.rya.periodic.notification.api.LifeCycle;
-import org.apache.rya.periodic.notification.api.NodeBin;
-import org.apache.rya.periodic.notification.notification.TimestampedNotification;
-
-import com.google.common.base.Preconditions;
-
-/**
- * Executor service that runs {@link TimestampedNotificationProcessor}s with basic
- * functionality for starting, stopping, and determining whether notification processors are
- * being executed.
- *
- */
-public class NotificationProcessorExecutor implements LifeCycle {
-
- private static final Logger log = Logger.getLogger(TimestampedNotificationProcessor.class);
- private BlockingQueue<TimestampedNotification> notifications; // notifications
- private BlockingQueue<NodeBin> bins; // entries to delete from Fluo
- private BlockingQueue<BindingSetRecord> bindingSets; // query results to
- // export
- private PeriodicQueryResultStorage periodicStorage;
- private List<TimestampedNotificationProcessor> processors;
- private int numberThreads;
- private ExecutorService executor;
- private boolean running = false;
-
- /**
- * Creates NotificationProcessorExecutor.
- * @param periodicStorage - storage layer that periodic results are read from
- * @param notifications - notifications are pulled from this queue, and the timestamp indicates which bin of results to query for
- * @param bins - after notifications are processed, they are added to the bin to be deleted
- * @param bindingSets - results read from the storage layer to be exported
- * @param numberThreads - number of threads used for processing
- */
- public NotificationProcessorExecutor(PeriodicQueryResultStorage periodicStorage, BlockingQueue<TimestampedNotification> notifications,
- BlockingQueue<NodeBin> bins, BlockingQueue<BindingSetRecord> bindingSets, int numberThreads) {
- this.notifications = Preconditions.checkNotNull(notifications);
- this.bins = Preconditions.checkNotNull(bins);
- this.bindingSets = Preconditions.checkNotNull(bindingSets);
- this.periodicStorage = periodicStorage;
- this.numberThreads = numberThreads;
- processors = new ArrayList<>();
- }
-
- @Override
- public void start() {
- if (!running) {
- executor = Executors.newFixedThreadPool(numberThreads);
- for (int threadNumber = 0; threadNumber < numberThreads; threadNumber++) {
- log.info("Creating exporter:" + threadNumber);
- TimestampedNotificationProcessor processor = TimestampedNotificationProcessor.builder().setBindingSets(bindingSets)
- .setBins(bins).setPeriodicStorage(periodicStorage).setNotifications(notifications).setThreadNumber(threadNumber)
- .build();
- processors.add(processor);
- executor.submit(processor);
- }
- running = true;
- }
- }
-
- @Override
- public void stop() {
- if (processors != null && processors.size() > 0) {
- processors.forEach(x -> x.shutdown());
- }
- if (executor != null) {
- executor.shutdown();
- }
- running = false;
- try {
- if (!executor.awaitTermination(5000, TimeUnit.MILLISECONDS)) {
- log.info("Timed out waiting for consumer threads to shut down, exiting uncleanly");
- executor.shutdownNow();
- }
- } catch (InterruptedException e) {
- log.info("Interrupted during shutdown, exiting uncleanly");
- }
- }
-
- @Override
- public boolean currentlyRunning() {
- return running;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/processor/TimestampedNotificationProcessor.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/processor/TimestampedNotificationProcessor.java b/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/processor/TimestampedNotificationProcessor.java
deleted file mode 100644
index 8b65683..0000000
--- a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/processor/TimestampedNotificationProcessor.java
+++ /dev/null
@@ -1,203 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.processor;
-
-import java.util.Optional;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.log4j.Logger;
-import org.apache.rya.indexing.pcj.storage.PeriodicQueryResultStorage;
-import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
-import org.apache.rya.periodic.notification.api.BinPruner;
-import org.apache.rya.periodic.notification.api.BindingSetRecord;
-import org.apache.rya.periodic.notification.api.NodeBin;
-import org.apache.rya.periodic.notification.api.NotificationProcessor;
-import org.apache.rya.periodic.notification.exporter.KafkaPeriodicBindingSetExporter;
-import org.apache.rya.periodic.notification.notification.TimestampedNotification;
-import org.openrdf.query.BindingSet;
-
-import com.google.common.base.Preconditions;
-
-/**
- * Implementation of {@link NotificationProcessor} that uses the id indicated by
- * the {@link TimestampedNotification} to obtain results from the
- * {@link PeriodicQueryResultStorage} layer containing the results of the
- * Periodic Query. The TimestampedNotificationProcessor then parses the results
- * and adds them to work queues to be processed by the {@link BinPruner} and the
- * {@link KafkaPeriodicBindingSetExporter}.
- *
- */
-public class TimestampedNotificationProcessor implements NotificationProcessor, Runnable {
-
- private static final Logger log = Logger.getLogger(TimestampedNotificationProcessor.class);
- private PeriodicQueryResultStorage periodicStorage;
- private BlockingQueue<TimestampedNotification> notifications; // notifications
- // to process
- private BlockingQueue<NodeBin> bins; // entries to delete from Fluo
- private BlockingQueue<BindingSetRecord> bindingSets; // query results to export
- private AtomicBoolean closed = new AtomicBoolean(false);
- private int threadNumber;
-
-
- public TimestampedNotificationProcessor(PeriodicQueryResultStorage periodicStorage,
- BlockingQueue<TimestampedNotification> notifications, BlockingQueue<NodeBin> bins, BlockingQueue<BindingSetRecord> bindingSets,
- int threadNumber) {
- this.notifications = Preconditions.checkNotNull(notifications);
- this.bins = Preconditions.checkNotNull(bins);
- this.bindingSets = Preconditions.checkNotNull(bindingSets);
- this.periodicStorage = periodicStorage;
- this.threadNumber = threadNumber;
- }
-
- /**
- * Processes the TimestampNotifications by scanning the PCJ tables for
- * entries in the bin corresponding to
- * {@link TimestampedNotification#getTimestamp()} and adding them to the
- * export BlockingQueue. The TimestampNotification is then used to form a
- * {@link NodeBin} that is passed to the BinPruner BlockingQueue so that the
- * bins can be deleted from Fluo and Accumulo.
- */
- @Override
- public void processNotification(TimestampedNotification notification) {
-
- String id = notification.getId();
- long ts = notification.getTimestamp().getTime();
- long period = notification.getPeriod();
- long bin = getBinFromTimestamp(ts, period);
- NodeBin nodeBin = new NodeBin(id, bin);
-
- try (CloseableIterator<BindingSet> iter = periodicStorage.listResults(id, Optional.of(bin));) {
-
- while(iter.hasNext()) {
- bindingSets.add(new BindingSetRecord(iter.next(), id));
- }
- // add NodeBin to BinPruner queue so that bin can be deleted from
- // Fluo and Accumulo
- bins.add(nodeBin);
- } catch (Exception e) {
- log.debug("Encountered error: " + e.getMessage() + " while accessing periodic results for bin: " + bin + " for query: " + id);
- }
- }
-
- /**
- * Computes left bin end point containing event time ts
- *
- * @param ts - event time
- * @param start - time that periodic event began
- * @param period - length of period
- * @return left bin end point containing event time ts
- */
- private long getBinFromTimestamp(long ts, long period) {
- Preconditions.checkArgument(period > 0);
- return (ts / period) * period;
- }
-
- @Override
- public void run() {
- try {
- while(!closed.get()) {
- processNotification(notifications.take());
- }
- } catch (Exception e) {
- log.trace("Thread_" + threadNumber + " is unable to process next notification.");
- throw new RuntimeException(e);
- }
-
- }
-
- public void shutdown() {
- closed.set(true);
- }
-
- public static Builder builder() {
- return new Builder();
- }
-
-
-
- public static class Builder {
-
- private PeriodicQueryResultStorage periodicStorage;
- private BlockingQueue<TimestampedNotification> notifications; // notifications to process
- private BlockingQueue<NodeBin> bins; // entries to delete from Fluo
- private BlockingQueue<BindingSetRecord> bindingSets; // query results to export
-
- private int threadNumber;
-
- /**
- * Set notification queue
- * @param notifications - work queue containing notifications to be processed
- * @return this Builder for chaining method calls
- */
- public Builder setNotifications(BlockingQueue<TimestampedNotification> notifications) {
- this.notifications = notifications;
- return this;
- }
-
- /**
- * Set nodeBin queue
- * @param bins - work queue containing NodeBins to be pruned
- * @return this Builder for chaining method calls
- */
- public Builder setBins(BlockingQueue<NodeBin> bins) {
- this.bins = bins;
- return this;
- }
-
- /**
- * Set BindingSet queue
- * @param bindingSets - work queue containing BindingSets to be exported
- * @return this Builder for chaining method calls
- */
- public Builder setBindingSets(BlockingQueue<BindingSetRecord> bindingSets) {
- this.bindingSets = bindingSets;
- return this;
- }
-
- /**
- * Sets the number of threads used by this processor
- * @param threadNumber - number of threads used by this processor
- * @return - number of threads used by this processor
- */
- public Builder setThreadNumber(int threadNumber) {
- this.threadNumber = threadNumber;
- return this;
- }
-
- /**
- * Set the PeriodicStorage layer
- * @param periodicStorage - periodic storage layer that periodic results are read from
- * @return - this Builder for chaining method calls
- */
- public Builder setPeriodicStorage(PeriodicQueryResultStorage periodicStorage) {
- this.periodicStorage = periodicStorage;
- return this;
- }
-
- /**
- * Builds a TimestampedNotificationProcessor
- * @return - TimestampedNotificationProcessor built from arguments passed to this Builder
- */
- public TimestampedNotificationProcessor build() {
- return new TimestampedNotificationProcessor(periodicStorage, notifications, bins, bindingSets, threadNumber);
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/pruner/AccumuloBinPruner.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/pruner/AccumuloBinPruner.java b/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/pruner/AccumuloBinPruner.java
deleted file mode 100644
index 4dac64c..0000000
--- a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/pruner/AccumuloBinPruner.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.pruner;
-
-import org.apache.log4j.Logger;
-import org.apache.rya.indexing.pcj.storage.PeriodicQueryResultStorage;
-import org.apache.rya.indexing.pcj.storage.PeriodicQueryStorageException;
-import org.apache.rya.periodic.notification.api.BinPruner;
-import org.apache.rya.periodic.notification.api.NodeBin;
-
-import jline.internal.Preconditions;
-
-/**
- * Deletes BindingSets from time bins in the indicated PCJ table
- */
-public class AccumuloBinPruner implements BinPruner {
-
- private static final Logger log = Logger.getLogger(AccumuloBinPruner.class);
- private PeriodicQueryResultStorage periodicStorage;
-
- public AccumuloBinPruner(PeriodicQueryResultStorage periodicStorage) {
- Preconditions.checkNotNull(periodicStorage);
- this.periodicStorage = periodicStorage;
- }
-
- /**
- * This method deletes all BindingSets in the indicated bin from the PCJ
- * table indicated by the id. It is assumed that all BindingSet entries for
- * the corresponding bin are written to the PCJ table so that the bin Id
- * occurs first.
- *
- * @param id
- * - pcj table id
- * @param bin
- * - temporal bin the BindingSets are contained in
- */
- @Override
- public void pruneBindingSetBin(NodeBin nodeBin) {
- Preconditions.checkNotNull(nodeBin);
- String id = nodeBin.getNodeId();
- long bin = nodeBin.getBin();
- try {
- periodicStorage.deletePeriodicQueryResults(id, bin);
- } catch (PeriodicQueryStorageException e) {
- log.trace("Unable to delete results from Peroidic Table: " + id + " for bin: " + bin);
- throw new RuntimeException(e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/pruner/FluoBinPruner.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/pruner/FluoBinPruner.java b/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/pruner/FluoBinPruner.java
deleted file mode 100644
index bee9c02..0000000
--- a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/pruner/FluoBinPruner.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.pruner;
-
-import org.apache.fluo.api.client.FluoClient;
-import org.apache.fluo.api.client.Transaction;
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.api.data.Column;
-import org.apache.fluo.api.data.Span;
-import org.apache.log4j.Logger;
-import org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants;
-import org.apache.rya.indexing.pcj.fluo.app.NodeType;
-import org.apache.rya.indexing.pcj.fluo.app.batch.BatchInformationDAO;
-import org.apache.rya.indexing.pcj.fluo.app.batch.SpanBatchDeleteInformation;
-import org.apache.rya.periodic.notification.api.BinPruner;
-import org.apache.rya.periodic.notification.api.NodeBin;
-
-import com.google.common.base.Optional;
-
-/**
- * Deletes {@link BindingSet}s from the indicated Fluo table.
- */
-public class FluoBinPruner implements BinPruner {
-
- private static final Logger log = Logger.getLogger(FluoBinPruner.class);
- private FluoClient client;
-
- public FluoBinPruner(FluoClient client) {
- this.client = client;
- }
-
- /**
- * This method deletes BindingSets in the specified bin from the BindingSet
- * Column of the indicated Fluo nodeId
- *
- * @param id
- * - Fluo nodeId
- * @param bin
- * - bin id
- */
- @Override
- public void pruneBindingSetBin(NodeBin nodeBin) {
- String id = nodeBin.getNodeId();
- long bin = nodeBin.getBin();
- try (Transaction tx = client.newTransaction()) {
- Optional<NodeType> type = NodeType.fromNodeId(id);
- if (!type.isPresent()) {
- log.trace("Unable to determine NodeType from id: " + id);
- throw new RuntimeException();
- }
- Column batchInfoColumn = type.get().getResultColumn();
- String batchInfoSpanPrefix = id + IncrementalUpdateConstants.NODEID_BS_DELIM + bin;
- SpanBatchDeleteInformation batchInfo = SpanBatchDeleteInformation.builder().setColumn(batchInfoColumn)
- .setSpan(Span.prefix(Bytes.of(batchInfoSpanPrefix))).build();
- BatchInformationDAO.addBatch(tx, id, batchInfo);
- tx.commit();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/pruner/PeriodicQueryPruner.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/pruner/PeriodicQueryPruner.java b/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/pruner/PeriodicQueryPruner.java
deleted file mode 100644
index 516690e..0000000
--- a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/pruner/PeriodicQueryPruner.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.pruner;
-
-import java.util.HashSet;
-import java.util.Set;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.fluo.api.client.FluoClient;
-import org.apache.fluo.api.client.Snapshot;
-import org.apache.fluo.api.client.SnapshotBase;
-import org.apache.log4j.Logger;
-import org.apache.rya.indexing.pcj.fluo.app.NodeType;
-import org.apache.rya.indexing.pcj.fluo.app.util.PeriodicQueryUtil;
-import org.apache.rya.periodic.notification.api.BinPruner;
-import org.apache.rya.periodic.notification.api.NodeBin;
-
-import jline.internal.Preconditions;
-
-/**
- * Implementation of {@link BinPruner} that deletes old, already processed
- * Periodic Query results from Fluo and the PCJ table to which the Fluo results
- * are exported.
- *
- */
-public class PeriodicQueryPruner implements BinPruner, Runnable {
-
- private static final Logger log = Logger.getLogger(PeriodicQueryPruner.class);
- private FluoClient client;
- private AccumuloBinPruner accPruner;
- private FluoBinPruner fluoPruner;
- private BlockingQueue<NodeBin> bins;
- private AtomicBoolean closed = new AtomicBoolean(false);
- private int threadNumber;
-
- public PeriodicQueryPruner(FluoBinPruner fluoPruner, AccumuloBinPruner accPruner, FluoClient client, BlockingQueue<NodeBin> bins, int threadNumber) {
- this.fluoPruner = Preconditions.checkNotNull(fluoPruner);
- this.accPruner = Preconditions.checkNotNull(accPruner);
- this.client = Preconditions.checkNotNull(client);
- this.bins = Preconditions.checkNotNull(bins);
- this.threadNumber = threadNumber;
- }
-
- @Override
- public void run() {
- try {
- while (!closed.get()) {
- pruneBindingSetBin(bins.take());
- }
- } catch (InterruptedException e) {
- log.trace("Thread " + threadNumber + " is unable to prune the next message.");
- throw new RuntimeException(e);
- }
- }
-
- /**
- * Prunes BindingSet bins from the Rya Fluo Application in addition to the BindingSet
- * bins created in the PCJ tables associated with the give query id.
- * @param id - QueryResult Id for the Rya Fluo application
- * @param bin - bin id for bins to be deleted
- */
- @Override
- public void pruneBindingSetBin(NodeBin nodeBin) {
- String pcjId = nodeBin.getNodeId();
- long bin = nodeBin.getBin();
- try(Snapshot sx = client.newSnapshot()) {
- String queryId = NodeType.generateNewIdForType(NodeType.QUERY, pcjId);
- Set<String> fluoIds = getNodeIdsFromResultId(sx, queryId);
- accPruner.pruneBindingSetBin(nodeBin);
- for(String fluoId: fluoIds) {
- fluoPruner.pruneBindingSetBin(new NodeBin(fluoId, bin));
- }
- } catch (Exception e) {
- log.trace("Could not successfully initialize PeriodicQueryBinPruner.");
- }
- }
-
-
- public void shutdown() {
- closed.set(true);
- }
-
- private Set<String> getNodeIdsFromResultId(SnapshotBase sx, String id) {
- Set<String> ids = new HashSet<>();
- PeriodicQueryUtil.getPeriodicQueryNodeAncestorIds(sx, id, ids);
- return ids;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/pruner/PeriodicQueryPrunerExecutor.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/pruner/PeriodicQueryPrunerExecutor.java b/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/pruner/PeriodicQueryPrunerExecutor.java
deleted file mode 100644
index 1c11f96..0000000
--- a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/pruner/PeriodicQueryPrunerExecutor.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.pruner;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.fluo.api.client.FluoClient;
-import org.apache.log4j.Logger;
-import org.apache.rya.indexing.pcj.storage.PeriodicQueryResultStorage;
-import org.apache.rya.periodic.notification.api.LifeCycle;
-import org.apache.rya.periodic.notification.api.NodeBin;
-
-import com.google.common.base.Preconditions;
-
-/**
- * Executor service that runs {@link PeriodicQueryPruner}s with added functionality
- * for starting, stopping, and determining if the query pruners are running.
- */
-public class PeriodicQueryPrunerExecutor implements LifeCycle {
-
- private static final Logger log = Logger.getLogger(PeriodicQueryPrunerExecutor.class);
- private FluoClient client;
- private int numThreads;
- private ExecutorService executor;
- private BlockingQueue<NodeBin> bins;
- private PeriodicQueryResultStorage periodicStorage;
- private List<PeriodicQueryPruner> pruners;
- private boolean running = false;
-
- public PeriodicQueryPrunerExecutor(PeriodicQueryResultStorage periodicStorage, FluoClient client, int numThreads,
- BlockingQueue<NodeBin> bins) {
- Preconditions.checkArgument(numThreads > 0);
- this.periodicStorage = periodicStorage;
- this.numThreads = numThreads;
- executor = Executors.newFixedThreadPool(numThreads);
- this.bins = bins;
- this.client = client;
- this.pruners = new ArrayList<>();
- }
-
- @Override
- public void start() {
- if (!running) {
- AccumuloBinPruner accPruner = new AccumuloBinPruner(periodicStorage);
- FluoBinPruner fluoPruner = new FluoBinPruner(client);
-
- for (int threadNumber = 0; threadNumber < numThreads; threadNumber++) {
- PeriodicQueryPruner pruner = new PeriodicQueryPruner(fluoPruner, accPruner, client, bins, threadNumber);
- pruners.add(pruner);
- executor.submit(pruner);
- }
- running = true;
- }
- }
-
- @Override
- public void stop() {
- if (pruners != null && pruners.size() > 0) {
- pruners.forEach(x -> x.shutdown());
- }
- if(client != null) {
- client.close();
- }
- if (executor != null) {
- executor.shutdown();
- running = false;
- }
- try {
- if (!executor.awaitTermination(5000, TimeUnit.MILLISECONDS)) {
- log.info("Timed out waiting for consumer threads to shut down, exiting uncleanly");
- executor.shutdownNow();
- }
- } catch (InterruptedException e) {
- log.info("Interrupted during shutdown, exiting uncleanly");
- }
- }
-
- @Override
- public boolean currentlyRunning() {
- return running;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/recovery/PeriodicNotificationProvider.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/recovery/PeriodicNotificationProvider.java b/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/recovery/PeriodicNotificationProvider.java
deleted file mode 100644
index 69bd39c..0000000
--- a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/recovery/PeriodicNotificationProvider.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.recovery;
-
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.fluo.api.client.Snapshot;
-import org.apache.fluo.api.client.scanner.ColumnScanner;
-import org.apache.fluo.api.client.scanner.RowScanner;
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.api.data.ColumnValue;
-import org.apache.fluo.api.data.Span;
-import org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants;
-import org.apache.rya.indexing.pcj.fluo.app.NodeType;
-import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryColumns;
-import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryMetadataDAO;
-import org.apache.rya.indexing.pcj.fluo.app.query.PeriodicQueryMetadata;
-import org.apache.rya.indexing.pcj.fluo.app.util.FluoQueryUtils;
-import org.apache.rya.periodic.notification.api.NotificationCoordinatorExecutor;
-import org.apache.rya.periodic.notification.coordinator.PeriodicNotificationCoordinatorExecutor;
-import org.apache.rya.periodic.notification.notification.CommandNotification;
-import org.apache.rya.periodic.notification.notification.CommandNotification.Command;
-import org.apache.rya.periodic.notification.notification.PeriodicNotification;
-
-/**
- * This class is used by the {@link PeriodicNotificationCoordinatorExecutor}
- * to add all existing {@link PeriodicNotification}s stored in Fluo when it is
- * initialized. This enables the the {@link PeriodicServiceApplication} to be
- * recovered from failure by restoring it original state.
- *
- */
-public class PeriodicNotificationProvider {
-
- private FluoQueryMetadataDAO dao;
-
- public PeriodicNotificationProvider() {
- this.dao = new FluoQueryMetadataDAO();
- }
-
- /**
- * Retrieve all of the information about Periodic Query results already registered
- * with Fluo. This is returned in the form of {@link CommandNotification}s that
- * can be registered with the {@link NotificationCoordinatorExecutor}.
- * @param sx - snapshot for reading results from Fluo
- * @return - collection of CommandNotifications that indicate Periodic Query information registered with system
- */
- public Collection<CommandNotification> getNotifications(Snapshot sx) {
- Set<PeriodicQueryMetadata> periodicMetadata = new HashSet<>();
- RowScanner scanner = sx.scanner().fetch(FluoQueryColumns.PERIODIC_QUERY_NODE_ID)
- .over(Span.prefix(IncrementalUpdateConstants.PERIODIC_QUERY_PREFIX)).byRow().build();
- Iterator<ColumnScanner> colScannerIter = scanner.iterator();
- while (colScannerIter.hasNext()) {
- ColumnScanner colScanner = colScannerIter.next();
- Iterator<ColumnValue> values = colScanner.iterator();
- while (values.hasNext()) {
- PeriodicQueryMetadata metadata = dao.readPeriodicQueryMetadata(sx, values.next().getsValue());
- periodicMetadata.add(metadata);
- }
- }
- return getCommandNotifications(sx, periodicMetadata);
- }
-
- /**
- * Registers all of Periodic Query information already contained within Fluo to the
- * {@link NotificationCoordinatorExecutor}.
- * @param coordinator - coordinator that periodic info will be registered with
- * @param sx - snapshot for reading results from Fluo
- */
- public void processRegisteredNotifications(NotificationCoordinatorExecutor coordinator, Snapshot sx) {
- coordinator.start();
- Collection<CommandNotification> notifications = getNotifications(sx);
- for(CommandNotification notification: notifications) {
- coordinator.processNextCommandNotification(notification);
- }
- }
-
- private Collection<CommandNotification> getCommandNotifications(Snapshot sx, Collection<PeriodicQueryMetadata> metadata) {
- Set<CommandNotification> notifications = new HashSet<>();
- int i = 1;
- for(PeriodicQueryMetadata meta:metadata) {
- //offset initial wait to avoid overloading system
- PeriodicNotification periodic = new PeriodicNotification(getQueryId(meta.getNodeId(), sx), meta.getPeriod(),TimeUnit.MILLISECONDS,i*5000);
- notifications.add(new CommandNotification(Command.ADD, periodic));
- i++;
- }
- return notifications;
- }
-
- private String getQueryId(String periodicNodeId, Snapshot sx) {
- return getQueryIdFromPeriodicId(sx, periodicNodeId);
- }
-
- private String getQueryIdFromPeriodicId(Snapshot sx, String nodeId) {
- NodeType nodeType = NodeType.fromNodeId(nodeId).orNull();
- String id = null;
- switch (nodeType) {
- case FILTER:
- id = getQueryIdFromPeriodicId(sx, sx.get(Bytes.of(nodeId), FluoQueryColumns.FILTER_PARENT_NODE_ID).toString());
- break;
- case PERIODIC_QUERY:
- id = getQueryIdFromPeriodicId(sx, sx.get(Bytes.of(nodeId), FluoQueryColumns.PERIODIC_QUERY_PARENT_NODE_ID).toString());
- break;
- case QUERY:
- id = FluoQueryUtils.convertFluoQueryIdToPcjId(nodeId);
- break;
- case AGGREGATION:
- id = getQueryIdFromPeriodicId(sx, sx.get(Bytes.of(nodeId), FluoQueryColumns.AGGREGATION_PARENT_NODE_ID).toString());
- break;
- case CONSTRUCT:
- id = getQueryIdFromPeriodicId(sx, sx.get(Bytes.of(nodeId), FluoQueryColumns.CONSTRUCT_PARENT_NODE_ID).toString());
- break;
- case PROJECTION:
- id = getQueryIdFromPeriodicId(sx, sx.get(Bytes.of(nodeId), FluoQueryColumns.PROJECTION_PARENT_NODE_ID).toString());
- break;
- default:
- throw new IllegalArgumentException("Invalid node type");
-
- }
- return id;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/registration/kafka/KafkaNotificationProvider.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/registration/kafka/KafkaNotificationProvider.java b/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/registration/kafka/KafkaNotificationProvider.java
deleted file mode 100644
index f5cd13a..0000000
--- a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/registration/kafka/KafkaNotificationProvider.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.registration.kafka;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Properties;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.kafka.clients.consumer.KafkaConsumer;
-import org.apache.kafka.common.serialization.Deserializer;
-import org.apache.rya.periodic.notification.api.LifeCycle;
-import org.apache.rya.periodic.notification.api.Notification;
-import org.apache.rya.periodic.notification.api.NotificationCoordinatorExecutor;
-import org.apache.rya.periodic.notification.notification.CommandNotification;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Consumer group to pull all requests for adding and deleting {@link Notification}s
- * from Kafka. This Object executes {@link PeriodicNotificationConsumer}s that retrieve
- * the {@link CommandNotification}s and register them with the {@link NotificationCoordinatorExecutor}.
- *
- */
-public class KafkaNotificationProvider implements LifeCycle {
- private static final Logger LOG = LoggerFactory.getLogger(KafkaNotificationProvider.class);
- private String topic;
- private ExecutorService executor;
- private NotificationCoordinatorExecutor coord;
- private Properties props;
- private int numThreads;
- private boolean running = false;
- Deserializer<String> keyDe;
- Deserializer<CommandNotification> valDe;
- List<PeriodicNotificationConsumer> consumers;
-
- /**
- * Create KafkaNotificationProvider for reading new notification requests form Kafka
- * @param topic - notification topic
- * @param keyDe - Kafka message key deserializer
- * @param valDe - Kafka message value deserializer
- * @param props - properties used to creates a {@link KafkaConsumer}
- * @param coord - {@link NotificationCoordinatorExecutor} for managing and generating notifications
- * @param numThreads - number of threads used by this notification provider
- */
- public KafkaNotificationProvider(String topic, Deserializer<String> keyDe, Deserializer<CommandNotification> valDe, Properties props,
- NotificationCoordinatorExecutor coord, int numThreads) {
- this.coord = coord;
- this.numThreads = numThreads;
- this.topic = topic;
- this.props = props;
- this.consumers = new ArrayList<>();
- this.keyDe = keyDe;
- this.valDe = valDe;
- }
-
- @Override
- public void stop() {
- if (consumers != null && consumers.size() > 0) {
- for (PeriodicNotificationConsumer consumer : consumers) {
- consumer.shutdown();
- }
- }
- if (executor != null) {
- executor.shutdown();
- }
- running = false;
- try {
- if (!executor.awaitTermination(5000, TimeUnit.MILLISECONDS)) {
- LOG.info("Timed out waiting for consumer threads to shut down, exiting uncleanly");
- executor.shutdownNow();
- }
- } catch (InterruptedException e) {
- LOG.info("Interrupted during shutdown, exiting uncleanly");
- }
- }
-
- public void start() {
- if (!running) {
- if (!coord.currentlyRunning()) {
- coord.start();
- }
- // now launch all the threads
- executor = Executors.newFixedThreadPool(numThreads);
-
- // now create consumers to consume the messages
- int threadNumber = 0;
- for (int i = 0; i < numThreads; i++) {
- LOG.info("Creating consumer:" + threadNumber);
- KafkaConsumer<String, CommandNotification> consumer = new KafkaConsumer<String, CommandNotification>(props, keyDe, valDe);
- PeriodicNotificationConsumer periodicConsumer = new PeriodicNotificationConsumer(topic, consumer, threadNumber, coord);
- consumers.add(periodicConsumer);
- executor.submit(periodicConsumer);
- threadNumber++;
- }
- running = true;
- }
- }
-
- @Override
- public boolean currentlyRunning() {
- return running;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/registration/kafka/PeriodicNotificationConsumer.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/registration/kafka/PeriodicNotificationConsumer.java b/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/registration/kafka/PeriodicNotificationConsumer.java
deleted file mode 100644
index 6785ce8..0000000
--- a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/registration/kafka/PeriodicNotificationConsumer.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.registration.kafka;
-
-import java.util.Arrays;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.kafka.clients.consumer.ConsumerRecord;
-import org.apache.kafka.clients.consumer.ConsumerRecords;
-import org.apache.kafka.clients.consumer.KafkaConsumer;
-import org.apache.kafka.common.errors.WakeupException;
-import org.apache.log4j.Logger;
-import org.apache.rya.periodic.notification.api.NotificationCoordinatorExecutor;
-import org.apache.rya.periodic.notification.notification.CommandNotification;
-
-/**
- * Consumer for the {@link KafkaNotificationProvider}. This consumer pull messages
- * from Kafka and registers them with the {@link NotificationCoordinatorExecutor}.
- *
- */
-public class PeriodicNotificationConsumer implements Runnable {
- private KafkaConsumer<String, CommandNotification> consumer;
- private int m_threadNumber;
- private String topic;
- private final AtomicBoolean closed = new AtomicBoolean(false);
- private NotificationCoordinatorExecutor coord;
- private static final Logger LOG = Logger.getLogger(PeriodicNotificationConsumer.class);
-
- /**
- * Creates a new PeriodicNotificationConsumer for consuming new notification requests from
- * Kafka.
- * @param topic - new notification topic
- * @param consumer - consumer for pulling new requests from Kafka
- * @param a_threadNumber - number of consumer threads to be used
- * @param coord - notification coordinator for managing and generating notifications
- */
- public PeriodicNotificationConsumer(String topic, KafkaConsumer<String, CommandNotification> consumer, int a_threadNumber,
- NotificationCoordinatorExecutor coord) {
- this.topic = topic;
- m_threadNumber = a_threadNumber;
- this.consumer = consumer;
- this.coord = coord;
- }
-
- public void run() {
-
- try {
- LOG.info("Creating kafka stream for consumer:" + m_threadNumber);
- consumer.subscribe(Arrays.asList(topic));
- while (!closed.get()) {
- ConsumerRecords<String, CommandNotification> records = consumer.poll(10000);
- // Handle new records
- for(ConsumerRecord<String, CommandNotification> record: records) {
- CommandNotification notification = record.value();
- LOG.info("Thread " + m_threadNumber + " is adding notification " + notification + " to queue.");
- LOG.info("Message: " + notification);
- coord.processNextCommandNotification(notification);
- }
- }
- } catch (WakeupException e) {
- // Ignore exception if closing
- if (!closed.get()) throw e;
- } finally {
- consumer.close();
- }
- }
-
- public void shutdown() {
- closed.set(true);
- consumer.wakeup();
- }
-}
[6/7] incubator-rya git commit: RYA-355 Refactored the periodic
notification service structure. Closes #221.
Posted by ca...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplication.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplication.java b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplication.java
new file mode 100644
index 0000000..92a7d18
--- /dev/null
+++ b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplication.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.application;
+
+import org.apache.log4j.Logger;
+import org.apache.rya.indexing.pcj.fluo.app.util.PeriodicQueryUtil;
+import org.apache.rya.periodic.notification.api.BinPruner;
+import org.apache.rya.periodic.notification.api.BindingSetRecord;
+import org.apache.rya.periodic.notification.api.LifeCycle;
+import org.apache.rya.periodic.notification.api.NodeBin;
+import org.apache.rya.periodic.notification.api.NotificationCoordinatorExecutor;
+import org.apache.rya.periodic.notification.exporter.KafkaExporterExecutor;
+import org.apache.rya.periodic.notification.processor.NotificationProcessorExecutor;
+import org.apache.rya.periodic.notification.pruner.PeriodicQueryPrunerExecutor;
+import org.apache.rya.periodic.notification.registration.kafka.KafkaNotificationProvider;
+import org.openrdf.query.algebra.evaluation.function.Function;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * The PeriodicNotificationApplication runs the key components of the Periodic
+ * Query Service. It consists of a {@link KafkaNotificationProvider}, a
+ * {@link NotificationCoordinatorExecutor}, a
+ * {@link NotificationProcessorExecutor}, a {@link KafkaExporterExecutor}, and a
+ * {@link PeriodicQueryPrunerExecutor}. These services run in coordination with
+ * one another to perform the following tasks in the indicated order: <br>
+ * <li>Retrieve new requests to generate periodic notifications from Kafka
+ * <li>Register them with the {@link NotificationCoordinatorExecutor} to
+ * generate the periodic notifications
+ * <li>As notifications are generated, they are added to a work queue that is
+ * monitored by the {@link NotificationProcessorExecutor}.
+ * <li>The processor processes the notifications by reading all of the query
+ * results corresponding to the bin and query id indicated by the notification.
+ * <li>After reading the results, the processor adds a {@link BindingSetRecord}
+ * to a work queue monitored by the {@link KafkaExporterExecutor}.
+ * <li>The processor then adds a {@link NodeBin} to a workqueue monitored by the
+ * {@link BinPruner}
+ * <li>The exporter processes the BindingSetRecord by exporing the result to
+ * Kafka
+ * <li>The BinPruner processes the NodeBin by cleaning up the results for the
+ * indicated bin and query in Accumulo and Fluo. <br>
+ * <br>
+ * The purpose of this Periodic Query Service is to facilitate the ability to
+ * answer Periodic Queries using the Rya Fluo application, where a Periodic
+ * Query is any query requesting periodic updates about events that occurred
+ * within a given window of time of this instant. This is also known as a
+ * rolling window query. Period Queries can be expressed using SPARQL by
+ * including the {@link Function} indicated by the URI
+ * {@link PeriodicQueryUtil#PeriodicQueryURI}. The user must provide this
+ * Function with the following arguments: the temporal variable in the query
+ * that will be filtered on, the window of time that events must occur within,
+ * the period at which the user wants to receive updates, and the time unit. The
+ * following query requests all observations that occurred within the last
+ * minute and requests updates every 15 seconds. It also performs a count on
+ * those observations. <br>
+ * <br>
+ * <li>prefix function: http://org.apache.rya/function#
+ * <li>"prefix time: http://www.w3.org/2006/time#
+ * <li>"select (count(?obs) as ?total) where {
+ * <li>"Filter(function:periodic(?time, 1, .25, time:minutes))
+ * <li>"?obs uri:hasTime ?time.
+ * <li>"?obs uri:hasId ?id }
+ * <li>
+ */
+public class PeriodicNotificationApplication implements LifeCycle {
+
+ private static final Logger log = Logger.getLogger(PeriodicNotificationApplication.class);
+ private NotificationCoordinatorExecutor coordinator;
+ private KafkaNotificationProvider provider;
+ private PeriodicQueryPrunerExecutor pruner;
+ private NotificationProcessorExecutor processor;
+ private KafkaExporterExecutor exporter;
+ private boolean running = false;
+
+ /**
+ * Creates a PeriodicNotificationApplication
+ * @param provider - {@link KafkaNotificationProvider} that retrieves new Notificaiton requests from Kafka
+ * @param coordinator - {NotificationCoordinator} that manages PeriodicNotifications.
+ * @param processor - {@link NotificationProcessorExecutor} that processes PeriodicNotifications
+ * @param exporter - {@link KafkaExporterExecutor} that exports periodic results
+ * @param pruner - {@link PeriodicQueryPrunerExecutor} that cleans up old periodic bins
+ */
+ public PeriodicNotificationApplication(KafkaNotificationProvider provider, NotificationCoordinatorExecutor coordinator,
+ NotificationProcessorExecutor processor, KafkaExporterExecutor exporter, PeriodicQueryPrunerExecutor pruner) {
+ this.provider = Preconditions.checkNotNull(provider);
+ this.coordinator = Preconditions.checkNotNull(coordinator);
+ this.processor = Preconditions.checkNotNull(processor);
+ this.exporter = Preconditions.checkNotNull(exporter);
+ this.pruner = Preconditions.checkNotNull(pruner);
+ }
+
+ @Override
+ public void start() {
+ if (!running) {
+ log.info("Starting PeriodicNotificationApplication.");
+ coordinator.start();
+ provider.start();
+ processor.start();
+ pruner.start();
+ exporter.start();
+ running = true;
+ }
+ }
+
+ @Override
+ public void stop() {
+ log.info("Stopping PeriodicNotificationApplication.");
+ provider.stop();
+ coordinator.stop();
+ processor.stop();
+ pruner.stop();
+ exporter.stop();
+ running = false;
+ }
+
+ /**
+ * @return boolean indicating whether the application is running
+ */
+ @Override
+ public boolean currentlyRunning() {
+ return running;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static class Builder {
+
+ private PeriodicQueryPrunerExecutor pruner;
+ private KafkaNotificationProvider provider;
+ private NotificationProcessorExecutor processor;
+ private KafkaExporterExecutor exporter;
+ private NotificationCoordinatorExecutor coordinator;
+
+ /**
+ * Sets the PeriodicQueryPrunerExecutor.
+ * @param pruner - PeriodicQueryPrunerExecutor for cleaning up old periodic bins
+ * @return this Builder for chaining method calls
+ */
+ public Builder setPruner(PeriodicQueryPrunerExecutor pruner) {
+ this.pruner = pruner;
+ return this;
+ }
+
+ /**
+ * Sets the KafkaNotificationProvider
+ * @param provider - KafkaNotificationProvider for retrieving new periodic notification requests from Kafka
+ * @return this Builder for chaining method calls
+ */
+ public Builder setProvider(KafkaNotificationProvider provider) {
+ this.provider = provider;
+ return this;
+ }
+
+ public Builder setProcessor(NotificationProcessorExecutor processor) {
+ this.processor = processor;
+ return this;
+ }
+
+ /**
+ * Sets KafkaExporterExecutor
+ * @param exporter for exporting periodic query results to Kafka
+ * @return this Builder for chaining method calls
+ */
+ public Builder setExporter(KafkaExporterExecutor exporter) {
+ this.exporter = exporter;
+ return this;
+ }
+
+ /**
+ * Sets NotificationCoordinatorExecutor
+ * @param coordinator for managing and generating periodic notifications
+ * @return this Builder for chaining method calls
+ */
+ public Builder setCoordinator(NotificationCoordinatorExecutor coordinator) {
+ this.coordinator = coordinator;
+ return this;
+ }
+
+ /**
+ * Creates a PeriodicNotificationApplication
+ * @return PeriodicNotificationApplication for periodically polling Rya Fluo Application
+ */
+ public PeriodicNotificationApplication build() {
+ return new PeriodicNotificationApplication(provider, coordinator, processor, exporter, pruner);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplicationConfiguration.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplicationConfiguration.java b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplicationConfiguration.java
new file mode 100644
index 0000000..d69efe5
--- /dev/null
+++ b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplicationConfiguration.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.application;
+
+import java.util.Properties;
+
+import org.apache.rya.accumulo.AccumuloRdfConfiguration;
+
+import jline.internal.Preconditions;
+
+/**
+ * Configuration object for creating a {@link PeriodicNotificationApplication}.
+ */
+public class PeriodicNotificationApplicationConfiguration extends AccumuloRdfConfiguration {
+
+ public static String FLUO_APP_NAME = "fluo.app.name";
+ public static String FLUO_TABLE_NAME = "fluo.table.name";
+ public static String KAFKA_BOOTSTRAP_SERVERS = "kafka.bootstrap.servers";
+ public static String NOTIFICATION_TOPIC = "kafka.notification.topic";
+ public static String NOTIFICATION_GROUP_ID = "kafka.notification.group.id";
+ public static String NOTIFICATION_CLIENT_ID = "kafka.notification.client.id";
+ public static String COORDINATOR_THREADS = "cep.coordinator.threads";
+ public static String PRODUCER_THREADS = "cep.producer.threads";
+ public static String EXPORTER_THREADS = "cep.exporter.threads";
+ public static String PROCESSOR_THREADS = "cep.processor.threads";
+ public static String PRUNER_THREADS = "cep.pruner.threads";
+
+ public PeriodicNotificationApplicationConfiguration() {}
+
+ /**
+ * Creates an PeriodicNotificationApplicationConfiguration object from a Properties file. This method assumes
+ * that all values in the Properties file are Strings and that the Properties file uses the keys below.
+ * See rya.cep/cep.integration.tests/src/test/resources/properties/notification.properties for an example.
+ * <br>
+ * <ul>
+ * <li>"accumulo.auths" - String of Accumulo authorizations. Default is empty String.
+ * <li>"accumulo.instance" - Accumulo instance name (required)
+ * <li>"accumulo.user" - Accumulo user (required)
+ * <li>"accumulo.password" - Accumulo password (required)
+ * <li>"accumulo.rya.prefix" - Prefix for Accumulo backed Rya instance. Default is "rya_"
+ * <li>"accumulo.zookeepers" - Zookeepers for underlying Accumulo instance (required)
+ * <li>"fluo.app.name" - Name of Fluo Application (required)
+ * <li>"fluo.table.name" - Name of Fluo Table (required)
+ * <li>"kafka.bootstrap.servers" - Kafka Bootstrap servers for Producers and Consumers (required)
+ * <li>"kafka.notification.topic" - Topic to which new Periodic Notifications are published. Default is "notifications".
+ * <li>"kafka.notification.client.id" - Client Id for notification topic. Default is "consumer0"
+ * <li>"kafka.notification.group.id" - Group Id for notification topic. Default is "group0"
+ * <li>"cep.coordinator.threads" - Number of threads used by coordinator. Default is 1.
+ * <li>"cep.producer.threads" - Number of threads used by producer. Default is 1.
+ * <li>"cep.exporter.threads" - Number of threads used by exporter. Default is 1.
+ * <li>"cep.processor.threads" - Number of threads used by processor. Default is 1.
+ * <li>"cep.pruner.threads" - Number of threads used by pruner. Default is 1.
+ * </ul>
+ * <br>
+ * @param props - Properties file containing Accumulo specific configuration parameters
+ * @return AccumumuloRdfConfiguration with properties set
+ */
+ public PeriodicNotificationApplicationConfiguration(Properties props) {
+ super(fromProperties(props));
+ setFluoAppName(props.getProperty(FLUO_APP_NAME));
+ setFluoTableName(props.getProperty(FLUO_TABLE_NAME));
+ setBootStrapServers(props.getProperty(KAFKA_BOOTSTRAP_SERVERS));
+ setNotificationClientId(props.getProperty(NOTIFICATION_CLIENT_ID, "consumer0"));
+ setNotificationTopic(props.getProperty(NOTIFICATION_TOPIC, "notifications"));
+ setNotificationGroupId(props.getProperty(NOTIFICATION_GROUP_ID, "group0"));
+ setProducerThreads(Integer.parseInt(props.getProperty(PRODUCER_THREADS, "1")));
+ setProcessorThreads(Integer.parseInt(props.getProperty(PROCESSOR_THREADS, "1")));
+ setExporterThreads(Integer.parseInt(props.getProperty(EXPORTER_THREADS, "1")));
+ setPrunerThreads(Integer.parseInt(props.getProperty(PRUNER_THREADS, "1")));
+ setCoordinatorThreads(Integer.parseInt(props.getProperty(COORDINATOR_THREADS, "1")));
+ }
+
+ /**
+ * Sets the name of the Fluo Application
+ * @param fluoAppName
+ */
+ public void setFluoAppName(String fluoAppName) {
+ set(FLUO_APP_NAME, Preconditions.checkNotNull(fluoAppName));
+ }
+
+ /**
+ * Sets the name of the Fluo table
+ * @param fluoTableName
+ */
+ public void setFluoTableName(String fluoTableName) {
+ set(FLUO_TABLE_NAME, Preconditions.checkNotNull(fluoTableName));
+ }
+
+ /**
+ * Sets the Kafka bootstrap servers
+ * @param bootStrapServers
+ */
+ public void setBootStrapServers(String bootStrapServers) {
+ set(KAFKA_BOOTSTRAP_SERVERS, Preconditions.checkNotNull(bootStrapServers));
+ }
+
+ /**
+ * Sets the Kafka topic name for new notification requests
+ * @param notificationTopic
+ */
+ public void setNotificationTopic(String notificationTopic) {
+ set(NOTIFICATION_TOPIC, Preconditions.checkNotNull(notificationTopic));
+ }
+
+ /**
+ * Sets the GroupId for new notification request topic
+ * @param notificationGroupId
+ */
+ public void setNotificationGroupId(String notificationGroupId) {
+ set(NOTIFICATION_GROUP_ID, Preconditions.checkNotNull(notificationGroupId));
+ }
+
+ /**
+ * Sets the ClientId for the Kafka notification topic
+ * @param notificationClientId
+ */
+ public void setNotificationClientId(String notificationClientId) {
+ set(NOTIFICATION_GROUP_ID, Preconditions.checkNotNull(notificationClientId));
+ }
+
+ /**
+ * Sets the number of threads for the coordinator
+ * @param threads
+ */
+ public void setCoordinatorThreads(int threads) {
+ setInt(COORDINATOR_THREADS, threads);
+ }
+
+ /**
+ * Sets the number of threads for the exporter
+ * @param threads
+ */
+ public void setExporterThreads(int threads) {
+ setInt(EXPORTER_THREADS, threads);
+ }
+
+ /**
+ * Sets the number of threads for the producer for reading new periodic notifications
+ * @param threads
+ */
+ public void setProducerThreads(int threads) {
+ setInt(PRODUCER_THREADS, threads);
+ }
+
+ /**
+ * Sets the number of threads for the bin pruner
+ * @param threads
+ */
+ public void setPrunerThreads(int threads) {
+ setInt(PRUNER_THREADS, threads);
+ }
+
+ /**
+ * Sets the number of threads for the Notification processor
+ * @param threads
+ */
+ public void setProcessorThreads(int threads) {
+ setInt(PROCESSOR_THREADS, threads);
+ }
+
+ /**
+ * @return name of the Fluo application
+ */
+ public String getFluoAppName() {
+ return get(FLUO_APP_NAME);
+ }
+
+ /**
+ * @return name of the Fluo table
+ */
+ public String getFluoTableName() {
+ return get(FLUO_TABLE_NAME);
+ }
+
+ /**
+ * @return Kafka bootstrap servers
+ */
+ public String getBootStrapServers() {
+ return get(KAFKA_BOOTSTRAP_SERVERS);
+ }
+
+ /**
+ * @return notification topic
+ */
+ public String getNotificationTopic() {
+ return get(NOTIFICATION_TOPIC, "notifications");
+ }
+
+ /**
+ * @return Kafka GroupId for the notificaton topic
+ */
+ public String getNotificationGroupId() {
+ return get(NOTIFICATION_GROUP_ID, "group0");
+ }
+
+ /**
+ * @return Kafka ClientId for the notification topic
+ */
+ public String getNotificationClientId() {
+ return get(NOTIFICATION_CLIENT_ID, "consumer0");
+ }
+
+ /**
+ * @return the number of threads for the coordinator
+ */
+ public int getCoordinatorThreads() {
+ return getInt(COORDINATOR_THREADS, 1);
+ }
+
+ /**
+ * @return the number of threads for the exporter
+ */
+ public int getExporterThreads() {
+ return getInt(EXPORTER_THREADS, 1);
+ }
+
+ /**
+ * @return the number of threads for the notification producer
+ */
+ public int getProducerThreads() {
+ return getInt(PRODUCER_THREADS, 1);
+ }
+
+ /**
+ * @return the number of threads for the bin pruner
+ */
+ public int getPrunerThreads() {
+ return getInt(PRUNER_THREADS, 1);
+ }
+
+ /**
+ * @return number of threads for the processor
+ */
+ public int getProcessorThreads() {
+ return getInt(PROCESSOR_THREADS, 1);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplicationFactory.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplicationFactory.java b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplicationFactory.java
new file mode 100644
index 0000000..771a4ab
--- /dev/null
+++ b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplicationFactory.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.application;
+
+import java.util.Optional;
+import java.util.Properties;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.fluo.api.client.FluoClient;
+import org.apache.fluo.api.client.Snapshot;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.apache.kafka.common.serialization.StringSerializer;
+import org.apache.rya.indexing.pcj.fluo.app.util.FluoClientFactory;
+import org.apache.rya.indexing.pcj.storage.PeriodicQueryResultStorage;
+import org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPeriodicQueryResultStorage;
+import org.apache.rya.periodic.notification.api.BindingSetRecord;
+import org.apache.rya.periodic.notification.api.NodeBin;
+import org.apache.rya.periodic.notification.api.NotificationCoordinatorExecutor;
+import org.apache.rya.periodic.notification.coordinator.PeriodicNotificationCoordinatorExecutor;
+import org.apache.rya.periodic.notification.exporter.KafkaExporterExecutor;
+import org.apache.rya.periodic.notification.notification.TimestampedNotification;
+import org.apache.rya.periodic.notification.processor.NotificationProcessorExecutor;
+import org.apache.rya.periodic.notification.pruner.PeriodicQueryPrunerExecutor;
+import org.apache.rya.periodic.notification.recovery.PeriodicNotificationProvider;
+import org.apache.rya.periodic.notification.registration.kafka.KafkaNotificationProvider;
+import org.apache.rya.periodic.notification.serialization.BindingSetSerDe;
+import org.apache.rya.periodic.notification.serialization.CommandNotificationSerializer;
+import org.openrdf.query.BindingSet;
+
+/**
+ * Factory for creating a {@link PeriodicNotificationApplication}.
+ */
+public class PeriodicNotificationApplicationFactory {
+
+ /**
+ * Create a PeriodicNotificationApplication.
+ * @param props - Properties file that specifies the parameters needed to create the application
+ * @return PeriodicNotificationApplication to periodically poll Rya Fluo for new results
+ * @throws PeriodicApplicationException
+ */
+ public static PeriodicNotificationApplication getPeriodicApplication(Properties props) throws PeriodicApplicationException {
+ PeriodicNotificationApplicationConfiguration conf = new PeriodicNotificationApplicationConfiguration(props);
+ Properties kafkaProps = getKafkaProperties(conf);
+
+ BlockingQueue<TimestampedNotification> notifications = new LinkedBlockingQueue<>();
+ BlockingQueue<NodeBin> bins = new LinkedBlockingQueue<>();
+ BlockingQueue<BindingSetRecord> bindingSets = new LinkedBlockingQueue<>();
+
+ FluoClient fluo = null;
+ try {
+ PeriodicQueryResultStorage storage = getPeriodicQueryResultStorage(conf);
+ fluo = FluoClientFactory.getFluoClient(conf.getFluoAppName(), Optional.of(conf.getFluoTableName()), conf);
+ NotificationCoordinatorExecutor coordinator = getCoordinator(conf.getCoordinatorThreads(), notifications);
+ addRegisteredNotices(coordinator, fluo.newSnapshot());
+ KafkaExporterExecutor exporter = getExporter(conf.getExporterThreads(), kafkaProps, bindingSets);
+ PeriodicQueryPrunerExecutor pruner = getPruner(storage, fluo, conf.getPrunerThreads(), bins);
+ NotificationProcessorExecutor processor = getProcessor(storage, notifications, bins, bindingSets, conf.getProcessorThreads());
+ KafkaNotificationProvider provider = getProvider(conf.getProducerThreads(), conf.getNotificationTopic(), coordinator, kafkaProps);
+ return PeriodicNotificationApplication.builder().setCoordinator(coordinator).setProvider(provider).setExporter(exporter)
+ .setProcessor(processor).setPruner(pruner).build();
+ } catch (AccumuloException | AccumuloSecurityException e) {
+ throw new PeriodicApplicationException(e.getMessage());
+ }
+ }
+
+ private static void addRegisteredNotices(NotificationCoordinatorExecutor coord, Snapshot sx) {
+ coord.start();
+ PeriodicNotificationProvider provider = new PeriodicNotificationProvider();
+ provider.processRegisteredNotifications(coord, sx);
+ }
+
+ private static NotificationCoordinatorExecutor getCoordinator(int numThreads, BlockingQueue<TimestampedNotification> notifications) {
+ return new PeriodicNotificationCoordinatorExecutor(numThreads, notifications);
+ }
+
+ private static KafkaExporterExecutor getExporter(int numThreads, Properties props, BlockingQueue<BindingSetRecord> bindingSets) {
+ KafkaProducer<String, BindingSet> producer = new KafkaProducer<>(props, new StringSerializer(), new BindingSetSerDe());
+ return new KafkaExporterExecutor(producer, numThreads, bindingSets);
+ }
+
+ private static PeriodicQueryPrunerExecutor getPruner(PeriodicQueryResultStorage storage, FluoClient fluo, int numThreads,
+ BlockingQueue<NodeBin> bins) {
+ return new PeriodicQueryPrunerExecutor(storage, fluo, numThreads, bins);
+ }
+
+ private static NotificationProcessorExecutor getProcessor(PeriodicQueryResultStorage periodicStorage,
+ BlockingQueue<TimestampedNotification> notifications, BlockingQueue<NodeBin> bins, BlockingQueue<BindingSetRecord> bindingSets,
+ int numThreads) {
+ return new NotificationProcessorExecutor(periodicStorage, notifications, bins, bindingSets, numThreads);
+ }
+
+ private static KafkaNotificationProvider getProvider(int numThreads, String topic, NotificationCoordinatorExecutor coord,
+ Properties props) {
+ return new KafkaNotificationProvider(topic, new StringDeserializer(), new CommandNotificationSerializer(), props, coord,
+ numThreads);
+ }
+
+ private static PeriodicQueryResultStorage getPeriodicQueryResultStorage(PeriodicNotificationApplicationConfiguration conf)
+ throws AccumuloException, AccumuloSecurityException {
+ Instance instance = new ZooKeeperInstance(conf.getAccumuloInstance(), conf.getAccumuloZookeepers());
+ Connector conn = instance.getConnector(conf.getAccumuloUser(), new PasswordToken(conf.getAccumuloPassword()));
+ String ryaInstance = conf.getTablePrefix();
+ return new AccumuloPeriodicQueryResultStorage(conn, ryaInstance);
+ }
+
+ private static Properties getKafkaProperties(PeriodicNotificationApplicationConfiguration conf) {
+ Properties kafkaProps = new Properties();
+ kafkaProps.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, conf.getBootStrapServers());
+ kafkaProps.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, conf.getNotificationClientId());
+ kafkaProps.setProperty(ConsumerConfig.GROUP_ID_CONFIG, conf.getNotificationGroupId());
+ kafkaProps.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+ return kafkaProps;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/coordinator/PeriodicNotificationCoordinatorExecutor.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/coordinator/PeriodicNotificationCoordinatorExecutor.java b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/coordinator/PeriodicNotificationCoordinatorExecutor.java
new file mode 100644
index 0000000..0486244
--- /dev/null
+++ b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/coordinator/PeriodicNotificationCoordinatorExecutor.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.coordinator;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.rya.periodic.notification.api.Notification;
+import org.apache.rya.periodic.notification.api.NotificationCoordinatorExecutor;
+import org.apache.rya.periodic.notification.api.NotificationProcessor;
+import org.apache.rya.periodic.notification.notification.CommandNotification;
+import org.apache.rya.periodic.notification.notification.PeriodicNotification;
+import org.apache.rya.periodic.notification.notification.TimestampedNotification;
+import org.apache.rya.periodic.notification.notification.CommandNotification.Command;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Implementation of {@link NotificationCoordinatorExecutor} that generates regular notifications
+ * as indicated by {@link PeriodicNotification}s that are registered with this Object. When notifications
+ * are generated they are placed on a work queue to be processed by the {@link NotificationProcessor}.
+ *
+ */
+public class PeriodicNotificationCoordinatorExecutor implements NotificationCoordinatorExecutor {
+
+ private static final Logger LOG = LoggerFactory.getLogger(PeriodicNotificationCoordinatorExecutor.class);
+ private int numThreads;
+ private ScheduledExecutorService producerThreadPool;
+ private Map<String, ScheduledFuture<?>> serviceMap = new HashMap<>();
+ private BlockingQueue<TimestampedNotification> notifications;
+ private final ReentrantLock lock = new ReentrantLock(true);
+ private boolean running = false;
+
+ public PeriodicNotificationCoordinatorExecutor(int numThreads, BlockingQueue<TimestampedNotification> notifications) {
+ this.numThreads = numThreads;
+ this.notifications = notifications;
+ }
+
+ @Override
+ public void processNextCommandNotification(CommandNotification notification) {
+ lock.lock();
+ try {
+ processNotification(notification);
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ @Override
+ public void start() {
+ if (!running) {
+ producerThreadPool = Executors.newScheduledThreadPool(numThreads);
+ running = true;
+ }
+ }
+
+ @Override
+ public void stop() {
+
+ if (producerThreadPool != null) {
+ producerThreadPool.shutdown();
+ }
+
+ running = false;
+
+ try {
+ if (!producerThreadPool.awaitTermination(5000, TimeUnit.MILLISECONDS)) {
+ producerThreadPool.shutdownNow();
+ }
+ } catch (Exception e) {
+ LOG.info("Service Executor Shutdown has been called. Terminating NotificationRunnable");
+ }
+ }
+
+ private void processNotification(CommandNotification notification) {
+ Command command = notification.getCommand();
+ Notification periodic = notification.getNotification();
+ switch (command) {
+ case ADD:
+ addNotification(periodic);
+ break;
+ case DELETE:
+ deleteNotification(periodic);
+ break;
+ }
+ }
+
+ private void addNotification(Notification notification) {
+ Preconditions.checkArgument(notification instanceof PeriodicNotification);
+ PeriodicNotification notify = (PeriodicNotification) notification;
+ if (!serviceMap.containsKey(notification.getId())) {
+ ScheduledFuture<?> future = producerThreadPool.scheduleAtFixedRate(new NotificationProducer(notify), notify.getInitialDelay(),
+ notify.getPeriod(), notify.getTimeUnit());
+ serviceMap.put(notify.getId(), future);
+ }
+ }
+
+ private boolean deleteNotification(Notification notification) {
+ if (serviceMap.containsKey(notification.getId())) {
+ ScheduledFuture<?> future = serviceMap.remove(notification.getId());
+ future.cancel(true);
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Scheduled Task that places a {@link PeriodicNotification}
+ * in the work queue at regular intervals.
+ *
+ */
+ class NotificationProducer implements Runnable {
+
+ private PeriodicNotification notification;
+
+ public NotificationProducer(PeriodicNotification notification) {
+ this.notification = notification;
+ }
+
+ public void run() {
+ try {
+ notifications.put(new TimestampedNotification(notification));
+ } catch (InterruptedException e) {
+ LOG.info("Unable to add notification. Process interrupted. ");
+ throw new RuntimeException(e);
+ }
+ }
+
+ }
+
+ @Override
+ public boolean currentlyRunning() {
+ return running;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/exporter/KafkaExporterExecutor.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/exporter/KafkaExporterExecutor.java b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/exporter/KafkaExporterExecutor.java
new file mode 100644
index 0000000..c2e5ebf
--- /dev/null
+++ b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/exporter/KafkaExporterExecutor.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.exporter;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.log4j.Logger;
+import org.apache.rya.periodic.notification.api.BindingSetExporter;
+import org.apache.rya.periodic.notification.api.BindingSetRecord;
+import org.apache.rya.periodic.notification.api.LifeCycle;
+import org.openrdf.query.BindingSet;
+
+import jline.internal.Preconditions;
+
+/**
+ * Executor service that runs {@link KafkaPeriodicBindingSetExporter}s.
+ *
+ */
+public class KafkaExporterExecutor implements LifeCycle {
+
+ private static final Logger log = Logger.getLogger(BindingSetExporter.class);
+ private KafkaProducer<String, BindingSet> producer;
+ private BlockingQueue<BindingSetRecord> bindingSets;
+ private ExecutorService executor;
+ private List<KafkaPeriodicBindingSetExporter> exporters;
+ private int num_Threads;
+ private boolean running = false;
+
+ /**
+ * Creates a KafkaExporterExecutor for exporting periodic query results to Kafka.
+ * @param producer for publishing results to Kafka
+ * @param num_Threads number of threads used to publish results
+ * @param bindingSets - work queue containing {@link BindingSet}s to be published
+ */
+ public KafkaExporterExecutor(KafkaProducer<String, BindingSet> producer, int num_Threads, BlockingQueue<BindingSetRecord> bindingSets) {
+ Preconditions.checkNotNull(producer);
+ Preconditions.checkNotNull(bindingSets);
+ this.producer = producer;
+ this.bindingSets = bindingSets;
+ this.num_Threads = num_Threads;
+ this.exporters = new ArrayList<>();
+ }
+
+ @Override
+ public void start() {
+ if (!running) {
+ executor = Executors.newFixedThreadPool(num_Threads);
+
+ for (int threadNumber = 0; threadNumber < num_Threads; threadNumber++) {
+ log.info("Creating exporter:" + threadNumber);
+ KafkaPeriodicBindingSetExporter exporter = new KafkaPeriodicBindingSetExporter(producer, threadNumber, bindingSets);
+ exporters.add(exporter);
+ executor.submit(exporter);
+ }
+ running = true;
+ }
+ }
+
+ @Override
+ public void stop() {
+ if (executor != null) {
+ executor.shutdown();
+ }
+
+ if (exporters != null && exporters.size() > 0) {
+ exporters.forEach(x -> x.shutdown());
+ }
+
+ if (producer != null) {
+ producer.close();
+ }
+
+ running = false;
+ try {
+ if (!executor.awaitTermination(5000, TimeUnit.MILLISECONDS)) {
+ log.info("Timed out waiting for consumer threads to shut down, exiting uncleanly");
+ executor.shutdownNow();
+ }
+ } catch (InterruptedException e) {
+ log.info("Interrupted during shutdown, exiting uncleanly");
+ }
+ }
+
+ @Override
+ public boolean currentlyRunning() {
+ return running;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/exporter/KafkaPeriodicBindingSetExporter.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/exporter/KafkaPeriodicBindingSetExporter.java b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/exporter/KafkaPeriodicBindingSetExporter.java
new file mode 100644
index 0000000..8a0322f
--- /dev/null
+++ b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/exporter/KafkaPeriodicBindingSetExporter.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.exporter;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.log4j.Logger;
+import org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants;
+import org.apache.rya.periodic.notification.api.BindingSetExporter;
+import org.apache.rya.periodic.notification.api.BindingSetRecord;
+import org.apache.rya.periodic.notification.api.BindingSetRecordExportException;
+import org.openrdf.model.Literal;
+import org.openrdf.query.BindingSet;
+
+import jline.internal.Preconditions;
+
+/**
+ * Object that exports {@link BindingSet}s to the Kafka topic indicated by
+ * the {@link BindingSetRecord}.
+ *
+ */
+public class KafkaPeriodicBindingSetExporter implements BindingSetExporter, Runnable {
+
+ private static final Logger log = Logger.getLogger(BindingSetExporter.class);
+ private KafkaProducer<String, BindingSet> producer;
+ private BlockingQueue<BindingSetRecord> bindingSets;
+ private AtomicBoolean closed = new AtomicBoolean(false);
+ private int threadNumber;
+
+ public KafkaPeriodicBindingSetExporter(KafkaProducer<String, BindingSet> producer, int threadNumber,
+ BlockingQueue<BindingSetRecord> bindingSets) {
+ Preconditions.checkNotNull(producer);
+ Preconditions.checkNotNull(bindingSets);
+ this.threadNumber = threadNumber;
+ this.producer = producer;
+ this.bindingSets = bindingSets;
+ }
+
+ /**
+ * Exports BindingSets to Kafka. The BindingSet and topic are extracted from
+ * the indicated BindingSetRecord and the BindingSet is then exported to the topic.
+ */
+ @Override
+ public void exportNotification(BindingSetRecord record) throws BindingSetRecordExportException {
+ String bindingName = IncrementalUpdateConstants.PERIODIC_BIN_ID;
+ BindingSet bindingSet = record.getBindingSet();
+ String topic = record.getTopic();
+ long binId = ((Literal) bindingSet.getValue(bindingName)).longValue();
+ final Future<RecordMetadata> future = producer
+ .send(new ProducerRecord<String, BindingSet>(topic, Long.toString(binId), bindingSet));
+ try {
+ //wait for confirmation that results have been received
+ future.get(5, TimeUnit.SECONDS);
+ } catch (InterruptedException | ExecutionException | TimeoutException e) {
+ throw new BindingSetRecordExportException(e.getMessage());
+ }
+ }
+
+ @Override
+ public void run() {
+ try {
+ while (!closed.get()) {
+ exportNotification(bindingSets.take());
+ }
+ } catch (InterruptedException | BindingSetRecordExportException e) {
+ log.trace("Thread " + threadNumber + " is unable to process message.");
+ }
+ }
+
+
+ public void shutdown() {
+ closed.set(true);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/processor/NotificationProcessorExecutor.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/processor/NotificationProcessorExecutor.java b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/processor/NotificationProcessorExecutor.java
new file mode 100644
index 0000000..a9a5ad1
--- /dev/null
+++ b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/processor/NotificationProcessorExecutor.java
@@ -0,0 +1,114 @@
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */package org.apache.rya.periodic.notification.processor;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.log4j.Logger;
+import org.apache.rya.indexing.pcj.storage.PeriodicQueryResultStorage;
+import org.apache.rya.periodic.notification.api.BindingSetRecord;
+import org.apache.rya.periodic.notification.api.LifeCycle;
+import org.apache.rya.periodic.notification.api.NodeBin;
+import org.apache.rya.periodic.notification.notification.TimestampedNotification;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Executor service that runs {@link TimestampedNotificationProcessor}s with basic
+ * functionality for starting, stopping, and determining whether notification processors are
+ * being executed.
+ *
+ */
+public class NotificationProcessorExecutor implements LifeCycle {
+
+ private static final Logger log = Logger.getLogger(TimestampedNotificationProcessor.class);
+ private BlockingQueue<TimestampedNotification> notifications; // notifications
+ private BlockingQueue<NodeBin> bins; // entries to delete from Fluo
+ private BlockingQueue<BindingSetRecord> bindingSets; // query results to
+ // export
+ private PeriodicQueryResultStorage periodicStorage;
+ private List<TimestampedNotificationProcessor> processors;
+ private int numberThreads;
+ private ExecutorService executor;
+ private boolean running = false;
+
+ /**
+ * Creates NotificationProcessorExecutor.
+ * @param periodicStorage - storage layer that periodic results are read from
+ * @param notifications - notifications are pulled from this queue, and the timestamp indicates which bin of results to query for
+ * @param bins - after notifications are processed, they are added to the bin to be deleted
+ * @param bindingSets - results read from the storage layer to be exported
+ * @param numberThreads - number of threads used for processing
+ */
+ public NotificationProcessorExecutor(PeriodicQueryResultStorage periodicStorage, BlockingQueue<TimestampedNotification> notifications,
+ BlockingQueue<NodeBin> bins, BlockingQueue<BindingSetRecord> bindingSets, int numberThreads) {
+ this.notifications = Preconditions.checkNotNull(notifications);
+ this.bins = Preconditions.checkNotNull(bins);
+ this.bindingSets = Preconditions.checkNotNull(bindingSets);
+ this.periodicStorage = periodicStorage;
+ this.numberThreads = numberThreads;
+ processors = new ArrayList<>();
+ }
+
+ @Override
+ public void start() {
+ if (!running) {
+ executor = Executors.newFixedThreadPool(numberThreads);
+ for (int threadNumber = 0; threadNumber < numberThreads; threadNumber++) {
+ log.info("Creating exporter:" + threadNumber);
+ TimestampedNotificationProcessor processor = TimestampedNotificationProcessor.builder().setBindingSets(bindingSets)
+ .setBins(bins).setPeriodicStorage(periodicStorage).setNotifications(notifications).setThreadNumber(threadNumber)
+ .build();
+ processors.add(processor);
+ executor.submit(processor);
+ }
+ running = true;
+ }
+ }
+
+ @Override
+ public void stop() {
+ if (processors != null && processors.size() > 0) {
+ processors.forEach(x -> x.shutdown());
+ }
+ if (executor != null) {
+ executor.shutdown();
+ }
+ running = false;
+ try {
+ if (!executor.awaitTermination(5000, TimeUnit.MILLISECONDS)) {
+ log.info("Timed out waiting for consumer threads to shut down, exiting uncleanly");
+ executor.shutdownNow();
+ }
+ } catch (InterruptedException e) {
+ log.info("Interrupted during shutdown, exiting uncleanly");
+ }
+ }
+
+ @Override
+ public boolean currentlyRunning() {
+ return running;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/processor/TimestampedNotificationProcessor.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/processor/TimestampedNotificationProcessor.java b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/processor/TimestampedNotificationProcessor.java
new file mode 100644
index 0000000..8b65683
--- /dev/null
+++ b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/processor/TimestampedNotificationProcessor.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.processor;
+
+import java.util.Optional;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.log4j.Logger;
+import org.apache.rya.indexing.pcj.storage.PeriodicQueryResultStorage;
+import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import org.apache.rya.periodic.notification.api.BinPruner;
+import org.apache.rya.periodic.notification.api.BindingSetRecord;
+import org.apache.rya.periodic.notification.api.NodeBin;
+import org.apache.rya.periodic.notification.api.NotificationProcessor;
+import org.apache.rya.periodic.notification.exporter.KafkaPeriodicBindingSetExporter;
+import org.apache.rya.periodic.notification.notification.TimestampedNotification;
+import org.openrdf.query.BindingSet;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Implementation of {@link NotificationProcessor} that uses the id indicated by
+ * the {@link TimestampedNotification} to obtain results from the
+ * {@link PeriodicQueryResultStorage} layer containing the results of the
+ * Periodic Query. The TimestampedNotificationProcessor then parses the results
+ * and adds them to work queues to be processed by the {@link BinPruner} and the
+ * {@link KafkaPeriodicBindingSetExporter}.
+ *
+ */
+public class TimestampedNotificationProcessor implements NotificationProcessor, Runnable {
+
+ private static final Logger log = Logger.getLogger(TimestampedNotificationProcessor.class);
+ private PeriodicQueryResultStorage periodicStorage;
+ private BlockingQueue<TimestampedNotification> notifications; // notifications
+ // to process
+ private BlockingQueue<NodeBin> bins; // entries to delete from Fluo
+ private BlockingQueue<BindingSetRecord> bindingSets; // query results to export
+ private AtomicBoolean closed = new AtomicBoolean(false);
+ private int threadNumber;
+
+
+ public TimestampedNotificationProcessor(PeriodicQueryResultStorage periodicStorage,
+ BlockingQueue<TimestampedNotification> notifications, BlockingQueue<NodeBin> bins, BlockingQueue<BindingSetRecord> bindingSets,
+ int threadNumber) {
+ this.notifications = Preconditions.checkNotNull(notifications);
+ this.bins = Preconditions.checkNotNull(bins);
+ this.bindingSets = Preconditions.checkNotNull(bindingSets);
+ this.periodicStorage = periodicStorage;
+ this.threadNumber = threadNumber;
+ }
+
+ /**
+ * Processes the TimestampNotifications by scanning the PCJ tables for
+ * entries in the bin corresponding to
+ * {@link TimestampedNotification#getTimestamp()} and adding them to the
+ * export BlockingQueue. The TimestampNotification is then used to form a
+ * {@link NodeBin} that is passed to the BinPruner BlockingQueue so that the
+ * bins can be deleted from Fluo and Accumulo.
+ */
+ @Override
+ public void processNotification(TimestampedNotification notification) {
+
+ String id = notification.getId();
+ long ts = notification.getTimestamp().getTime();
+ long period = notification.getPeriod();
+ long bin = getBinFromTimestamp(ts, period);
+ NodeBin nodeBin = new NodeBin(id, bin);
+
+ try (CloseableIterator<BindingSet> iter = periodicStorage.listResults(id, Optional.of(bin));) {
+
+ while(iter.hasNext()) {
+ bindingSets.add(new BindingSetRecord(iter.next(), id));
+ }
+ // add NodeBin to BinPruner queue so that bin can be deleted from
+ // Fluo and Accumulo
+ bins.add(nodeBin);
+ } catch (Exception e) {
+ log.debug("Encountered error: " + e.getMessage() + " while accessing periodic results for bin: " + bin + " for query: " + id);
+ }
+ }
+
+ /**
+ * Computes left bin end point containing event time ts
+ *
+ * @param ts - event time
+ * @param start - time that periodic event began
+ * @param period - length of period
+ * @return left bin end point containing event time ts
+ */
+ private long getBinFromTimestamp(long ts, long period) {
+ Preconditions.checkArgument(period > 0);
+ return (ts / period) * period;
+ }
+
+ @Override
+ public void run() {
+ try {
+ while(!closed.get()) {
+ processNotification(notifications.take());
+ }
+ } catch (Exception e) {
+ log.trace("Thread_" + threadNumber + " is unable to process next notification.");
+ throw new RuntimeException(e);
+ }
+
+ }
+
+ public void shutdown() {
+ closed.set(true);
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+
+
+ public static class Builder {
+
+ private PeriodicQueryResultStorage periodicStorage;
+ private BlockingQueue<TimestampedNotification> notifications; // notifications to process
+ private BlockingQueue<NodeBin> bins; // entries to delete from Fluo
+ private BlockingQueue<BindingSetRecord> bindingSets; // query results to export
+
+ private int threadNumber;
+
+ /**
+ * Set notification queue
+ * @param notifications - work queue containing notifications to be processed
+ * @return this Builder for chaining method calls
+ */
+ public Builder setNotifications(BlockingQueue<TimestampedNotification> notifications) {
+ this.notifications = notifications;
+ return this;
+ }
+
+ /**
+ * Set nodeBin queue
+ * @param bins - work queue containing NodeBins to be pruned
+ * @return this Builder for chaining method calls
+ */
+ public Builder setBins(BlockingQueue<NodeBin> bins) {
+ this.bins = bins;
+ return this;
+ }
+
+ /**
+ * Set BindingSet queue
+ * @param bindingSets - work queue containing BindingSets to be exported
+ * @return this Builder for chaining method calls
+ */
+ public Builder setBindingSets(BlockingQueue<BindingSetRecord> bindingSets) {
+ this.bindingSets = bindingSets;
+ return this;
+ }
+
+ /**
+ * Sets the number of threads used by this processor
+ * @param threadNumber - number of threads used by this processor
+ * @return - number of threads used by this processor
+ */
+ public Builder setThreadNumber(int threadNumber) {
+ this.threadNumber = threadNumber;
+ return this;
+ }
+
+ /**
+ * Set the PeriodicStorage layer
+ * @param periodicStorage - periodic storage layer that periodic results are read from
+ * @return - this Builder for chaining method calls
+ */
+ public Builder setPeriodicStorage(PeriodicQueryResultStorage periodicStorage) {
+ this.periodicStorage = periodicStorage;
+ return this;
+ }
+
+ /**
+ * Builds a TimestampedNotificationProcessor
+ * @return - TimestampedNotificationProcessor built from arguments passed to this Builder
+ */
+ public TimestampedNotificationProcessor build() {
+ return new TimestampedNotificationProcessor(periodicStorage, notifications, bins, bindingSets, threadNumber);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/pruner/AccumuloBinPruner.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/pruner/AccumuloBinPruner.java b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/pruner/AccumuloBinPruner.java
new file mode 100644
index 0000000..4dac64c
--- /dev/null
+++ b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/pruner/AccumuloBinPruner.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.pruner;
+
+import org.apache.log4j.Logger;
+import org.apache.rya.indexing.pcj.storage.PeriodicQueryResultStorage;
+import org.apache.rya.indexing.pcj.storage.PeriodicQueryStorageException;
+import org.apache.rya.periodic.notification.api.BinPruner;
+import org.apache.rya.periodic.notification.api.NodeBin;
+
+import jline.internal.Preconditions;
+
+/**
+ * Deletes BindingSets from time bins in the indicated PCJ table
+ */
+public class AccumuloBinPruner implements BinPruner {
+
+ private static final Logger log = Logger.getLogger(AccumuloBinPruner.class);
+ private PeriodicQueryResultStorage periodicStorage;
+
+ public AccumuloBinPruner(PeriodicQueryResultStorage periodicStorage) {
+ Preconditions.checkNotNull(periodicStorage);
+ this.periodicStorage = periodicStorage;
+ }
+
+ /**
+ * This method deletes all BindingSets in the indicated bin from the PCJ
+ * table indicated by the id. It is assumed that all BindingSet entries for
+ * the corresponding bin are written to the PCJ table so that the bin Id
+ * occurs first.
+ *
+ * @param id
+ * - pcj table id
+ * @param bin
+ * - temporal bin the BindingSets are contained in
+ */
+ @Override
+ public void pruneBindingSetBin(NodeBin nodeBin) {
+ Preconditions.checkNotNull(nodeBin);
+ String id = nodeBin.getNodeId();
+ long bin = nodeBin.getBin();
+ try {
+ periodicStorage.deletePeriodicQueryResults(id, bin);
+ } catch (PeriodicQueryStorageException e) {
+ log.trace("Unable to delete results from Peroidic Table: " + id + " for bin: " + bin);
+ throw new RuntimeException(e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/pruner/FluoBinPruner.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/pruner/FluoBinPruner.java b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/pruner/FluoBinPruner.java
new file mode 100644
index 0000000..bee9c02
--- /dev/null
+++ b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/pruner/FluoBinPruner.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.pruner;
+
+import org.apache.fluo.api.client.FluoClient;
+import org.apache.fluo.api.client.Transaction;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.api.data.Column;
+import org.apache.fluo.api.data.Span;
+import org.apache.log4j.Logger;
+import org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants;
+import org.apache.rya.indexing.pcj.fluo.app.NodeType;
+import org.apache.rya.indexing.pcj.fluo.app.batch.BatchInformationDAO;
+import org.apache.rya.indexing.pcj.fluo.app.batch.SpanBatchDeleteInformation;
+import org.apache.rya.periodic.notification.api.BinPruner;
+import org.apache.rya.periodic.notification.api.NodeBin;
+
+import com.google.common.base.Optional;
+
+/**
+ * Deletes {@link BindingSet}s from the indicated Fluo table.
+ */
+public class FluoBinPruner implements BinPruner {
+
+ private static final Logger log = Logger.getLogger(FluoBinPruner.class);
+ private FluoClient client;
+
+ public FluoBinPruner(FluoClient client) {
+ this.client = client;
+ }
+
+ /**
+ * This method deletes BindingSets in the specified bin from the BindingSet
+ * Column of the indicated Fluo nodeId
+ *
+ * @param id
+ * - Fluo nodeId
+ * @param bin
+ * - bin id
+ */
+ @Override
+ public void pruneBindingSetBin(NodeBin nodeBin) {
+ String id = nodeBin.getNodeId();
+ long bin = nodeBin.getBin();
+ try (Transaction tx = client.newTransaction()) {
+ Optional<NodeType> type = NodeType.fromNodeId(id);
+ if (!type.isPresent()) {
+ log.trace("Unable to determine NodeType from id: " + id);
+ throw new RuntimeException();
+ }
+ Column batchInfoColumn = type.get().getResultColumn();
+ String batchInfoSpanPrefix = id + IncrementalUpdateConstants.NODEID_BS_DELIM + bin;
+ SpanBatchDeleteInformation batchInfo = SpanBatchDeleteInformation.builder().setColumn(batchInfoColumn)
+ .setSpan(Span.prefix(Bytes.of(batchInfoSpanPrefix))).build();
+ BatchInformationDAO.addBatch(tx, id, batchInfo);
+ tx.commit();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/pruner/PeriodicQueryPruner.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/pruner/PeriodicQueryPruner.java b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/pruner/PeriodicQueryPruner.java
new file mode 100644
index 0000000..516690e
--- /dev/null
+++ b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/pruner/PeriodicQueryPruner.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.pruner;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.fluo.api.client.FluoClient;
+import org.apache.fluo.api.client.Snapshot;
+import org.apache.fluo.api.client.SnapshotBase;
+import org.apache.log4j.Logger;
+import org.apache.rya.indexing.pcj.fluo.app.NodeType;
+import org.apache.rya.indexing.pcj.fluo.app.util.PeriodicQueryUtil;
+import org.apache.rya.periodic.notification.api.BinPruner;
+import org.apache.rya.periodic.notification.api.NodeBin;
+
+import jline.internal.Preconditions;
+
+/**
+ * Implementation of {@link BinPruner} that deletes old, already processed
+ * Periodic Query results from Fluo and the PCJ table to which the Fluo results
+ * are exported.
+ *
+ */
+public class PeriodicQueryPruner implements BinPruner, Runnable {
+
+ private static final Logger log = Logger.getLogger(PeriodicQueryPruner.class);
+ private FluoClient client;
+ private AccumuloBinPruner accPruner;
+ private FluoBinPruner fluoPruner;
+ private BlockingQueue<NodeBin> bins;
+ private AtomicBoolean closed = new AtomicBoolean(false);
+ private int threadNumber;
+
+ public PeriodicQueryPruner(FluoBinPruner fluoPruner, AccumuloBinPruner accPruner, FluoClient client, BlockingQueue<NodeBin> bins, int threadNumber) {
+ this.fluoPruner = Preconditions.checkNotNull(fluoPruner);
+ this.accPruner = Preconditions.checkNotNull(accPruner);
+ this.client = Preconditions.checkNotNull(client);
+ this.bins = Preconditions.checkNotNull(bins);
+ this.threadNumber = threadNumber;
+ }
+
+ @Override
+ public void run() {
+ try {
+ while (!closed.get()) {
+ pruneBindingSetBin(bins.take());
+ }
+ } catch (InterruptedException e) {
+ log.trace("Thread " + threadNumber + " is unable to prune the next message.");
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Prunes BindingSet bins from the Rya Fluo Application in addition to the BindingSet
+ * bins created in the PCJ tables associated with the give query id.
+ * @param id - QueryResult Id for the Rya Fluo application
+ * @param bin - bin id for bins to be deleted
+ */
+ @Override
+ public void pruneBindingSetBin(NodeBin nodeBin) {
+ String pcjId = nodeBin.getNodeId();
+ long bin = nodeBin.getBin();
+ try(Snapshot sx = client.newSnapshot()) {
+ String queryId = NodeType.generateNewIdForType(NodeType.QUERY, pcjId);
+ Set<String> fluoIds = getNodeIdsFromResultId(sx, queryId);
+ accPruner.pruneBindingSetBin(nodeBin);
+ for(String fluoId: fluoIds) {
+ fluoPruner.pruneBindingSetBin(new NodeBin(fluoId, bin));
+ }
+ } catch (Exception e) {
+ log.trace("Could not successfully initialize PeriodicQueryBinPruner.");
+ }
+ }
+
+
+ public void shutdown() {
+ closed.set(true);
+ }
+
+ private Set<String> getNodeIdsFromResultId(SnapshotBase sx, String id) {
+ Set<String> ids = new HashSet<>();
+ PeriodicQueryUtil.getPeriodicQueryNodeAncestorIds(sx, id, ids);
+ return ids;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/pruner/PeriodicQueryPrunerExecutor.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/pruner/PeriodicQueryPrunerExecutor.java b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/pruner/PeriodicQueryPrunerExecutor.java
new file mode 100644
index 0000000..1c11f96
--- /dev/null
+++ b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/pruner/PeriodicQueryPrunerExecutor.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.pruner;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.fluo.api.client.FluoClient;
+import org.apache.log4j.Logger;
+import org.apache.rya.indexing.pcj.storage.PeriodicQueryResultStorage;
+import org.apache.rya.periodic.notification.api.LifeCycle;
+import org.apache.rya.periodic.notification.api.NodeBin;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Executor service that runs {@link PeriodicQueryPruner}s with added functionality
+ * for starting, stopping, and determining if the query pruners are running.
+ */
+public class PeriodicQueryPrunerExecutor implements LifeCycle {
+
+ private static final Logger log = Logger.getLogger(PeriodicQueryPrunerExecutor.class);
+ private FluoClient client;
+ private int numThreads;
+ private ExecutorService executor;
+ private BlockingQueue<NodeBin> bins;
+ private PeriodicQueryResultStorage periodicStorage;
+ private List<PeriodicQueryPruner> pruners;
+ private boolean running = false;
+
+ public PeriodicQueryPrunerExecutor(PeriodicQueryResultStorage periodicStorage, FluoClient client, int numThreads,
+ BlockingQueue<NodeBin> bins) {
+ Preconditions.checkArgument(numThreads > 0);
+ this.periodicStorage = periodicStorage;
+ this.numThreads = numThreads;
+ executor = Executors.newFixedThreadPool(numThreads);
+ this.bins = bins;
+ this.client = client;
+ this.pruners = new ArrayList<>();
+ }
+
+ @Override
+ public void start() {
+ if (!running) {
+ AccumuloBinPruner accPruner = new AccumuloBinPruner(periodicStorage);
+ FluoBinPruner fluoPruner = new FluoBinPruner(client);
+
+ for (int threadNumber = 0; threadNumber < numThreads; threadNumber++) {
+ PeriodicQueryPruner pruner = new PeriodicQueryPruner(fluoPruner, accPruner, client, bins, threadNumber);
+ pruners.add(pruner);
+ executor.submit(pruner);
+ }
+ running = true;
+ }
+ }
+
+ @Override
+ public void stop() {
+ if (pruners != null && pruners.size() > 0) {
+ pruners.forEach(x -> x.shutdown());
+ }
+ if(client != null) {
+ client.close();
+ }
+ if (executor != null) {
+ executor.shutdown();
+ running = false;
+ }
+ try {
+ if (!executor.awaitTermination(5000, TimeUnit.MILLISECONDS)) {
+ log.info("Timed out waiting for consumer threads to shut down, exiting uncleanly");
+ executor.shutdownNow();
+ }
+ } catch (InterruptedException e) {
+ log.info("Interrupted during shutdown, exiting uncleanly");
+ }
+ }
+
+ @Override
+ public boolean currentlyRunning() {
+ return running;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/recovery/PeriodicNotificationProvider.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/recovery/PeriodicNotificationProvider.java b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/recovery/PeriodicNotificationProvider.java
new file mode 100644
index 0000000..69bd39c
--- /dev/null
+++ b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/recovery/PeriodicNotificationProvider.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.recovery;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.fluo.api.client.Snapshot;
+import org.apache.fluo.api.client.scanner.ColumnScanner;
+import org.apache.fluo.api.client.scanner.RowScanner;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.api.data.ColumnValue;
+import org.apache.fluo.api.data.Span;
+import org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants;
+import org.apache.rya.indexing.pcj.fluo.app.NodeType;
+import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryColumns;
+import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryMetadataDAO;
+import org.apache.rya.indexing.pcj.fluo.app.query.PeriodicQueryMetadata;
+import org.apache.rya.indexing.pcj.fluo.app.util.FluoQueryUtils;
+import org.apache.rya.periodic.notification.api.NotificationCoordinatorExecutor;
+import org.apache.rya.periodic.notification.coordinator.PeriodicNotificationCoordinatorExecutor;
+import org.apache.rya.periodic.notification.notification.CommandNotification;
+import org.apache.rya.periodic.notification.notification.CommandNotification.Command;
+import org.apache.rya.periodic.notification.notification.PeriodicNotification;
+
+/**
+ * This class is used by the {@link PeriodicNotificationCoordinatorExecutor}
+ * to add all existing {@link PeriodicNotification}s stored in Fluo when it is
+ * initialized. This enables the the {@link PeriodicServiceApplication} to be
+ * recovered from failure by restoring it original state.
+ *
+ */
+public class PeriodicNotificationProvider {
+
+ private FluoQueryMetadataDAO dao;
+
+ public PeriodicNotificationProvider() {
+ this.dao = new FluoQueryMetadataDAO();
+ }
+
+ /**
+ * Retrieve all of the information about Periodic Query results already registered
+ * with Fluo. This is returned in the form of {@link CommandNotification}s that
+ * can be registered with the {@link NotificationCoordinatorExecutor}.
+ * @param sx - snapshot for reading results from Fluo
+ * @return - collection of CommandNotifications that indicate Periodic Query information registered with system
+ */
+ public Collection<CommandNotification> getNotifications(Snapshot sx) {
+ Set<PeriodicQueryMetadata> periodicMetadata = new HashSet<>();
+ RowScanner scanner = sx.scanner().fetch(FluoQueryColumns.PERIODIC_QUERY_NODE_ID)
+ .over(Span.prefix(IncrementalUpdateConstants.PERIODIC_QUERY_PREFIX)).byRow().build();
+ Iterator<ColumnScanner> colScannerIter = scanner.iterator();
+ while (colScannerIter.hasNext()) {
+ ColumnScanner colScanner = colScannerIter.next();
+ Iterator<ColumnValue> values = colScanner.iterator();
+ while (values.hasNext()) {
+ PeriodicQueryMetadata metadata = dao.readPeriodicQueryMetadata(sx, values.next().getsValue());
+ periodicMetadata.add(metadata);
+ }
+ }
+ return getCommandNotifications(sx, periodicMetadata);
+ }
+
+ /**
+ * Registers all of Periodic Query information already contained within Fluo to the
+ * {@link NotificationCoordinatorExecutor}.
+ * @param coordinator - coordinator that periodic info will be registered with
+ * @param sx - snapshot for reading results from Fluo
+ */
+ public void processRegisteredNotifications(NotificationCoordinatorExecutor coordinator, Snapshot sx) {
+ coordinator.start();
+ Collection<CommandNotification> notifications = getNotifications(sx);
+ for(CommandNotification notification: notifications) {
+ coordinator.processNextCommandNotification(notification);
+ }
+ }
+
+ private Collection<CommandNotification> getCommandNotifications(Snapshot sx, Collection<PeriodicQueryMetadata> metadata) {
+ Set<CommandNotification> notifications = new HashSet<>();
+ int i = 1;
+ for(PeriodicQueryMetadata meta:metadata) {
+ //offset initial wait to avoid overloading system
+ PeriodicNotification periodic = new PeriodicNotification(getQueryId(meta.getNodeId(), sx), meta.getPeriod(),TimeUnit.MILLISECONDS,i*5000);
+ notifications.add(new CommandNotification(Command.ADD, periodic));
+ i++;
+ }
+ return notifications;
+ }
+
+ private String getQueryId(String periodicNodeId, Snapshot sx) {
+ return getQueryIdFromPeriodicId(sx, periodicNodeId);
+ }
+
+ private String getQueryIdFromPeriodicId(Snapshot sx, String nodeId) {
+ NodeType nodeType = NodeType.fromNodeId(nodeId).orNull();
+ String id = null;
+ switch (nodeType) {
+ case FILTER:
+ id = getQueryIdFromPeriodicId(sx, sx.get(Bytes.of(nodeId), FluoQueryColumns.FILTER_PARENT_NODE_ID).toString());
+ break;
+ case PERIODIC_QUERY:
+ id = getQueryIdFromPeriodicId(sx, sx.get(Bytes.of(nodeId), FluoQueryColumns.PERIODIC_QUERY_PARENT_NODE_ID).toString());
+ break;
+ case QUERY:
+ id = FluoQueryUtils.convertFluoQueryIdToPcjId(nodeId);
+ break;
+ case AGGREGATION:
+ id = getQueryIdFromPeriodicId(sx, sx.get(Bytes.of(nodeId), FluoQueryColumns.AGGREGATION_PARENT_NODE_ID).toString());
+ break;
+ case CONSTRUCT:
+ id = getQueryIdFromPeriodicId(sx, sx.get(Bytes.of(nodeId), FluoQueryColumns.CONSTRUCT_PARENT_NODE_ID).toString());
+ break;
+ case PROJECTION:
+ id = getQueryIdFromPeriodicId(sx, sx.get(Bytes.of(nodeId), FluoQueryColumns.PROJECTION_PARENT_NODE_ID).toString());
+ break;
+ default:
+ throw new IllegalArgumentException("Invalid node type");
+
+ }
+ return id;
+ }
+
+}
[3/7] incubator-rya git commit: RYA-355 Refactored the periodic
notification service structure. Closes #221.
Posted by ca...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplicationIT.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplicationIT.java b/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplicationIT.java
deleted file mode 100644
index 9109775..0000000
--- a/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplicationIT.java
+++ /dev/null
@@ -1,493 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.application;
-
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.time.ZonedDateTime;
-import java.time.format.DateTimeFormatter;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Properties;
-import java.util.Set;
-import java.util.UUID;
-
-import javax.xml.datatype.DatatypeConfigurationException;
-import javax.xml.datatype.DatatypeFactory;
-
-import org.apache.accumulo.core.client.Connector;
-import org.apache.fluo.api.client.FluoClient;
-import org.apache.fluo.api.config.FluoConfiguration;
-import org.apache.fluo.core.client.FluoClientImpl;
-import org.apache.kafka.clients.CommonClientConfigs;
-import org.apache.kafka.clients.consumer.ConsumerConfig;
-import org.apache.kafka.clients.consumer.ConsumerRecord;
-import org.apache.kafka.clients.consumer.ConsumerRecords;
-import org.apache.kafka.clients.consumer.KafkaConsumer;
-import org.apache.kafka.clients.producer.KafkaProducer;
-import org.apache.kafka.common.serialization.StringDeserializer;
-import org.apache.kafka.common.serialization.StringSerializer;
-import org.apache.rya.api.resolver.RdfToRyaConversions;
-import org.apache.rya.indexing.accumulo.ConfigUtils;
-import org.apache.rya.indexing.pcj.fluo.api.CreatePeriodicQuery;
-import org.apache.rya.indexing.pcj.fluo.api.InsertTriples;
-import org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants;
-import org.apache.rya.indexing.pcj.fluo.app.util.FluoClientFactory;
-import org.apache.rya.indexing.pcj.fluo.app.util.FluoQueryUtils;
-import org.apache.rya.indexing.pcj.storage.PeriodicQueryResultStorage;
-import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
-import org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPeriodicQueryResultStorage;
-import org.apache.rya.kafka.base.EmbeddedKafkaInstance;
-import org.apache.rya.kafka.base.EmbeddedKafkaSingleton;
-import org.apache.rya.kafka.base.KafkaTestInstanceRule;
-import org.apache.rya.pcj.fluo.test.base.RyaExportITBase;
-import org.apache.rya.periodic.notification.notification.CommandNotification;
-import org.apache.rya.periodic.notification.registration.KafkaNotificationRegistrationClient;
-import org.apache.rya.periodic.notification.serialization.BindingSetSerDe;
-import org.apache.rya.periodic.notification.serialization.CommandNotificationSerializer;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.openrdf.model.Statement;
-import org.openrdf.model.Value;
-import org.openrdf.model.ValueFactory;
-import org.openrdf.model.impl.LiteralImpl;
-import org.openrdf.model.impl.ValueFactoryImpl;
-import org.openrdf.model.vocabulary.XMLSchema;
-import org.openrdf.query.BindingSet;
-import org.openrdf.query.algebra.evaluation.QueryBindingSet;
-
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Multimap;
-import com.google.common.collect.Sets;
-
-import static org.apache.rya.periodic.notification.application.PeriodicNotificationApplicationConfiguration.NOTIFICATION_TOPIC;
-import static org.apache.rya.periodic.notification.application.PeriodicNotificationApplicationConfiguration.KAFKA_BOOTSTRAP_SERVERS;;
-
-
-public class PeriodicNotificationApplicationIT extends RyaExportITBase {
-
- private PeriodicNotificationApplication app;
- private KafkaNotificationRegistrationClient registrar;
- private KafkaProducer<String, CommandNotification> producer;
- private Properties props;
- private Properties kafkaProps;
- private PeriodicNotificationApplicationConfiguration conf;
- private static EmbeddedKafkaInstance embeddedKafka = EmbeddedKafkaSingleton.getInstance();
- private static String bootstrapServers;
-
- @Rule
- public KafkaTestInstanceRule rule = new KafkaTestInstanceRule(false);
-
- @BeforeClass
- public static void initClass() {
- bootstrapServers = embeddedKafka.createBootstrapServerConfig().getProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG);
- }
-
- @Before
- public void init() throws Exception {
- String topic = rule.getKafkaTopicName();
- rule.createTopic(topic);
-
- //get user specified props and update with the embedded kafka bootstrap servers and rule generated topic
- props = getProps();
- props.setProperty(NOTIFICATION_TOPIC, topic);
- props.setProperty(KAFKA_BOOTSTRAP_SERVERS, bootstrapServers);
- conf = new PeriodicNotificationApplicationConfiguration(props);
-
- //create Kafka Producer
- kafkaProps = getKafkaProperties(conf);
- producer = new KafkaProducer<>(kafkaProps, new StringSerializer(), new CommandNotificationSerializer());
-
- //extract kafka specific properties from application config
- app = PeriodicNotificationApplicationFactory.getPeriodicApplication(props);
- registrar = new KafkaNotificationRegistrationClient(conf.getNotificationTopic(), producer);
- }
-
- @Test
- public void periodicApplicationWithAggAndGroupByTest() throws Exception {
-
- String sparql = "prefix function: <http://org.apache.rya/function#> " // n
- + "prefix time: <http://www.w3.org/2006/time#> " // n
- + "select ?type (count(?obs) as ?total) where {" // n
- + "Filter(function:periodic(?time, 1, .25, time:minutes)) " // n
- + "?obs <uri:hasTime> ?time. " // n
- + "?obs <uri:hasObsType> ?type } group by ?type"; // n
-
- //make data
- int periodMult = 15;
- final ValueFactory vf = new ValueFactoryImpl();
- final DatatypeFactory dtf = DatatypeFactory.newInstance();
- //Sleep until current time aligns nicely with period to makell
- //results more predictable
- while(System.currentTimeMillis() % (periodMult*1000) > 500);
- ZonedDateTime time = ZonedDateTime.now();
-
- ZonedDateTime zTime1 = time.minusSeconds(2*periodMult);
- String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT);
-
- ZonedDateTime zTime2 = zTime1.minusSeconds(periodMult);
- String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT);
-
- ZonedDateTime zTime3 = zTime2.minusSeconds(periodMult);
- String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT);
-
- final Collection<Statement> statements = Sets.newHashSet(
- vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"),
- vf.createLiteral(dtf.newXMLGregorianCalendar(time1))),
- vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasObsType"), vf.createLiteral("ship")),
- vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasTime"),
- vf.createLiteral(dtf.newXMLGregorianCalendar(time1))),
- vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasObsType"), vf.createLiteral("airplane")),
- vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasTime"),
- vf.createLiteral(dtf.newXMLGregorianCalendar(time2))),
- vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasObsType"), vf.createLiteral("ship")),
- vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasTime"),
- vf.createLiteral(dtf.newXMLGregorianCalendar(time2))),
- vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasObsType"), vf.createLiteral("airplane")),
- vf.createStatement(vf.createURI("urn:obs_5"), vf.createURI("uri:hasTime"),
- vf.createLiteral(dtf.newXMLGregorianCalendar(time3))),
- vf.createStatement(vf.createURI("urn:obs_5"), vf.createURI("uri:hasObsType"), vf.createLiteral("automobile")));
-
- try (FluoClient fluo = FluoClientFactory.getFluoClient(conf.getFluoAppName(), Optional.of(conf.getFluoTableName()), conf)) {
- Connector connector = ConfigUtils.getConnector(conf);
- PeriodicQueryResultStorage storage = new AccumuloPeriodicQueryResultStorage(connector, conf.getTablePrefix());
- CreatePeriodicQuery periodicQuery = new CreatePeriodicQuery(fluo, storage);
- String id = FluoQueryUtils.convertFluoQueryIdToPcjId(periodicQuery.createPeriodicQuery(sparql, registrar).getQueryId());
- addData(statements);
- app.start();
-
- Multimap<Long, BindingSet> actual = HashMultimap.create();
- try (KafkaConsumer<String, BindingSet> consumer = new KafkaConsumer<>(kafkaProps, new StringDeserializer(), new BindingSetSerDe())) {
- consumer.subscribe(Arrays.asList(id));
- long end = System.currentTimeMillis() + 4*periodMult*1000;
- long lastBinId = 0L;
- long binId = 0L;
- List<Long> ids = new ArrayList<>();
- while (System.currentTimeMillis() < end) {
- ConsumerRecords<String, BindingSet> records = consumer.poll(periodMult*1000);
- for(ConsumerRecord<String, BindingSet> record: records){
- BindingSet result = record.value();
- binId = Long.parseLong(result.getBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID).getValue().stringValue());
- if(lastBinId != binId) {
- lastBinId = binId;
- ids.add(binId);
- }
- actual.put(binId, result);
- }
- }
-
- Map<Long, Set<BindingSet>> expected = new HashMap<>();
-
- Set<BindingSet> expected1 = new HashSet<>();
- QueryBindingSet bs1 = new QueryBindingSet();
- bs1.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(ids.get(0)));
- bs1.addBinding("total", new LiteralImpl("2", XMLSchema.INTEGER));
- bs1.addBinding("type", vf.createLiteral("airplane"));
-
- QueryBindingSet bs2 = new QueryBindingSet();
- bs2.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(ids.get(0)));
- bs2.addBinding("total", new LiteralImpl("2", XMLSchema.INTEGER));
- bs2.addBinding("type", vf.createLiteral("ship"));
-
- QueryBindingSet bs3 = new QueryBindingSet();
- bs3.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(ids.get(0)));
- bs3.addBinding("total", new LiteralImpl("1", XMLSchema.INTEGER));
- bs3.addBinding("type", vf.createLiteral("automobile"));
-
- expected1.add(bs1);
- expected1.add(bs2);
- expected1.add(bs3);
-
- Set<BindingSet> expected2 = new HashSet<>();
- QueryBindingSet bs4 = new QueryBindingSet();
- bs4.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(ids.get(1)));
- bs4.addBinding("total", new LiteralImpl("2", XMLSchema.INTEGER));
- bs4.addBinding("type", vf.createLiteral("airplane"));
-
- QueryBindingSet bs5 = new QueryBindingSet();
- bs5.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(ids.get(1)));
- bs5.addBinding("total", new LiteralImpl("2", XMLSchema.INTEGER));
- bs5.addBinding("type", vf.createLiteral("ship"));
-
- expected2.add(bs4);
- expected2.add(bs5);
-
- Set<BindingSet> expected3 = new HashSet<>();
- QueryBindingSet bs6 = new QueryBindingSet();
- bs6.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(ids.get(2)));
- bs6.addBinding("total", new LiteralImpl("1", XMLSchema.INTEGER));
- bs6.addBinding("type", vf.createLiteral("ship"));
-
- QueryBindingSet bs7 = new QueryBindingSet();
- bs7.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(ids.get(2)));
- bs7.addBinding("total", new LiteralImpl("1", XMLSchema.INTEGER));
- bs7.addBinding("type", vf.createLiteral("airplane"));
-
- expected3.add(bs6);
- expected3.add(bs7);
-
- expected.put(ids.get(0), expected1);
- expected.put(ids.get(1), expected2);
- expected.put(ids.get(2), expected3);
-
- Assert.assertEquals(3, actual.asMap().size());
- for(Long ident: ids) {
- Assert.assertEquals(expected.get(ident), actual.get(ident));
- }
- }
-
- Set<BindingSet> expectedResults = new HashSet<>();
- try (CloseableIterator<BindingSet> results = storage.listResults(id, Optional.empty())) {
- results.forEachRemaining(x -> expectedResults.add(x));
- Assert.assertEquals(0, expectedResults.size());
- }
- }
- }
-
-
- @Test
- public void periodicApplicationWithAggTest() throws Exception {
-
- String sparql = "prefix function: <http://org.apache.rya/function#> " // n
- + "prefix time: <http://www.w3.org/2006/time#> " // n
- + "select (count(?obs) as ?total) where {" // n
- + "Filter(function:periodic(?time, 1, .25, time:minutes)) " // n
- + "?obs <uri:hasTime> ?time. " // n
- + "?obs <uri:hasId> ?id } "; // n
-
- //make data
- int periodMult = 15;
- final ValueFactory vf = new ValueFactoryImpl();
- final DatatypeFactory dtf = DatatypeFactory.newInstance();
- //Sleep until current time aligns nicely with period to make
- //results more predictable
- while(System.currentTimeMillis() % (periodMult*1000) > 500);
- ZonedDateTime time = ZonedDateTime.now();
-
- ZonedDateTime zTime1 = time.minusSeconds(2*periodMult);
- String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT);
-
- ZonedDateTime zTime2 = zTime1.minusSeconds(periodMult);
- String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT);
-
- ZonedDateTime zTime3 = zTime2.minusSeconds(periodMult);
- String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT);
-
- final Collection<Statement> statements = Sets.newHashSet(
- vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"),
- vf.createLiteral(dtf.newXMLGregorianCalendar(time1))),
- vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasId"), vf.createLiteral("id_1")),
- vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasTime"),
- vf.createLiteral(dtf.newXMLGregorianCalendar(time2))),
- vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasId"), vf.createLiteral("id_2")),
- vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasTime"),
- vf.createLiteral(dtf.newXMLGregorianCalendar(time3))),
- vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasId"), vf.createLiteral("id_3")));
-
- try (FluoClient fluo = FluoClientFactory.getFluoClient(conf.getFluoAppName(), Optional.of(conf.getFluoTableName()), conf)) {
- Connector connector = ConfigUtils.getConnector(conf);
- PeriodicQueryResultStorage storage = new AccumuloPeriodicQueryResultStorage(connector, conf.getTablePrefix());
- CreatePeriodicQuery periodicQuery = new CreatePeriodicQuery(fluo, storage);
- String id = FluoQueryUtils.convertFluoQueryIdToPcjId(periodicQuery.createPeriodicQuery(sparql, registrar).getQueryId());
- addData(statements);
- app.start();
-
- Multimap<Long, BindingSet> expected = HashMultimap.create();
- try (KafkaConsumer<String, BindingSet> consumer = new KafkaConsumer<>(kafkaProps, new StringDeserializer(), new BindingSetSerDe())) {
- consumer.subscribe(Arrays.asList(id));
- long end = System.currentTimeMillis() + 4*periodMult*1000;
- long lastBinId = 0L;
- long binId = 0L;
- List<Long> ids = new ArrayList<>();
- while (System.currentTimeMillis() < end) {
- ConsumerRecords<String, BindingSet> records = consumer.poll(periodMult*1000);
- for(ConsumerRecord<String, BindingSet> record: records){
- BindingSet result = record.value();
- binId = Long.parseLong(result.getBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID).getValue().stringValue());
- if(lastBinId != binId) {
- lastBinId = binId;
- ids.add(binId);
- }
- expected.put(binId, result);
- }
- }
-
- Assert.assertEquals(3, expected.asMap().size());
- int i = 0;
- for(Long ident: ids) {
- Assert.assertEquals(1, expected.get(ident).size());
- BindingSet bs = expected.get(ident).iterator().next();
- Value val = bs.getValue("total");
- int total = Integer.parseInt(val.stringValue());
- Assert.assertEquals(3-i, total);
- i++;
- }
- }
-
-
- Set<BindingSet> expectedResults = new HashSet<>();
- try (CloseableIterator<BindingSet> results = storage.listResults(id, Optional.empty())) {
- results.forEachRemaining(x -> expectedResults.add(x));
- Assert.assertEquals(0, expectedResults.size());
- }
- }
-
- }
-
-
- @Test
- public void periodicApplicationTest() throws Exception {
-
- String sparql = "prefix function: <http://org.apache.rya/function#> " // n
- + "prefix time: <http://www.w3.org/2006/time#> " // n
- + "select ?obs ?id where {" // n
- + "Filter(function:periodic(?time, 1, .25, time:minutes)) " // n
- + "?obs <uri:hasTime> ?time. " // n
- + "?obs <uri:hasId> ?id } "; // n
-
- //make data
- int periodMult = 15;
- final ValueFactory vf = new ValueFactoryImpl();
- final DatatypeFactory dtf = DatatypeFactory.newInstance();
- //Sleep until current time aligns nicely with period to make
- //results more predictable
- while(System.currentTimeMillis() % (periodMult*1000) > 500);
- ZonedDateTime time = ZonedDateTime.now();
-
- ZonedDateTime zTime1 = time.minusSeconds(2*periodMult);
- String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT);
-
- ZonedDateTime zTime2 = zTime1.minusSeconds(periodMult);
- String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT);
-
- ZonedDateTime zTime3 = zTime2.minusSeconds(periodMult);
- String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT);
-
- final Collection<Statement> statements = Sets.newHashSet(
- vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"),
- vf.createLiteral(dtf.newXMLGregorianCalendar(time1))),
- vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasId"), vf.createLiteral("id_1")),
- vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasTime"),
- vf.createLiteral(dtf.newXMLGregorianCalendar(time2))),
- vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasId"), vf.createLiteral("id_2")),
- vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasTime"),
- vf.createLiteral(dtf.newXMLGregorianCalendar(time3))),
- vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasId"), vf.createLiteral("id_3")));
-
- try (FluoClient fluo = FluoClientFactory.getFluoClient(conf.getFluoAppName(), Optional.of(conf.getFluoTableName()), conf)) {
- Connector connector = ConfigUtils.getConnector(conf);
- PeriodicQueryResultStorage storage = new AccumuloPeriodicQueryResultStorage(connector, conf.getTablePrefix());
- CreatePeriodicQuery periodicQuery = new CreatePeriodicQuery(fluo, storage);
- String id = FluoQueryUtils.convertFluoQueryIdToPcjId(periodicQuery.createPeriodicQuery(sparql, registrar).getQueryId());
- addData(statements);
- app.start();
-
- Multimap<Long, BindingSet> expected = HashMultimap.create();
- try (KafkaConsumer<String, BindingSet> consumer = new KafkaConsumer<>(kafkaProps, new StringDeserializer(), new BindingSetSerDe())) {
- consumer.subscribe(Arrays.asList(id));
- long end = System.currentTimeMillis() + 4*periodMult*1000;
- long lastBinId = 0L;
- long binId = 0L;
- List<Long> ids = new ArrayList<>();
- while (System.currentTimeMillis() < end) {
- ConsumerRecords<String, BindingSet> records = consumer.poll(periodMult*1000);
- for(ConsumerRecord<String, BindingSet> record: records){
- BindingSet result = record.value();
- binId = Long.parseLong(result.getBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID).getValue().stringValue());
- if(lastBinId != binId) {
- lastBinId = binId;
- ids.add(binId);
- }
- expected.put(binId, result);
- }
- }
-
- Assert.assertEquals(3, expected.asMap().size());
- int i = 0;
- for(Long ident: ids) {
- Assert.assertEquals(3-i, expected.get(ident).size());
- i++;
- }
- }
-
-
- Set<BindingSet> expectedResults = new HashSet<>();
- try (CloseableIterator<BindingSet> results = storage.listResults(id, Optional.empty())) {
- results.forEachRemaining(x -> expectedResults.add(x));
- Assert.assertEquals(0, expectedResults.size());
- }
- }
-
- }
-
-
- @After
- public void shutdown() {
- registrar.close();
- app.stop();
- }
-
- private void addData(Collection<Statement> statements) throws DatatypeConfigurationException {
- // add statements to Fluo
- try (FluoClient fluo = new FluoClientImpl(getFluoConfiguration())) {
- InsertTriples inserter = new InsertTriples();
- statements.forEach(x -> inserter.insert(fluo, RdfToRyaConversions.convertStatement(x)));
- getMiniFluo().waitForObservers();
- }
- }
-
- private static Properties getKafkaProperties(PeriodicNotificationApplicationConfiguration conf) {
- Properties kafkaProps = new Properties();
- kafkaProps.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
- kafkaProps.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, UUID.randomUUID().toString());
- kafkaProps.setProperty(ConsumerConfig.GROUP_ID_CONFIG, conf.getNotificationGroupId());
- kafkaProps.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
- return kafkaProps;
- }
-
- private Properties getProps() throws IOException {
-
- Properties props = new Properties();
- try(InputStream in = new FileInputStream("src/test/resources/notification.properties")) {
- props.load(in);
- }
-
- FluoConfiguration fluoConf = getFluoConfiguration();
- props.setProperty("accumulo.user", getUsername());
- props.setProperty("accumulo.password", getPassword());
- props.setProperty("accumulo.instance", getMiniAccumuloCluster().getInstanceName());
- props.setProperty("accumulo.zookeepers", getMiniAccumuloCluster().getZooKeepers());
- props.setProperty("accumulo.rya.prefix", getRyaInstanceName());
- props.setProperty(PeriodicNotificationApplicationConfiguration.FLUO_APP_NAME, fluoConf.getApplicationName());
- props.setProperty(PeriodicNotificationApplicationConfiguration.FLUO_TABLE_NAME, fluoConf.getAccumuloTable());
- return props;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/application/PeriodicNotificationProviderIT.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/application/PeriodicNotificationProviderIT.java b/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/application/PeriodicNotificationProviderIT.java
deleted file mode 100644
index e05ca6f..0000000
--- a/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/application/PeriodicNotificationProviderIT.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.application;
-
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.fluo.api.client.FluoClient;
-import org.apache.fluo.core.client.FluoClientImpl;
-import org.apache.fluo.recipes.test.AccumuloExportITBase;
-import org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj;
-import org.apache.rya.indexing.pcj.fluo.app.query.UnsupportedQueryException;
-import org.apache.rya.indexing.pcj.fluo.app.util.FluoQueryUtils;
-import org.apache.rya.periodic.notification.coordinator.PeriodicNotificationCoordinatorExecutor;
-import org.apache.rya.periodic.notification.notification.TimestampedNotification;
-import org.apache.rya.periodic.notification.recovery.PeriodicNotificationProvider;
-import org.junit.Assert;
-import org.junit.Test;
-import org.openrdf.query.MalformedQueryException;
-
-import com.google.common.collect.Sets;
-
-public class PeriodicNotificationProviderIT extends AccumuloExportITBase {
-
- @Test
- public void testProvider() throws MalformedQueryException, InterruptedException, UnsupportedQueryException {
-
- String sparql = "prefix function: <http://org.apache.rya/function#> " // n
- + "prefix time: <http://www.w3.org/2006/time#> " // n
- + "select ?id (count(?obs) as ?total) where {" // n
- + "Filter(function:periodic(?time, 1, .25, time:minutes)) " // n
- + "?obs <uri:hasTime> ?time. " // n
- + "?obs <uri:hasId> ?id } group by ?id"; // n
-
- BlockingQueue<TimestampedNotification> notifications = new LinkedBlockingQueue<>();
- PeriodicNotificationCoordinatorExecutor coord = new PeriodicNotificationCoordinatorExecutor(2, notifications);
- PeriodicNotificationProvider provider = new PeriodicNotificationProvider();
- CreateFluoPcj pcj = new CreateFluoPcj();
-
- String id = null;
- try(FluoClient fluo = new FluoClientImpl(getFluoConfiguration())) {
- id = pcj.createPcj(FluoQueryUtils.createNewPcjId(), sparql, Sets.newHashSet(), fluo).getQueryId();
- provider.processRegisteredNotifications(coord, fluo.newSnapshot());
- }
-
- TimestampedNotification notification = notifications.take();
- Assert.assertEquals(5000, notification.getInitialDelay());
- Assert.assertEquals(15000, notification.getPeriod());
- Assert.assertEquals(TimeUnit.MILLISECONDS, notification.getTimeUnit());
- Assert.assertEquals(FluoQueryUtils.convertFluoQueryIdToPcjId(id), notification.getId());
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/exporter/PeriodicNotificationExporterIT.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/exporter/PeriodicNotificationExporterIT.java b/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/exporter/PeriodicNotificationExporterIT.java
deleted file mode 100644
index 874e7e2..0000000
--- a/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/exporter/PeriodicNotificationExporterIT.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.exporter;
-
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Properties;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import org.apache.kafka.clients.consumer.ConsumerConfig;
-import org.apache.kafka.clients.consumer.ConsumerRecords;
-import org.apache.kafka.clients.consumer.KafkaConsumer;
-import org.apache.kafka.clients.producer.KafkaProducer;
-import org.apache.kafka.clients.producer.ProducerConfig;
-import org.apache.kafka.common.serialization.StringDeserializer;
-import org.apache.kafka.common.serialization.StringSerializer;
-import org.apache.rya.indexing.pcj.storage.PeriodicQueryResultStorage;
-import org.apache.rya.kafka.base.KafkaITBase;
-import org.apache.rya.kafka.base.KafkaTestInstanceRule;
-import org.apache.rya.periodic.notification.api.BindingSetRecord;
-import org.apache.rya.periodic.notification.serialization.BindingSetSerDe;
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.openrdf.model.ValueFactory;
-import org.openrdf.model.impl.ValueFactoryImpl;
-import org.openrdf.query.BindingSet;
-import org.openrdf.query.algebra.evaluation.QueryBindingSet;
-
-public class PeriodicNotificationExporterIT extends KafkaITBase {
-
-
- @Rule
- public KafkaTestInstanceRule kafkaTestInstanceRule = new KafkaTestInstanceRule(false);
-
-
- private static final ValueFactory vf = new ValueFactoryImpl();
-
- @Test
- public void testExporter() throws InterruptedException {
-
- final String topic1 = kafkaTestInstanceRule.getKafkaTopicName() + "1";
- final String topic2 = kafkaTestInstanceRule.getKafkaTopicName() + "2";
-
- kafkaTestInstanceRule.createTopic(topic1);
- kafkaTestInstanceRule.createTopic(topic2);
-
- final BlockingQueue<BindingSetRecord> records = new LinkedBlockingQueue<>();
-
- final KafkaExporterExecutor exporter = new KafkaExporterExecutor(new KafkaProducer<String, BindingSet>(createKafkaProducerConfig()), 1, records);
- exporter.start();
- final QueryBindingSet bs1 = new QueryBindingSet();
- bs1.addBinding(PeriodicQueryResultStorage.PeriodicBinId, vf.createLiteral(1L));
- bs1.addBinding("name", vf.createURI("uri:Bob"));
- final BindingSetRecord record1 = new BindingSetRecord(bs1, topic1);
-
- final QueryBindingSet bs2 = new QueryBindingSet();
- bs2.addBinding(PeriodicQueryResultStorage.PeriodicBinId, vf.createLiteral(2L));
- bs2.addBinding("name", vf.createURI("uri:Joe"));
- final BindingSetRecord record2 = new BindingSetRecord(bs2, topic2);
-
- records.add(record1);
- records.add(record2);
-
- final Set<BindingSet> expected1 = new HashSet<>();
- expected1.add(bs1);
- final Set<BindingSet> expected2 = new HashSet<>();
- expected2.add(bs2);
-
- final Set<BindingSet> actual1 = getBindingSetsFromKafka(topic1);
- final Set<BindingSet> actual2 = getBindingSetsFromKafka(topic2);
-
- Assert.assertEquals(expected1, actual1);
- Assert.assertEquals(expected2, actual2);
-
- exporter.stop();
- }
-
-
- private Properties createKafkaProducerConfig() {
- final Properties props = createBootstrapServerConfig();
- props.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
- props.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, BindingSetSerDe.class.getName());
- return props;
- }
- private Properties createKafkaConsumerConfig() {
- final Properties props = createBootstrapServerConfig();
- props.setProperty(ConsumerConfig.GROUP_ID_CONFIG, UUID.randomUUID().toString());
- props.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, "consumer0");
- props.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
- props.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
- props.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, BindingSetSerDe.class.getName());
- return props;
- }
-
-
- private KafkaConsumer<String, BindingSet> makeBindingSetConsumer(final String topicName) {
- // setup consumer
- final KafkaConsumer<String, BindingSet> consumer = new KafkaConsumer<>(createKafkaConsumerConfig());
- consumer.subscribe(Arrays.asList(topicName));
- return consumer;
- }
-
- private Set<BindingSet> getBindingSetsFromKafka(final String topicName) {
- KafkaConsumer<String, BindingSet> consumer = null;
-
- try {
- consumer = makeBindingSetConsumer(topicName);
- final ConsumerRecords<String, BindingSet> records = consumer.poll(20000); // Wait up to 20 seconds for a result to be published.
-
- final Set<BindingSet> bindingSets = new HashSet<>();
- records.forEach(x -> bindingSets.add(x.value()));
-
- return bindingSets;
-
- } catch (final Exception e) {
- throw new RuntimeException(e);
- } finally {
- if (consumer != null) {
- consumer.close();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/processor/PeriodicNotificationProcessorIT.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/processor/PeriodicNotificationProcessorIT.java b/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/processor/PeriodicNotificationProcessorIT.java
deleted file mode 100644
index 21109ae..0000000
--- a/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/processor/PeriodicNotificationProcessorIT.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.processor;
-
-import java.util.HashSet;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.fluo.recipes.test.AccumuloExportITBase;
-import org.apache.rya.indexing.pcj.storage.PeriodicQueryResultStorage;
-import org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPeriodicQueryResultStorage;
-import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
-import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
-import org.apache.rya.periodic.notification.api.BindingSetRecord;
-import org.apache.rya.periodic.notification.api.NodeBin;
-import org.apache.rya.periodic.notification.notification.PeriodicNotification;
-import org.apache.rya.periodic.notification.notification.TimestampedNotification;
-import org.junit.Assert;
-import org.junit.Test;
-import org.openrdf.model.ValueFactory;
-import org.openrdf.model.impl.ValueFactoryImpl;
-import org.openrdf.query.BindingSet;
-import org.openrdf.query.algebra.evaluation.QueryBindingSet;
-
-public class PeriodicNotificationProcessorIT extends AccumuloExportITBase {
-
- private static final ValueFactory vf = new ValueFactoryImpl();
- private static final String RYA_INSTANCE_NAME = "rya_";
-
- @Test
- public void periodicProcessorTest() throws Exception {
-
- String id = UUID.randomUUID().toString().replace("-", "");
- BlockingQueue<TimestampedNotification> notifications = new LinkedBlockingQueue<>();
- BlockingQueue<NodeBin> bins = new LinkedBlockingQueue<>();
- BlockingQueue<BindingSetRecord> bindingSets = new LinkedBlockingQueue<>();
-
- TimestampedNotification ts1 = new TimestampedNotification(
- PeriodicNotification.builder().id(id).initialDelay(0).period(2000).timeUnit(TimeUnit.SECONDS).build());
- long binId1 = (ts1.getTimestamp().getTime()/ts1.getPeriod())*ts1.getPeriod();
-
- Thread.sleep(2000);
-
- TimestampedNotification ts2 = new TimestampedNotification(
- PeriodicNotification.builder().id(id).initialDelay(0).period(2000).timeUnit(TimeUnit.SECONDS).build());
- long binId2 = (ts2.getTimestamp().getTime()/ts2.getPeriod())*ts2.getPeriod();
-
- Set<NodeBin> expectedBins = new HashSet<>();
- expectedBins.add(new NodeBin(id, binId1));
- expectedBins.add(new NodeBin(id, binId2));
-
- Set<BindingSet> expected = new HashSet<>();
- Set<VisibilityBindingSet> storageResults = new HashSet<>();
-
- QueryBindingSet bs1 = new QueryBindingSet();
- bs1.addBinding("periodicBinId", vf.createLiteral(binId1));
- bs1.addBinding("id", vf.createLiteral(1));
- expected.add(bs1);
- storageResults.add(new VisibilityBindingSet(bs1));
-
- QueryBindingSet bs2 = new QueryBindingSet();
- bs2.addBinding("periodicBinId", vf.createLiteral(binId1));
- bs2.addBinding("id", vf.createLiteral(2));
- expected.add(bs2);
- storageResults.add(new VisibilityBindingSet(bs2));
-
- QueryBindingSet bs3 = new QueryBindingSet();
- bs3.addBinding("periodicBinId", vf.createLiteral(binId2));
- bs3.addBinding("id", vf.createLiteral(3));
- expected.add(bs3);
- storageResults.add(new VisibilityBindingSet(bs3));
-
- QueryBindingSet bs4 = new QueryBindingSet();
- bs4.addBinding("periodicBinId", vf.createLiteral(binId2));
- bs4.addBinding("id", vf.createLiteral(4));
- expected.add(bs4);
- storageResults.add(new VisibilityBindingSet(bs4));
-
- PeriodicQueryResultStorage periodicStorage = new AccumuloPeriodicQueryResultStorage(super.getAccumuloConnector(),
- RYA_INSTANCE_NAME);
- periodicStorage.createPeriodicQuery(id, "select ?id where {?obs <urn:hasId> ?id.}", new VariableOrder("periodicBinId", "id"));
- periodicStorage.addPeriodicQueryResults(id, storageResults);
-
- NotificationProcessorExecutor processor = new NotificationProcessorExecutor(periodicStorage, notifications, bins, bindingSets, 1);
- processor.start();
-
- notifications.add(ts1);
- notifications.add(ts2);
-
- Thread.sleep(5000);
-
- Assert.assertEquals(expectedBins.size(), bins.size());
- Assert.assertEquals(true, bins.containsAll(expectedBins));
-
- Set<BindingSet> actual = new HashSet<>();
- bindingSets.forEach(x -> actual.add(x.getBindingSet()));
- Assert.assertEquals(expected, actual);
-
- processor.stop();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/pruner/PeriodicNotificationBinPrunerIT.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/pruner/PeriodicNotificationBinPrunerIT.java b/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/pruner/PeriodicNotificationBinPrunerIT.java
deleted file mode 100644
index 830fa46..0000000
--- a/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/pruner/PeriodicNotificationBinPrunerIT.java
+++ /dev/null
@@ -1,283 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.pruner;
-
-import java.time.ZonedDateTime;
-import java.time.format.DateTimeFormatter;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import javax.xml.datatype.DatatypeFactory;
-
-import org.apache.fluo.api.client.FluoClient;
-import org.apache.fluo.api.client.Snapshot;
-import org.apache.fluo.api.client.scanner.ColumnScanner;
-import org.apache.fluo.api.client.scanner.RowScanner;
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.api.data.ColumnValue;
-import org.apache.fluo.api.data.Span;
-import org.apache.fluo.core.client.FluoClientImpl;
-import org.apache.rya.api.resolver.RdfToRyaConversions;
-import org.apache.rya.indexing.pcj.fluo.api.CreatePeriodicQuery;
-import org.apache.rya.indexing.pcj.fluo.api.InsertTriples;
-import org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants;
-import org.apache.rya.indexing.pcj.fluo.app.NodeType;
-import org.apache.rya.indexing.pcj.fluo.app.util.FluoQueryUtils;
-import org.apache.rya.indexing.pcj.fluo.app.util.PeriodicQueryUtil;
-import org.apache.rya.indexing.pcj.fluo.app.util.RowKeyUtil;
-import org.apache.rya.indexing.pcj.storage.PeriodicQueryResultStorage;
-import org.apache.rya.indexing.pcj.storage.PeriodicQueryStorageException;
-import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
-import org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPeriodicQueryResultStorage;
-import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
-import org.apache.rya.pcj.fluo.test.base.RyaExportITBase;
-import org.apache.rya.periodic.notification.api.NodeBin;
-import org.junit.Assert;
-import org.junit.Test;
-import org.openrdf.model.Statement;
-import org.openrdf.model.ValueFactory;
-import org.openrdf.model.impl.LiteralImpl;
-import org.openrdf.model.impl.ValueFactoryImpl;
-import org.openrdf.model.vocabulary.XMLSchema;
-import org.openrdf.query.BindingSet;
-import org.openrdf.query.algebra.evaluation.QueryBindingSet;
-import org.openrdf.query.impl.MapBindingSet;
-
-import com.google.common.collect.Sets;
-
-public class PeriodicNotificationBinPrunerIT extends RyaExportITBase {
-
-
- @Test
- public void periodicPrunerTest() throws Exception {
-
- String sparql = "prefix function: <http://org.apache.rya/function#> " // n
- + "prefix time: <http://www.w3.org/2006/time#> " // n
- + "select ?id (count(?obs) as ?total) where {" // n
- + "Filter(function:periodic(?time, 2, .5, time:hours)) " // n
- + "?obs <uri:hasTime> ?time. " // n
- + "?obs <uri:hasId> ?id } group by ?id"; // n
-
- FluoClient fluo = new FluoClientImpl(super.getFluoConfiguration());
-
- // initialize resources and create pcj
- PeriodicQueryResultStorage periodicStorage = new AccumuloPeriodicQueryResultStorage(super.getAccumuloConnector(),
- getRyaInstanceName());
- CreatePeriodicQuery createPeriodicQuery = new CreatePeriodicQuery(fluo, periodicStorage);
- String queryId = FluoQueryUtils.convertFluoQueryIdToPcjId(createPeriodicQuery.createPeriodicQuery(sparql).getQueryId());
-
- // create statements to ingest into Fluo
- final ValueFactory vf = new ValueFactoryImpl();
- final DatatypeFactory dtf = DatatypeFactory.newInstance();
- ZonedDateTime time = ZonedDateTime.now();
- long currentTime = time.toInstant().toEpochMilli();
-
- ZonedDateTime zTime1 = time.minusMinutes(30);
- String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT);
-
- ZonedDateTime zTime2 = zTime1.minusMinutes(30);
- String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT);
-
- ZonedDateTime zTime3 = zTime2.minusMinutes(30);
- String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT);
-
- ZonedDateTime zTime4 = zTime3.minusMinutes(30);
- String time4 = zTime4.format(DateTimeFormatter.ISO_INSTANT);
-
- final Collection<Statement> statements = Sets.newHashSet(
- vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"),
- vf.createLiteral(dtf.newXMLGregorianCalendar(time1))),
- vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasId"), vf.createLiteral("id_1")),
- vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasTime"),
- vf.createLiteral(dtf.newXMLGregorianCalendar(time2))),
- vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasId"), vf.createLiteral("id_2")),
- vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasTime"),
- vf.createLiteral(dtf.newXMLGregorianCalendar(time3))),
- vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasId"), vf.createLiteral("id_3")),
- vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasTime"),
- vf.createLiteral(dtf.newXMLGregorianCalendar(time4))),
- vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasId"), vf.createLiteral("id_4")),
- vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"),
- vf.createLiteral(dtf.newXMLGregorianCalendar(time4))),
- vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasId"), vf.createLiteral("id_1")),
- vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasTime"),
- vf.createLiteral(dtf.newXMLGregorianCalendar(time3))),
- vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasId"), vf.createLiteral("id_2")));
-
- // add statements to Fluo
- InsertTriples inserter = new InsertTriples();
- statements.forEach(x -> inserter.insert(fluo, RdfToRyaConversions.convertStatement(x)));
-
- super.getMiniFluo().waitForObservers();
-
- // FluoITHelper.printFluoTable(fluo);
-
- // Create the expected results of the SPARQL query once the PCJ has been
- // computed.
- final Set<BindingSet> expected1 = new HashSet<>();
- final Set<BindingSet> expected2 = new HashSet<>();
- final Set<BindingSet> expected3 = new HashSet<>();
- final Set<BindingSet> expected4 = new HashSet<>();
-
- long period = 1800000;
- long binId = (currentTime / period) * period;
-
- long bin1 = binId;
- long bin2 = binId + period;
- long bin3 = binId + 2 * period;
- long bin4 = binId + 3 * period;
-
- MapBindingSet bs = new MapBindingSet();
- bs.addBinding("total", vf.createLiteral("2", XMLSchema.INTEGER));
- bs.addBinding("id", vf.createLiteral("id_1", XMLSchema.STRING));
- bs.addBinding("periodicBinId", vf.createLiteral(bin1));
- expected1.add(bs);
-
- bs = new MapBindingSet();
- bs.addBinding("total", vf.createLiteral("2", XMLSchema.INTEGER));
- bs.addBinding("id", vf.createLiteral("id_2", XMLSchema.STRING));
- bs.addBinding("periodicBinId", vf.createLiteral(bin1));
- expected1.add(bs);
-
- bs = new MapBindingSet();
- bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
- bs.addBinding("id", vf.createLiteral("id_3", XMLSchema.STRING));
- bs.addBinding("periodicBinId", vf.createLiteral(bin1));
- expected1.add(bs);
-
- bs = new MapBindingSet();
- bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
- bs.addBinding("id", vf.createLiteral("id_4", XMLSchema.STRING));
- bs.addBinding("periodicBinId", vf.createLiteral(bin1));
- expected1.add(bs);
-
- bs = new MapBindingSet();
- bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
- bs.addBinding("id", vf.createLiteral("id_1", XMLSchema.STRING));
- bs.addBinding("periodicBinId", vf.createLiteral(bin2));
- expected2.add(bs);
-
- bs = new MapBindingSet();
- bs.addBinding("total", vf.createLiteral("2", XMLSchema.INTEGER));
- bs.addBinding("id", vf.createLiteral("id_2", XMLSchema.STRING));
- bs.addBinding("periodicBinId", vf.createLiteral(bin2));
- expected2.add(bs);
-
- bs = new MapBindingSet();
- bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
- bs.addBinding("id", vf.createLiteral("id_3", XMLSchema.STRING));
- bs.addBinding("periodicBinId", vf.createLiteral(bin2));
- expected2.add(bs);
-
- bs = new MapBindingSet();
- bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
- bs.addBinding("id", vf.createLiteral("id_1", XMLSchema.STRING));
- bs.addBinding("periodicBinId", vf.createLiteral(bin3));
- expected3.add(bs);
-
- bs = new MapBindingSet();
- bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
- bs.addBinding("id", vf.createLiteral("id_2", XMLSchema.STRING));
- bs.addBinding("periodicBinId", vf.createLiteral(bin3));
- expected3.add(bs);
-
- bs = new MapBindingSet();
- bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
- bs.addBinding("id", vf.createLiteral("id_1", XMLSchema.STRING));
- bs.addBinding("periodicBinId", vf.createLiteral(bin4));
- expected4.add(bs);
-
- // make sure that expected and actual results align after ingest
- compareResults(periodicStorage, queryId, bin1, expected1);
- compareResults(periodicStorage, queryId, bin2, expected2);
- compareResults(periodicStorage, queryId, bin3, expected3);
- compareResults(periodicStorage, queryId, bin4, expected4);
-
- BlockingQueue<NodeBin> bins = new LinkedBlockingQueue<>();
- PeriodicQueryPrunerExecutor pruner = new PeriodicQueryPrunerExecutor(periodicStorage, fluo, 1, bins);
- pruner.start();
-
- bins.add(new NodeBin(queryId, bin1));
- bins.add(new NodeBin(queryId, bin2));
- bins.add(new NodeBin(queryId, bin3));
- bins.add(new NodeBin(queryId, bin4));
-
- Thread.sleep(10000);
-
- compareResults(periodicStorage, queryId, bin1, new HashSet<>());
- compareResults(periodicStorage, queryId, bin2, new HashSet<>());
- compareResults(periodicStorage, queryId, bin3, new HashSet<>());
- compareResults(periodicStorage, queryId, bin4, new HashSet<>());
-
- compareFluoCounts(fluo, queryId, bin1);
- compareFluoCounts(fluo, queryId, bin2);
- compareFluoCounts(fluo, queryId, bin3);
- compareFluoCounts(fluo, queryId, bin4);
-
- pruner.stop();
-
- }
-
- private void compareResults(PeriodicQueryResultStorage periodicStorage, String queryId, long bin, Set<BindingSet> expected) throws PeriodicQueryStorageException, Exception {
- try(CloseableIterator<BindingSet> iter = periodicStorage.listResults(queryId, Optional.of(bin))) {
- Set<BindingSet> actual = new HashSet<>();
- while(iter.hasNext()) {
- actual.add(iter.next());
- }
- Assert.assertEquals(expected, actual);
- }
- }
-
- private void compareFluoCounts(FluoClient client, String pcjId, long bin) {
- QueryBindingSet bs = new QueryBindingSet();
- bs.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, new LiteralImpl(Long.toString(bin), XMLSchema.LONG));
-
- VariableOrder varOrder = new VariableOrder(IncrementalUpdateConstants.PERIODIC_BIN_ID);
-
- try(Snapshot sx = client.newSnapshot()) {
- String fluoQueryId = NodeType.generateNewIdForType(NodeType.QUERY, pcjId);
- Set<String> ids = new HashSet<>();
- PeriodicQueryUtil.getPeriodicQueryNodeAncestorIds(sx, fluoQueryId, ids);
- for(String id: ids) {
- NodeType optNode = NodeType.fromNodeId(id).orNull();
- if(optNode == null) throw new RuntimeException("Invalid NodeType.");
- Bytes prefix = RowKeyUtil.makeRowKey(id,varOrder, bs);
- RowScanner scanner = sx.scanner().fetch(optNode.getResultColumn()).over(Span.prefix(prefix)).byRow().build();
- int count = 0;
- Iterator<ColumnScanner> colScannerIter = scanner.iterator();
- while(colScannerIter.hasNext()) {
- ColumnScanner colScanner = colScannerIter.next();
- String row = colScanner.getRow().toString();
- Iterator<ColumnValue> values = colScanner.iterator();
- while(values.hasNext()) {
- values.next();
- count++;
- }
- }
- Assert.assertEquals(0, count);
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/registration/kafka/PeriodicCommandNotificationConsumerIT.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/registration/kafka/PeriodicCommandNotificationConsumerIT.java b/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/registration/kafka/PeriodicCommandNotificationConsumerIT.java
deleted file mode 100644
index 522e69d..0000000
--- a/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/registration/kafka/PeriodicCommandNotificationConsumerIT.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */package org.apache.rya.periodic.notification.registration.kafka;
-
-import java.util.Properties;
-import java.util.UUID;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.kafka.clients.CommonClientConfigs;
-import org.apache.kafka.clients.consumer.ConsumerConfig;
-import org.apache.kafka.clients.producer.KafkaProducer;
-import org.apache.kafka.clients.producer.ProducerConfig;
-import org.apache.kafka.common.serialization.StringDeserializer;
-import org.apache.kafka.common.serialization.StringSerializer;
-import org.apache.log4j.BasicConfigurator;
-import org.apache.rya.kafka.base.KafkaITBase;
-import org.apache.rya.kafka.base.KafkaTestInstanceRule;
-import org.apache.rya.periodic.notification.coordinator.PeriodicNotificationCoordinatorExecutor;
-import org.apache.rya.periodic.notification.notification.CommandNotification;
-import org.apache.rya.periodic.notification.notification.TimestampedNotification;
-import org.apache.rya.periodic.notification.registration.KafkaNotificationRegistrationClient;
-import org.apache.rya.periodic.notification.serialization.CommandNotificationSerializer;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-
-public class PeriodicCommandNotificationConsumerIT extends KafkaITBase {
-
- private KafkaNotificationRegistrationClient registration;
- private PeriodicNotificationCoordinatorExecutor coord;
- private KafkaNotificationProvider provider;
- private String bootstrapServer;
-
- @Rule
- public KafkaTestInstanceRule rule = new KafkaTestInstanceRule(false);
-
- @Before
- public void init() throws Exception {
- bootstrapServer = createBootstrapServerConfig().getProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG);
- }
-
- @Test
- public void kafkaNotificationProviderTest() throws InterruptedException {
-
- BasicConfigurator.configure();
-
- BlockingQueue<TimestampedNotification> notifications = new LinkedBlockingQueue<>();
- Properties props = createKafkaConfig();
- KafkaProducer<String, CommandNotification> producer = new KafkaProducer<>(props);
- String topic = rule.getKafkaTopicName();
- rule.createTopic(topic);
-
- registration = new KafkaNotificationRegistrationClient(topic, producer);
- coord = new PeriodicNotificationCoordinatorExecutor(1, notifications);
- provider = new KafkaNotificationProvider(topic, new StringDeserializer(), new CommandNotificationSerializer(), props, coord, 1);
- provider.start();
-
- registration.addNotification("1", 1, 0, TimeUnit.SECONDS);
- Thread.sleep(4000);
- // check that notifications are being added to the blocking queue
- Assert.assertEquals(true, notifications.size() > 0);
-
- registration.deleteNotification("1");
- Thread.sleep(2000);
- int size = notifications.size();
- // sleep for 2 seconds to ensure no more messages being produced
- Thread.sleep(2000);
- Assert.assertEquals(size, notifications.size());
-
- tearDown();
- }
-
- @Test
- public void kafkaNotificationMillisProviderTest() throws InterruptedException {
-
- BasicConfigurator.configure();
-
- BlockingQueue<TimestampedNotification> notifications = new LinkedBlockingQueue<>();
- Properties props = createKafkaConfig();
- KafkaProducer<String, CommandNotification> producer = new KafkaProducer<>(props);
- String topic = rule.getKafkaTopicName();
- rule.createTopic(topic);
-
- registration = new KafkaNotificationRegistrationClient(topic, producer);
- coord = new PeriodicNotificationCoordinatorExecutor(1, notifications);
- provider = new KafkaNotificationProvider(topic, new StringDeserializer(), new CommandNotificationSerializer(), props, coord, 1);
- provider.start();
-
- registration.addNotification("1", 1000, 0, TimeUnit.MILLISECONDS);
- Thread.sleep(4000);
- // check that notifications are being added to the blocking queue
- Assert.assertEquals(true, notifications.size() > 0);
-
- registration.deleteNotification("1");
- Thread.sleep(2000);
- int size = notifications.size();
- // sleep for 2 seconds to ensure no more messages being produced
- Thread.sleep(2000);
- Assert.assertEquals(size, notifications.size());
-
- tearDown();
- }
-
- private void tearDown() {
- registration.close();
- provider.stop();
- coord.stop();
- }
-
- private Properties createKafkaConfig() {
- Properties props = new Properties();
- props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServer);
- props.setProperty(ConsumerConfig.GROUP_ID_CONFIG, UUID.randomUUID().toString());
- props.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, "consumer0");
- props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
- props.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
- props.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, CommandNotificationSerializer.class.getName());
-
- return props;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.integration.tests/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.integration.tests/src/test/resources/log4j.properties b/extras/rya.periodic.service/periodic.service.integration.tests/src/test/resources/log4j.properties
deleted file mode 100644
index 19cc13c..0000000
--- a/extras/rya.periodic.service/periodic.service.integration.tests/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,37 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-# Valid levels:
-# TRACE, DEBUG, INFO, WARN, ERROR and FATAL
-log4j.rootLogger=INFO, CONSOLE
-
-# Set independent logging levels
-log4j.logger.org.apache.zookeeper=WARN
-log4j.logger.kafka=WARN
-log4j.logger.org.apache.kafka=WARN
-
-# LOGFILE is set to be a File appender using a PatternLayout.
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-#log4j.appender.CONSOLE.Threshold=DEBUG
-
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%d [%t] %-5p %c - %m%n
-
-#log4j.appender.CONSOLE.layout=org.apache.log4j.EnhancedPatternLayout
-#log4j.appender.CONSOLE.layout.ConversionPattern=%d [%t] %-5p %c{1.} - %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.integration.tests/src/test/resources/notification.properties
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.integration.tests/src/test/resources/notification.properties b/extras/rya.periodic.service/periodic.service.integration.tests/src/test/resources/notification.properties
deleted file mode 100644
index 4b25b93..0000000
--- a/extras/rya.periodic.service/periodic.service.integration.tests/src/test/resources/notification.properties
+++ /dev/null
@@ -1,35 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#/
-accumulo.auths=
-accumulo.instance="instance"
-accumulo.user="root"
-accumulo.password="secret"
-accumulo.rya.prefix="rya_"
-accumulo.zookeepers=
-fluo.app.name="fluo_app"
-fluo.table.name="fluo_table"
-kafka.bootstrap.servers=127.0.0.1:9092
-kafka.notification.topic=notifications
-kafka.notification.client.id=consumer0
-kafka.notification.group.id=group0
-cep.coordinator.threads=1
-cep.producer.threads=1
-cep.exporter.threads=1
-cep.processor.threads=1
-cep.pruner.threads=1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.notification/pom.xml
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.notification/pom.xml b/extras/rya.periodic.service/periodic.service.notification/pom.xml
deleted file mode 100644
index 1e59e15..0000000
--- a/extras/rya.periodic.service/periodic.service.notification/pom.xml
+++ /dev/null
@@ -1,112 +0,0 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <!-- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with this
- work for additional information regarding copyright ownership. The ASF licenses
- this file to you under the Apache License, Version 2.0 (the "License"); you
- may not use this file except in compliance with the License. You may obtain
- a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless
- required by applicable law or agreed to in writing, software distributed
- under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
- OR CONDITIONS OF ANY KIND, either express or implied. See the License for
- the specific language governing permissions and limitations under the License. -->
- <parent>
- <groupId>org.apache.rya</groupId>
- <artifactId>rya.periodic.service</artifactId>
- <version>3.2.11-incubating-SNAPSHOT</version>
- </parent>
-
- <artifactId>rya.periodic.service.notification</artifactId>
-
- <name>Apache Rya Periodic Service Notification</name>
- <description>Notifications for Rya Periodic Service</description>
-
- <dependencies>
-
- <dependency>
- <groupId>org.apache.twill</groupId>
- <artifactId>twill-api</artifactId>
- <version>0.11.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.twill</groupId>
- <artifactId>twill-yarn</artifactId>
- <version>0.11.0</version>
- <exclusions>
- <exclusion>
- <artifactId>kafka_2.10</artifactId>
- <groupId>org.apache.kafka</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- <version>2.8.0</version>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.fluo</groupId>
- <artifactId>fluo-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.fluo</groupId>
- <artifactId>fluo-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.rya</groupId>
- <artifactId>rya.indexing</artifactId>
- </dependency>
- <dependency>
- <groupId>org.openrdf.sesame</groupId>
- <artifactId>sesame-query</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.rya</groupId>
- <artifactId>rya.indexing.pcj</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.rya</groupId>
- <artifactId>rya.pcj.fluo.app</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.rya</groupId>
- <artifactId>rya.periodic.service.api</artifactId>
- </dependency>
-
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <encoding>UTF-8</encoding>
- <source>1.8</source>
- <target>1.8</target>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>3.0.0</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
-
-</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/application/PeriodicApplicationException.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/application/PeriodicApplicationException.java b/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/application/PeriodicApplicationException.java
deleted file mode 100644
index b2c3709..0000000
--- a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/application/PeriodicApplicationException.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.application;
-
-/**
- * Exception thrown when attempting to create a {@link PeriodicNotificationApplication}.
- * Indicates that a factory was unable to create some component of the application
- * because something was configured incorrectly.
- *
- */
-public class PeriodicApplicationException extends Exception {
-
- private static final long serialVersionUID = 1L;
-
- /**
- * Creates a PeriodicApplicationException.
- * @param message - message contained in Exception
- */
- public PeriodicApplicationException(String message) {
- super(message);
- }
-
- /**
- * Creates a PeriodicApplicationException.
- * @param message - message contained in Exception
- * @param t - Exception that spawned this PeriodicApplicationException
- */
- public PeriodicApplicationException(String message, Throwable t) {
- super(message, t);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplication.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplication.java b/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplication.java
deleted file mode 100644
index 92a7d18..0000000
--- a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplication.java
+++ /dev/null
@@ -1,207 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.application;
-
-import org.apache.log4j.Logger;
-import org.apache.rya.indexing.pcj.fluo.app.util.PeriodicQueryUtil;
-import org.apache.rya.periodic.notification.api.BinPruner;
-import org.apache.rya.periodic.notification.api.BindingSetRecord;
-import org.apache.rya.periodic.notification.api.LifeCycle;
-import org.apache.rya.periodic.notification.api.NodeBin;
-import org.apache.rya.periodic.notification.api.NotificationCoordinatorExecutor;
-import org.apache.rya.periodic.notification.exporter.KafkaExporterExecutor;
-import org.apache.rya.periodic.notification.processor.NotificationProcessorExecutor;
-import org.apache.rya.periodic.notification.pruner.PeriodicQueryPrunerExecutor;
-import org.apache.rya.periodic.notification.registration.kafka.KafkaNotificationProvider;
-import org.openrdf.query.algebra.evaluation.function.Function;
-
-import com.google.common.base.Preconditions;
-
-/**
- * The PeriodicNotificationApplication runs the key components of the Periodic
- * Query Service. It consists of a {@link KafkaNotificationProvider}, a
- * {@link NotificationCoordinatorExecutor}, a
- * {@link NotificationProcessorExecutor}, a {@link KafkaExporterExecutor}, and a
- * {@link PeriodicQueryPrunerExecutor}. These services run in coordination with
- * one another to perform the following tasks in the indicated order: <br>
- * <li>Retrieve new requests to generate periodic notifications from Kafka
- * <li>Register them with the {@link NotificationCoordinatorExecutor} to
- * generate the periodic notifications
- * <li>As notifications are generated, they are added to a work queue that is
- * monitored by the {@link NotificationProcessorExecutor}.
- * <li>The processor processes the notifications by reading all of the query
- * results corresponding to the bin and query id indicated by the notification.
- * <li>After reading the results, the processor adds a {@link BindingSetRecord}
- * to a work queue monitored by the {@link KafkaExporterExecutor}.
- * <li>The processor then adds a {@link NodeBin} to a workqueue monitored by the
- * {@link BinPruner}
- * <li>The exporter processes the BindingSetRecord by exporing the result to
- * Kafka
- * <li>The BinPruner processes the NodeBin by cleaning up the results for the
- * indicated bin and query in Accumulo and Fluo. <br>
- * <br>
- * The purpose of this Periodic Query Service is to facilitate the ability to
- * answer Periodic Queries using the Rya Fluo application, where a Periodic
- * Query is any query requesting periodic updates about events that occurred
- * within a given window of time of this instant. This is also known as a
- * rolling window query. Period Queries can be expressed using SPARQL by
- * including the {@link Function} indicated by the URI
- * {@link PeriodicQueryUtil#PeriodicQueryURI}. The user must provide this
- * Function with the following arguments: the temporal variable in the query
- * that will be filtered on, the window of time that events must occur within,
- * the period at which the user wants to receive updates, and the time unit. The
- * following query requests all observations that occurred within the last
- * minute and requests updates every 15 seconds. It also performs a count on
- * those observations. <br>
- * <br>
- * <li>prefix function: http://org.apache.rya/function#
- * <li>"prefix time: http://www.w3.org/2006/time#
- * <li>"select (count(?obs) as ?total) where {
- * <li>"Filter(function:periodic(?time, 1, .25, time:minutes))
- * <li>"?obs uri:hasTime ?time.
- * <li>"?obs uri:hasId ?id }
- * <li>
- */
-public class PeriodicNotificationApplication implements LifeCycle {
-
- private static final Logger log = Logger.getLogger(PeriodicNotificationApplication.class);
- private NotificationCoordinatorExecutor coordinator;
- private KafkaNotificationProvider provider;
- private PeriodicQueryPrunerExecutor pruner;
- private NotificationProcessorExecutor processor;
- private KafkaExporterExecutor exporter;
- private boolean running = false;
-
- /**
- * Creates a PeriodicNotificationApplication
- * @param provider - {@link KafkaNotificationProvider} that retrieves new Notificaiton requests from Kafka
- * @param coordinator - {NotificationCoordinator} that manages PeriodicNotifications.
- * @param processor - {@link NotificationProcessorExecutor} that processes PeriodicNotifications
- * @param exporter - {@link KafkaExporterExecutor} that exports periodic results
- * @param pruner - {@link PeriodicQueryPrunerExecutor} that cleans up old periodic bins
- */
- public PeriodicNotificationApplication(KafkaNotificationProvider provider, NotificationCoordinatorExecutor coordinator,
- NotificationProcessorExecutor processor, KafkaExporterExecutor exporter, PeriodicQueryPrunerExecutor pruner) {
- this.provider = Preconditions.checkNotNull(provider);
- this.coordinator = Preconditions.checkNotNull(coordinator);
- this.processor = Preconditions.checkNotNull(processor);
- this.exporter = Preconditions.checkNotNull(exporter);
- this.pruner = Preconditions.checkNotNull(pruner);
- }
-
- @Override
- public void start() {
- if (!running) {
- log.info("Starting PeriodicNotificationApplication.");
- coordinator.start();
- provider.start();
- processor.start();
- pruner.start();
- exporter.start();
- running = true;
- }
- }
-
- @Override
- public void stop() {
- log.info("Stopping PeriodicNotificationApplication.");
- provider.stop();
- coordinator.stop();
- processor.stop();
- pruner.stop();
- exporter.stop();
- running = false;
- }
-
- /**
- * @return boolean indicating whether the application is running
- */
- @Override
- public boolean currentlyRunning() {
- return running;
- }
-
- public static Builder builder() {
- return new Builder();
- }
-
- public static class Builder {
-
- private PeriodicQueryPrunerExecutor pruner;
- private KafkaNotificationProvider provider;
- private NotificationProcessorExecutor processor;
- private KafkaExporterExecutor exporter;
- private NotificationCoordinatorExecutor coordinator;
-
- /**
- * Sets the PeriodicQueryPrunerExecutor.
- * @param pruner - PeriodicQueryPrunerExecutor for cleaning up old periodic bins
- * @return this Builder for chaining method calls
- */
- public Builder setPruner(PeriodicQueryPrunerExecutor pruner) {
- this.pruner = pruner;
- return this;
- }
-
- /**
- * Sets the KafkaNotificationProvider
- * @param provider - KafkaNotificationProvider for retrieving new periodic notification requests from Kafka
- * @return this Builder for chaining method calls
- */
- public Builder setProvider(KafkaNotificationProvider provider) {
- this.provider = provider;
- return this;
- }
-
- public Builder setProcessor(NotificationProcessorExecutor processor) {
- this.processor = processor;
- return this;
- }
-
- /**
- * Sets KafkaExporterExecutor
- * @param exporter for exporting periodic query results to Kafka
- * @return this Builder for chaining method calls
- */
- public Builder setExporter(KafkaExporterExecutor exporter) {
- this.exporter = exporter;
- return this;
- }
-
- /**
- * Sets NotificationCoordinatorExecutor
- * @param coordinator for managing and generating periodic notifications
- * @return this Builder for chaining method calls
- */
- public Builder setCoordinator(NotificationCoordinatorExecutor coordinator) {
- this.coordinator = coordinator;
- return this;
- }
-
- /**
- * Creates a PeriodicNotificationApplication
- * @return PeriodicNotificationApplication for periodically polling Rya Fluo Application
- */
- public PeriodicNotificationApplication build() {
- return new PeriodicNotificationApplication(provider, coordinator, processor, exporter, pruner);
- }
-
- }
-
-}
[4/7] incubator-rya git commit: RYA-355 Refactored the periodic
notification service structure. Closes #221.
Posted by ca...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.pcj.fluo/pcj.fluo.api/pom.xml
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.api/pom.xml b/extras/rya.pcj.fluo/pcj.fluo.api/pom.xml
index 16d33b2..439ea16 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.api/pom.xml
+++ b/extras/rya.pcj.fluo/pcj.fluo.api/pom.xml
@@ -34,17 +34,17 @@ under the License.
that allow other applications to interact with the Rya PCJ Fluo
application while it is running on a cluster.
</description>
-
+
<dependencies>
- <!-- Rya Runtime Dependencies. -->
- <dependency>
+ <!-- Rya Runtime Dependencies. -->
+ <dependency>
<groupId>org.apache.rya</groupId>
<artifactId>rya.pcj.fluo.app</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.rya</groupId>
- <artifactId>rya.periodic.service.api</artifactId>
- </dependency>
+ <groupId>org.apache.rya</groupId>
+ <artifactId>rya.periodic.notification.api</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.rya</groupId>
<artifactId>rya.sail</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.api/.gitignore
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.api/.gitignore b/extras/rya.periodic.service/periodic.service.api/.gitignore
deleted file mode 100644
index b83d222..0000000
--- a/extras/rya.periodic.service/periodic.service.api/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-/target/
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.api/pom.xml
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.api/pom.xml b/extras/rya.periodic.service/periodic.service.api/pom.xml
deleted file mode 100644
index b57beaf..0000000
--- a/extras/rya.periodic.service/periodic.service.api/pom.xml
+++ /dev/null
@@ -1,52 +0,0 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <!-- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with this
- work for additional information regarding copyright ownership. The ASF licenses
- this file to you under the Apache License, Version 2.0 (the "License"); you
- may not use this file except in compliance with the License. You may obtain
- a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless
- required by applicable law or agreed to in writing, software distributed
- under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
- OR CONDITIONS OF ANY KIND, either express or implied. See the License for
- the specific language governing permissions and limitations under the License. -->
-
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.rya</groupId>
- <artifactId>rya.periodic.service</artifactId>
- <version>3.2.11-incubating-SNAPSHOT</version>
- </parent>
-
- <artifactId>rya.periodic.service.api</artifactId>
-
- <name>Apache Rya Periodic Service API</name>
- <description>API for Periodic Service Application</description>
-
- <dependencies>
-
- <dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- <version>2.8.0</version>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.openrdf.sesame</groupId>
- <artifactId>sesame-query</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka-clients</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.rya</groupId>
- <artifactId>rya.indexing.pcj</artifactId>
- </dependency>
- </dependencies>
-
-</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/BinPruner.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/BinPruner.java b/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/BinPruner.java
deleted file mode 100644
index f4a083c..0000000
--- a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/BinPruner.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.api;
-
-import org.openrdf.query.Binding;
-import org.openrdf.query.BindingSet;
-
-/**
- * Object that cleans up old {@link BindingSet}s corresponding to the specified
- * {@link NodeBin}. This class deletes all BindingSets with the bin
- * indicated by {@link NodeBin#getBin()}. A BindingSet corresponds to a given
- * bin if it contains a {@link Binding} with name {@link IncrementalUpdateConstants#PERIODIC_BIN_ID}
- * and value equal to the given bin.
- *
- */
-public interface BinPruner {
-
- /**
- * Cleans up all {@link BindingSet}s associated with the indicated {@link NodeBin}.
- * @param bin - NodeBin that indicates which BindingSets to delete..
- */
- public void pruneBindingSetBin(NodeBin bin);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/BindingSetExporter.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/BindingSetExporter.java b/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/BindingSetExporter.java
deleted file mode 100644
index 491576b..0000000
--- a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/BindingSetExporter.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.api;
-
-import org.openrdf.query.BindingSet;
-
-/**
- * An Object that is used to export {@link BindingSet}s to an external repository or queuing system.
- *
- */
-public interface BindingSetExporter {
-
- /**
- * This method exports the BindingSet to the external repository or queuing system
- * that this BindingSetExporter is configured to export to.
- * @param bindingSet - {@link BindingSet} to be exported
- * @throws ResultExportException
- */
- public void exportNotification(BindingSetRecord bindingSet) throws BindingSetRecordExportException;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/BindingSetRecord.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/BindingSetRecord.java b/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/BindingSetRecord.java
deleted file mode 100644
index c3f70f1..0000000
--- a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/BindingSetRecord.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.api;
-
-import org.openrdf.query.BindingSet;
-
-import com.google.common.base.Objects;
-
-/**
- * Object that associates a {@link BindingSet} with a given Kafka topic.
- * This ensures that the {@link KafkaPeriodicBindingSetExporter} can export
- * each BindingSet to its appropriate topic.
- *
- */
-public class BindingSetRecord {
-
- private BindingSet bs;
- private String topic;
-
- public BindingSetRecord(BindingSet bs, String topic) {
- this.bs = bs;
- this.topic = topic;
- }
-
- /**
- * @return BindingSet in this BindingSetRecord
- */
- public BindingSet getBindingSet() {
- return bs;
- }
-
- /**
- * @return Kafka topic for this BindingSetRecord
- */
- public String getTopic() {
- return topic;
- }
-
- @Override
- public boolean equals(Object o) {
- if(this == o) {
- return true;
- }
-
- if(o instanceof BindingSetRecord) {
- BindingSetRecord record = (BindingSetRecord) o;
- return Objects.equal(this.bs, record.bs)&&Objects.equal(this.topic,record.topic);
- }
-
- return false;
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(bs, topic);
- }
-
- @Override
- public String toString() {
- return new StringBuilder().append("Binding Set Record \n").append(" Topic: " + topic + "\n").append(" BindingSet: " + bs + "\n")
- .toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/BindingSetRecordExportException.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/BindingSetRecordExportException.java b/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/BindingSetRecordExportException.java
deleted file mode 100644
index 94e4980..0000000
--- a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/BindingSetRecordExportException.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.api;
-
-/**
- * A result could not be exported.
- */
-public class BindingSetRecordExportException extends Exception {
- private static final long serialVersionUID = 1L;
-
- /**
- * Constructs an instance of {@link BindingSetRecordExportException}.
- *
- * @param message - Explains why the exception was thrown.
- */
- public BindingSetRecordExportException(final String message) {
- super(message);
- }
-
- /**
- * Constructs an instance of {@link BindingSetRecordExportException}.
- *
- * @param message - Explains why the exception was thrown.
- * @param cause - The exception that caused this one to be thrown.
- */
- public BindingSetRecordExportException(final String message, final Throwable cause) {
- super(message, cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/LifeCycle.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/LifeCycle.java b/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/LifeCycle.java
deleted file mode 100644
index b1e8bad..0000000
--- a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/LifeCycle.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.api;
-
-/**
- * Interface providing basic life cycle functionality,
- * including stopping and starting any class implementing this
- * interface and checking whether is it running.
- *
- */
-public interface LifeCycle {
-
- /**
- * Starts a running application.
- */
- public void start();
-
- /**
- * Stops a running application.
- */
- public void stop();
-
- /**
- * Determine if application is currently running.
- * @return true if application is running and false otherwise.
- */
- public boolean currentlyRunning();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/NodeBin.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/NodeBin.java b/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/NodeBin.java
deleted file mode 100644
index 3ed7979..0000000
--- a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/NodeBin.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.api;
-
-import java.util.Objects;
-
-/**
- * Object used to indicate the id of a given Periodic Query
- * along with a particular bin of results. This Object is used
- * by the {@link BinPruner} to clean up old query results after
- * they have been processed.
- *
- */
-public class NodeBin {
-
- private long bin;
- private String nodeId;
-
- public NodeBin(String nodeId, long bin) {
- this.bin = bin;
- this.nodeId = nodeId;
- }
-
- /**
- * @return id of Periodic Query
- */
- public String getNodeId() {
- return nodeId;
- }
-/**
- * @return bin id of results for a given Periodic Query
- */
- public long getBin() {
- return bin;
- }
-
- @Override
- public boolean equals(Object other) {
- if (this == other) {
- return true;
- }
-
- if (other instanceof NodeBin) {
- NodeBin bin = (NodeBin) other;
- return this.bin == bin.bin && this.nodeId.equals(bin.nodeId);
- }
-
- return false;
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(bin, nodeId);
- }
-
- @Override
- public String toString() {
- return new StringBuilder().append("Node Bin \n").append(" QueryId: " + nodeId + "\n").append(" Bin: " + bin + "\n").toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/Notification.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/Notification.java b/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/Notification.java
deleted file mode 100644
index 3e9e0d1..0000000
--- a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/Notification.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.api;
-
-/**
- * Notification Object used by the Periodic Query Service
- * to inform workers to process results for a given Periodic
- * Query with the indicated id.
- *
- */
-public interface Notification {
-
- /**
- * @return id of a Periodic Query
- */
- public String getId();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/NotificationCoordinatorExecutor.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/NotificationCoordinatorExecutor.java b/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/NotificationCoordinatorExecutor.java
deleted file mode 100644
index d53dc17..0000000
--- a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/NotificationCoordinatorExecutor.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.api;
-
-import java.util.concurrent.ScheduledExecutorService;
-
-import org.apache.rya.periodic.notification.notification.CommandNotification;
-
-/**
- * Object that manages the periodic notifications for the Periodic Query Service.
- * This Object processes new requests for periodic updates by registering them with
- * some sort of service that generates periodic updates (such as a {@link ScheduledExecutorService}).
- *
- */
-public interface NotificationCoordinatorExecutor extends LifeCycle {
-
- /**
- * Registers or deletes a {@link CommandNotification}s with the periodic service to
- * generate notifications at a regular interval indicated by the CommandNotification.
- * @param notification - CommandNotification to be registered or deleted from the periodic update
- * service.
- */
- public void processNextCommandNotification(CommandNotification notification);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/NotificationProcessor.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/NotificationProcessor.java b/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/NotificationProcessor.java
deleted file mode 100644
index 4ac9089..0000000
--- a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/NotificationProcessor.java
+++ /dev/null
@@ -1,41 +0,0 @@
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.api;
-
-import org.apache.rya.periodic.notification.notification.TimestampedNotification;
-
-/**
- * Object that processes new {@link TimestampedNotification}s generated by {@link NotificationCoordinatorExecutor}.
- * It is expected that the NotificationCoordinatorExecutor will this Object with notifications to perform work via some sort
- * sort of queuing service such as a BlockingQueue or Kafka. This Object processes the notifications by retrieving
- * query results associated with the Periodic Query id given by {@link TimestampedNotification#getId()}, parsing them
- * and then providing them to another service to be exported.
- *
- */
-public interface NotificationProcessor {
-
- /**
- * Processes {@link TimestampedNotification}s by retrieving the Periodic Query results
- * associated the query id given by {@link TimestampedNotification#getId()}.
- * @param notification - contains information about which query results to retrieve
- */
- public void processNotification(TimestampedNotification notification);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/PeriodicNotificationClient.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/PeriodicNotificationClient.java b/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/PeriodicNotificationClient.java
deleted file mode 100644
index ff08733..0000000
--- a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/api/PeriodicNotificationClient.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.api;
-
-import java.util.concurrent.TimeUnit;
-
-import org.apache.rya.periodic.notification.notification.BasicNotification;
-import org.apache.rya.periodic.notification.notification.PeriodicNotification;
-
-/**
- * Object to register {@link PeriodicNotification}s with an external queuing
- * service to be handled by a {@link NotificationCoordinatorExecutor} service.
- * The service will generate notifications to process Periodic Query results at regular
- * intervals corresponding the period of the PeriodicNotification.
- *
- */
-public interface PeriodicNotificationClient extends AutoCloseable {
-
- /**
- * Adds a new notification to be registered with the {@link NotificationCoordinatorExecutor}
- * @param notification - notification to be added
- */
- public void addNotification(PeriodicNotification notification);
-
- /**
- * Deletes a notification from the {@link NotificationCoordinatorExecutor}.
- * @param notification - notification to be deleted
- */
- public void deleteNotification(BasicNotification notification);
-
- /**
- * Deletes a notification from the {@link NotificationCoordinatorExecutor}.
- * @param notification - id corresponding to the notification to be deleted
- */
- public void deleteNotification(String notificationId);
-
- /**
- * Adds a new notification with the indicated id and period to the {@link NotificationCoordinatorExecutor}
- * @param id - Periodic Query id
- * @param period - period indicating frequency at which notifications will be generated
- * @param delay - initial delay for starting periodic notifications
- * @param unit - time unit of delay and period
- */
- public void addNotification(String id, long period, long delay, TimeUnit unit);
-
- public void close();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/notification/BasicNotification.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/notification/BasicNotification.java b/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/notification/BasicNotification.java
deleted file mode 100644
index c31a5c0..0000000
--- a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/notification/BasicNotification.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.notification;
-
-import org.apache.rya.periodic.notification.api.Notification;
-
-import com.google.common.base.Objects;
-
-/**
- * Notification Object used by the Periodic Query Service
- * to inform workers to process results for a given Periodic
- * Query with the indicated id.
- *
- */
-public class BasicNotification implements Notification {
-
- private String id;
-
- /**
- * Creates a BasicNotification
- * @param id - Fluo query id associated with this Notification
- */
- public BasicNotification(String id) {
- this.id = id;
- }
-
- /**
- * @return the Fluo Query Id that this notification will generate results for
- */
- @Override
- public String getId() {
- return id;
- }
-
- @Override
- public boolean equals(Object other) {
- if (this == other) {
- return true;
- }
-
- if (other instanceof BasicNotification) {
- BasicNotification not = (BasicNotification) other;
- return Objects.equal(this.id, not.id);
- }
-
- return false;
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(id);
- }
-
- @Override
- public String toString() {
- StringBuilder builder = new StringBuilder();
- return builder.append("id").append("=").append(id).toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/notification/CommandNotification.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/notification/CommandNotification.java b/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/notification/CommandNotification.java
deleted file mode 100644
index 597b228..0000000
--- a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/notification/CommandNotification.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.notification;
-
-import org.apache.rya.periodic.notification.api.Notification;
-
-import com.google.common.base.Objects;
-import com.google.common.base.Preconditions;
-
-/**
- * This Object contains a Notification Object used by the Periodic Query Service
- * to inform workers to process results for a given Periodic Query with the
- * indicated id. Additionally, the CommandNotification contains a
- * {@link Command} about which action the
- * {@link NotificationCoordinatorExecutor} should take (adding or deleting).
- * CommandNotifications are meant to be added to an external work queue (such as
- * Kafka) to be processed by the NotificationCoordinatorExecutor.
- *
- */
-public class CommandNotification implements Notification {
-
- private Notification notification;
- private Command command;
-
- public enum Command {
- ADD, DELETE
- };
-
- /**
- * Creates a new CommandNotification
- * @param command - the command associated with this notification (either add, update, or delete)
- * @param notification - the underlying notification associated with this command
- */
- public CommandNotification(Command command, Notification notification) {
- this.notification = Preconditions.checkNotNull(notification);
- this.command = Preconditions.checkNotNull(command);
- }
-
- @Override
- public String getId() {
- return notification.getId();
- }
-
- /**
- * Returns {@link Notification} contained by this CommmandNotification.
- * @return - Notification contained by this Object
- */
- public Notification getNotification() {
- return this.notification;
- }
-
- /**
- * @return Command contained by this Object (either add or delete)
- */
- public Command getCommand() {
- return this.command;
- }
-
- @Override
- public boolean equals(Object other) {
- if (this == other) {
- return true;
- }
- if (other instanceof CommandNotification) {
- CommandNotification cn = (CommandNotification) other;
- return Objects.equal(this.command, cn.command) && Objects.equal(this.notification, cn.notification);
- } else {
- return false;
- }
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(command, notification);
- }
-
- @Override
- public String toString() {
- return new StringBuilder().append("command").append("=").append(command.toString()).append(";")
- .append(notification.toString()).toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/notification/PeriodicNotification.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/notification/PeriodicNotification.java b/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/notification/PeriodicNotification.java
deleted file mode 100644
index aa9e581..0000000
--- a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/notification/PeriodicNotification.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.notification;
-
-import java.util.Objects;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.rya.periodic.notification.api.Notification;
-
-import com.google.common.base.Preconditions;
-
-/**
- * Notification Object used by the Periodic Query Service to inform workers to
- * process results for a given Periodic Query with the indicated id.
- * Additionally, this Object contains a period that indicates a frequency at
- * which regular updates are generated.
- *
- */
-public class PeriodicNotification implements Notification {
-
- private String id;
- private long period;
- private TimeUnit periodTimeUnit;
- private long initialDelay;
-
- /**
- * Creates a PeriodicNotification.
- * @param id - Fluo Query Id that this notification is associated with
- * @param period - period at which notifications are generated
- * @param periodTimeUnit - time unit associated with the period and delay
- * @param initialDelay - amount of time to wait before generating the first notification
- */
- public PeriodicNotification(String id, long period, TimeUnit periodTimeUnit, long initialDelay) {
- this.id = Preconditions.checkNotNull(id);
- this.periodTimeUnit = Preconditions.checkNotNull(periodTimeUnit);
- Preconditions.checkArgument(period > 0 && initialDelay >= 0);
- this.period = period;
- this.initialDelay = initialDelay;
- }
-
-
- /**
- * Create a PeriodicNotification
- * @param other - other PeriodicNotification used in copy constructor
- */
- public PeriodicNotification(PeriodicNotification other) {
- this(other.id, other.period, other.periodTimeUnit, other.initialDelay);
- }
-
- public String getId() {
- return id;
- }
-
- /**
- * @return - period at which regular notifications are generated
- */
- public long getPeriod() {
- return period;
- }
-
- /**
- * @return time unit of period and initial delay
- */
- public TimeUnit getTimeUnit() {
- return periodTimeUnit;
- }
-
- /**
- * @return amount of time to delay before beginning to generate notifications
- */
- public long getInitialDelay() {
- return initialDelay;
- }
-
- @Override
- public String toString() {
- StringBuilder builder = new StringBuilder();
- String delim = "=";
- String delim2 = ";";
- return builder.append("id").append(delim).append(id).append(delim2).append("period").append(delim).append(period).append(delim2)
- .append("periodTimeUnit").append(delim).append(periodTimeUnit).append(delim2).append("initialDelay").append(delim)
- .append(initialDelay).toString();
- }
-
- @Override
- public boolean equals(Object other) {
- if (this == other) {
- return true;
- }
-
- if (!(other instanceof PeriodicNotification)) {
- return false;
- }
-
- PeriodicNotification notification = (PeriodicNotification) other;
- return Objects.equals(this.id, notification.id) && (this.period == notification.period)
- && Objects.equals(this.periodTimeUnit, notification.periodTimeUnit) && (this.initialDelay == notification.initialDelay);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(id, period, periodTimeUnit, initialDelay);
- }
-
- public static Builder builder() {
- return new Builder();
- }
-
- public static class Builder {
-
- private String id;
- private long period;
- private TimeUnit periodTimeUnit;
- private long initialDelay = 0;
-
- /**
- * @param id - periodic query id
- * @return - builder to chain method calls
- */
- public Builder id(String id) {
- this.id = id;
- return this;
- }
-
- /**
- * @param period of the periodic notification for generating regular notifications
- * @return - builder to chain method calls
- */
- public Builder period(long period) {
- this.period = period;
- return this;
- }
-
- /**
- * @param timeUnit of period and initial delay
- * @return - builder to chain method calls
- */
- public Builder timeUnit(TimeUnit timeUnit) {
- this.periodTimeUnit = timeUnit;
- return this;
- }
-
- /**
- * @param initialDelay - amount of time to wait before generating notifications
- * @return - builder to chain method calls
- */
- public Builder initialDelay(long initialDelay) {
- this.initialDelay = initialDelay;
- return this;
- }
-
- /**
- * Builds PeriodicNotification
- * @return PeriodicNotification constructed from Builder specified parameters
- */
- public PeriodicNotification build() {
- return new PeriodicNotification(id, period, periodTimeUnit, initialDelay);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/notification/TimestampedNotification.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/notification/TimestampedNotification.java b/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/notification/TimestampedNotification.java
deleted file mode 100644
index 38073ce..0000000
--- a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/notification/TimestampedNotification.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.notification;
-
-import java.util.Date;
-import java.util.concurrent.TimeUnit;
-
-/**
- * {@link PeriodicNotification} Object used by the Periodic Query Service to inform workers to
- * process results for a given Periodic Query with the indicated id. Additionally
- * this Object contains a {@link Date} object to indicate the date time at which this
- * notification was generated.
- *
- */
-public class TimestampedNotification extends PeriodicNotification {
-
- private Date date;
-
- /**
- * Constructs a TimestampedNotification
- * @param id - Fluo Query Id associated with this Notification
- * @param period - period at which notifications are generated
- * @param periodTimeUnit - time unit associated with period and initial delay
- * @param initialDelay - amount of time to wait before generating first notification
- */
- public TimestampedNotification(String id, long period, TimeUnit periodTimeUnit, long initialDelay) {
- super(id, period, periodTimeUnit, initialDelay);
- date = new Date();
- }
-
- /**
- * Creates a TimestampedNotification
- * @param notification - PeriodicNotification used to create this TimestampedNotification.
- * This constructor creates a time stamp for the TimestampedNotification.
- */
- public TimestampedNotification(PeriodicNotification notification) {
- super(notification);
- date = new Date();
- }
-
- /**
- * @return timestamp at which this notification was generated
- */
- public Date getTimestamp() {
- return date;
- }
-
- @Override
- public String toString() {
- return super.toString() + ";date=" + date;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/registration/KafkaNotificationRegistrationClient.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/registration/KafkaNotificationRegistrationClient.java b/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/registration/KafkaNotificationRegistrationClient.java
deleted file mode 100644
index bb438be..0000000
--- a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/registration/KafkaNotificationRegistrationClient.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.registration;
-
-import java.util.concurrent.TimeUnit;
-
-import org.apache.kafka.clients.producer.KafkaProducer;
-import org.apache.kafka.clients.producer.ProducerRecord;
-import org.apache.rya.periodic.notification.api.Notification;
-import org.apache.rya.periodic.notification.api.PeriodicNotificationClient;
-import org.apache.rya.periodic.notification.notification.BasicNotification;
-import org.apache.rya.periodic.notification.notification.CommandNotification;
-import org.apache.rya.periodic.notification.notification.CommandNotification.Command;
-import org.apache.rya.periodic.notification.notification.PeriodicNotification;
-
-/**
- * Implementation of {@link PeriodicNotificaitonClient} used to register new notification
- * requests with the PeriodicQueryService.
- *
- */
-public class KafkaNotificationRegistrationClient implements PeriodicNotificationClient {
-
- private KafkaProducer<String, CommandNotification> producer;
- private String topic;
-
- public KafkaNotificationRegistrationClient(String topic, KafkaProducer<String, CommandNotification> producer) {
- this.topic = topic;
- this.producer = producer;
- }
-
- @Override
- public void addNotification(PeriodicNotification notification) {
- processNotification(new CommandNotification(Command.ADD, notification));
-
- }
-
- @Override
- public void deleteNotification(BasicNotification notification) {
- processNotification(new CommandNotification(Command.DELETE, notification));
- }
-
- @Override
- public void deleteNotification(String notificationId) {
- processNotification(new CommandNotification(Command.DELETE, new BasicNotification(notificationId)));
- }
-
- @Override
- public void addNotification(String id, long period, long delay, TimeUnit unit) {
- Notification notification = PeriodicNotification.builder().id(id).period(period).initialDelay(delay).timeUnit(unit).build();
- processNotification(new CommandNotification(Command.ADD, notification));
- }
-
-
- private void processNotification(CommandNotification notification) {
- producer.send(new ProducerRecord<String, CommandNotification>(topic, notification.getId(), notification));
- }
-
- @Override
- public void close() {
- producer.close();
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/serialization/BasicNotificationTypeAdapter.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/serialization/BasicNotificationTypeAdapter.java b/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/serialization/BasicNotificationTypeAdapter.java
deleted file mode 100644
index bd29d29..0000000
--- a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/serialization/BasicNotificationTypeAdapter.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.serialization;
-
-import java.lang.reflect.Type;
-
-import org.apache.rya.periodic.notification.notification.BasicNotification;
-
-import com.google.gson.JsonDeserializationContext;
-import com.google.gson.JsonDeserializer;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParseException;
-import com.google.gson.JsonPrimitive;
-import com.google.gson.JsonSerializationContext;
-import com.google.gson.JsonSerializer;
-
-/**
- * {@link TypeAdapter} for {@link BasicNotification}s. Used in {@link CommandNotificationTypeAdapter} to
- * serialize {@link CommandNotification}s.
- *
- */
-public class BasicNotificationTypeAdapter implements JsonDeserializer<BasicNotification>, JsonSerializer<BasicNotification> {
-
- @Override
- public JsonElement serialize(BasicNotification arg0, Type arg1, JsonSerializationContext arg2) {
- JsonObject result = new JsonObject();
- result.add("id", new JsonPrimitive(arg0.getId()));
- return result;
- }
-
- @Override
- public BasicNotification deserialize(JsonElement arg0, Type arg1, JsonDeserializationContext arg2) throws JsonParseException {
- JsonObject json = arg0.getAsJsonObject();
- String id = json.get("id").getAsString();
- return new BasicNotification(id);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/serialization/BindingSetSerDe.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/serialization/BindingSetSerDe.java b/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/serialization/BindingSetSerDe.java
deleted file mode 100644
index 50180ad..0000000
--- a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/serialization/BindingSetSerDe.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.serialization;
-
-import java.io.UnsupportedEncodingException;
-import java.util.Arrays;
-import java.util.Map;
-
-import org.apache.kafka.common.serialization.Deserializer;
-import org.apache.kafka.common.serialization.Serializer;
-import org.apache.log4j.Logger;
-import org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjSerializer;
-import org.apache.rya.indexing.pcj.storage.accumulo.BindingSetConverter.BindingSetConversionException;
-import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
-import org.openrdf.query.BindingSet;
-import org.openrdf.query.algebra.evaluation.QueryBindingSet;
-
-import com.google.common.base.Joiner;
-import com.google.common.primitives.Bytes;
-
-/**
- * Kafka {@link Serializer} and {@link Deserializer} for producing and consuming messages
- * from Kafka.
- *
- */
-public class BindingSetSerDe implements Serializer<BindingSet>, Deserializer<BindingSet> {
-
- private static final Logger log = Logger.getLogger(BindingSetSerDe.class);
- private static final AccumuloPcjSerializer serializer = new AccumuloPcjSerializer();
- private static final byte[] DELIM_BYTE = "\u0002".getBytes();
-
- private byte[] toBytes(BindingSet bindingSet) {
- try {
- return getBytes(getVarOrder(bindingSet), bindingSet);
- } catch(Exception e) {
- log.trace("Unable to serialize BindingSet: " + bindingSet);
- return new byte[0];
- }
- }
-
- private BindingSet fromBytes(byte[] bsBytes) {
- try{
- int firstIndex = Bytes.indexOf(bsBytes, DELIM_BYTE);
- byte[] varOrderBytes = Arrays.copyOf(bsBytes, firstIndex);
- byte[] bsBytesNoVarOrder = Arrays.copyOfRange(bsBytes, firstIndex + 1, bsBytes.length);
- VariableOrder varOrder = new VariableOrder(new String(varOrderBytes,"UTF-8").split(";"));
- return getBindingSet(varOrder, bsBytesNoVarOrder);
- } catch(Exception e) {
- log.trace("Unable to deserialize BindingSet: " + bsBytes);
- return new QueryBindingSet();
- }
- }
-
- private VariableOrder getVarOrder(BindingSet bs) {
- return new VariableOrder(bs.getBindingNames());
- }
-
- private byte[] getBytes(VariableOrder varOrder, BindingSet bs) throws UnsupportedEncodingException, BindingSetConversionException {
- byte[] bsBytes = serializer.convert(bs, varOrder);
- String varOrderString = Joiner.on(";").join(varOrder.getVariableOrders());
- byte[] varOrderBytes = varOrderString.getBytes("UTF-8");
- return Bytes.concat(varOrderBytes, DELIM_BYTE, bsBytes);
- }
-
- private BindingSet getBindingSet(VariableOrder varOrder, byte[] bsBytes) throws BindingSetConversionException {
- return serializer.convert(bsBytes, varOrder);
- }
-
- @Override
- public BindingSet deserialize(String topic, byte[] bytes) {
- return fromBytes(bytes);
- }
-
- @Override
- public void close() {
- // Do nothing. Nothing to close.
- }
-
- @Override
- public void configure(Map<String, ?> arg0, boolean arg1) {
- // Do nothing. Nothing to configure.
- }
-
- @Override
- public byte[] serialize(String topic, BindingSet bs) {
- return toBytes(bs);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/serialization/CommandNotificationSerializer.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/serialization/CommandNotificationSerializer.java b/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/serialization/CommandNotificationSerializer.java
deleted file mode 100644
index 302e1be..0000000
--- a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/serialization/CommandNotificationSerializer.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.serialization;
-
-import java.io.UnsupportedEncodingException;
-import java.util.Map;
-
-import org.apache.kafka.common.serialization.Deserializer;
-import org.apache.kafka.common.serialization.Serializer;
-import org.apache.rya.periodic.notification.api.Notification;
-import org.apache.rya.periodic.notification.notification.CommandNotification;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-
-/**
- * Kafka {@link Serializer} and {@link Deserializer} for producing and consuming {@link CommandNotification}s
- * to and from Kafka.
- *
- */
-public class CommandNotificationSerializer implements Serializer<CommandNotification>, Deserializer<CommandNotification> {
-
- private static Gson gson = new GsonBuilder()
- .registerTypeHierarchyAdapter(Notification.class, new CommandNotificationTypeAdapter()).create();
- private static final Logger LOG = LoggerFactory.getLogger(CommandNotificationSerializer.class);
-
- @Override
- public CommandNotification deserialize(String topic, byte[] bytes) {
- String json = null;
- try {
- json = new String(bytes, "UTF-8");
- } catch (UnsupportedEncodingException e) {
- LOG.info("Unable to deserialize notification for topic: " + topic);
- }
- return gson.fromJson(json, CommandNotification.class);
- }
-
- @Override
- public byte[] serialize(String topic, CommandNotification command) {
- try {
- return gson.toJson(command).getBytes("UTF-8");
- } catch (UnsupportedEncodingException e) {
- LOG.info("Unable to serialize notification: " + command + "for topic: " + topic);
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void close() {
- // Do nothing. Nothing to close
- }
-
- @Override
- public void configure(Map<String, ?> arg0, boolean arg1) {
- // Do nothing. Nothing to configure
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/serialization/CommandNotificationTypeAdapter.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/serialization/CommandNotificationTypeAdapter.java b/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/serialization/CommandNotificationTypeAdapter.java
deleted file mode 100644
index a9fb7e1..0000000
--- a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/serialization/CommandNotificationTypeAdapter.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.serialization;
-
-import java.lang.reflect.Type;
-
-import org.apache.rya.periodic.notification.api.Notification;
-import org.apache.rya.periodic.notification.notification.BasicNotification;
-import org.apache.rya.periodic.notification.notification.CommandNotification;
-import org.apache.rya.periodic.notification.notification.PeriodicNotification;
-import org.apache.rya.periodic.notification.notification.CommandNotification.Command;
-
-import com.google.gson.JsonDeserializationContext;
-import com.google.gson.JsonDeserializer;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParseException;
-import com.google.gson.JsonPrimitive;
-import com.google.gson.JsonSerializationContext;
-import com.google.gson.JsonSerializer;
-
-/**
- * {@link TypeAdapter} used to serialize and deserialize {@link CommandNotification}s.
- * This TypeAdapter is used in {@link CommandNotificationSerializer} for producing and
- * consuming messages to and from Kafka.
- *
- */
-public class CommandNotificationTypeAdapter
- implements JsonDeserializer<CommandNotification>, JsonSerializer<CommandNotification> {
-
- @Override
- public JsonElement serialize(CommandNotification arg0, Type arg1, JsonSerializationContext arg2) {
- JsonObject result = new JsonObject();
- result.add("command", new JsonPrimitive(arg0.getCommand().name()));
- Notification notification = arg0.getNotification();
- if (notification instanceof PeriodicNotification) {
- result.add("type", new JsonPrimitive(PeriodicNotification.class.getSimpleName()));
- PeriodicNotificationTypeAdapter adapter = new PeriodicNotificationTypeAdapter();
- result.add("notification",
- adapter.serialize((PeriodicNotification) notification, PeriodicNotification.class, arg2));
- } else if (notification instanceof BasicNotification) {
- result.add("type", new JsonPrimitive(BasicNotification.class.getSimpleName()));
- BasicNotificationTypeAdapter adapter = new BasicNotificationTypeAdapter();
- result.add("notification",
- adapter.serialize((BasicNotification) notification, BasicNotification.class, arg2));
- } else {
- throw new IllegalArgumentException("Invalid notification type.");
- }
- return result;
- }
-
- @Override
- public CommandNotification deserialize(JsonElement arg0, Type arg1, JsonDeserializationContext arg2)
- throws JsonParseException {
-
- JsonObject json = arg0.getAsJsonObject();
- Command command = Command.valueOf(json.get("command").getAsString());
- String type = json.get("type").getAsString();
- Notification notification = null;
- if (type.equals(PeriodicNotification.class.getSimpleName())) {
- notification = (new PeriodicNotificationTypeAdapter()).deserialize(json.get("notification"),
- PeriodicNotification.class, arg2);
- } else if (type.equals(BasicNotification.class.getSimpleName())) {
- notification = (new BasicNotificationTypeAdapter()).deserialize(json.get("notification"),
- BasicNotification.class, arg2);
- } else {
- throw new JsonParseException("Cannot deserialize Json");
- }
-
- return new CommandNotification(command, notification);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/serialization/PeriodicNotificationTypeAdapter.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/serialization/PeriodicNotificationTypeAdapter.java b/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/serialization/PeriodicNotificationTypeAdapter.java
deleted file mode 100644
index fcc0ba2..0000000
--- a/extras/rya.periodic.service/periodic.service.api/src/main/java/org/apache/rya/periodic/notification/serialization/PeriodicNotificationTypeAdapter.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.periodic.notification.serialization;
-
-import java.lang.reflect.Type;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.rya.periodic.notification.notification.PeriodicNotification;
-import org.apache.rya.periodic.notification.notification.PeriodicNotification.Builder;
-
-import com.google.gson.JsonDeserializationContext;
-import com.google.gson.JsonDeserializer;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParseException;
-import com.google.gson.JsonPrimitive;
-import com.google.gson.JsonSerializationContext;
-import com.google.gson.JsonSerializer;
-
-/**
- * {@link TypeAdapter} used to serialize and deserialize {@link PeriodicNotification}s.
- * This TypeAdapter is used in {@link CommandNotificationTypeAdapter} which is used in
- * {@link CommandNotificationSerializer} for producing and consuming messages to and from
- * Kafka.
- *
- */
-public class PeriodicNotificationTypeAdapter
- implements JsonSerializer<PeriodicNotification>, JsonDeserializer<PeriodicNotification> {
-
- @Override
- public PeriodicNotification deserialize(JsonElement arg0, Type arg1, JsonDeserializationContext arg2)
- throws JsonParseException {
-
- JsonObject json = arg0.getAsJsonObject();
- String id = json.get("id").getAsString();
- long period = json.get("period").getAsLong();
- TimeUnit periodTimeUnit = TimeUnit.valueOf(json.get("timeUnit").getAsString());
- long initialDelay = json.get("initialDelay").getAsLong();
- Builder builder = PeriodicNotification.builder().id(id).period(period)
- .initialDelay(initialDelay).timeUnit(periodTimeUnit);
-
- return builder.build();
- }
-
- @Override
- public JsonElement serialize(PeriodicNotification arg0, Type arg1, JsonSerializationContext arg2) {
-
- JsonObject result = new JsonObject();
- result.add("id", new JsonPrimitive(arg0.getId()));
- result.add("period", new JsonPrimitive(arg0.getPeriod()));
- result.add("initialDelay", new JsonPrimitive(arg0.getInitialDelay()));
- result.add("timeUnit", new JsonPrimitive(arg0.getTimeUnit().name()));
-
- return result;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/rya.periodic.service/periodic.service.integration.tests/pom.xml
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.integration.tests/pom.xml b/extras/rya.periodic.service/periodic.service.integration.tests/pom.xml
deleted file mode 100644
index 402f81d..0000000
--- a/extras/rya.periodic.service/periodic.service.integration.tests/pom.xml
+++ /dev/null
@@ -1,62 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
- license agreements. See the NOTICE file distributed with this work for additional
- information regarding copyright ownership. The ASF licenses this file to
- you under the Apache License, Version 2.0 (the "License"); you may not use
- this file except in compliance with the License. You may obtain a copy of
- the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
- by applicable law or agreed to in writing, software distributed under the
- License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
- OF ANY KIND, either express or implied. See the License for the specific
- language governing permissions and limitations under the License. -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.rya</groupId>
- <artifactId>rya.periodic.service</artifactId>
- <version>3.2.11-incubating-SNAPSHOT</version>
- </parent>
-
- <artifactId>rya.periodic.service.integration.tests</artifactId>
-
- <name>Apache Rya Periodic Service Integration Tests</name>
- <description>Integration Tests for Rya Periodic Service</description>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.rya</groupId>
- <artifactId>rya.pcj.fluo.test.base</artifactId>
- <exclusions>
- <exclusion>
- <artifactId>log4j-1.2-api</artifactId>
- <groupId>org.apache.logging.log4j</groupId>
- </exclusion>
- <exclusion>
- <artifactId>log4j-api</artifactId>
- <groupId>org.apache.logging.log4j</groupId>
- </exclusion>
- <exclusion>
- <artifactId>log4j-core</artifactId>
- <groupId>org.apache.logging.log4j</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.rya</groupId>
- <artifactId>rya.periodic.service.notification</artifactId>
- <exclusions>
- <exclusion>
- <artifactId>logback-classic</artifactId>
- <groupId>ch.qos.logback</groupId>
- </exclusion>
- <exclusion>
- <artifactId>logback-core</artifactId>
- <groupId>ch.qos.logback</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
-
-</project>
\ No newline at end of file
[5/7] incubator-rya git commit: RYA-355 Refactored the periodic
notification service structure. Closes #221.
Posted by ca...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/registration/kafka/KafkaNotificationProvider.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/registration/kafka/KafkaNotificationProvider.java b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/registration/kafka/KafkaNotificationProvider.java
new file mode 100644
index 0000000..f5cd13a
--- /dev/null
+++ b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/registration/kafka/KafkaNotificationProvider.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.registration.kafka;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.serialization.Deserializer;
+import org.apache.rya.periodic.notification.api.LifeCycle;
+import org.apache.rya.periodic.notification.api.Notification;
+import org.apache.rya.periodic.notification.api.NotificationCoordinatorExecutor;
+import org.apache.rya.periodic.notification.notification.CommandNotification;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Consumer group to pull all requests for adding and deleting {@link Notification}s
+ * from Kafka. This Object executes {@link PeriodicNotificationConsumer}s that retrieve
+ * the {@link CommandNotification}s and register them with the {@link NotificationCoordinatorExecutor}.
+ *
+ */
+public class KafkaNotificationProvider implements LifeCycle {
+ private static final Logger LOG = LoggerFactory.getLogger(KafkaNotificationProvider.class);
+ private String topic;
+ private ExecutorService executor;
+ private NotificationCoordinatorExecutor coord;
+ private Properties props;
+ private int numThreads;
+ private boolean running = false;
+ Deserializer<String> keyDe;
+ Deserializer<CommandNotification> valDe;
+ List<PeriodicNotificationConsumer> consumers;
+
+ /**
+ * Create KafkaNotificationProvider for reading new notification requests form Kafka
+ * @param topic - notification topic
+ * @param keyDe - Kafka message key deserializer
+ * @param valDe - Kafka message value deserializer
+ * @param props - properties used to creates a {@link KafkaConsumer}
+ * @param coord - {@link NotificationCoordinatorExecutor} for managing and generating notifications
+ * @param numThreads - number of threads used by this notification provider
+ */
+ public KafkaNotificationProvider(String topic, Deserializer<String> keyDe, Deserializer<CommandNotification> valDe, Properties props,
+ NotificationCoordinatorExecutor coord, int numThreads) {
+ this.coord = coord;
+ this.numThreads = numThreads;
+ this.topic = topic;
+ this.props = props;
+ this.consumers = new ArrayList<>();
+ this.keyDe = keyDe;
+ this.valDe = valDe;
+ }
+
+ @Override
+ public void stop() {
+ if (consumers != null && consumers.size() > 0) {
+ for (PeriodicNotificationConsumer consumer : consumers) {
+ consumer.shutdown();
+ }
+ }
+ if (executor != null) {
+ executor.shutdown();
+ }
+ running = false;
+ try {
+ if (!executor.awaitTermination(5000, TimeUnit.MILLISECONDS)) {
+ LOG.info("Timed out waiting for consumer threads to shut down, exiting uncleanly");
+ executor.shutdownNow();
+ }
+ } catch (InterruptedException e) {
+ LOG.info("Interrupted during shutdown, exiting uncleanly");
+ }
+ }
+
+ public void start() {
+ if (!running) {
+ if (!coord.currentlyRunning()) {
+ coord.start();
+ }
+ // now launch all the threads
+ executor = Executors.newFixedThreadPool(numThreads);
+
+ // now create consumers to consume the messages
+ int threadNumber = 0;
+ for (int i = 0; i < numThreads; i++) {
+ LOG.info("Creating consumer:" + threadNumber);
+ KafkaConsumer<String, CommandNotification> consumer = new KafkaConsumer<String, CommandNotification>(props, keyDe, valDe);
+ PeriodicNotificationConsumer periodicConsumer = new PeriodicNotificationConsumer(topic, consumer, threadNumber, coord);
+ consumers.add(periodicConsumer);
+ executor.submit(periodicConsumer);
+ threadNumber++;
+ }
+ running = true;
+ }
+ }
+
+ @Override
+ public boolean currentlyRunning() {
+ return running;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/registration/kafka/PeriodicNotificationConsumer.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/registration/kafka/PeriodicNotificationConsumer.java b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/registration/kafka/PeriodicNotificationConsumer.java
new file mode 100644
index 0000000..6785ce8
--- /dev/null
+++ b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/registration/kafka/PeriodicNotificationConsumer.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.registration.kafka;
+
+import java.util.Arrays;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.errors.WakeupException;
+import org.apache.log4j.Logger;
+import org.apache.rya.periodic.notification.api.NotificationCoordinatorExecutor;
+import org.apache.rya.periodic.notification.notification.CommandNotification;
+
+/**
+ * Consumer for the {@link KafkaNotificationProvider}. This consumer pull messages
+ * from Kafka and registers them with the {@link NotificationCoordinatorExecutor}.
+ *
+ */
+public class PeriodicNotificationConsumer implements Runnable {
+ private KafkaConsumer<String, CommandNotification> consumer;
+ private int m_threadNumber;
+ private String topic;
+ private final AtomicBoolean closed = new AtomicBoolean(false);
+ private NotificationCoordinatorExecutor coord;
+ private static final Logger LOG = Logger.getLogger(PeriodicNotificationConsumer.class);
+
+ /**
+ * Creates a new PeriodicNotificationConsumer for consuming new notification requests from
+ * Kafka.
+ * @param topic - new notification topic
+ * @param consumer - consumer for pulling new requests from Kafka
+ * @param a_threadNumber - number of consumer threads to be used
+ * @param coord - notification coordinator for managing and generating notifications
+ */
+ public PeriodicNotificationConsumer(String topic, KafkaConsumer<String, CommandNotification> consumer, int a_threadNumber,
+ NotificationCoordinatorExecutor coord) {
+ this.topic = topic;
+ m_threadNumber = a_threadNumber;
+ this.consumer = consumer;
+ this.coord = coord;
+ }
+
+ public void run() {
+
+ try {
+ LOG.info("Creating kafka stream for consumer:" + m_threadNumber);
+ consumer.subscribe(Arrays.asList(topic));
+ while (!closed.get()) {
+ ConsumerRecords<String, CommandNotification> records = consumer.poll(10000);
+ // Handle new records
+ for(ConsumerRecord<String, CommandNotification> record: records) {
+ CommandNotification notification = record.value();
+ LOG.info("Thread " + m_threadNumber + " is adding notification " + notification + " to queue.");
+ LOG.info("Message: " + notification);
+ coord.processNextCommandNotification(notification);
+ }
+ }
+ } catch (WakeupException e) {
+ // Ignore exception if closing
+ if (!closed.get()) throw e;
+ } finally {
+ consumer.close();
+ }
+ }
+
+ public void shutdown() {
+ closed.set(true);
+ consumer.wakeup();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/service/src/test/java/org/apache/rya/periodic/notification/serialization/CommandNotificationSerializerTest.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/service/src/test/java/org/apache/rya/periodic/notification/serialization/CommandNotificationSerializerTest.java b/extras/periodic.notification/service/src/test/java/org/apache/rya/periodic/notification/serialization/CommandNotificationSerializerTest.java
new file mode 100644
index 0000000..4aad1c6
--- /dev/null
+++ b/extras/periodic.notification/service/src/test/java/org/apache/rya/periodic/notification/serialization/CommandNotificationSerializerTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.serialization;
+
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.rya.periodic.notification.notification.BasicNotification;
+import org.apache.rya.periodic.notification.notification.CommandNotification;
+import org.apache.rya.periodic.notification.notification.CommandNotification.Command;
+import org.apache.rya.periodic.notification.notification.PeriodicNotification;
+import org.apache.rya.periodic.notification.serialization.CommandNotificationSerializer;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class CommandNotificationSerializerTest {
+
+ private CommandNotificationSerializer serializer = new CommandNotificationSerializer();
+ private static final String topic = "topic";
+
+ @Test
+ public void basicSerializationTest() {
+ PeriodicNotification notification = PeriodicNotification.builder().id(UUID.randomUUID().toString()).period(24)
+ .timeUnit(TimeUnit.DAYS).initialDelay(1).build();
+ CommandNotification command = new CommandNotification(Command.ADD, notification);
+ Assert.assertEquals(command, serializer.deserialize(topic,serializer.serialize(topic, command)));
+
+ PeriodicNotification notification1 = PeriodicNotification.builder().id(UUID.randomUUID().toString()).period(32)
+ .timeUnit(TimeUnit.SECONDS).initialDelay(15).build();
+ CommandNotification command1 = new CommandNotification(Command.ADD, notification1);
+ Assert.assertEquals(command1, serializer.deserialize(topic,serializer.serialize(topic,command1)));
+
+ PeriodicNotification notification2 = PeriodicNotification.builder().id(UUID.randomUUID().toString()).period(32)
+ .timeUnit(TimeUnit.SECONDS).initialDelay(15).build();
+ CommandNotification command2 = new CommandNotification(Command.ADD, notification2);
+ Assert.assertEquals(command2, serializer.deserialize(topic,serializer.serialize(topic,command2)));
+
+ BasicNotification notification3 = new BasicNotification(UUID.randomUUID().toString());
+ CommandNotification command3 = new CommandNotification(Command.ADD, notification3);
+ Assert.assertEquals(command3, serializer.deserialize(topic,serializer.serialize(topic,command3)));
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/tests/pom.xml
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/tests/pom.xml b/extras/periodic.notification/tests/pom.xml
new file mode 100644
index 0000000..31a6c0e
--- /dev/null
+++ b/extras/periodic.notification/tests/pom.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ you under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.rya</groupId>
+ <artifactId>rya.periodic.notification.parent</artifactId>
+ <version>3.2.11-incubating-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>rya.periodic.notification.tests</artifactId>
+
+ <name>Apache Rya Periodic Notification Service Integration Tests</name>
+ <description>Integration Tests for Rya Periodic Notification Service</description>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.rya</groupId>
+ <artifactId>rya.pcj.fluo.test.base</artifactId>
+ <exclusions>
+ <exclusion>
+ <artifactId>log4j-1.2-api</artifactId>
+ <groupId>org.apache.logging.log4j</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>log4j-api</artifactId>
+ <groupId>org.apache.logging.log4j</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>log4j-core</artifactId>
+ <groupId>org.apache.logging.log4j</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rya</groupId>
+ <artifactId>rya.periodic.notification.service</artifactId>
+ <exclusions>
+ <exclusion>
+ <artifactId>logback-classic</artifactId>
+ <groupId>ch.qos.logback</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>logback-core</artifactId>
+ <groupId>ch.qos.logback</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/tests/src/test/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplicationIT.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/tests/src/test/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplicationIT.java b/extras/periodic.notification/tests/src/test/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplicationIT.java
new file mode 100644
index 0000000..9109775
--- /dev/null
+++ b/extras/periodic.notification/tests/src/test/java/org/apache/rya/periodic/notification/application/PeriodicNotificationApplicationIT.java
@@ -0,0 +1,493 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.application;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Properties;
+import java.util.Set;
+import java.util.UUID;
+
+import javax.xml.datatype.DatatypeConfigurationException;
+import javax.xml.datatype.DatatypeFactory;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.fluo.api.client.FluoClient;
+import org.apache.fluo.api.config.FluoConfiguration;
+import org.apache.fluo.core.client.FluoClientImpl;
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.apache.kafka.common.serialization.StringSerializer;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.indexing.accumulo.ConfigUtils;
+import org.apache.rya.indexing.pcj.fluo.api.CreatePeriodicQuery;
+import org.apache.rya.indexing.pcj.fluo.api.InsertTriples;
+import org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants;
+import org.apache.rya.indexing.pcj.fluo.app.util.FluoClientFactory;
+import org.apache.rya.indexing.pcj.fluo.app.util.FluoQueryUtils;
+import org.apache.rya.indexing.pcj.storage.PeriodicQueryResultStorage;
+import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPeriodicQueryResultStorage;
+import org.apache.rya.kafka.base.EmbeddedKafkaInstance;
+import org.apache.rya.kafka.base.EmbeddedKafkaSingleton;
+import org.apache.rya.kafka.base.KafkaTestInstanceRule;
+import org.apache.rya.pcj.fluo.test.base.RyaExportITBase;
+import org.apache.rya.periodic.notification.notification.CommandNotification;
+import org.apache.rya.periodic.notification.registration.KafkaNotificationRegistrationClient;
+import org.apache.rya.periodic.notification.serialization.BindingSetSerDe;
+import org.apache.rya.periodic.notification.serialization.CommandNotificationSerializer;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.openrdf.model.Statement;
+import org.openrdf.model.Value;
+import org.openrdf.model.ValueFactory;
+import org.openrdf.model.impl.LiteralImpl;
+import org.openrdf.model.impl.ValueFactoryImpl;
+import org.openrdf.model.vocabulary.XMLSchema;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.algebra.evaluation.QueryBindingSet;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
+
+import static org.apache.rya.periodic.notification.application.PeriodicNotificationApplicationConfiguration.NOTIFICATION_TOPIC;
+import static org.apache.rya.periodic.notification.application.PeriodicNotificationApplicationConfiguration.KAFKA_BOOTSTRAP_SERVERS;;
+
+
+public class PeriodicNotificationApplicationIT extends RyaExportITBase {
+
+ private PeriodicNotificationApplication app;
+ private KafkaNotificationRegistrationClient registrar;
+ private KafkaProducer<String, CommandNotification> producer;
+ private Properties props;
+ private Properties kafkaProps;
+ private PeriodicNotificationApplicationConfiguration conf;
+ private static EmbeddedKafkaInstance embeddedKafka = EmbeddedKafkaSingleton.getInstance();
+ private static String bootstrapServers;
+
+ @Rule
+ public KafkaTestInstanceRule rule = new KafkaTestInstanceRule(false);
+
+ @BeforeClass
+ public static void initClass() {
+ bootstrapServers = embeddedKafka.createBootstrapServerConfig().getProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG);
+ }
+
+ @Before
+ public void init() throws Exception {
+ String topic = rule.getKafkaTopicName();
+ rule.createTopic(topic);
+
+ //get user specified props and update with the embedded kafka bootstrap servers and rule generated topic
+ props = getProps();
+ props.setProperty(NOTIFICATION_TOPIC, topic);
+ props.setProperty(KAFKA_BOOTSTRAP_SERVERS, bootstrapServers);
+ conf = new PeriodicNotificationApplicationConfiguration(props);
+
+ //create Kafka Producer
+ kafkaProps = getKafkaProperties(conf);
+ producer = new KafkaProducer<>(kafkaProps, new StringSerializer(), new CommandNotificationSerializer());
+
+ //extract kafka specific properties from application config
+ app = PeriodicNotificationApplicationFactory.getPeriodicApplication(props);
+ registrar = new KafkaNotificationRegistrationClient(conf.getNotificationTopic(), producer);
+ }
+
+ @Test
+ public void periodicApplicationWithAggAndGroupByTest() throws Exception {
+
+ String sparql = "prefix function: <http://org.apache.rya/function#> " // n
+ + "prefix time: <http://www.w3.org/2006/time#> " // n
+ + "select ?type (count(?obs) as ?total) where {" // n
+ + "Filter(function:periodic(?time, 1, .25, time:minutes)) " // n
+ + "?obs <uri:hasTime> ?time. " // n
+ + "?obs <uri:hasObsType> ?type } group by ?type"; // n
+
+ //make data
+ int periodMult = 15;
+ final ValueFactory vf = new ValueFactoryImpl();
+ final DatatypeFactory dtf = DatatypeFactory.newInstance();
+ //Sleep until current time aligns nicely with period to makell
+ //results more predictable
+ while(System.currentTimeMillis() % (periodMult*1000) > 500);
+ ZonedDateTime time = ZonedDateTime.now();
+
+ ZonedDateTime zTime1 = time.minusSeconds(2*periodMult);
+ String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT);
+
+ ZonedDateTime zTime2 = zTime1.minusSeconds(periodMult);
+ String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT);
+
+ ZonedDateTime zTime3 = zTime2.minusSeconds(periodMult);
+ String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT);
+
+ final Collection<Statement> statements = Sets.newHashSet(
+ vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time1))),
+ vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasObsType"), vf.createLiteral("ship")),
+ vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time1))),
+ vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasObsType"), vf.createLiteral("airplane")),
+ vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time2))),
+ vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasObsType"), vf.createLiteral("ship")),
+ vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time2))),
+ vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasObsType"), vf.createLiteral("airplane")),
+ vf.createStatement(vf.createURI("urn:obs_5"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time3))),
+ vf.createStatement(vf.createURI("urn:obs_5"), vf.createURI("uri:hasObsType"), vf.createLiteral("automobile")));
+
+ try (FluoClient fluo = FluoClientFactory.getFluoClient(conf.getFluoAppName(), Optional.of(conf.getFluoTableName()), conf)) {
+ Connector connector = ConfigUtils.getConnector(conf);
+ PeriodicQueryResultStorage storage = new AccumuloPeriodicQueryResultStorage(connector, conf.getTablePrefix());
+ CreatePeriodicQuery periodicQuery = new CreatePeriodicQuery(fluo, storage);
+ String id = FluoQueryUtils.convertFluoQueryIdToPcjId(periodicQuery.createPeriodicQuery(sparql, registrar).getQueryId());
+ addData(statements);
+ app.start();
+
+ Multimap<Long, BindingSet> actual = HashMultimap.create();
+ try (KafkaConsumer<String, BindingSet> consumer = new KafkaConsumer<>(kafkaProps, new StringDeserializer(), new BindingSetSerDe())) {
+ consumer.subscribe(Arrays.asList(id));
+ long end = System.currentTimeMillis() + 4*periodMult*1000;
+ long lastBinId = 0L;
+ long binId = 0L;
+ List<Long> ids = new ArrayList<>();
+ while (System.currentTimeMillis() < end) {
+ ConsumerRecords<String, BindingSet> records = consumer.poll(periodMult*1000);
+ for(ConsumerRecord<String, BindingSet> record: records){
+ BindingSet result = record.value();
+ binId = Long.parseLong(result.getBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID).getValue().stringValue());
+ if(lastBinId != binId) {
+ lastBinId = binId;
+ ids.add(binId);
+ }
+ actual.put(binId, result);
+ }
+ }
+
+ Map<Long, Set<BindingSet>> expected = new HashMap<>();
+
+ Set<BindingSet> expected1 = new HashSet<>();
+ QueryBindingSet bs1 = new QueryBindingSet();
+ bs1.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(ids.get(0)));
+ bs1.addBinding("total", new LiteralImpl("2", XMLSchema.INTEGER));
+ bs1.addBinding("type", vf.createLiteral("airplane"));
+
+ QueryBindingSet bs2 = new QueryBindingSet();
+ bs2.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(ids.get(0)));
+ bs2.addBinding("total", new LiteralImpl("2", XMLSchema.INTEGER));
+ bs2.addBinding("type", vf.createLiteral("ship"));
+
+ QueryBindingSet bs3 = new QueryBindingSet();
+ bs3.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(ids.get(0)));
+ bs3.addBinding("total", new LiteralImpl("1", XMLSchema.INTEGER));
+ bs3.addBinding("type", vf.createLiteral("automobile"));
+
+ expected1.add(bs1);
+ expected1.add(bs2);
+ expected1.add(bs3);
+
+ Set<BindingSet> expected2 = new HashSet<>();
+ QueryBindingSet bs4 = new QueryBindingSet();
+ bs4.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(ids.get(1)));
+ bs4.addBinding("total", new LiteralImpl("2", XMLSchema.INTEGER));
+ bs4.addBinding("type", vf.createLiteral("airplane"));
+
+ QueryBindingSet bs5 = new QueryBindingSet();
+ bs5.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(ids.get(1)));
+ bs5.addBinding("total", new LiteralImpl("2", XMLSchema.INTEGER));
+ bs5.addBinding("type", vf.createLiteral("ship"));
+
+ expected2.add(bs4);
+ expected2.add(bs5);
+
+ Set<BindingSet> expected3 = new HashSet<>();
+ QueryBindingSet bs6 = new QueryBindingSet();
+ bs6.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(ids.get(2)));
+ bs6.addBinding("total", new LiteralImpl("1", XMLSchema.INTEGER));
+ bs6.addBinding("type", vf.createLiteral("ship"));
+
+ QueryBindingSet bs7 = new QueryBindingSet();
+ bs7.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(ids.get(2)));
+ bs7.addBinding("total", new LiteralImpl("1", XMLSchema.INTEGER));
+ bs7.addBinding("type", vf.createLiteral("airplane"));
+
+ expected3.add(bs6);
+ expected3.add(bs7);
+
+ expected.put(ids.get(0), expected1);
+ expected.put(ids.get(1), expected2);
+ expected.put(ids.get(2), expected3);
+
+ Assert.assertEquals(3, actual.asMap().size());
+ for(Long ident: ids) {
+ Assert.assertEquals(expected.get(ident), actual.get(ident));
+ }
+ }
+
+ Set<BindingSet> expectedResults = new HashSet<>();
+ try (CloseableIterator<BindingSet> results = storage.listResults(id, Optional.empty())) {
+ results.forEachRemaining(x -> expectedResults.add(x));
+ Assert.assertEquals(0, expectedResults.size());
+ }
+ }
+ }
+
+
+ @Test
+ public void periodicApplicationWithAggTest() throws Exception {
+
+ String sparql = "prefix function: <http://org.apache.rya/function#> " // n
+ + "prefix time: <http://www.w3.org/2006/time#> " // n
+ + "select (count(?obs) as ?total) where {" // n
+ + "Filter(function:periodic(?time, 1, .25, time:minutes)) " // n
+ + "?obs <uri:hasTime> ?time. " // n
+ + "?obs <uri:hasId> ?id } "; // n
+
+ //make data
+ int periodMult = 15;
+ final ValueFactory vf = new ValueFactoryImpl();
+ final DatatypeFactory dtf = DatatypeFactory.newInstance();
+ //Sleep until current time aligns nicely with period to make
+ //results more predictable
+ while(System.currentTimeMillis() % (periodMult*1000) > 500);
+ ZonedDateTime time = ZonedDateTime.now();
+
+ ZonedDateTime zTime1 = time.minusSeconds(2*periodMult);
+ String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT);
+
+ ZonedDateTime zTime2 = zTime1.minusSeconds(periodMult);
+ String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT);
+
+ ZonedDateTime zTime3 = zTime2.minusSeconds(periodMult);
+ String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT);
+
+ final Collection<Statement> statements = Sets.newHashSet(
+ vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time1))),
+ vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasId"), vf.createLiteral("id_1")),
+ vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time2))),
+ vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasId"), vf.createLiteral("id_2")),
+ vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time3))),
+ vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasId"), vf.createLiteral("id_3")));
+
+ try (FluoClient fluo = FluoClientFactory.getFluoClient(conf.getFluoAppName(), Optional.of(conf.getFluoTableName()), conf)) {
+ Connector connector = ConfigUtils.getConnector(conf);
+ PeriodicQueryResultStorage storage = new AccumuloPeriodicQueryResultStorage(connector, conf.getTablePrefix());
+ CreatePeriodicQuery periodicQuery = new CreatePeriodicQuery(fluo, storage);
+ String id = FluoQueryUtils.convertFluoQueryIdToPcjId(periodicQuery.createPeriodicQuery(sparql, registrar).getQueryId());
+ addData(statements);
+ app.start();
+
+ Multimap<Long, BindingSet> expected = HashMultimap.create();
+ try (KafkaConsumer<String, BindingSet> consumer = new KafkaConsumer<>(kafkaProps, new StringDeserializer(), new BindingSetSerDe())) {
+ consumer.subscribe(Arrays.asList(id));
+ long end = System.currentTimeMillis() + 4*periodMult*1000;
+ long lastBinId = 0L;
+ long binId = 0L;
+ List<Long> ids = new ArrayList<>();
+ while (System.currentTimeMillis() < end) {
+ ConsumerRecords<String, BindingSet> records = consumer.poll(periodMult*1000);
+ for(ConsumerRecord<String, BindingSet> record: records){
+ BindingSet result = record.value();
+ binId = Long.parseLong(result.getBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID).getValue().stringValue());
+ if(lastBinId != binId) {
+ lastBinId = binId;
+ ids.add(binId);
+ }
+ expected.put(binId, result);
+ }
+ }
+
+ Assert.assertEquals(3, expected.asMap().size());
+ int i = 0;
+ for(Long ident: ids) {
+ Assert.assertEquals(1, expected.get(ident).size());
+ BindingSet bs = expected.get(ident).iterator().next();
+ Value val = bs.getValue("total");
+ int total = Integer.parseInt(val.stringValue());
+ Assert.assertEquals(3-i, total);
+ i++;
+ }
+ }
+
+
+ Set<BindingSet> expectedResults = new HashSet<>();
+ try (CloseableIterator<BindingSet> results = storage.listResults(id, Optional.empty())) {
+ results.forEachRemaining(x -> expectedResults.add(x));
+ Assert.assertEquals(0, expectedResults.size());
+ }
+ }
+
+ }
+
+
+ @Test
+ public void periodicApplicationTest() throws Exception {
+
+ String sparql = "prefix function: <http://org.apache.rya/function#> " // n
+ + "prefix time: <http://www.w3.org/2006/time#> " // n
+ + "select ?obs ?id where {" // n
+ + "Filter(function:periodic(?time, 1, .25, time:minutes)) " // n
+ + "?obs <uri:hasTime> ?time. " // n
+ + "?obs <uri:hasId> ?id } "; // n
+
+ //make data
+ int periodMult = 15;
+ final ValueFactory vf = new ValueFactoryImpl();
+ final DatatypeFactory dtf = DatatypeFactory.newInstance();
+ //Sleep until current time aligns nicely with period to make
+ //results more predictable
+ while(System.currentTimeMillis() % (periodMult*1000) > 500);
+ ZonedDateTime time = ZonedDateTime.now();
+
+ ZonedDateTime zTime1 = time.minusSeconds(2*periodMult);
+ String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT);
+
+ ZonedDateTime zTime2 = zTime1.minusSeconds(periodMult);
+ String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT);
+
+ ZonedDateTime zTime3 = zTime2.minusSeconds(periodMult);
+ String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT);
+
+ final Collection<Statement> statements = Sets.newHashSet(
+ vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time1))),
+ vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasId"), vf.createLiteral("id_1")),
+ vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time2))),
+ vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasId"), vf.createLiteral("id_2")),
+ vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time3))),
+ vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasId"), vf.createLiteral("id_3")));
+
+ try (FluoClient fluo = FluoClientFactory.getFluoClient(conf.getFluoAppName(), Optional.of(conf.getFluoTableName()), conf)) {
+ Connector connector = ConfigUtils.getConnector(conf);
+ PeriodicQueryResultStorage storage = new AccumuloPeriodicQueryResultStorage(connector, conf.getTablePrefix());
+ CreatePeriodicQuery periodicQuery = new CreatePeriodicQuery(fluo, storage);
+ String id = FluoQueryUtils.convertFluoQueryIdToPcjId(periodicQuery.createPeriodicQuery(sparql, registrar).getQueryId());
+ addData(statements);
+ app.start();
+
+ Multimap<Long, BindingSet> expected = HashMultimap.create();
+ try (KafkaConsumer<String, BindingSet> consumer = new KafkaConsumer<>(kafkaProps, new StringDeserializer(), new BindingSetSerDe())) {
+ consumer.subscribe(Arrays.asList(id));
+ long end = System.currentTimeMillis() + 4*periodMult*1000;
+ long lastBinId = 0L;
+ long binId = 0L;
+ List<Long> ids = new ArrayList<>();
+ while (System.currentTimeMillis() < end) {
+ ConsumerRecords<String, BindingSet> records = consumer.poll(periodMult*1000);
+ for(ConsumerRecord<String, BindingSet> record: records){
+ BindingSet result = record.value();
+ binId = Long.parseLong(result.getBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID).getValue().stringValue());
+ if(lastBinId != binId) {
+ lastBinId = binId;
+ ids.add(binId);
+ }
+ expected.put(binId, result);
+ }
+ }
+
+ Assert.assertEquals(3, expected.asMap().size());
+ int i = 0;
+ for(Long ident: ids) {
+ Assert.assertEquals(3-i, expected.get(ident).size());
+ i++;
+ }
+ }
+
+
+ Set<BindingSet> expectedResults = new HashSet<>();
+ try (CloseableIterator<BindingSet> results = storage.listResults(id, Optional.empty())) {
+ results.forEachRemaining(x -> expectedResults.add(x));
+ Assert.assertEquals(0, expectedResults.size());
+ }
+ }
+
+ }
+
+
+ @After
+ public void shutdown() {
+ registrar.close();
+ app.stop();
+ }
+
+ private void addData(Collection<Statement> statements) throws DatatypeConfigurationException {
+ // add statements to Fluo
+ try (FluoClient fluo = new FluoClientImpl(getFluoConfiguration())) {
+ InsertTriples inserter = new InsertTriples();
+ statements.forEach(x -> inserter.insert(fluo, RdfToRyaConversions.convertStatement(x)));
+ getMiniFluo().waitForObservers();
+ }
+ }
+
+ private static Properties getKafkaProperties(PeriodicNotificationApplicationConfiguration conf) {
+ Properties kafkaProps = new Properties();
+ kafkaProps.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
+ kafkaProps.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, UUID.randomUUID().toString());
+ kafkaProps.setProperty(ConsumerConfig.GROUP_ID_CONFIG, conf.getNotificationGroupId());
+ kafkaProps.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+ return kafkaProps;
+ }
+
+ private Properties getProps() throws IOException {
+
+ Properties props = new Properties();
+ try(InputStream in = new FileInputStream("src/test/resources/notification.properties")) {
+ props.load(in);
+ }
+
+ FluoConfiguration fluoConf = getFluoConfiguration();
+ props.setProperty("accumulo.user", getUsername());
+ props.setProperty("accumulo.password", getPassword());
+ props.setProperty("accumulo.instance", getMiniAccumuloCluster().getInstanceName());
+ props.setProperty("accumulo.zookeepers", getMiniAccumuloCluster().getZooKeepers());
+ props.setProperty("accumulo.rya.prefix", getRyaInstanceName());
+ props.setProperty(PeriodicNotificationApplicationConfiguration.FLUO_APP_NAME, fluoConf.getApplicationName());
+ props.setProperty(PeriodicNotificationApplicationConfiguration.FLUO_TABLE_NAME, fluoConf.getAccumuloTable());
+ return props;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/tests/src/test/java/org/apache/rya/periodic/notification/application/PeriodicNotificationProviderIT.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/tests/src/test/java/org/apache/rya/periodic/notification/application/PeriodicNotificationProviderIT.java b/extras/periodic.notification/tests/src/test/java/org/apache/rya/periodic/notification/application/PeriodicNotificationProviderIT.java
new file mode 100644
index 0000000..e05ca6f
--- /dev/null
+++ b/extras/periodic.notification/tests/src/test/java/org/apache/rya/periodic/notification/application/PeriodicNotificationProviderIT.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.application;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.fluo.api.client.FluoClient;
+import org.apache.fluo.core.client.FluoClientImpl;
+import org.apache.fluo.recipes.test.AccumuloExportITBase;
+import org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj;
+import org.apache.rya.indexing.pcj.fluo.app.query.UnsupportedQueryException;
+import org.apache.rya.indexing.pcj.fluo.app.util.FluoQueryUtils;
+import org.apache.rya.periodic.notification.coordinator.PeriodicNotificationCoordinatorExecutor;
+import org.apache.rya.periodic.notification.notification.TimestampedNotification;
+import org.apache.rya.periodic.notification.recovery.PeriodicNotificationProvider;
+import org.junit.Assert;
+import org.junit.Test;
+import org.openrdf.query.MalformedQueryException;
+
+import com.google.common.collect.Sets;
+
+public class PeriodicNotificationProviderIT extends AccumuloExportITBase {
+
+ @Test
+ public void testProvider() throws MalformedQueryException, InterruptedException, UnsupportedQueryException {
+
+ String sparql = "prefix function: <http://org.apache.rya/function#> " // n
+ + "prefix time: <http://www.w3.org/2006/time#> " // n
+ + "select ?id (count(?obs) as ?total) where {" // n
+ + "Filter(function:periodic(?time, 1, .25, time:minutes)) " // n
+ + "?obs <uri:hasTime> ?time. " // n
+ + "?obs <uri:hasId> ?id } group by ?id"; // n
+
+ BlockingQueue<TimestampedNotification> notifications = new LinkedBlockingQueue<>();
+ PeriodicNotificationCoordinatorExecutor coord = new PeriodicNotificationCoordinatorExecutor(2, notifications);
+ PeriodicNotificationProvider provider = new PeriodicNotificationProvider();
+ CreateFluoPcj pcj = new CreateFluoPcj();
+
+ String id = null;
+ try(FluoClient fluo = new FluoClientImpl(getFluoConfiguration())) {
+ id = pcj.createPcj(FluoQueryUtils.createNewPcjId(), sparql, Sets.newHashSet(), fluo).getQueryId();
+ provider.processRegisteredNotifications(coord, fluo.newSnapshot());
+ }
+
+ TimestampedNotification notification = notifications.take();
+ Assert.assertEquals(5000, notification.getInitialDelay());
+ Assert.assertEquals(15000, notification.getPeriod());
+ Assert.assertEquals(TimeUnit.MILLISECONDS, notification.getTimeUnit());
+ Assert.assertEquals(FluoQueryUtils.convertFluoQueryIdToPcjId(id), notification.getId());
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/tests/src/test/java/org/apache/rya/periodic/notification/exporter/PeriodicNotificationExporterIT.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/tests/src/test/java/org/apache/rya/periodic/notification/exporter/PeriodicNotificationExporterIT.java b/extras/periodic.notification/tests/src/test/java/org/apache/rya/periodic/notification/exporter/PeriodicNotificationExporterIT.java
new file mode 100644
index 0000000..874e7e2
--- /dev/null
+++ b/extras/periodic.notification/tests/src/test/java/org/apache/rya/periodic/notification/exporter/PeriodicNotificationExporterIT.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.exporter;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Properties;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.apache.kafka.common.serialization.StringSerializer;
+import org.apache.rya.indexing.pcj.storage.PeriodicQueryResultStorage;
+import org.apache.rya.kafka.base.KafkaITBase;
+import org.apache.rya.kafka.base.KafkaTestInstanceRule;
+import org.apache.rya.periodic.notification.api.BindingSetRecord;
+import org.apache.rya.periodic.notification.serialization.BindingSetSerDe;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.openrdf.model.ValueFactory;
+import org.openrdf.model.impl.ValueFactoryImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.algebra.evaluation.QueryBindingSet;
+
+public class PeriodicNotificationExporterIT extends KafkaITBase {
+
+
+ @Rule
+ public KafkaTestInstanceRule kafkaTestInstanceRule = new KafkaTestInstanceRule(false);
+
+
+ private static final ValueFactory vf = new ValueFactoryImpl();
+
+ @Test
+ public void testExporter() throws InterruptedException {
+
+ final String topic1 = kafkaTestInstanceRule.getKafkaTopicName() + "1";
+ final String topic2 = kafkaTestInstanceRule.getKafkaTopicName() + "2";
+
+ kafkaTestInstanceRule.createTopic(topic1);
+ kafkaTestInstanceRule.createTopic(topic2);
+
+ final BlockingQueue<BindingSetRecord> records = new LinkedBlockingQueue<>();
+
+ final KafkaExporterExecutor exporter = new KafkaExporterExecutor(new KafkaProducer<String, BindingSet>(createKafkaProducerConfig()), 1, records);
+ exporter.start();
+ final QueryBindingSet bs1 = new QueryBindingSet();
+ bs1.addBinding(PeriodicQueryResultStorage.PeriodicBinId, vf.createLiteral(1L));
+ bs1.addBinding("name", vf.createURI("uri:Bob"));
+ final BindingSetRecord record1 = new BindingSetRecord(bs1, topic1);
+
+ final QueryBindingSet bs2 = new QueryBindingSet();
+ bs2.addBinding(PeriodicQueryResultStorage.PeriodicBinId, vf.createLiteral(2L));
+ bs2.addBinding("name", vf.createURI("uri:Joe"));
+ final BindingSetRecord record2 = new BindingSetRecord(bs2, topic2);
+
+ records.add(record1);
+ records.add(record2);
+
+ final Set<BindingSet> expected1 = new HashSet<>();
+ expected1.add(bs1);
+ final Set<BindingSet> expected2 = new HashSet<>();
+ expected2.add(bs2);
+
+ final Set<BindingSet> actual1 = getBindingSetsFromKafka(topic1);
+ final Set<BindingSet> actual2 = getBindingSetsFromKafka(topic2);
+
+ Assert.assertEquals(expected1, actual1);
+ Assert.assertEquals(expected2, actual2);
+
+ exporter.stop();
+ }
+
+
+ private Properties createKafkaProducerConfig() {
+ final Properties props = createBootstrapServerConfig();
+ props.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
+ props.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, BindingSetSerDe.class.getName());
+ return props;
+ }
+ private Properties createKafkaConsumerConfig() {
+ final Properties props = createBootstrapServerConfig();
+ props.setProperty(ConsumerConfig.GROUP_ID_CONFIG, UUID.randomUUID().toString());
+ props.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, "consumer0");
+ props.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+ props.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
+ props.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, BindingSetSerDe.class.getName());
+ return props;
+ }
+
+
+ private KafkaConsumer<String, BindingSet> makeBindingSetConsumer(final String topicName) {
+ // setup consumer
+ final KafkaConsumer<String, BindingSet> consumer = new KafkaConsumer<>(createKafkaConsumerConfig());
+ consumer.subscribe(Arrays.asList(topicName));
+ return consumer;
+ }
+
+ private Set<BindingSet> getBindingSetsFromKafka(final String topicName) {
+ KafkaConsumer<String, BindingSet> consumer = null;
+
+ try {
+ consumer = makeBindingSetConsumer(topicName);
+ final ConsumerRecords<String, BindingSet> records = consumer.poll(20000); // Wait up to 20 seconds for a result to be published.
+
+ final Set<BindingSet> bindingSets = new HashSet<>();
+ records.forEach(x -> bindingSets.add(x.value()));
+
+ return bindingSets;
+
+ } catch (final Exception e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (consumer != null) {
+ consumer.close();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/tests/src/test/java/org/apache/rya/periodic/notification/processor/PeriodicNotificationProcessorIT.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/tests/src/test/java/org/apache/rya/periodic/notification/processor/PeriodicNotificationProcessorIT.java b/extras/periodic.notification/tests/src/test/java/org/apache/rya/periodic/notification/processor/PeriodicNotificationProcessorIT.java
new file mode 100644
index 0000000..21109ae
--- /dev/null
+++ b/extras/periodic.notification/tests/src/test/java/org/apache/rya/periodic/notification/processor/PeriodicNotificationProcessorIT.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.processor;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.fluo.recipes.test.AccumuloExportITBase;
+import org.apache.rya.indexing.pcj.storage.PeriodicQueryResultStorage;
+import org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPeriodicQueryResultStorage;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.apache.rya.periodic.notification.api.BindingSetRecord;
+import org.apache.rya.periodic.notification.api.NodeBin;
+import org.apache.rya.periodic.notification.notification.PeriodicNotification;
+import org.apache.rya.periodic.notification.notification.TimestampedNotification;
+import org.junit.Assert;
+import org.junit.Test;
+import org.openrdf.model.ValueFactory;
+import org.openrdf.model.impl.ValueFactoryImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.algebra.evaluation.QueryBindingSet;
+
+public class PeriodicNotificationProcessorIT extends AccumuloExportITBase {
+
+ private static final ValueFactory vf = new ValueFactoryImpl();
+ private static final String RYA_INSTANCE_NAME = "rya_";
+
+ @Test
+ public void periodicProcessorTest() throws Exception {
+
+ String id = UUID.randomUUID().toString().replace("-", "");
+ BlockingQueue<TimestampedNotification> notifications = new LinkedBlockingQueue<>();
+ BlockingQueue<NodeBin> bins = new LinkedBlockingQueue<>();
+ BlockingQueue<BindingSetRecord> bindingSets = new LinkedBlockingQueue<>();
+
+ TimestampedNotification ts1 = new TimestampedNotification(
+ PeriodicNotification.builder().id(id).initialDelay(0).period(2000).timeUnit(TimeUnit.SECONDS).build());
+ long binId1 = (ts1.getTimestamp().getTime()/ts1.getPeriod())*ts1.getPeriod();
+
+ Thread.sleep(2000);
+
+ TimestampedNotification ts2 = new TimestampedNotification(
+ PeriodicNotification.builder().id(id).initialDelay(0).period(2000).timeUnit(TimeUnit.SECONDS).build());
+ long binId2 = (ts2.getTimestamp().getTime()/ts2.getPeriod())*ts2.getPeriod();
+
+ Set<NodeBin> expectedBins = new HashSet<>();
+ expectedBins.add(new NodeBin(id, binId1));
+ expectedBins.add(new NodeBin(id, binId2));
+
+ Set<BindingSet> expected = new HashSet<>();
+ Set<VisibilityBindingSet> storageResults = new HashSet<>();
+
+ QueryBindingSet bs1 = new QueryBindingSet();
+ bs1.addBinding("periodicBinId", vf.createLiteral(binId1));
+ bs1.addBinding("id", vf.createLiteral(1));
+ expected.add(bs1);
+ storageResults.add(new VisibilityBindingSet(bs1));
+
+ QueryBindingSet bs2 = new QueryBindingSet();
+ bs2.addBinding("periodicBinId", vf.createLiteral(binId1));
+ bs2.addBinding("id", vf.createLiteral(2));
+ expected.add(bs2);
+ storageResults.add(new VisibilityBindingSet(bs2));
+
+ QueryBindingSet bs3 = new QueryBindingSet();
+ bs3.addBinding("periodicBinId", vf.createLiteral(binId2));
+ bs3.addBinding("id", vf.createLiteral(3));
+ expected.add(bs3);
+ storageResults.add(new VisibilityBindingSet(bs3));
+
+ QueryBindingSet bs4 = new QueryBindingSet();
+ bs4.addBinding("periodicBinId", vf.createLiteral(binId2));
+ bs4.addBinding("id", vf.createLiteral(4));
+ expected.add(bs4);
+ storageResults.add(new VisibilityBindingSet(bs4));
+
+ PeriodicQueryResultStorage periodicStorage = new AccumuloPeriodicQueryResultStorage(super.getAccumuloConnector(),
+ RYA_INSTANCE_NAME);
+ periodicStorage.createPeriodicQuery(id, "select ?id where {?obs <urn:hasId> ?id.}", new VariableOrder("periodicBinId", "id"));
+ periodicStorage.addPeriodicQueryResults(id, storageResults);
+
+ NotificationProcessorExecutor processor = new NotificationProcessorExecutor(periodicStorage, notifications, bins, bindingSets, 1);
+ processor.start();
+
+ notifications.add(ts1);
+ notifications.add(ts2);
+
+ Thread.sleep(5000);
+
+ Assert.assertEquals(expectedBins.size(), bins.size());
+ Assert.assertEquals(true, bins.containsAll(expectedBins));
+
+ Set<BindingSet> actual = new HashSet<>();
+ bindingSets.forEach(x -> actual.add(x.getBindingSet()));
+ Assert.assertEquals(expected, actual);
+
+ processor.stop();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/tests/src/test/java/org/apache/rya/periodic/notification/pruner/PeriodicNotificationBinPrunerIT.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/tests/src/test/java/org/apache/rya/periodic/notification/pruner/PeriodicNotificationBinPrunerIT.java b/extras/periodic.notification/tests/src/test/java/org/apache/rya/periodic/notification/pruner/PeriodicNotificationBinPrunerIT.java
new file mode 100644
index 0000000..830fa46
--- /dev/null
+++ b/extras/periodic.notification/tests/src/test/java/org/apache/rya/periodic/notification/pruner/PeriodicNotificationBinPrunerIT.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.pruner;
+
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import javax.xml.datatype.DatatypeFactory;
+
+import org.apache.fluo.api.client.FluoClient;
+import org.apache.fluo.api.client.Snapshot;
+import org.apache.fluo.api.client.scanner.ColumnScanner;
+import org.apache.fluo.api.client.scanner.RowScanner;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.api.data.ColumnValue;
+import org.apache.fluo.api.data.Span;
+import org.apache.fluo.core.client.FluoClientImpl;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.indexing.pcj.fluo.api.CreatePeriodicQuery;
+import org.apache.rya.indexing.pcj.fluo.api.InsertTriples;
+import org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants;
+import org.apache.rya.indexing.pcj.fluo.app.NodeType;
+import org.apache.rya.indexing.pcj.fluo.app.util.FluoQueryUtils;
+import org.apache.rya.indexing.pcj.fluo.app.util.PeriodicQueryUtil;
+import org.apache.rya.indexing.pcj.fluo.app.util.RowKeyUtil;
+import org.apache.rya.indexing.pcj.storage.PeriodicQueryResultStorage;
+import org.apache.rya.indexing.pcj.storage.PeriodicQueryStorageException;
+import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
+import org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPeriodicQueryResultStorage;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.pcj.fluo.test.base.RyaExportITBase;
+import org.apache.rya.periodic.notification.api.NodeBin;
+import org.junit.Assert;
+import org.junit.Test;
+import org.openrdf.model.Statement;
+import org.openrdf.model.ValueFactory;
+import org.openrdf.model.impl.LiteralImpl;
+import org.openrdf.model.impl.ValueFactoryImpl;
+import org.openrdf.model.vocabulary.XMLSchema;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.algebra.evaluation.QueryBindingSet;
+import org.openrdf.query.impl.MapBindingSet;
+
+import com.google.common.collect.Sets;
+
+public class PeriodicNotificationBinPrunerIT extends RyaExportITBase {
+
+
+ @Test
+ public void periodicPrunerTest() throws Exception {
+
+ String sparql = "prefix function: <http://org.apache.rya/function#> " // n
+ + "prefix time: <http://www.w3.org/2006/time#> " // n
+ + "select ?id (count(?obs) as ?total) where {" // n
+ + "Filter(function:periodic(?time, 2, .5, time:hours)) " // n
+ + "?obs <uri:hasTime> ?time. " // n
+ + "?obs <uri:hasId> ?id } group by ?id"; // n
+
+ FluoClient fluo = new FluoClientImpl(super.getFluoConfiguration());
+
+ // initialize resources and create pcj
+ PeriodicQueryResultStorage periodicStorage = new AccumuloPeriodicQueryResultStorage(super.getAccumuloConnector(),
+ getRyaInstanceName());
+ CreatePeriodicQuery createPeriodicQuery = new CreatePeriodicQuery(fluo, periodicStorage);
+ String queryId = FluoQueryUtils.convertFluoQueryIdToPcjId(createPeriodicQuery.createPeriodicQuery(sparql).getQueryId());
+
+ // create statements to ingest into Fluo
+ final ValueFactory vf = new ValueFactoryImpl();
+ final DatatypeFactory dtf = DatatypeFactory.newInstance();
+ ZonedDateTime time = ZonedDateTime.now();
+ long currentTime = time.toInstant().toEpochMilli();
+
+ ZonedDateTime zTime1 = time.minusMinutes(30);
+ String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT);
+
+ ZonedDateTime zTime2 = zTime1.minusMinutes(30);
+ String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT);
+
+ ZonedDateTime zTime3 = zTime2.minusMinutes(30);
+ String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT);
+
+ ZonedDateTime zTime4 = zTime3.minusMinutes(30);
+ String time4 = zTime4.format(DateTimeFormatter.ISO_INSTANT);
+
+ final Collection<Statement> statements = Sets.newHashSet(
+ vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time1))),
+ vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasId"), vf.createLiteral("id_1")),
+ vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time2))),
+ vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasId"), vf.createLiteral("id_2")),
+ vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time3))),
+ vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasId"), vf.createLiteral("id_3")),
+ vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time4))),
+ vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasId"), vf.createLiteral("id_4")),
+ vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time4))),
+ vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasId"), vf.createLiteral("id_1")),
+ vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time3))),
+ vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasId"), vf.createLiteral("id_2")));
+
+ // add statements to Fluo
+ InsertTriples inserter = new InsertTriples();
+ statements.forEach(x -> inserter.insert(fluo, RdfToRyaConversions.convertStatement(x)));
+
+ super.getMiniFluo().waitForObservers();
+
+ // FluoITHelper.printFluoTable(fluo);
+
+ // Create the expected results of the SPARQL query once the PCJ has been
+ // computed.
+ final Set<BindingSet> expected1 = new HashSet<>();
+ final Set<BindingSet> expected2 = new HashSet<>();
+ final Set<BindingSet> expected3 = new HashSet<>();
+ final Set<BindingSet> expected4 = new HashSet<>();
+
+ long period = 1800000;
+ long binId = (currentTime / period) * period;
+
+ long bin1 = binId;
+ long bin2 = binId + period;
+ long bin3 = binId + 2 * period;
+ long bin4 = binId + 3 * period;
+
+ MapBindingSet bs = new MapBindingSet();
+ bs.addBinding("total", vf.createLiteral("2", XMLSchema.INTEGER));
+ bs.addBinding("id", vf.createLiteral("id_1", XMLSchema.STRING));
+ bs.addBinding("periodicBinId", vf.createLiteral(bin1));
+ expected1.add(bs);
+
+ bs = new MapBindingSet();
+ bs.addBinding("total", vf.createLiteral("2", XMLSchema.INTEGER));
+ bs.addBinding("id", vf.createLiteral("id_2", XMLSchema.STRING));
+ bs.addBinding("periodicBinId", vf.createLiteral(bin1));
+ expected1.add(bs);
+
+ bs = new MapBindingSet();
+ bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
+ bs.addBinding("id", vf.createLiteral("id_3", XMLSchema.STRING));
+ bs.addBinding("periodicBinId", vf.createLiteral(bin1));
+ expected1.add(bs);
+
+ bs = new MapBindingSet();
+ bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
+ bs.addBinding("id", vf.createLiteral("id_4", XMLSchema.STRING));
+ bs.addBinding("periodicBinId", vf.createLiteral(bin1));
+ expected1.add(bs);
+
+ bs = new MapBindingSet();
+ bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
+ bs.addBinding("id", vf.createLiteral("id_1", XMLSchema.STRING));
+ bs.addBinding("periodicBinId", vf.createLiteral(bin2));
+ expected2.add(bs);
+
+ bs = new MapBindingSet();
+ bs.addBinding("total", vf.createLiteral("2", XMLSchema.INTEGER));
+ bs.addBinding("id", vf.createLiteral("id_2", XMLSchema.STRING));
+ bs.addBinding("periodicBinId", vf.createLiteral(bin2));
+ expected2.add(bs);
+
+ bs = new MapBindingSet();
+ bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
+ bs.addBinding("id", vf.createLiteral("id_3", XMLSchema.STRING));
+ bs.addBinding("periodicBinId", vf.createLiteral(bin2));
+ expected2.add(bs);
+
+ bs = new MapBindingSet();
+ bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
+ bs.addBinding("id", vf.createLiteral("id_1", XMLSchema.STRING));
+ bs.addBinding("periodicBinId", vf.createLiteral(bin3));
+ expected3.add(bs);
+
+ bs = new MapBindingSet();
+ bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
+ bs.addBinding("id", vf.createLiteral("id_2", XMLSchema.STRING));
+ bs.addBinding("periodicBinId", vf.createLiteral(bin3));
+ expected3.add(bs);
+
+ bs = new MapBindingSet();
+ bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
+ bs.addBinding("id", vf.createLiteral("id_1", XMLSchema.STRING));
+ bs.addBinding("periodicBinId", vf.createLiteral(bin4));
+ expected4.add(bs);
+
+ // make sure that expected and actual results align after ingest
+ compareResults(periodicStorage, queryId, bin1, expected1);
+ compareResults(periodicStorage, queryId, bin2, expected2);
+ compareResults(periodicStorage, queryId, bin3, expected3);
+ compareResults(periodicStorage, queryId, bin4, expected4);
+
+ BlockingQueue<NodeBin> bins = new LinkedBlockingQueue<>();
+ PeriodicQueryPrunerExecutor pruner = new PeriodicQueryPrunerExecutor(periodicStorage, fluo, 1, bins);
+ pruner.start();
+
+ bins.add(new NodeBin(queryId, bin1));
+ bins.add(new NodeBin(queryId, bin2));
+ bins.add(new NodeBin(queryId, bin3));
+ bins.add(new NodeBin(queryId, bin4));
+
+ Thread.sleep(10000);
+
+ compareResults(periodicStorage, queryId, bin1, new HashSet<>());
+ compareResults(periodicStorage, queryId, bin2, new HashSet<>());
+ compareResults(periodicStorage, queryId, bin3, new HashSet<>());
+ compareResults(periodicStorage, queryId, bin4, new HashSet<>());
+
+ compareFluoCounts(fluo, queryId, bin1);
+ compareFluoCounts(fluo, queryId, bin2);
+ compareFluoCounts(fluo, queryId, bin3);
+ compareFluoCounts(fluo, queryId, bin4);
+
+ pruner.stop();
+
+ }
+
+ private void compareResults(PeriodicQueryResultStorage periodicStorage, String queryId, long bin, Set<BindingSet> expected) throws PeriodicQueryStorageException, Exception {
+ try(CloseableIterator<BindingSet> iter = periodicStorage.listResults(queryId, Optional.of(bin))) {
+ Set<BindingSet> actual = new HashSet<>();
+ while(iter.hasNext()) {
+ actual.add(iter.next());
+ }
+ Assert.assertEquals(expected, actual);
+ }
+ }
+
+ private void compareFluoCounts(FluoClient client, String pcjId, long bin) {
+ QueryBindingSet bs = new QueryBindingSet();
+ bs.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, new LiteralImpl(Long.toString(bin), XMLSchema.LONG));
+
+ VariableOrder varOrder = new VariableOrder(IncrementalUpdateConstants.PERIODIC_BIN_ID);
+
+ try(Snapshot sx = client.newSnapshot()) {
+ String fluoQueryId = NodeType.generateNewIdForType(NodeType.QUERY, pcjId);
+ Set<String> ids = new HashSet<>();
+ PeriodicQueryUtil.getPeriodicQueryNodeAncestorIds(sx, fluoQueryId, ids);
+ for(String id: ids) {
+ NodeType optNode = NodeType.fromNodeId(id).orNull();
+ if(optNode == null) throw new RuntimeException("Invalid NodeType.");
+ Bytes prefix = RowKeyUtil.makeRowKey(id,varOrder, bs);
+ RowScanner scanner = sx.scanner().fetch(optNode.getResultColumn()).over(Span.prefix(prefix)).byRow().build();
+ int count = 0;
+ Iterator<ColumnScanner> colScannerIter = scanner.iterator();
+ while(colScannerIter.hasNext()) {
+ ColumnScanner colScanner = colScannerIter.next();
+ String row = colScanner.getRow().toString();
+ Iterator<ColumnValue> values = colScanner.iterator();
+ while(values.hasNext()) {
+ values.next();
+ count++;
+ }
+ }
+ Assert.assertEquals(0, count);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/tests/src/test/java/org/apache/rya/periodic/notification/registration/kafka/PeriodicCommandNotificationConsumerIT.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/tests/src/test/java/org/apache/rya/periodic/notification/registration/kafka/PeriodicCommandNotificationConsumerIT.java b/extras/periodic.notification/tests/src/test/java/org/apache/rya/periodic/notification/registration/kafka/PeriodicCommandNotificationConsumerIT.java
new file mode 100644
index 0000000..522e69d
--- /dev/null
+++ b/extras/periodic.notification/tests/src/test/java/org/apache/rya/periodic/notification/registration/kafka/PeriodicCommandNotificationConsumerIT.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */package org.apache.rya.periodic.notification.registration.kafka;
+
+import java.util.Properties;
+import java.util.UUID;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.apache.kafka.common.serialization.StringSerializer;
+import org.apache.log4j.BasicConfigurator;
+import org.apache.rya.kafka.base.KafkaITBase;
+import org.apache.rya.kafka.base.KafkaTestInstanceRule;
+import org.apache.rya.periodic.notification.coordinator.PeriodicNotificationCoordinatorExecutor;
+import org.apache.rya.periodic.notification.notification.CommandNotification;
+import org.apache.rya.periodic.notification.notification.TimestampedNotification;
+import org.apache.rya.periodic.notification.registration.KafkaNotificationRegistrationClient;
+import org.apache.rya.periodic.notification.serialization.CommandNotificationSerializer;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+public class PeriodicCommandNotificationConsumerIT extends KafkaITBase {
+
+ private KafkaNotificationRegistrationClient registration;
+ private PeriodicNotificationCoordinatorExecutor coord;
+ private KafkaNotificationProvider provider;
+ private String bootstrapServer;
+
+ @Rule
+ public KafkaTestInstanceRule rule = new KafkaTestInstanceRule(false);
+
+ @Before
+ public void init() throws Exception {
+ bootstrapServer = createBootstrapServerConfig().getProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG);
+ }
+
+ @Test
+ public void kafkaNotificationProviderTest() throws InterruptedException {
+
+ BasicConfigurator.configure();
+
+ BlockingQueue<TimestampedNotification> notifications = new LinkedBlockingQueue<>();
+ Properties props = createKafkaConfig();
+ KafkaProducer<String, CommandNotification> producer = new KafkaProducer<>(props);
+ String topic = rule.getKafkaTopicName();
+ rule.createTopic(topic);
+
+ registration = new KafkaNotificationRegistrationClient(topic, producer);
+ coord = new PeriodicNotificationCoordinatorExecutor(1, notifications);
+ provider = new KafkaNotificationProvider(topic, new StringDeserializer(), new CommandNotificationSerializer(), props, coord, 1);
+ provider.start();
+
+ registration.addNotification("1", 1, 0, TimeUnit.SECONDS);
+ Thread.sleep(4000);
+ // check that notifications are being added to the blocking queue
+ Assert.assertEquals(true, notifications.size() > 0);
+
+ registration.deleteNotification("1");
+ Thread.sleep(2000);
+ int size = notifications.size();
+ // sleep for 2 seconds to ensure no more messages being produced
+ Thread.sleep(2000);
+ Assert.assertEquals(size, notifications.size());
+
+ tearDown();
+ }
+
+ @Test
+ public void kafkaNotificationMillisProviderTest() throws InterruptedException {
+
+ BasicConfigurator.configure();
+
+ BlockingQueue<TimestampedNotification> notifications = new LinkedBlockingQueue<>();
+ Properties props = createKafkaConfig();
+ KafkaProducer<String, CommandNotification> producer = new KafkaProducer<>(props);
+ String topic = rule.getKafkaTopicName();
+ rule.createTopic(topic);
+
+ registration = new KafkaNotificationRegistrationClient(topic, producer);
+ coord = new PeriodicNotificationCoordinatorExecutor(1, notifications);
+ provider = new KafkaNotificationProvider(topic, new StringDeserializer(), new CommandNotificationSerializer(), props, coord, 1);
+ provider.start();
+
+ registration.addNotification("1", 1000, 0, TimeUnit.MILLISECONDS);
+ Thread.sleep(4000);
+ // check that notifications are being added to the blocking queue
+ Assert.assertEquals(true, notifications.size() > 0);
+
+ registration.deleteNotification("1");
+ Thread.sleep(2000);
+ int size = notifications.size();
+ // sleep for 2 seconds to ensure no more messages being produced
+ Thread.sleep(2000);
+ Assert.assertEquals(size, notifications.size());
+
+ tearDown();
+ }
+
+ private void tearDown() {
+ registration.close();
+ provider.stop();
+ coord.stop();
+ }
+
+ private Properties createKafkaConfig() {
+ Properties props = new Properties();
+ props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServer);
+ props.setProperty(ConsumerConfig.GROUP_ID_CONFIG, UUID.randomUUID().toString());
+ props.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, "consumer0");
+ props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+ props.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
+ props.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, CommandNotificationSerializer.class.getName());
+
+ return props;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/tests/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/tests/src/test/resources/log4j.properties b/extras/periodic.notification/tests/src/test/resources/log4j.properties
new file mode 100644
index 0000000..19cc13c
--- /dev/null
+++ b/extras/periodic.notification/tests/src/test/resources/log4j.properties
@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Valid levels:
+# TRACE, DEBUG, INFO, WARN, ERROR and FATAL
+log4j.rootLogger=INFO, CONSOLE
+
+# Set independent logging levels
+log4j.logger.org.apache.zookeeper=WARN
+log4j.logger.kafka=WARN
+log4j.logger.org.apache.kafka=WARN
+
+# LOGFILE is set to be a File appender using a PatternLayout.
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+#log4j.appender.CONSOLE.Threshold=DEBUG
+
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d [%t] %-5p %c - %m%n
+
+#log4j.appender.CONSOLE.layout=org.apache.log4j.EnhancedPatternLayout
+#log4j.appender.CONSOLE.layout.ConversionPattern=%d [%t] %-5p %c{1.} - %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/tests/src/test/resources/notification.properties
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/tests/src/test/resources/notification.properties b/extras/periodic.notification/tests/src/test/resources/notification.properties
new file mode 100644
index 0000000..4b25b93
--- /dev/null
+++ b/extras/periodic.notification/tests/src/test/resources/notification.properties
@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#/
+accumulo.auths=
+accumulo.instance="instance"
+accumulo.user="root"
+accumulo.password="secret"
+accumulo.rya.prefix="rya_"
+accumulo.zookeepers=
+fluo.app.name="fluo_app"
+fluo.table.name="fluo_table"
+kafka.bootstrap.servers=127.0.0.1:9092
+kafka.notification.topic=notifications
+kafka.notification.client.id=consumer0
+kafka.notification.group.id=group0
+cep.coordinator.threads=1
+cep.producer.threads=1
+cep.exporter.threads=1
+cep.processor.threads=1
+cep.pruner.threads=1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/pom.xml
----------------------------------------------------------------------
diff --git a/extras/pom.xml b/extras/pom.xml
index 82930bc..53c7b4f 100644
--- a/extras/pom.xml
+++ b/extras/pom.xml
@@ -33,7 +33,7 @@ under the License.
<modules>
<module>rya.prospector</module>
<module>rya.manual</module>
- <module>rya.periodic.service</module>
+ <module>periodic.notification</module>
<module>shell</module>
<module>indexing</module>
<module>rya.indexing.pcj</module>
[7/7] incubator-rya git commit: RYA-355 Refactored the periodic
notification service structure. Closes #221.
Posted by ca...@apache.org.
RYA-355 Refactored the periodic notification service structure. Closes #221.
New artifactIds align better with code packaging, semantics.
Light pom cleaning.
Eliminates redundant text in file paths.
Makes adding the twill app for RYA-356 a little cleaner.
Project: http://git-wip-us.apache.org/repos/asf/incubator-rya/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rya/commit/de365c17
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rya/tree/de365c17
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rya/diff/de365c17
Branch: refs/heads/master
Commit: de365c1795b700cc5383be1927c4e598c37d948a
Parents: 28b0a52
Author: jdasch <hc...@gmail.com>
Authored: Fri Sep 1 20:51:25 2017 -0400
Committer: Caleb Meier <ca...@parsons.com>
Committed: Mon Sep 11 09:13:53 2017 -0700
----------------------------------------------------------------------
extras/indexing/pom.xml | 2 +-
extras/periodic.notification/api/.gitignore | 1 +
extras/periodic.notification/api/pom.xml | 52 ++
.../periodic/notification/api/BinPruner.java | 40 ++
.../notification/api/BindingSetExporter.java | 37 ++
.../notification/api/BindingSetRecord.java | 80 +++
.../api/BindingSetRecordExportException.java | 45 ++
.../periodic/notification/api/LifeCycle.java | 45 ++
.../rya/periodic/notification/api/NodeBin.java | 77 +++
.../periodic/notification/api/Notification.java | 34 ++
.../api/NotificationCoordinatorExecutor.java | 41 ++
.../notification/api/NotificationProcessor.java | 41 ++
.../api/PeriodicNotificationClient.java | 64 +++
.../notification/BasicNotification.java | 76 +++
.../notification/CommandNotification.java | 99 ++++
.../notification/PeriodicNotification.java | 178 +++++++
.../notification/TimestampedNotification.java | 69 +++
.../KafkaNotificationRegistrationClient.java | 80 +++
.../BasicNotificationTypeAdapter.java | 55 +++
.../serialization/BindingSetSerDe.java | 105 ++++
.../CommandNotificationSerializer.java | 76 +++
.../CommandNotificationTypeAdapter.java | 89 ++++
.../PeriodicNotificationTypeAdapter.java | 73 +++
extras/periodic.notification/pom.xml | 40 ++
extras/periodic.notification/service/pom.xml | 102 ++++
.../PeriodicApplicationException.java | 47 ++
.../PeriodicNotificationApplication.java | 207 ++++++++
...dicNotificationApplicationConfiguration.java | 254 ++++++++++
.../PeriodicNotificationApplicationFactory.java | 140 ++++++
...PeriodicNotificationCoordinatorExecutor.java | 159 ++++++
.../exporter/KafkaExporterExecutor.java | 110 +++++
.../KafkaPeriodicBindingSetExporter.java | 99 ++++
.../NotificationProcessorExecutor.java | 114 +++++
.../TimestampedNotificationProcessor.java | 203 ++++++++
.../notification/pruner/AccumuloBinPruner.java | 66 +++
.../notification/pruner/FluoBinPruner.java | 76 +++
.../pruner/PeriodicQueryPruner.java | 107 ++++
.../pruner/PeriodicQueryPrunerExecutor.java | 104 ++++
.../recovery/PeriodicNotificationProvider.java | 142 ++++++
.../kafka/KafkaNotificationProvider.java | 123 +++++
.../kafka/PeriodicNotificationConsumer.java | 88 ++++
.../CommandNotificationSerializerTest.java | 60 +++
extras/periodic.notification/tests/pom.xml | 62 +++
.../PeriodicNotificationApplicationIT.java | 493 +++++++++++++++++++
.../PeriodicNotificationProviderIT.java | 71 +++
.../PeriodicNotificationExporterIT.java | 143 ++++++
.../PeriodicNotificationProcessorIT.java | 121 +++++
.../pruner/PeriodicNotificationBinPrunerIT.java | 283 +++++++++++
.../PeriodicCommandNotificationConsumerIT.java | 139 ++++++
.../tests/src/test/resources/log4j.properties | 37 ++
.../src/test/resources/notification.properties | 35 ++
extras/pom.xml | 2 +-
extras/rya.pcj.fluo/pcj.fluo.api/pom.xml | 12 +-
.../periodic.service.api/.gitignore | 1 -
.../periodic.service.api/pom.xml | 52 --
.../periodic/notification/api/BinPruner.java | 40 --
.../notification/api/BindingSetExporter.java | 37 --
.../notification/api/BindingSetRecord.java | 80 ---
.../api/BindingSetRecordExportException.java | 45 --
.../periodic/notification/api/LifeCycle.java | 45 --
.../rya/periodic/notification/api/NodeBin.java | 77 ---
.../periodic/notification/api/Notification.java | 34 --
.../api/NotificationCoordinatorExecutor.java | 41 --
.../notification/api/NotificationProcessor.java | 41 --
.../api/PeriodicNotificationClient.java | 64 ---
.../notification/BasicNotification.java | 76 ---
.../notification/CommandNotification.java | 99 ----
.../notification/PeriodicNotification.java | 178 -------
.../notification/TimestampedNotification.java | 69 ---
.../KafkaNotificationRegistrationClient.java | 80 ---
.../BasicNotificationTypeAdapter.java | 55 ---
.../serialization/BindingSetSerDe.java | 105 ----
.../CommandNotificationSerializer.java | 76 ---
.../CommandNotificationTypeAdapter.java | 89 ----
.../PeriodicNotificationTypeAdapter.java | 73 ---
.../periodic.service.integration.tests/pom.xml | 62 ---
.../PeriodicNotificationApplicationIT.java | 493 -------------------
.../PeriodicNotificationProviderIT.java | 71 ---
.../PeriodicNotificationExporterIT.java | 143 ------
.../PeriodicNotificationProcessorIT.java | 121 -----
.../pruner/PeriodicNotificationBinPrunerIT.java | 283 -----------
.../PeriodicCommandNotificationConsumerIT.java | 139 ------
.../src/test/resources/log4j.properties | 37 --
.../src/test/resources/notification.properties | 35 --
.../periodic.service.notification/pom.xml | 112 -----
.../PeriodicApplicationException.java | 47 --
.../PeriodicNotificationApplication.java | 207 --------
...dicNotificationApplicationConfiguration.java | 254 ----------
.../PeriodicNotificationApplicationFactory.java | 140 ------
...PeriodicNotificationCoordinatorExecutor.java | 159 ------
.../exporter/KafkaExporterExecutor.java | 110 -----
.../KafkaPeriodicBindingSetExporter.java | 99 ----
.../NotificationProcessorExecutor.java | 114 -----
.../TimestampedNotificationProcessor.java | 203 --------
.../notification/pruner/AccumuloBinPruner.java | 66 ---
.../notification/pruner/FluoBinPruner.java | 76 ---
.../pruner/PeriodicQueryPruner.java | 107 ----
.../pruner/PeriodicQueryPrunerExecutor.java | 104 ----
.../recovery/PeriodicNotificationProvider.java | 142 ------
.../kafka/KafkaNotificationProvider.java | 123 -----
.../kafka/PeriodicNotificationConsumer.java | 88 ----
.../CommandNotificationSerializerTest.java | 60 ---
extras/rya.periodic.service/pom.xml | 40 --
pom.xml | 11 +-
104 files changed, 5093 insertions(+), 5108 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/indexing/pom.xml
----------------------------------------------------------------------
diff --git a/extras/indexing/pom.xml b/extras/indexing/pom.xml
index 16a205f..1d3a32b 100644
--- a/extras/indexing/pom.xml
+++ b/extras/indexing/pom.xml
@@ -83,7 +83,7 @@
</dependency>
<dependency>
<groupId>org.apache.rya</groupId>
- <artifactId>rya.periodic.service.api</artifactId>
+ <artifactId>rya.periodic.notification.api</artifactId>
</dependency>
<!-- OpenRDF -->
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/api/.gitignore
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/api/.gitignore b/extras/periodic.notification/api/.gitignore
new file mode 100644
index 0000000..b83d222
--- /dev/null
+++ b/extras/periodic.notification/api/.gitignore
@@ -0,0 +1 @@
+/target/
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/api/pom.xml
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/api/pom.xml b/extras/periodic.notification/api/pom.xml
new file mode 100644
index 0000000..aecd723
--- /dev/null
+++ b/extras/periodic.notification/api/pom.xml
@@ -0,0 +1,52 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <!-- Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with this
+ work for additional information regarding copyright ownership. The ASF licenses
+ this file to you under the Apache License, Version 2.0 (the "License"); you
+ may not use this file except in compliance with the License. You may obtain
+ a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless
+ required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License. -->
+
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.rya</groupId>
+ <artifactId>rya.periodic.notification.parent</artifactId>
+ <version>3.2.11-incubating-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>rya.periodic.notification.api</artifactId>
+
+ <name>Apache Rya Periodic Notification API</name>
+ <description>API for Periodic Notification Applications</description>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>2.8.0</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.openrdf.sesame</groupId>
+ <artifactId>sesame-query</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rya</groupId>
+ <artifactId>rya.indexing.pcj</artifactId>
+ </dependency>
+ </dependencies>
+
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/BinPruner.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/BinPruner.java b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/BinPruner.java
new file mode 100644
index 0000000..f4a083c
--- /dev/null
+++ b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/BinPruner.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.api;
+
+import org.openrdf.query.Binding;
+import org.openrdf.query.BindingSet;
+
+/**
+ * Object that cleans up old {@link BindingSet}s corresponding to the specified
+ * {@link NodeBin}. This class deletes all BindingSets with the bin
+ * indicated by {@link NodeBin#getBin()}. A BindingSet corresponds to a given
+ * bin if it contains a {@link Binding} with name {@link IncrementalUpdateConstants#PERIODIC_BIN_ID}
+ * and value equal to the given bin.
+ *
+ */
+public interface BinPruner {
+
+ /**
+ * Cleans up all {@link BindingSet}s associated with the indicated {@link NodeBin}.
+ * @param bin - NodeBin that indicates which BindingSets to delete..
+ */
+ public void pruneBindingSetBin(NodeBin bin);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/BindingSetExporter.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/BindingSetExporter.java b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/BindingSetExporter.java
new file mode 100644
index 0000000..491576b
--- /dev/null
+++ b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/BindingSetExporter.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.api;
+
+import org.openrdf.query.BindingSet;
+
+/**
+ * An Object that is used to export {@link BindingSet}s to an external repository or queuing system.
+ *
+ */
+public interface BindingSetExporter {
+
+ /**
+ * This method exports the BindingSet to the external repository or queuing system
+ * that this BindingSetExporter is configured to export to.
+ * @param bindingSet - {@link BindingSet} to be exported
+ * @throws ResultExportException
+ */
+ public void exportNotification(BindingSetRecord bindingSet) throws BindingSetRecordExportException;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/BindingSetRecord.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/BindingSetRecord.java b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/BindingSetRecord.java
new file mode 100644
index 0000000..c3f70f1
--- /dev/null
+++ b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/BindingSetRecord.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.api;
+
+import org.openrdf.query.BindingSet;
+
+import com.google.common.base.Objects;
+
+/**
+ * Object that associates a {@link BindingSet} with a given Kafka topic.
+ * This ensures that the {@link KafkaPeriodicBindingSetExporter} can export
+ * each BindingSet to its appropriate topic.
+ *
+ */
+public class BindingSetRecord {
+
+ private BindingSet bs;
+ private String topic;
+
+ public BindingSetRecord(BindingSet bs, String topic) {
+ this.bs = bs;
+ this.topic = topic;
+ }
+
+ /**
+ * @return BindingSet in this BindingSetRecord
+ */
+ public BindingSet getBindingSet() {
+ return bs;
+ }
+
+ /**
+ * @return Kafka topic for this BindingSetRecord
+ */
+ public String getTopic() {
+ return topic;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if(this == o) {
+ return true;
+ }
+
+ if(o instanceof BindingSetRecord) {
+ BindingSetRecord record = (BindingSetRecord) o;
+ return Objects.equal(this.bs, record.bs)&&Objects.equal(this.topic,record.topic);
+ }
+
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(bs, topic);
+ }
+
+ @Override
+ public String toString() {
+ return new StringBuilder().append("Binding Set Record \n").append(" Topic: " + topic + "\n").append(" BindingSet: " + bs + "\n")
+ .toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/BindingSetRecordExportException.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/BindingSetRecordExportException.java b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/BindingSetRecordExportException.java
new file mode 100644
index 0000000..94e4980
--- /dev/null
+++ b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/BindingSetRecordExportException.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.api;
+
+/**
+ * A result could not be exported.
+ */
+public class BindingSetRecordExportException extends Exception {
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * Constructs an instance of {@link BindingSetRecordExportException}.
+ *
+ * @param message - Explains why the exception was thrown.
+ */
+ public BindingSetRecordExportException(final String message) {
+ super(message);
+ }
+
+ /**
+ * Constructs an instance of {@link BindingSetRecordExportException}.
+ *
+ * @param message - Explains why the exception was thrown.
+ * @param cause - The exception that caused this one to be thrown.
+ */
+ public BindingSetRecordExportException(final String message, final Throwable cause) {
+ super(message, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/LifeCycle.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/LifeCycle.java b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/LifeCycle.java
new file mode 100644
index 0000000..b1e8bad
--- /dev/null
+++ b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/LifeCycle.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.api;
+
+/**
+ * Interface providing basic life cycle functionality,
+ * including stopping and starting any class implementing this
+ * interface and checking whether is it running.
+ *
+ */
+public interface LifeCycle {
+
+ /**
+ * Starts a running application.
+ */
+ public void start();
+
+ /**
+ * Stops a running application.
+ */
+ public void stop();
+
+ /**
+ * Determine if application is currently running.
+ * @return true if application is running and false otherwise.
+ */
+ public boolean currentlyRunning();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/NodeBin.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/NodeBin.java b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/NodeBin.java
new file mode 100644
index 0000000..3ed7979
--- /dev/null
+++ b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/NodeBin.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.api;
+
+import java.util.Objects;
+
+/**
+ * Object used to indicate the id of a given Periodic Query
+ * along with a particular bin of results. This Object is used
+ * by the {@link BinPruner} to clean up old query results after
+ * they have been processed.
+ *
+ */
+public class NodeBin {
+
+ private long bin;
+ private String nodeId;
+
+ public NodeBin(String nodeId, long bin) {
+ this.bin = bin;
+ this.nodeId = nodeId;
+ }
+
+ /**
+ * @return id of Periodic Query
+ */
+ public String getNodeId() {
+ return nodeId;
+ }
+/**
+ * @return bin id of results for a given Periodic Query
+ */
+ public long getBin() {
+ return bin;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+
+ if (other instanceof NodeBin) {
+ NodeBin bin = (NodeBin) other;
+ return this.bin == bin.bin && this.nodeId.equals(bin.nodeId);
+ }
+
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(bin, nodeId);
+ }
+
+ @Override
+ public String toString() {
+ return new StringBuilder().append("Node Bin \n").append(" QueryId: " + nodeId + "\n").append(" Bin: " + bin + "\n").toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/Notification.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/Notification.java b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/Notification.java
new file mode 100644
index 0000000..3e9e0d1
--- /dev/null
+++ b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/Notification.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.api;
+
+/**
+ * Notification Object used by the Periodic Query Service
+ * to inform workers to process results for a given Periodic
+ * Query with the indicated id.
+ *
+ */
+public interface Notification {
+
+ /**
+ * @return id of a Periodic Query
+ */
+ public String getId();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/NotificationCoordinatorExecutor.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/NotificationCoordinatorExecutor.java b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/NotificationCoordinatorExecutor.java
new file mode 100644
index 0000000..d53dc17
--- /dev/null
+++ b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/NotificationCoordinatorExecutor.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.api;
+
+import java.util.concurrent.ScheduledExecutorService;
+
+import org.apache.rya.periodic.notification.notification.CommandNotification;
+
+/**
+ * Object that manages the periodic notifications for the Periodic Query Service.
+ * This Object processes new requests for periodic updates by registering them with
+ * some sort of service that generates periodic updates (such as a {@link ScheduledExecutorService}).
+ *
+ */
+public interface NotificationCoordinatorExecutor extends LifeCycle {
+
+ /**
+ * Registers or deletes a {@link CommandNotification}s with the periodic service to
+ * generate notifications at a regular interval indicated by the CommandNotification.
+ * @param notification - CommandNotification to be registered or deleted from the periodic update
+ * service.
+ */
+ public void processNextCommandNotification(CommandNotification notification);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/NotificationProcessor.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/NotificationProcessor.java b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/NotificationProcessor.java
new file mode 100644
index 0000000..4ac9089
--- /dev/null
+++ b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/NotificationProcessor.java
@@ -0,0 +1,41 @@
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.api;
+
+import org.apache.rya.periodic.notification.notification.TimestampedNotification;
+
+/**
+ * Object that processes new {@link TimestampedNotification}s generated by {@link NotificationCoordinatorExecutor}.
+ * It is expected that the NotificationCoordinatorExecutor will this Object with notifications to perform work via some sort
+ * sort of queuing service such as a BlockingQueue or Kafka. This Object processes the notifications by retrieving
+ * query results associated with the Periodic Query id given by {@link TimestampedNotification#getId()}, parsing them
+ * and then providing them to another service to be exported.
+ *
+ */
+public interface NotificationProcessor {
+
+ /**
+ * Processes {@link TimestampedNotification}s by retrieving the Periodic Query results
+ * associated the query id given by {@link TimestampedNotification#getId()}.
+ * @param notification - contains information about which query results to retrieve
+ */
+ public void processNotification(TimestampedNotification notification);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/PeriodicNotificationClient.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/PeriodicNotificationClient.java b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/PeriodicNotificationClient.java
new file mode 100644
index 0000000..ff08733
--- /dev/null
+++ b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/api/PeriodicNotificationClient.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.api;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.rya.periodic.notification.notification.BasicNotification;
+import org.apache.rya.periodic.notification.notification.PeriodicNotification;
+
+/**
+ * Object to register {@link PeriodicNotification}s with an external queuing
+ * service to be handled by a {@link NotificationCoordinatorExecutor} service.
+ * The service will generate notifications to process Periodic Query results at regular
+ * intervals corresponding the period of the PeriodicNotification.
+ *
+ */
+public interface PeriodicNotificationClient extends AutoCloseable {
+
+ /**
+ * Adds a new notification to be registered with the {@link NotificationCoordinatorExecutor}
+ * @param notification - notification to be added
+ */
+ public void addNotification(PeriodicNotification notification);
+
+ /**
+ * Deletes a notification from the {@link NotificationCoordinatorExecutor}.
+ * @param notification - notification to be deleted
+ */
+ public void deleteNotification(BasicNotification notification);
+
+ /**
+ * Deletes a notification from the {@link NotificationCoordinatorExecutor}.
+ * @param notification - id corresponding to the notification to be deleted
+ */
+ public void deleteNotification(String notificationId);
+
+ /**
+ * Adds a new notification with the indicated id and period to the {@link NotificationCoordinatorExecutor}
+ * @param id - Periodic Query id
+ * @param period - period indicating frequency at which notifications will be generated
+ * @param delay - initial delay for starting periodic notifications
+ * @param unit - time unit of delay and period
+ */
+ public void addNotification(String id, long period, long delay, TimeUnit unit);
+
+ public void close();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/notification/BasicNotification.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/notification/BasicNotification.java b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/notification/BasicNotification.java
new file mode 100644
index 0000000..c31a5c0
--- /dev/null
+++ b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/notification/BasicNotification.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.notification;
+
+import org.apache.rya.periodic.notification.api.Notification;
+
+import com.google.common.base.Objects;
+
+/**
+ * Notification Object used by the Periodic Query Service
+ * to inform workers to process results for a given Periodic
+ * Query with the indicated id.
+ *
+ */
+public class BasicNotification implements Notification {
+
+ private String id;
+
+ /**
+ * Creates a BasicNotification
+ * @param id - Fluo query id associated with this Notification
+ */
+ public BasicNotification(String id) {
+ this.id = id;
+ }
+
+ /**
+ * @return the Fluo Query Id that this notification will generate results for
+ */
+ @Override
+ public String getId() {
+ return id;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+
+ if (other instanceof BasicNotification) {
+ BasicNotification not = (BasicNotification) other;
+ return Objects.equal(this.id, not.id);
+ }
+
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(id);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ return builder.append("id").append("=").append(id).toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/notification/CommandNotification.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/notification/CommandNotification.java b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/notification/CommandNotification.java
new file mode 100644
index 0000000..597b228
--- /dev/null
+++ b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/notification/CommandNotification.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.notification;
+
+import org.apache.rya.periodic.notification.api.Notification;
+
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+
+/**
+ * This Object contains a Notification Object used by the Periodic Query Service
+ * to inform workers to process results for a given Periodic Query with the
+ * indicated id. Additionally, the CommandNotification contains a
+ * {@link Command} about which action the
+ * {@link NotificationCoordinatorExecutor} should take (adding or deleting).
+ * CommandNotifications are meant to be added to an external work queue (such as
+ * Kafka) to be processed by the NotificationCoordinatorExecutor.
+ *
+ */
+public class CommandNotification implements Notification {
+
+ private Notification notification;
+ private Command command;
+
+ public enum Command {
+ ADD, DELETE
+ };
+
+ /**
+ * Creates a new CommandNotification
+ * @param command - the command associated with this notification (either add, update, or delete)
+ * @param notification - the underlying notification associated with this command
+ */
+ public CommandNotification(Command command, Notification notification) {
+ this.notification = Preconditions.checkNotNull(notification);
+ this.command = Preconditions.checkNotNull(command);
+ }
+
+ @Override
+ public String getId() {
+ return notification.getId();
+ }
+
+ /**
+ * Returns {@link Notification} contained by this CommmandNotification.
+ * @return - Notification contained by this Object
+ */
+ public Notification getNotification() {
+ return this.notification;
+ }
+
+ /**
+ * @return Command contained by this Object (either add or delete)
+ */
+ public Command getCommand() {
+ return this.command;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other instanceof CommandNotification) {
+ CommandNotification cn = (CommandNotification) other;
+ return Objects.equal(this.command, cn.command) && Objects.equal(this.notification, cn.notification);
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(command, notification);
+ }
+
+ @Override
+ public String toString() {
+ return new StringBuilder().append("command").append("=").append(command.toString()).append(";")
+ .append(notification.toString()).toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/notification/PeriodicNotification.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/notification/PeriodicNotification.java b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/notification/PeriodicNotification.java
new file mode 100644
index 0000000..aa9e581
--- /dev/null
+++ b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/notification/PeriodicNotification.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.notification;
+
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.rya.periodic.notification.api.Notification;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Notification Object used by the Periodic Query Service to inform workers to
+ * process results for a given Periodic Query with the indicated id.
+ * Additionally, this Object contains a period that indicates a frequency at
+ * which regular updates are generated.
+ *
+ */
+public class PeriodicNotification implements Notification {
+
+ private String id;
+ private long period;
+ private TimeUnit periodTimeUnit;
+ private long initialDelay;
+
+ /**
+ * Creates a PeriodicNotification.
+ * @param id - Fluo Query Id that this notification is associated with
+ * @param period - period at which notifications are generated
+ * @param periodTimeUnit - time unit associated with the period and delay
+ * @param initialDelay - amount of time to wait before generating the first notification
+ */
+ public PeriodicNotification(String id, long period, TimeUnit periodTimeUnit, long initialDelay) {
+ this.id = Preconditions.checkNotNull(id);
+ this.periodTimeUnit = Preconditions.checkNotNull(periodTimeUnit);
+ Preconditions.checkArgument(period > 0 && initialDelay >= 0);
+ this.period = period;
+ this.initialDelay = initialDelay;
+ }
+
+
+ /**
+ * Create a PeriodicNotification
+ * @param other - other PeriodicNotification used in copy constructor
+ */
+ public PeriodicNotification(PeriodicNotification other) {
+ this(other.id, other.period, other.periodTimeUnit, other.initialDelay);
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ /**
+ * @return - period at which regular notifications are generated
+ */
+ public long getPeriod() {
+ return period;
+ }
+
+ /**
+ * @return time unit of period and initial delay
+ */
+ public TimeUnit getTimeUnit() {
+ return periodTimeUnit;
+ }
+
+ /**
+ * @return amount of time to delay before beginning to generate notifications
+ */
+ public long getInitialDelay() {
+ return initialDelay;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ String delim = "=";
+ String delim2 = ";";
+ return builder.append("id").append(delim).append(id).append(delim2).append("period").append(delim).append(period).append(delim2)
+ .append("periodTimeUnit").append(delim).append(periodTimeUnit).append(delim2).append("initialDelay").append(delim)
+ .append(initialDelay).toString();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+
+ if (!(other instanceof PeriodicNotification)) {
+ return false;
+ }
+
+ PeriodicNotification notification = (PeriodicNotification) other;
+ return Objects.equals(this.id, notification.id) && (this.period == notification.period)
+ && Objects.equals(this.periodTimeUnit, notification.periodTimeUnit) && (this.initialDelay == notification.initialDelay);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(id, period, periodTimeUnit, initialDelay);
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static class Builder {
+
+ private String id;
+ private long period;
+ private TimeUnit periodTimeUnit;
+ private long initialDelay = 0;
+
+ /**
+ * @param id - periodic query id
+ * @return - builder to chain method calls
+ */
+ public Builder id(String id) {
+ this.id = id;
+ return this;
+ }
+
+ /**
+ * @param period of the periodic notification for generating regular notifications
+ * @return - builder to chain method calls
+ */
+ public Builder period(long period) {
+ this.period = period;
+ return this;
+ }
+
+ /**
+ * @param timeUnit of period and initial delay
+ * @return - builder to chain method calls
+ */
+ public Builder timeUnit(TimeUnit timeUnit) {
+ this.periodTimeUnit = timeUnit;
+ return this;
+ }
+
+ /**
+ * @param initialDelay - amount of time to wait before generating notifications
+ * @return - builder to chain method calls
+ */
+ public Builder initialDelay(long initialDelay) {
+ this.initialDelay = initialDelay;
+ return this;
+ }
+
+ /**
+ * Builds PeriodicNotification
+ * @return PeriodicNotification constructed from Builder specified parameters
+ */
+ public PeriodicNotification build() {
+ return new PeriodicNotification(id, period, periodTimeUnit, initialDelay);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/notification/TimestampedNotification.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/notification/TimestampedNotification.java b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/notification/TimestampedNotification.java
new file mode 100644
index 0000000..38073ce
--- /dev/null
+++ b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/notification/TimestampedNotification.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.notification;
+
+import java.util.Date;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * {@link PeriodicNotification} Object used by the Periodic Query Service to inform workers to
+ * process results for a given Periodic Query with the indicated id. Additionally
+ * this Object contains a {@link Date} object to indicate the date time at which this
+ * notification was generated.
+ *
+ */
+public class TimestampedNotification extends PeriodicNotification {
+
+ private Date date;
+
+ /**
+ * Constructs a TimestampedNotification
+ * @param id - Fluo Query Id associated with this Notification
+ * @param period - period at which notifications are generated
+ * @param periodTimeUnit - time unit associated with period and initial delay
+ * @param initialDelay - amount of time to wait before generating first notification
+ */
+ public TimestampedNotification(String id, long period, TimeUnit periodTimeUnit, long initialDelay) {
+ super(id, period, periodTimeUnit, initialDelay);
+ date = new Date();
+ }
+
+ /**
+ * Creates a TimestampedNotification
+ * @param notification - PeriodicNotification used to create this TimestampedNotification.
+ * This constructor creates a time stamp for the TimestampedNotification.
+ */
+ public TimestampedNotification(PeriodicNotification notification) {
+ super(notification);
+ date = new Date();
+ }
+
+ /**
+ * @return timestamp at which this notification was generated
+ */
+ public Date getTimestamp() {
+ return date;
+ }
+
+ @Override
+ public String toString() {
+ return super.toString() + ";date=" + date;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/registration/KafkaNotificationRegistrationClient.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/registration/KafkaNotificationRegistrationClient.java b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/registration/KafkaNotificationRegistrationClient.java
new file mode 100644
index 0000000..bb438be
--- /dev/null
+++ b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/registration/KafkaNotificationRegistrationClient.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.registration;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.rya.periodic.notification.api.Notification;
+import org.apache.rya.periodic.notification.api.PeriodicNotificationClient;
+import org.apache.rya.periodic.notification.notification.BasicNotification;
+import org.apache.rya.periodic.notification.notification.CommandNotification;
+import org.apache.rya.periodic.notification.notification.CommandNotification.Command;
+import org.apache.rya.periodic.notification.notification.PeriodicNotification;
+
+/**
+ * Implementation of {@link PeriodicNotificaitonClient} used to register new notification
+ * requests with the PeriodicQueryService.
+ *
+ */
+public class KafkaNotificationRegistrationClient implements PeriodicNotificationClient {
+
+ private KafkaProducer<String, CommandNotification> producer;
+ private String topic;
+
+ public KafkaNotificationRegistrationClient(String topic, KafkaProducer<String, CommandNotification> producer) {
+ this.topic = topic;
+ this.producer = producer;
+ }
+
+ @Override
+ public void addNotification(PeriodicNotification notification) {
+ processNotification(new CommandNotification(Command.ADD, notification));
+
+ }
+
+ @Override
+ public void deleteNotification(BasicNotification notification) {
+ processNotification(new CommandNotification(Command.DELETE, notification));
+ }
+
+ @Override
+ public void deleteNotification(String notificationId) {
+ processNotification(new CommandNotification(Command.DELETE, new BasicNotification(notificationId)));
+ }
+
+ @Override
+ public void addNotification(String id, long period, long delay, TimeUnit unit) {
+ Notification notification = PeriodicNotification.builder().id(id).period(period).initialDelay(delay).timeUnit(unit).build();
+ processNotification(new CommandNotification(Command.ADD, notification));
+ }
+
+
+ private void processNotification(CommandNotification notification) {
+ producer.send(new ProducerRecord<String, CommandNotification>(topic, notification.getId(), notification));
+ }
+
+ @Override
+ public void close() {
+ producer.close();
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/serialization/BasicNotificationTypeAdapter.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/serialization/BasicNotificationTypeAdapter.java b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/serialization/BasicNotificationTypeAdapter.java
new file mode 100644
index 0000000..bd29d29
--- /dev/null
+++ b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/serialization/BasicNotificationTypeAdapter.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.serialization;
+
+import java.lang.reflect.Type;
+
+import org.apache.rya.periodic.notification.notification.BasicNotification;
+
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonPrimitive;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
+
+/**
+ * {@link TypeAdapter} for {@link BasicNotification}s. Used in {@link CommandNotificationTypeAdapter} to
+ * serialize {@link CommandNotification}s.
+ *
+ */
+public class BasicNotificationTypeAdapter implements JsonDeserializer<BasicNotification>, JsonSerializer<BasicNotification> {
+
+ @Override
+ public JsonElement serialize(BasicNotification arg0, Type arg1, JsonSerializationContext arg2) {
+ JsonObject result = new JsonObject();
+ result.add("id", new JsonPrimitive(arg0.getId()));
+ return result;
+ }
+
+ @Override
+ public BasicNotification deserialize(JsonElement arg0, Type arg1, JsonDeserializationContext arg2) throws JsonParseException {
+ JsonObject json = arg0.getAsJsonObject();
+ String id = json.get("id").getAsString();
+ return new BasicNotification(id);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/serialization/BindingSetSerDe.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/serialization/BindingSetSerDe.java b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/serialization/BindingSetSerDe.java
new file mode 100644
index 0000000..50180ad
--- /dev/null
+++ b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/serialization/BindingSetSerDe.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.serialization;
+
+import java.io.UnsupportedEncodingException;
+import java.util.Arrays;
+import java.util.Map;
+
+import org.apache.kafka.common.serialization.Deserializer;
+import org.apache.kafka.common.serialization.Serializer;
+import org.apache.log4j.Logger;
+import org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjSerializer;
+import org.apache.rya.indexing.pcj.storage.accumulo.BindingSetConverter.BindingSetConversionException;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.algebra.evaluation.QueryBindingSet;
+
+import com.google.common.base.Joiner;
+import com.google.common.primitives.Bytes;
+
+/**
+ * Kafka {@link Serializer} and {@link Deserializer} for producing and consuming messages
+ * from Kafka.
+ *
+ */
+public class BindingSetSerDe implements Serializer<BindingSet>, Deserializer<BindingSet> {
+
+ private static final Logger log = Logger.getLogger(BindingSetSerDe.class);
+ private static final AccumuloPcjSerializer serializer = new AccumuloPcjSerializer();
+ private static final byte[] DELIM_BYTE = "\u0002".getBytes();
+
+ private byte[] toBytes(BindingSet bindingSet) {
+ try {
+ return getBytes(getVarOrder(bindingSet), bindingSet);
+ } catch(Exception e) {
+ log.trace("Unable to serialize BindingSet: " + bindingSet);
+ return new byte[0];
+ }
+ }
+
+ private BindingSet fromBytes(byte[] bsBytes) {
+ try{
+ int firstIndex = Bytes.indexOf(bsBytes, DELIM_BYTE);
+ byte[] varOrderBytes = Arrays.copyOf(bsBytes, firstIndex);
+ byte[] bsBytesNoVarOrder = Arrays.copyOfRange(bsBytes, firstIndex + 1, bsBytes.length);
+ VariableOrder varOrder = new VariableOrder(new String(varOrderBytes,"UTF-8").split(";"));
+ return getBindingSet(varOrder, bsBytesNoVarOrder);
+ } catch(Exception e) {
+ log.trace("Unable to deserialize BindingSet: " + bsBytes);
+ return new QueryBindingSet();
+ }
+ }
+
+ private VariableOrder getVarOrder(BindingSet bs) {
+ return new VariableOrder(bs.getBindingNames());
+ }
+
+ private byte[] getBytes(VariableOrder varOrder, BindingSet bs) throws UnsupportedEncodingException, BindingSetConversionException {
+ byte[] bsBytes = serializer.convert(bs, varOrder);
+ String varOrderString = Joiner.on(";").join(varOrder.getVariableOrders());
+ byte[] varOrderBytes = varOrderString.getBytes("UTF-8");
+ return Bytes.concat(varOrderBytes, DELIM_BYTE, bsBytes);
+ }
+
+ private BindingSet getBindingSet(VariableOrder varOrder, byte[] bsBytes) throws BindingSetConversionException {
+ return serializer.convert(bsBytes, varOrder);
+ }
+
+ @Override
+ public BindingSet deserialize(String topic, byte[] bytes) {
+ return fromBytes(bytes);
+ }
+
+ @Override
+ public void close() {
+ // Do nothing. Nothing to close.
+ }
+
+ @Override
+ public void configure(Map<String, ?> arg0, boolean arg1) {
+ // Do nothing. Nothing to configure.
+ }
+
+ @Override
+ public byte[] serialize(String topic, BindingSet bs) {
+ return toBytes(bs);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/serialization/CommandNotificationSerializer.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/serialization/CommandNotificationSerializer.java b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/serialization/CommandNotificationSerializer.java
new file mode 100644
index 0000000..302e1be
--- /dev/null
+++ b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/serialization/CommandNotificationSerializer.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.serialization;
+
+import java.io.UnsupportedEncodingException;
+import java.util.Map;
+
+import org.apache.kafka.common.serialization.Deserializer;
+import org.apache.kafka.common.serialization.Serializer;
+import org.apache.rya.periodic.notification.api.Notification;
+import org.apache.rya.periodic.notification.notification.CommandNotification;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+
+/**
+ * Kafka {@link Serializer} and {@link Deserializer} for producing and consuming {@link CommandNotification}s
+ * to and from Kafka.
+ *
+ */
+public class CommandNotificationSerializer implements Serializer<CommandNotification>, Deserializer<CommandNotification> {
+
+ private static Gson gson = new GsonBuilder()
+ .registerTypeHierarchyAdapter(Notification.class, new CommandNotificationTypeAdapter()).create();
+ private static final Logger LOG = LoggerFactory.getLogger(CommandNotificationSerializer.class);
+
+ @Override
+ public CommandNotification deserialize(String topic, byte[] bytes) {
+ String json = null;
+ try {
+ json = new String(bytes, "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ LOG.info("Unable to deserialize notification for topic: " + topic);
+ }
+ return gson.fromJson(json, CommandNotification.class);
+ }
+
+ @Override
+ public byte[] serialize(String topic, CommandNotification command) {
+ try {
+ return gson.toJson(command).getBytes("UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ LOG.info("Unable to serialize notification: " + command + "for topic: " + topic);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void close() {
+ // Do nothing. Nothing to close
+ }
+
+ @Override
+ public void configure(Map<String, ?> arg0, boolean arg1) {
+ // Do nothing. Nothing to configure
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/serialization/CommandNotificationTypeAdapter.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/serialization/CommandNotificationTypeAdapter.java b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/serialization/CommandNotificationTypeAdapter.java
new file mode 100644
index 0000000..a9fb7e1
--- /dev/null
+++ b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/serialization/CommandNotificationTypeAdapter.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.serialization;
+
+import java.lang.reflect.Type;
+
+import org.apache.rya.periodic.notification.api.Notification;
+import org.apache.rya.periodic.notification.notification.BasicNotification;
+import org.apache.rya.periodic.notification.notification.CommandNotification;
+import org.apache.rya.periodic.notification.notification.PeriodicNotification;
+import org.apache.rya.periodic.notification.notification.CommandNotification.Command;
+
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonPrimitive;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
+
+/**
+ * {@link TypeAdapter} used to serialize and deserialize {@link CommandNotification}s.
+ * This TypeAdapter is used in {@link CommandNotificationSerializer} for producing and
+ * consuming messages to and from Kafka.
+ *
+ */
+public class CommandNotificationTypeAdapter
+ implements JsonDeserializer<CommandNotification>, JsonSerializer<CommandNotification> {
+
+ @Override
+ public JsonElement serialize(CommandNotification arg0, Type arg1, JsonSerializationContext arg2) {
+ JsonObject result = new JsonObject();
+ result.add("command", new JsonPrimitive(arg0.getCommand().name()));
+ Notification notification = arg0.getNotification();
+ if (notification instanceof PeriodicNotification) {
+ result.add("type", new JsonPrimitive(PeriodicNotification.class.getSimpleName()));
+ PeriodicNotificationTypeAdapter adapter = new PeriodicNotificationTypeAdapter();
+ result.add("notification",
+ adapter.serialize((PeriodicNotification) notification, PeriodicNotification.class, arg2));
+ } else if (notification instanceof BasicNotification) {
+ result.add("type", new JsonPrimitive(BasicNotification.class.getSimpleName()));
+ BasicNotificationTypeAdapter adapter = new BasicNotificationTypeAdapter();
+ result.add("notification",
+ adapter.serialize((BasicNotification) notification, BasicNotification.class, arg2));
+ } else {
+ throw new IllegalArgumentException("Invalid notification type.");
+ }
+ return result;
+ }
+
+ @Override
+ public CommandNotification deserialize(JsonElement arg0, Type arg1, JsonDeserializationContext arg2)
+ throws JsonParseException {
+
+ JsonObject json = arg0.getAsJsonObject();
+ Command command = Command.valueOf(json.get("command").getAsString());
+ String type = json.get("type").getAsString();
+ Notification notification = null;
+ if (type.equals(PeriodicNotification.class.getSimpleName())) {
+ notification = (new PeriodicNotificationTypeAdapter()).deserialize(json.get("notification"),
+ PeriodicNotification.class, arg2);
+ } else if (type.equals(BasicNotification.class.getSimpleName())) {
+ notification = (new BasicNotificationTypeAdapter()).deserialize(json.get("notification"),
+ BasicNotification.class, arg2);
+ } else {
+ throw new JsonParseException("Cannot deserialize Json");
+ }
+
+ return new CommandNotification(command, notification);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/serialization/PeriodicNotificationTypeAdapter.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/serialization/PeriodicNotificationTypeAdapter.java b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/serialization/PeriodicNotificationTypeAdapter.java
new file mode 100644
index 0000000..fcc0ba2
--- /dev/null
+++ b/extras/periodic.notification/api/src/main/java/org/apache/rya/periodic/notification/serialization/PeriodicNotificationTypeAdapter.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.serialization;
+
+import java.lang.reflect.Type;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.rya.periodic.notification.notification.PeriodicNotification;
+import org.apache.rya.periodic.notification.notification.PeriodicNotification.Builder;
+
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonPrimitive;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
+
+/**
+ * {@link TypeAdapter} used to serialize and deserialize {@link PeriodicNotification}s.
+ * This TypeAdapter is used in {@link CommandNotificationTypeAdapter} which is used in
+ * {@link CommandNotificationSerializer} for producing and consuming messages to and from
+ * Kafka.
+ *
+ */
+public class PeriodicNotificationTypeAdapter
+ implements JsonSerializer<PeriodicNotification>, JsonDeserializer<PeriodicNotification> {
+
+ @Override
+ public PeriodicNotification deserialize(JsonElement arg0, Type arg1, JsonDeserializationContext arg2)
+ throws JsonParseException {
+
+ JsonObject json = arg0.getAsJsonObject();
+ String id = json.get("id").getAsString();
+ long period = json.get("period").getAsLong();
+ TimeUnit periodTimeUnit = TimeUnit.valueOf(json.get("timeUnit").getAsString());
+ long initialDelay = json.get("initialDelay").getAsLong();
+ Builder builder = PeriodicNotification.builder().id(id).period(period)
+ .initialDelay(initialDelay).timeUnit(periodTimeUnit);
+
+ return builder.build();
+ }
+
+ @Override
+ public JsonElement serialize(PeriodicNotification arg0, Type arg1, JsonSerializationContext arg2) {
+
+ JsonObject result = new JsonObject();
+ result.add("id", new JsonPrimitive(arg0.getId()));
+ result.add("period", new JsonPrimitive(arg0.getPeriod()));
+ result.add("initialDelay", new JsonPrimitive(arg0.getInitialDelay()));
+ result.add("timeUnit", new JsonPrimitive(arg0.getTimeUnit().name()));
+
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/pom.xml
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/pom.xml b/extras/periodic.notification/pom.xml
new file mode 100644
index 0000000..814f14f
--- /dev/null
+++ b/extras/periodic.notification/pom.xml
@@ -0,0 +1,40 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>rya.periodic.notification.parent</artifactId>
+
+ <parent>
+ <groupId>org.apache.rya</groupId>
+ <artifactId>rya.extras</artifactId>
+ <version>3.2.11-incubating-SNAPSHOT</version>
+ </parent>
+
+ <name>Apache Rya Periodic Notification Parent</name>
+ <description>Parent POM for Rya Periodic Notification Projects</description>
+
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>api</module>
+ <module>service</module>
+ <module>tests</module>
+ </modules>
+
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/service/pom.xml
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/service/pom.xml b/extras/periodic.notification/service/pom.xml
new file mode 100644
index 0000000..cc78646
--- /dev/null
+++ b/extras/periodic.notification/service/pom.xml
@@ -0,0 +1,102 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <!-- Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with this
+ work for additional information regarding copyright ownership. The ASF licenses
+ this file to you under the Apache License, Version 2.0 (the "License"); you
+ may not use this file except in compliance with the License. You may obtain
+ a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless
+ required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License. -->
+ <parent>
+ <groupId>org.apache.rya</groupId>
+ <artifactId>rya.periodic.notification.parent</artifactId>
+ <version>3.2.11-incubating-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>rya.periodic.notification.service</artifactId>
+
+ <name>Apache Rya Periodic Notification Service</name>
+ <description>Notifications for Rya Periodic Service</description>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.twill</groupId>
+ <artifactId>twill-api</artifactId>
+ <version>0.11.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.twill</groupId>
+ <artifactId>twill-yarn</artifactId>
+ <version>0.11.0</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>kafka_2.10</artifactId>
+ <groupId>org.apache.kafka</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>2.8.0</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.fluo</groupId>
+ <artifactId>fluo-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.fluo</groupId>
+ <artifactId>fluo-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rya</groupId>
+ <artifactId>rya.indexing</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.openrdf.sesame</groupId>
+ <artifactId>sesame-query</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rya</groupId>
+ <artifactId>rya.indexing.pcj</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rya</groupId>
+ <artifactId>rya.pcj.fluo.app</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rya</groupId>
+ <artifactId>rya.periodic.notification.api</artifactId>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/de365c17/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/application/PeriodicApplicationException.java
----------------------------------------------------------------------
diff --git a/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/application/PeriodicApplicationException.java b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/application/PeriodicApplicationException.java
new file mode 100644
index 0000000..b2c3709
--- /dev/null
+++ b/extras/periodic.notification/service/src/main/java/org/apache/rya/periodic/notification/application/PeriodicApplicationException.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.periodic.notification.application;
+
+/**
+ * Exception thrown when attempting to create a {@link PeriodicNotificationApplication}.
+ * Indicates that a factory was unable to create some component of the application
+ * because something was configured incorrectly.
+ *
+ */
+public class PeriodicApplicationException extends Exception {
+
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * Creates a PeriodicApplicationException.
+ * @param message - message contained in Exception
+ */
+ public PeriodicApplicationException(String message) {
+ super(message);
+ }
+
+ /**
+ * Creates a PeriodicApplicationException.
+ * @param message - message contained in Exception
+ * @param t - Exception that spawned this PeriodicApplicationException
+ */
+ public PeriodicApplicationException(String message, Throwable t) {
+ super(message, t);
+ }
+}