You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@apex.apache.org by cs...@apache.org on 2016/05/17 18:55:14 UTC
[1/3] incubator-apex-malhar git commit: APEXMALHAR-2065 added
getWindowIds method to Window Data Manager
Repository: incubator-apex-malhar
Updated Branches:
refs/heads/master 1d3e20c69 -> 5d3b209fc
APEXMALHAR-2065 added getWindowIds method to Window Data Manager
Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/commit/2a6e1a6b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/tree/2a6e1a6b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/diff/2a6e1a6b
Branch: refs/heads/master
Commit: 2a6e1a6b50413f6e2c8c2ced55f4c10b562375f1
Parents: 4ef0700
Author: Timothy Farkas <ti...@datatorrent.com>
Authored: Sun Apr 24 18:39:24 2016 -0700
Committer: Chandni Singh <cs...@apache.org>
Committed: Tue May 17 11:41:47 2016 -0700
----------------------------------------------------------------------
.../apex/malhar/lib/wal/WindowDataManager.java | 33 ++++++++++++++++++++
.../malhar/lib/wal/FSWindowDataManagerTest.java | 19 +++++++++++
2 files changed, 52 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/blob/2a6e1a6b/library/src/main/java/org/apache/apex/malhar/lib/wal/WindowDataManager.java
----------------------------------------------------------------------
diff --git a/library/src/main/java/org/apache/apex/malhar/lib/wal/WindowDataManager.java b/library/src/main/java/org/apache/apex/malhar/lib/wal/WindowDataManager.java
index fd7948a..296238b 100644
--- a/library/src/main/java/org/apache/apex/malhar/lib/wal/WindowDataManager.java
+++ b/library/src/main/java/org/apache/apex/malhar/lib/wal/WindowDataManager.java
@@ -24,6 +24,7 @@ import java.util.Collection;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
+import java.util.SortedSet;
import javax.validation.constraints.NotNull;
@@ -99,6 +100,16 @@ public interface WindowDataManager extends StorageAgent, Component<Context.Opera
void partitioned(Collection<WindowDataManager> newManagers, Set<Integer> removedOperatorIds);
/**
+ * Returns an array of windowIds for which data was stored by atleast one partition. The array
+ * of winodwIds is sorted.
+ *
+ * @return An array of windowIds for which data was stored by atleast one partition. The array
+ * of winodwIds is sorted.
+ * @throws IOException
+ */
+ long[] getWindowIds() throws IOException;
+
+ /**
* An {@link WindowDataManager} that uses FS to persist state.
*/
class FSWindowDataManager implements WindowDataManager
@@ -227,6 +238,22 @@ public interface WindowDataManager extends StorageAgent, Component<Context.Opera
return storageAgent.getWindowIds(operatorId);
}
+ @Override
+ public long[] getWindowIds() throws IOException
+ {
+ SortedSet<Long> windowIds = replayState.keySet();
+ long[] windowIdsArray = new long[windowIds.size()];
+
+ int index = 0;
+
+ for (Long windowId: windowIds) {
+ windowIdsArray[index] = windowId;
+ index++;
+ }
+
+ return windowIdsArray;
+ }
+
/**
* This deletes all the recovery files of window ids <= windowId.
*
@@ -436,5 +463,11 @@ public interface WindowDataManager extends StorageAgent, Component<Context.Opera
{
return new long[0];
}
+
+ @Override
+ public long[] getWindowIds() throws IOException
+ {
+ return new long[0];
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/blob/2a6e1a6b/library/src/test/java/org/apache/apex/malhar/lib/wal/FSWindowDataManagerTest.java
----------------------------------------------------------------------
diff --git a/library/src/test/java/org/apache/apex/malhar/lib/wal/FSWindowDataManagerTest.java b/library/src/test/java/org/apache/apex/malhar/lib/wal/FSWindowDataManagerTest.java
index dff061a..7f3adc9 100644
--- a/library/src/test/java/org/apache/apex/malhar/lib/wal/FSWindowDataManagerTest.java
+++ b/library/src/test/java/org/apache/apex/malhar/lib/wal/FSWindowDataManagerTest.java
@@ -158,6 +158,25 @@ public class FSWindowDataManagerTest
}
@Test
+ public void testGetWindowIds() throws IOException
+ {
+ testMeta.storageManager.setup(testMeta.context);
+ Map<Integer, String> data = Maps.newHashMap();
+ data.put(1, "one");
+ data.put(2, "two");
+ data.put(3, "three");
+
+ testMeta.storageManager.save(data, 1, 1);
+ testMeta.storageManager.save(data, 2, 2);
+
+ testMeta.storageManager.setup(testMeta.context);
+
+ Assert.assertArrayEquals(new long[] {1, 2}, testMeta.storageManager.getWindowIds());
+
+ testMeta.storageManager.teardown();
+ }
+
+ @Test
public void testDelete() throws IOException
{
testMeta.storageManager.setup(testMeta.context);
[2/3] incubator-apex-malhar git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/incubator-apex-malhar into
APEXMALHAR-2065
Posted by cs...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-apex-malhar into APEXMALHAR-2065
Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/commit/543fc07e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/tree/543fc07e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/diff/543fc07e
Branch: refs/heads/master
Commit: 543fc07e9dae3c32fe08cf86541b9b67c9412628
Parents: 2a6e1a6 1d3e20c
Author: Chandni Singh <cs...@apache.org>
Authored: Tue May 17 11:44:06 2016 -0700
Committer: Chandni Singh <cs...@apache.org>
Committed: Tue May 17 11:44:06 2016 -0700
----------------------------------------------------------------------
DISCLAIMER | 11 -
NOTICE | 2 +-
README.md | 10 +-
apps/filecopy/pom.xml | 6 +-
.../datatorrent/apps/copy/HDFSFileCopyApp.java | 4 +-
apps/logstream/pom.xml | 4 +-
apps/pom.xml | 4 +-
benchmark/pom.xml | 4 +-
.../com/datatorrent/benchmark/package-info.java | 44 +-
contrib/pom.xml | 4 +-
.../contrib/enrich/AbstractEnricher.java | 321 +++++++++
.../contrib/enrich/BackendLoader.java | 42 ++
.../datatorrent/contrib/enrich/FSLoader.java | 184 +++++
.../datatorrent/contrib/enrich/JDBCLoader.java | 201 ++++++
.../datatorrent/contrib/enrich/MapEnricher.java | 138 ++++
.../contrib/enrich/NullValuesCacheManager.java | 60 ++
.../contrib/enrich/POJOEnricher.java | 287 ++++++++
.../contrib/enrich/ReadOnlyBackup.java | 61 ++
.../datatorrent/contrib/hbase/package-info.java | 46 +-
.../datatorrent/contrib/kafka/package-info.java | 44 +-
.../contrib/memcache/package-info.java | 46 +-
.../contrib/memcache_whalin/package-info.java | 46 +-
.../contrib/mongodb/package-info.java | 46 +-
.../contrib/rabbitmq/package-info.java | 46 +-
.../contrib/romesyndication/package-info.java | 46 +-
.../contrib/sqlite/package-info.java | 46 +-
.../datatorrent/contrib/zmq/package-info.java | 46 +-
.../contrib/enrich/EmployeeOrder.java | 114 +++
.../contrib/enrich/FileEnrichmentTest.java | 103 +++
.../contrib/enrich/JDBCLoaderTest.java | 209 ++++++
.../contrib/enrich/MapEnricherTest.java | 251 +++++++
.../com/datatorrent/contrib/enrich/Order.java | 71 ++
.../contrib/enrich/POJOEnricherTest.java | 232 +++++++
.../HBaseTransactionalPutOperatorTest.java | 18 +
.../datatorrent_feed_updated.rss | 688 +++++++++----------
contrib/src/test/resources/productmapping.txt | 100 +++
demos/distributedistinct/pom.xml | 4 +-
demos/echoserver/pom.xml | 4 +-
demos/frauddetect/pom.xml | 4 +-
demos/iteration/pom.xml | 4 +-
.../demos/iteration/package-info.java | 38 +-
demos/machinedata/pom.xml | 4 +-
demos/mobile/pom.xml | 4 +-
.../datatorrent/demos/mobile/package-info.java | 38 +-
demos/mrmonitor/pom.xml | 4 +-
demos/mroperator/pom.xml | 4 +-
demos/pi/pom.xml | 4 +-
.../com/datatorrent/demos/pi/package-info.java | 38 +-
demos/pom.xml | 6 +-
demos/r/pom.xml | 4 +-
demos/twitter/pom.xml | 4 +-
.../datatorrent/demos/twitter/package-info.java | 38 +-
demos/uniquecount/pom.xml | 4 +-
demos/wordcount/pom.xml | 4 +-
.../demos/wordcount/package-info.java | 38 +-
demos/yahoofinance/pom.xml | 4 +-
.../demos/yahoofinance/package-info.java | 38 +-
docs/README.md | 2 +-
docs/operators/block_reader.md | 2 +-
docs/operators/file_output.md | 2 +-
kafka/pom.xml | 4 +-
library/pom.xml | 14 +-
.../com/datatorrent/lib/db/jdbc/JdbcStore.java | 2 +
.../lib/io/PubSubWebSocketInputOperator.java | 3 +-
.../lib/io/PubSubWebSocketOutputOperator.java | 3 +-
.../lib/io/WebSocketInputOperator.java | 16 +-
.../lib/io/WebSocketOutputOperator.java | 38 +-
.../lib/io/WidgetOutputOperator.java | 71 +-
.../lib/io/block/AbstractBlockReader.java | 6 +
.../lib/io/block/AbstractFSBlockReader.java | 26 +-
.../datatorrent/lib/io/block/BlockMetadata.java | 28 -
.../datatorrent/lib/io/block/BlockReader.java | 66 --
.../lib/io/fs/AbstractFileOutputOperator.java | 15 +-
.../io/fs/AbstractSingleFileOutputOperator.java | 65 +-
.../datatorrent/lib/io/fs/FSInputModule.java | 262 +++++++
.../lib/io/fs/FileSplitterInput.java | 39 +-
.../datatorrent/lib/io/fs/HDFSFileSplitter.java | 120 ----
.../datatorrent/lib/io/fs/HDFSInputModule.java | 244 -------
.../lib/transform/TransformOperator.java | 85 +--
.../com/datatorrent/lib/util/FieldInfo.java | 15 +-
.../com/datatorrent/lib/util/package-info.java | 46 +-
.../malhar/lib/fs/BytesFileOutputOperator.java | 295 ++++++++
.../apex/malhar/lib/spillable/Spillable.java | 100 ---
.../spillable/SpillableComplexComponent.java | 126 ----
.../apex/malhar/lib/spillable/package-info.java | 20 -
.../malhar/lib/state/spillable/Spillable.java | 100 +++
.../spillable/SpillableComplexComponent.java | 126 ++++
.../lib/state/spillable/package-info.java | 20 +
.../org/apache/apex/malhar/lib/utils/Serde.java | 61 --
.../lib/utils/serde/PassThruByteArraySerde.java | 47 ++
.../apex/malhar/lib/utils/serde/Serde.java | 61 ++
.../malhar/lib/utils/serde/package-info.java | 20 +
.../lib/helper/OperatorContextTestHelper.java | 5 +
.../AbstractSingleFileOutputOperatorTest.java | 4 +-
.../lib/io/fs/FSInputModuleAppTest.java | 221 ++++++
.../lib/io/fs/FileSplitterInputTest.java | 1 -
.../lib/io/fs/HDFSInputModuleAppTest.java | 221 ------
.../lib/fs/BytesFileOutputOperatorTest.java | 151 ++++
.../utils/serde/PassThruByteArraySerdeTest.java | 72 ++
pom.xml | 12 +-
samples/pom.xml | 4 +-
101 files changed, 4788 insertions(+), 1928 deletions(-)
----------------------------------------------------------------------
[3/3] incubator-apex-malhar git commit: Merge branch 'APEXMALHAR-2065'
Posted by cs...@apache.org.
Merge branch 'APEXMALHAR-2065'
Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/commit/5d3b209f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/tree/5d3b209f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/diff/5d3b209f
Branch: refs/heads/master
Commit: 5d3b209fcd4d73084096ccddb6efe5be0f43b463
Parents: 1d3e20c 543fc07
Author: Chandni Singh <cs...@apache.org>
Authored: Tue May 17 11:54:20 2016 -0700
Committer: Chandni Singh <cs...@apache.org>
Committed: Tue May 17 11:54:20 2016 -0700
----------------------------------------------------------------------
.../apex/malhar/lib/wal/WindowDataManager.java | 33 ++++++++++++++++++++
.../malhar/lib/wal/FSWindowDataManagerTest.java | 19 +++++++++++
2 files changed, 52 insertions(+)
----------------------------------------------------------------------